From 263ca353a6b60d6e9bf1b8c695f184603247a486 Mon Sep 17 00:00:00 2001 From: BlackOvOoo <168417937+BlackOvOoo@users.noreply.github.com> Date: Sat, 16 May 2026 14:19:06 +0800 Subject: [PATCH 1/5] fix: shard parse cache persistence on large repos (#1580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: shard parse cache persistence on large repos * fix(parse-cache): validate shard keys, docs, and sharded-cache tests - Reject non-sha256-hex keys from index.json before path.join (path traversal). - saveParseCache: skip invalid keys defensively; try/catch per-shard JSON.stringify. - Clarify save comment (tmp dir + rename vs atomic). - Tests: hex keys throughout, traversal keys, multi-shard, version-mismatch+legacy, second save, legacy removal. - AGENTS.md / GUARDRAILS.md: document .gitnexus/parse-cache/ vs legacy parse-cache.json. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command --------- Co-authored-by: Gergő Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- AGENTS.md | 4 +- GUARDRAILS.md | 2 +- gitnexus/src/storage/parse-cache.ts | 124 +++++++++-- .../test/unit/incremental-parse-cache.test.ts | 204 +++++++++++++++++- 4 files changed, 310 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b9b9138b1..99fc68bd0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,9 +155,9 @@ npx gitnexus analyze --embeddings # also generate embeddings for new/changed npx gitnexus analyze --drop-embeddings # explicit opt-in to wipe existing embeddings ``` -`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** at `.gitnexus/parse-cache.json` for chunks whose file contents haven't changed since the last run. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). +`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** under `.gitnexus/parse-cache/` (per-chunk JSON shards plus `index.json`) for chunks whose file contents haven't changed since the last run. Older installs may still have a legacy single file `.gitnexus/parse-cache.json`, which is read for backward compatibility but no longer written. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). -The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete `.gitnexus/parse-cache.json` at any time — it'll be rebuilt on the next analyze. +The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete the whole `.gitnexus/parse-cache/` directory (and remove any legacy `.gitnexus/parse-cache.json` if present) at any time — it'll be rebuilt on the next analyze. Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). A plain `analyze` no longer drops existing vectors — pass `--drop-embeddings` to wipe. diff --git a/GUARDRAILS.md b/GUARDRAILS.md index c09f0319a..2e2b9db41 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -36,7 +36,7 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Index seems corrupt or "incremental" is misbehaving - **Trigger:** `analyze` produces unexpected results, or `meta.json.incrementalInProgress` is set, or the index is in a half-state after a crash. -- **Do:** `npx gitnexus analyze --force` to rebuild from scratch. The dirty-flag check forces this automatically when a previous incremental run didn't complete cleanly, but `--force` is the manual escape hatch. Safe to delete `.gitnexus/parse-cache.json` at any time — content-addressed, will be regenerated. +- **Do:** `npx gitnexus analyze --force` to rebuild from scratch. The dirty-flag check forces this automatically when a previous incremental run didn't complete cleanly, but `--force` is the manual escape hatch. Safe to delete the `.gitnexus/parse-cache/` directory (and any legacy `.gitnexus/parse-cache.json`) at any time — content-addressed, will be regenerated. - **Why:** Incremental writeback is selective DB row replacement; if the on-disk state is inconsistent for any reason, a full rebuild is the cheapest path back to a known-good index. ### Embeddings vanished after analyze diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index a1abf76fa..680ca1ab2 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -70,15 +70,28 @@ const GITNEXUS_PKG_VERSION = (() => { })(); export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`; -const CACHE_FILENAME = 'parse-cache.json'; +const LEGACY_CACHE_FILENAME = 'parse-cache.json'; +const CACHE_DIRNAME = 'parse-cache'; +const CACHE_INDEX_FILENAME = 'index.json'; -/** On-disk shape. */ +/** Keys on disk always come from `computeChunkHash` — 64-char lowercase hex. */ +const CHUNK_CACHE_KEY_HEX_RE = /^[a-f0-9]{64}$/; + +const isValidChunkCacheKey = (chunkHash: string): boolean => CHUNK_CACHE_KEY_HEX_RE.test(chunkHash); + +/** On-disk shape for the legacy single-file format. */ interface ParseCacheFile { version: string; /** key = chunk hash (hex) → cached chunk result list. */ entries: Record; } +/** On-disk shape for the sharded directory format. */ +interface ShardedParseCacheIndex { + version: string; + keys: string[]; +} + /** Runtime view: keyed Map for fast lookup; mutated in place during a run. */ export interface ParseCache { version: string; @@ -144,12 +157,19 @@ const mapReviver = (_key: string, value: unknown): unknown => { return value; }; -/** - * Load the parse cache. Returns an empty cache on any failure (missing - * file, corrupt JSON, version mismatch). Never throws on a normal load. - */ -export const loadParseCache = async (storagePath: string): Promise => { - const cachePath = path.join(storagePath, CACHE_FILENAME); +const getLegacyCachePath = (storagePath: string): string => + path.join(storagePath, LEGACY_CACHE_FILENAME); + +const getCacheDirPath = (storagePath: string): string => path.join(storagePath, CACHE_DIRNAME); + +const getCacheIndexPath = (storagePath: string): string => + path.join(getCacheDirPath(storagePath), CACHE_INDEX_FILENAME); + +const getCacheChunkPath = (storagePath: string, chunkHash: string): string => + path.join(getCacheDirPath(storagePath), `${chunkHash}.json`); + +const loadLegacyParseCache = async (storagePath: string): Promise => { + const cachePath = getLegacyCachePath(storagePath); try { const raw = await fs.readFile(cachePath, 'utf-8'); const data = JSON.parse(raw, mapReviver) as ParseCacheFile; @@ -172,22 +192,90 @@ export const loadParseCache = async (storagePath: string): Promise = } }; +const loadShardedParseCache = async (storagePath: string): Promise => { + const indexPath = getCacheIndexPath(storagePath); + try { + const raw = await fs.readFile(indexPath, 'utf-8'); + const data = JSON.parse(raw) as ShardedParseCacheIndex; + if ( + typeof data !== 'object' || + data === null || + data.version !== PARSE_CACHE_VERSION || + !Array.isArray(data.keys) + ) { + return emptyCache(); + } + + const entries = new Map(); + for (const chunkHash of data.keys) { + if (typeof chunkHash !== 'string' || !isValidChunkCacheKey(chunkHash)) continue; + try { + const chunkRaw = await fs.readFile(getCacheChunkPath(storagePath, chunkHash), 'utf-8'); + const chunkData = JSON.parse(chunkRaw, mapReviver) as ParseWorkerResult[]; + if (Array.isArray(chunkData)) entries.set(chunkHash, chunkData); + } catch { + /* skip corrupt or missing shard */ + } + } + + return { version: PARSE_CACHE_VERSION, entries, usedKeys: new Set() }; + } catch { + return null; + } +}; + /** - * Persist the cache to disk atomically (write-and-rename) so a crash - * mid-write doesn't leave a corrupt file. + * Load the parse cache. Returns an empty cache on any failure (missing + * file, corrupt JSON, version mismatch). Never throws on a normal load. + */ +export const loadParseCache = async (storagePath: string): Promise => { + const sharded = await loadShardedParseCache(storagePath); + if (sharded) return sharded; + return loadLegacyParseCache(storagePath); +}; + +/** + * Persist the cache to disk using a temp directory + rename. + * + * Writes shards under `${cacheDir}.tmp`, then removes the old `cacheDir` and + * renames the temp directory into place. There is a crash window after + * `rm(cacheDir)` and before `rename(tmpDir, cacheDir)` where no cache exists; + * that is acceptable — `loadParseCache` yields empty and the next run + * reparses. This is not a single atomic swap of the whole tree, but avoids + * leaving a half-written shard set visible to readers. */ export const saveParseCache = async (storagePath: string, cache: ParseCache): Promise => { await fs.mkdir(storagePath, { recursive: true }); - const cachePath = path.join(storagePath, CACHE_FILENAME); - const tmpPath = `${cachePath}.tmp`; - const out: ParseCacheFile = { + const cacheDir = getCacheDirPath(storagePath); + const tmpDir = `${cacheDir}.tmp`; + await fs.rm(tmpDir, { recursive: true, force: true }); + await fs.mkdir(tmpDir, { recursive: true }); + + const keys: string[] = []; + for (const [chunkHash, chunkResults] of cache.entries) { + if (!isValidChunkCacheKey(chunkHash)) continue; + let payload: string; + try { + payload = JSON.stringify(chunkResults, mapReplacer); + } catch { + // Extremely dense chunks could theoretically exceed string limits; skip + // rather than failing the entire save (orchestrator catches save errors). + continue; + } + keys.push(chunkHash); + const chunkPath = path.join(tmpDir, `${chunkHash}.json`); + await fs.writeFile(chunkPath, payload, 'utf-8'); + } + + const index: ShardedParseCacheIndex = { version: cache.version, - entries: Object.fromEntries(cache.entries), + keys, }; - // Compact JSON; this file can be tens of MB on a large repo and pretty- - // printing roughly doubles size for no value. - await fs.writeFile(tmpPath, JSON.stringify(out, mapReplacer), 'utf-8'); - await fs.rename(tmpPath, cachePath); + await fs.writeFile(path.join(tmpDir, CACHE_INDEX_FILENAME), JSON.stringify(index), 'utf-8'); + + await fs.rm(cacheDir, { recursive: true, force: true }); + await fs.rename(tmpDir, cacheDir); + await fs.rm(getLegacyCachePath(storagePath), { force: true }); }; /** diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index 757b9cf3a..17ec9c2e1 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -135,12 +135,15 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { it('round-trips an empty cache', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { + const fs = await import('fs/promises'); const cache: ParseCache = { version: PARSE_CACHE_VERSION, entries: new Map(), usedKeys: new Set(), }; await saveParseCache(dir, cache); + await expect(fs.access(path.join(dir, 'parse-cache', 'index.json'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(dir, 'parse-cache.json'))).rejects.toThrow(); const loaded = await loadParseCache(dir); expect(loaded.version).toBe(PARSE_CACHE_VERSION); expect(loaded.entries.size).toBe(0); @@ -189,6 +192,60 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { } }); + it('loads a legacy single-file cache for backwards compatibility', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { + legacyChunk: [minimalResult({ fileCount: 7 })], + }, + }), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get('legacyChunk')?.[0]?.fileCount).toBe(7); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('skips corrupt or missing shards while loading the sharded cache', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + const goodKey = 'a'.repeat(64); + const missingKey = 'b'.repeat(64); + const badKey = 'c'.repeat(64); + await fs.mkdir(cacheDir, { recursive: true }); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + keys: [goodKey, missingKey, badKey], + }), + 'utf-8', + ); + await fs.writeFile( + path.join(cacheDir, `${goodKey}.json`), + JSON.stringify([minimalResult({ fileCount: 3 })]), + 'utf-8', + ); + await fs.writeFile(path.join(cacheDir, `${badKey}.json`), '{not-json', 'utf-8'); + + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(goodKey)?.[0]?.fileCount).toBe(3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('round-trips Map and Set values through the JSON replacer/reviver', async () => { // ParsedFile.scopes[*].typeBindings is a ReadonlyMap. // Without the replacer/reviver pair, JSON.stringify collapses Maps to @@ -196,6 +253,7 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { // with "is not iterable". This test pins the round-trip behaviour. const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { + const fs = await import('fs/promises'); const innerMap = new Map([ ['k1', 'v1'], ['k2', 'v2'], @@ -218,14 +276,18 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { ], }); + const chunkKey = 'd'.repeat(64); const cache: ParseCache = { version: PARSE_CACHE_VERSION, - entries: new Map([['chunk-h', [fake]]]), - usedKeys: new Set(['chunk-h']), + entries: new Map([[chunkKey, [fake]]]), + usedKeys: new Set([chunkKey]), }; await saveParseCache(dir, cache); + const persisted = await fs.readdir(path.join(dir, 'parse-cache')); + expect(persisted).toContain('index.json'); + expect(persisted).toContain(`${chunkKey}.json`); const loaded = await loadParseCache(dir); - const reloaded = loaded.entries.get('chunk-h')?.[0]; + const reloaded = loaded.entries.get(chunkKey)?.[0]; expect(reloaded).toBeDefined(); const scope = (reloaded as ParseWorkerResult).parsedFiles[0]?.scopes[0] as unknown as { typeBindings?: unknown; @@ -240,4 +302,140 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { await rm(dir, { recursive: true, force: true }); } }); + + it('ignores traversal-like and non-hex keys in sharded index.json', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + await fs.mkdir(cacheDir, { recursive: true }); + const safeKey = 'e'.repeat(64); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + keys: ['../evil', '/absolute', 'G'.repeat(64), safeKey], + }), + 'utf-8', + ); + await fs.writeFile( + path.join(cacheDir, `${safeKey}.json`), + JSON.stringify([minimalResult({ fileCount: 9 })]), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(safeKey)?.[0]?.fileCount).toBe(9); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('writes one shard file per cache entry (three distinct keys)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const k1 = '1'.repeat(64); + const k2 = '2'.repeat(64); + const k3 = '3'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map([ + [k1, [minimalResult({ fileCount: 1 })]], + [k2, [minimalResult({ fileCount: 2 })]], + [k3, [minimalResult({ fileCount: 3 })]], + ]), + usedKeys: new Set([k1, k2, k3]), + }; + await saveParseCache(dir, cache); + const cacheDir = path.join(dir, 'parse-cache'); + const names = await fs.readdir(cacheDir); + expect(names).toContain('index.json'); + expect(names.filter((n) => n.endsWith('.json') && n !== 'index.json').length).toBe(3); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('returns empty when sharded index version mismatches even if legacy parse-cache.json is valid', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + await fs.mkdir(cacheDir, { recursive: true }); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ version: 'foreign-sharded-1', keys: [] }), + 'utf-8', + ); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { legacyChunk: [minimalResult({ fileCount: 42 })] }, + }), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('second saveParseCache replaces the first sharded cache', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const k1 = '4'.repeat(64); + const k2 = '5'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k1, [minimalResult()]]]), + usedKeys: new Set([k1]), + }); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k2, [minimalResult({ fileCount: 99 })]]]), + usedKeys: new Set([k2]), + }); + const names = await fs.readdir(path.join(dir, 'parse-cache')); + expect(names).not.toContain(`${k1}.json`); + expect(names).toContain(`${k2}.json`); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(k2)?.[0]?.fileCount).toBe(99); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('removes legacy parse-cache.json after a successful sharded save', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { oldLegacy: [minimalResult({ fileCount: 5 })] }, + }), + 'utf-8', + ); + const k = '6'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k, [minimalResult({ fileCount: 6 })]]]), + usedKeys: new Set([k]), + }); + await expect(fs.access(path.join(dir, 'parse-cache.json'))).rejects.toThrow(); + const loaded = await loadParseCache(dir); + expect(loaded.entries.get(k)?.[0]?.fileCount).toBe(6); + expect(loaded.entries.has('oldLegacy')).toBe(false); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); From 83fbd4be264b6f338a97edccd19a4752f69bcaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 16 May 2026 07:46:56 +0100 Subject: [PATCH 2/5] refactor(ci): unify release pipeline under publish.yml (#1610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse release-candidate.yml into publish.yml so there is exactly one workflow that publishes gitnexus to npm, creates GitHub Releases, and triggers Docker builds — for both release candidates and stable releases. Closes #1609 architecturally. A first-stage `route` job classifies push-to-main / push-tag / workflow_dispatch into `rc` / `stable` modes and fails closed on malformed shapes. RC path runs rc-guard → ci.yml → publish (mint GitHub App token → checkout with persist-credentials:false → resolve next rc version → atomic v-tag + rc/ marker push → vtag integrity gate → npm publish via OIDC → GitHub prerelease → if: failure() cleanup) → docker.yml. Stable path verifies package.json matches the tag and publishes to `latest` via OIDC (no docker). Hardening: • Self-trigger prevention via negative-glob `tags: ['v*', '!v*-rc.*']` — the bug class behind #1609 cannot recur. • Two distinct actions/checkout steps per mode (no conditional `token:` expression footgun). • Workflow-level `permissions: {}` deny-all + per-job grants; `id-token: write` only where OIDC is used. • npm Trusted Publishing replaces NPM_TOKEN (delete the secret after the first successful publish). • GitHub App installation token (actions/create-github-app-token@v3.2.0) replaces the long-lived RELEASE_PUSH_TOKEN PAT (delete after first successful RC). • vtag integrity gate fails closed on empty / mode-mismatched output (prevents Release named `main` from a github.ref fallback). • Annotation-injection sanitization on every logged ref. • Explicit `secrets:` passthrough on docker.yml (DOCKERHUB_USERNAME, DOCKERHUB_TOKEN); ci.yml no longer inherits anything. • `if: failure()` cleanup auto-deletes v-tag + rc-marker on partial failure (eliminates the external-consumer phantom-version ingestion window). • ACTIONS_STEP_DEBUG window closed via `set +x` wrap on the inline auth-header compute. • Curated retry-loud error handling on `gh api` bot-user-id lookup and `npx semver`. Pre-merge validation: • 10-reviewer multi-agent code-review pass; 14 findings fixed inline (commit 820cefae), 6 deferred to follow-ups. • End-to-end dry-run rehearsal via workflow_dispatch (run 25919563064) validated route classification, rc-guard, App token mint, RC checkout, version resolver, vtag synthetic-regex check, and faithful tarball pack at the bumped version. • All zizmor findings on the unification commits closed. • Branch-protection required checks all green. Post-merge actions: • After the first successful RC, delete the `NPM_TOKEN` and `RELEASE_PUSH_TOKEN` secrets — they are no longer used. • The first real RC after merge is the live-fire test for steps dry-run could not exercise (atomic tag push, real npm OIDC handshake, GitHub Release creation, docker.yml under explicit secrets passthrough). The if: failure() cleanup step handles the partial-failure recovery automatically; the Rollback Runbook in CONTRIBUTING.md covers the rare cases auto-cleanup can't reach. --- .github/workflows/ci.yml | 16 +- .github/workflows/docker.yml | 11 +- .github/workflows/publish.yml | 834 +++++++++++++++++++++++- .github/workflows/release-candidate.yml | 459 ------------- .github/zizmor.yml | 9 +- CONTRIBUTING.md | 84 ++- README.md | 4 +- 7 files changed, 884 insertions(+), 533 deletions(-) delete mode 100644 .github/workflows/release-candidate.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd07337b7..c28e49be9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,14 +11,14 @@ permissions: # Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". # Hardcoded `CI-` prefix (not `${{ github.workflow }}`) because this workflow is -# invoked as a reusable workflow from publish.yml and release-candidate.yml. In -# called-workflow context `github.workflow` evaluation is ambiguous across GitHub -# Actions versions, and a prefix that could resolve to the caller's name would -# share a concurrency group with the caller → deadlock. A literal prefix is -# immune. Direct `pull_request` invocations use `CI-`; invocations from a -# reusable-workflow caller fall into a per-run-unique group that never serializes -# with the caller. `push` to main is handled by release-candidate.yml, which -# calls this workflow once before publishing. +# invoked as a reusable workflow from publish.yml. In called-workflow context +# `github.workflow` evaluation is ambiguous across GitHub Actions versions, and a +# prefix that could resolve to the caller's name would share a concurrency group +# with the caller → deadlock. A literal prefix is immune. Direct `pull_request` +# invocations use `CI-`; invocations from a reusable-workflow caller fall +# into a per-run-unique group that never serializes with the caller. `push` to +# main is handled by publish.yml (RC mode), which calls this workflow once +# before publishing. concurrency: group: ${{ github.event_name == 'pull_request' && format('CI-{0}', github.ref) || format('CI-nested-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0cd526768..9c4ba0d8f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,6 +25,15 @@ on: a gitnexus/package.json whose version matches the tag. required: true type: string + # Explicit secret contract — callers pass these by name. Replaces the + # blanket `secrets: inherit` pattern (zizmor `secrets-inherit` audit). + # GHCR auth uses the implicit GITHUB_TOKEN; only Docker Hub credentials + # need to be passed through. + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true permissions: contents: read @@ -73,7 +82,7 @@ jobs: steps: # Only the workflow_call path requires a non-empty `inputs.tag` — callers - # (e.g. release-candidate.yml) must pass the RC tag explicitly. On direct + # (publish.yml in RC mode) must pass the RC tag explicitly. On direct # tag pushes the tag comes from `github.ref`, so `inputs.tag` is always # empty and validating it here would break every real release (#1064). # The downstream "Verify tag matches gitnexus/package.json version" step diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0694b5e4c..7ce377ca3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,62 +1,404 @@ -name: Publish to npm +name: Publish + +# ───────────────────────────────────────────────────────────────────────────── +# Sole publisher for the `gitnexus` npm package, GitHub Releases, and Docker +# images. Replaces the former two-workflow design — see issue #1609 for the +# double-publish race this unification closes. +# +# Two release modes, both routed through this file: +# • Release candidate (rc) — triggered by push to `main` or workflow_dispatch. +# The RC path computes the next rc version, applies it in-CI, pushes a +# detached release commit with v-rc. + rc/ marker +# atomically, then publishes to npm with --tag rc and creates a GitHub +# prerelease. RC-only docker.yml invocation follows. +# • Stable — triggered by push of a v tag (no -rc.* +# suffix). Verifies package.json matches the tag, publishes to npm with +# --tag latest, creates a stable GitHub Release. No docker (RC-only). +# +# ⚠️ SELF-TRIGGER INVARIANT — DO NOT WEAKEN ⚠️ +# The `tags:` filter below uses a negative glob `'!v*-rc.*'` to prevent the +# workflow from re-triggering itself when the RC path pushes its own v-tag. +# Without this exclusion, every RC publish double-fires (the bug fixed by +# #1609). If a NEW prerelease channel is introduced (e.g. `-beta.N`, +# `-alpha.N`, `-next.N`), the negative-glob list MUST be extended in +# lock-step or self-trigger returns. The same invariant applies to the +# `Classify` step further below — its accepted-tag regex must align with +# the trigger filter's exclusion list. +# ───────────────────────────────────────────────────────────────────────────── on: push: + branches: [main] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE' tags: + # Negative-globbed exclusion of RC tags this workflow itself produces + # (see the SELF-TRIGGER INVARIANT in the header comment). - 'v*' - -# No workflow-level permissions — scoped per job below. + - '!v*-rc.*' + workflow_dispatch: + inputs: + bump: + description: >- + Cycle policy. 'auto' (default) continues the active rc cycle on + this branch if there is one, otherwise bumps patch from latest. + Choose 'patch' / 'minor' / 'major' to explicitly start or reset + an rc cycle. + required: false + default: 'auto' + type: choice + options: + - auto + - patch + - minor + - major + force: + description: 'Publish even when HEAD already has an rc marker' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' +# Workflow-level deny-all; each job declares the minimum it needs. permissions: {} -# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". -# Tag refs are unique per release, so distinct tags run in parallel. Re-pushes of the -# same tag serialize. cancel-in-progress: false — never cancel a publish mid-flight. +# Distinct refs (refs/heads/main, refs/tags/v*) run in parallel. The +# release-PR-skip in rc-guard is the load-bearing invariant that prevents +# an RC main-push and a stable tag-push colliding on the same release commit. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: + # ── Phase 1: classify the triggering event into a release mode ───────────── + route: + name: Classify release event + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: + contents: read + outputs: + mode: ${{ steps.classify.outputs.mode }} + head_sha: ${{ steps.classify.outputs.head_sha }} + bump_input: ${{ inputs.bump }} + force_input: ${{ inputs.force }} + steps: + - name: Classify + id: classify + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + GH_REF: ${{ github.ref }} + GH_REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + + HEAD_SHA="${GITHUB_SHA}" + echo "head_sha=${HEAD_SHA}" >> "$GITHUB_OUTPUT" + + # Sanitize before logging (annotation-injection defense in depth). + REF_SAFE="${GH_REF//::/__}" + REF_NAME_SAFE="${GH_REF_NAME//::/__}" + echo "event=${EVENT_NAME} ref=${REF_SAFE} ref_name=${REF_NAME_SAFE}" + + MODE="" + case "${EVENT_NAME}" in + workflow_dispatch) + # Manual dispatch is only valid on main — that's the only ref + # where a real publish makes sense. + if [ "${GH_REF}" = "refs/heads/main" ]; then + MODE="rc" + else + echo "::error::workflow_dispatch is only permitted on refs/heads/main (got ${REF_SAFE})." + exit 1 + fi + ;; + push) + case "${GH_REF}" in + refs/heads/main) + MODE="rc" + ;; + refs/tags/v*) + # The trigger filter already excluded v*-rc.* tags. Anything + # reaching here is either a stable semver or a malformed v*. + TAG="${GH_REF#refs/tags/}" + if [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + MODE="stable" + else + echo "::error::malformed v* tag rejected: ${REF_NAME_SAFE}" + echo "::error::stable tags must match ^v[0-9]+\\.[0-9]+\\.[0-9]+\$" + exit 1 + fi + ;; + *) + echo "::error::unexpected push ref ${REF_SAFE} reached publish workflow." + exit 1 + ;; + esac + ;; + *) + echo "::error::unsupported event ${EVENT_NAME}." + exit 1 + ;; + esac + + echo "mode=${MODE}" >> "$GITHUB_OUTPUT" + echo "Classified as mode=${MODE}" + + # ── Phase 2 (RC only): dedup marker + release-PR skip ────────────────────── + rc-guard: + name: RC guard (marker + release-PR skip) + needs: route + if: needs.route.outputs.mode == 'rc' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.decide.outputs.should_run }} + head_sha: ${{ steps.decide.outputs.head_sha }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + fetch-tags: true + # rc-guard reads only — no git pushes from this job. Skip the + # default extraheader credential persistence (artipacked audit). + persist-credentials: false + + - name: Decide + id: decide + shell: bash + env: + FORCE: ${{ inputs.force }} + BUMP_INPUT: ${{ inputs.bump }} + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + HEAD_SHA=$(git rev-parse HEAD) + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + if [ "$FORCE" = "true" ]; then + echo "Force flag set — running regardless of marker tag." + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Explicit cycle reset on dispatch bypasses dedup. + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && [ -n "${BUMP_INPUT:-}" ] \ + && [ "${BUMP_INPUT:-auto}" != "auto" ]; then + echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup." + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # ── Skip when the merge commit corresponds to a release ─────────── + # This skip is load-bearing: it prevents an RC build firing on the + # release-PR commit from racing the imminent stable-tag push on the + # same SHA. Two complementary checks: + # 1. HEAD subject matches `chore: release vX.Y.Z` (the canonical + # release-PR title). Anchored to require the bare title or the + # squash-merge `(#NNNN)` suffix exactly. Case-insensitive so + # `Chore: Release v1.2.3` (IDE auto-capitalization) still + # matches — prior commit-author conventions left the door open. + # 2. Squash-merged PR carries the `release` label. + # Either match suppresses the rc build — stable releases publish on + # the v-tag instead. + HEAD_SUBJECT="$(git log -1 --pretty=%s HEAD)" + # Sanitize GitHub-Actions annotation prefixes before logging — even + # though %s strips newlines, a crafted subject containing `::error::` + # could forge log annotations. + HEAD_SUBJECT_SAFE="${HEAD_SUBJECT//::/__}" + RELEASE_SUBJECT_RE='^chore:[[:space:]]*release[[:space:]]+v[0-9]+\.[0-9]+\.[0-9]+([[:space:]]+\(#[0-9]+\))?$' + shopt -s nocasematch + if [[ "$HEAD_SUBJECT" =~ $RELEASE_SUBJECT_RE ]]; then + shopt -u nocasematch + echo "HEAD commit subject matches a release commit — skipping rc." + echo " subject (sanitised): $HEAD_SUBJECT_SAFE" + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + shopt -u nocasematch + + # Squash-merge commits include `(#NNNN)` at the end of the subject. + if [[ "$HEAD_SUBJECT" =~ \(#([0-9]+)\)[[:space:]]*$ ]]; then + PR_NUM="${BASH_REMATCH[1]}" + echo "Detected squash-merge of PR #$PR_NUM — checking labels." + if LABELS_JSON="$(gh pr view "$PR_NUM" --repo "$REPO" --json labels 2>/dev/null)"; then + if printf '%s' "$LABELS_JSON" | jq -e '.labels[] | select(.name == "release")' >/dev/null; then + echo "PR #$PR_NUM has the 'release' label — skipping rc." + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "PR #$PR_NUM has no 'release' label — proceeding." + else + # Lookup failure is not fatal — fall through to dedup check. + echo "::warning::Could not read labels for PR #${PR_NUM} — falling through." + fi + fi + + # Dedup: is there already an rc/ marker pointing at HEAD? + MARKER="rc/${HEAD_SHA}" + if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then + echo "HEAD already has marker $MARKER — skipping." + echo "should_run=false" >> "$GITHUB_OUTPUT" + else + echo "No marker on HEAD — proceeding." + echo "should_run=true" >> "$GITHUB_OUTPUT" + fi + + # ── Phase 3: reusable CI gate ────────────────────────────────────────────── + # Runs for both rc (when guard says go) and stable. No `secrets:` passed — + # ci.yml and its entire reusable-workflow chain (ci-quality, ci-tests, + # ci-e2e, ci-scope-parity, ci-report) reference zero `secrets.*` values; + # passing any would be unused surface. GITHUB_TOKEN is implicit. ci: + needs: [route, rc-guard] + if: ${{ always() && (needs.route.outputs.mode == 'stable' || needs.rc-guard.outputs.should_run == 'true') }} uses: ./.github/workflows/ci.yml permissions: contents: read actions: read - # No pull-requests:write — `ci.yml`'s save-pr-meta job is gated on - # `github.event_name == 'pull_request'`, so it never runs during a - # tag-triggered publish. Least-privilege for release-critical paths. + # ── Phase 4: publish to npm + push refs (RC path) ────────────────────────── + # INVARIANT: `timeout-minutes` MUST stay below the App-token TTL (~60 min + # for actions/create-github-app-token installation tokens). The atomic + # tag-push step relies on the token minted at job start; if the job ever + # runs longer than the TTL, the push fails with an opaque 401. If you + # need to raise the timeout, re-mint the token immediately before the + # `Create and push rc tags` step instead. publish: - needs: ci + name: Publish to npm + needs: [route, rc-guard, ci] + if: ${{ always() && needs.ci.result == 'success' && (needs.route.outputs.mode == 'stable' || needs.rc-guard.outputs.should_run == 'true') }} runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 permissions: + # contents: write — RC path needs it for `git push --atomic` (v-tag + + # marker). Stable path runs in the same job and inherits the grant; it + # never invokes `git push`, so the elevated scope is unused there. + # id-token: write — npm provenance attestation. contents: write id-token: write + outputs: + # Two distinct step IDs feed this output; exactly one fires per run. + vtag: ${{ steps.rc-tags.outputs.vtag || steps.stable-vtag.outputs.vtag }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # ── Mint short-lived GitHub App token (RC only) ────────────────────── + # Industry direction (2025-2026): GitHub Apps with + # `actions/create-github-app-token` over long-lived PATs for + # workflow-touching tag pushes. Same fine-grained permission surface, + # ~1h expiry, not tied to a user seat, organizationally auditable. + # Replaces a prior fine-grained PAT. + # + # Required secrets (set in repo Settings → Secrets and variables → Actions): + # secrets.RELEASE_APP_ID — the App's numeric ID + # secrets.RELEASE_APP_PRIVATE_KEY — the App's PEM private key + # (The App ID is technically not sensitive — it's visible on the App's + # settings page — but storing it as a secret is harmless and avoids + # mixing storage classes for the same App.) + # The App must be installed on this repository with: + # - Contents: write (push the v-tag and rc marker) + # - Workflows: write (because the v-tag's tree may touch + # .github/workflows/**, which the default + # GITHUB_TOKEN cannot author) + # - Metadata: read (required for the `gh api /users/[bot]` + # bot-identity lookup in the tag-push step) + - name: Mint GitHub App token (RC) + if: needs.route.outputs.mode == 'rc' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + # `client-id` is the renamed input that supersedes the deprecated + # `app-id` in v3.x. The action accepts the App's numeric ID or + # its Client ID under this name. We pass the numeric App ID, + # which the action resolves correctly. + client-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + # ── Separate checkout steps per mode ───────────────────────────────── + # Conditional `token:` expressions are footguns: empty string passed to + # actions/checkout fails opaquely, and `|| github.token` silently + # degrades a missing token to GITHUB_TOKEN, masking auth failures until + # the eventual `git push`. Two distinct steps make the auth contract + # explicit and fail loudly at checkout when the App token mint failed + # on the RC path. + - name: Checkout (RC) + if: needs.route.outputs.mode == 'rc' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + fetch-tags: true + # Short-lived GitHub App installation token. Required because the + # v-tag push lands at a SHA whose tree may touch + # `.github/workflows/**`, which the default GITHUB_TOKEN cannot + # author. + token: ${{ steps.app-token.outputs.token }} + # Do not persist the token in .git/config (artipacked audit). The + # RC tag push uses an inline `http.extraheader` at push time only; + # the credential never lands on disk. See the + # `Create and push rc tags` step below. + persist-credentials: false + + - name: Checkout (stable) + if: needs.route.outputs.mode == 'stable' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # No `token:` — actions/checkout uses GITHUB_TOKEN by default. Stable + # path performs no git pushes; the default scope is sufficient. + with: + # No git pushes from the stable path either. Skip credential + # persistence (artipacked audit). + persist-credentials: false + + - name: Working-tree sanity + # Defense in depth (mirrors the vtag integrity gate, but on the input side): + # if a route-mode regression skipped both checkout `if:` gates, all + # downstream steps would run on a bare runner and produce confusing + # ENOENT errors. Fail loudly and early here instead. + shell: bash + run: | + if [ ! -f gitnexus/package.json ]; then + echo "::error::no working tree at gitnexus/package.json — route classification likely failed silently." + exit 1 + fi + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 registry-url: https://registry.npmjs.org - # Hermetic install for the published artifact — no cache carry-over - # from non-tag contexts. setup-node v5+ caches by default when a - # packageManager field is present in package.json, so the explicit - # opt-out is required to clear the zizmor cache-poisoning audit. - # ~30s slower per release; runs rarely. + # Hermetic install for published artifacts — opt out of the v5+ + # default packageManager-based caching (clears the zizmor + # zizmor cache-poisoning audit). ~30s slower per + # release; runs rarely. package-manager-cache: false + - name: Build gitnexus-shared run: npm install && npm run build working-directory: gitnexus-shared - - run: npm ci + - name: Install gitnexus dependencies + run: npm ci working-directory: gitnexus - - name: Verify version consistency + # ── Stable-only: verify the tag and package.json agree ─────────────── + - name: Verify version consistency (stable) + if: needs.route.outputs.mode == 'stable' shell: bash + working-directory: gitnexus run: | + set -euo pipefail TAG_VERSION="${GITHUB_REF#refs/tags/v}" - if ! [[ "$TAG_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then - echo "::error::Tag does not follow semver: v$TAG_VERSION" + # Stable mode REJECTS prerelease suffixes — those are filtered at + # trigger by the negative-glob filter, but defend at the bash layer too. + if ! [[ "$TAG_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Stable tag must be ^v[0-9]+.[0-9]+.[0-9]+$ — got v$TAG_VERSION" exit 1 fi PKG_VERSION=$(node -p "require('./package.json').version") @@ -65,24 +407,367 @@ jobs: exit 1 fi echo "Version verified: $PKG_VERSION" - working-directory: gitnexus - - name: Build + # ── RC-only: compute the next rc version against the live registry ── + - name: Resolve rc version (rc) + id: rc-version + if: needs.route.outputs.mode == 'rc' + shell: bash + working-directory: gitnexus + env: + BUMP_INPUT: ${{ inputs.bump }} + EVENT_NAME: ${{ github.event_name }} + PKG_NAME: gitnexus + run: | + set -euo pipefail + + # 1. Current published `latest` — the floor for any new rc base. + # Only E404 ("never published") falls back to package.json; any + # other error (network, auth, malformed response) fails fast + # (retry-loud policy: never silently substitute on transient errors). + NPM_STDERR_LATEST="$(mktemp)" + if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then + : + else + if grep -qiE 'E404|not found' "$NPM_STDERR_LATEST"; then + CURRENT_LATEST="$(node -p "require('./package.json').version")" + echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST" + else + echo "::error::npm registry unreachable for 'view version':" >&2 + cat "$NPM_STDERR_LATEST" >&2 + rm -f "$NPM_STDERR_LATEST" + exit 1 + fi + fi + rm -f "$NPM_STDERR_LATEST" + CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}" + + # 2. Full version list — needed for the counter and active-cycle + # inference. Same E404-only fallback. + NPM_STDERR_VERSIONS="$(mktemp)" + if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then + : + else + if grep -qiE 'E404|not found' "$NPM_STDERR_VERSIONS"; then + VERSIONS_JSON='[]' + echo "No published versions for $PKG_NAME yet (E404)." + else + echo "::error::npm registry unreachable for 'view versions':" >&2 + cat "$NPM_STDERR_VERSIONS" >&2 + rm -f "$NPM_STDERR_VERSIONS" + exit 1 + fi + fi + rm -f "$NPM_STDERR_VERSIONS" + + # 3. Base selection. + # - workflow_dispatch + bump != auto → explicit cycle reset. + # - Otherwise (push, or dispatch with bump=auto) → continue the + # highest active rc base > latest if any; else patch from latest. + # Curated wrapper around `npx semver` — bare npx errors are noisy + # and don't distinguish registry-unreachable from invalid-bump-spec. + semver_bump() { + local kind="$1" current="$2" stderr_file out + stderr_file="$(mktemp)" + if out="$(npx --yes -p semver@7 semver -i "$kind" "$current" 2>"$stderr_file")"; then + rm -f "$stderr_file" + printf '%s' "$out" + return 0 + fi + echo "::error::semver bump failed (kind=${kind}, current=${current}):" >&2 + cat "$stderr_file" >&2 + rm -f "$stderr_file" + return 1 + } + + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && [ -n "${BUMP_INPUT:-}" ] \ + && [ "${BUMP_INPUT:-auto}" != "auto" ]; then + BASE="$(semver_bump "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")" + echo "Explicit bump=$BUMP_INPUT → BASE=$BASE" + else + cat > /tmp/active_base.mjs <<'NODESCRIPT' + const latest = process.env.LATEST; + let v; + try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } + if (!Array.isArray(v)) v = [v]; + const parse = s => s.split(".").map(n => parseInt(n, 10)); + const gt = (a, b) => { + const [A, B] = [parse(a), parse(b)]; + for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i]; + return false; + }; + const bases = new Set(); + for (const s of v) { + const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s); + if (m && gt(m[1], latest)) bases.add(m[1]); + } + if (!bases.size) { process.stdout.write(""); process.exit(0); } + const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1); + process.stdout.write(sorted[sorted.length - 1]); + NODESCRIPT + ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)" + if [ -n "$ACTIVE_BASE" ]; then + BASE="$ACTIVE_BASE" + echo "Continuing active rc cycle → BASE=$BASE" + else + BASE="$(semver_bump patch "$CURRENT_LATEST_CLEAN")" + echo "No active rc cycle → patch bump from latest → BASE=$BASE" + fi + fi + + # 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1. + cat > /tmp/next_rc.mjs <<'NODESCRIPT' + const base = process.env.BASE; + const prefix = base + "-rc."; + let v; + try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } + if (!Array.isArray(v)) v = [v]; + const ns = v + .filter(s => typeof s === "string" && s.startsWith(prefix)) + .map(s => parseInt(s.slice(prefix.length), 10)) + .filter(n => Number.isInteger(n) && n >= 0); + process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1)); + NODESCRIPT + NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)" + RC_VERSION="${BASE}-rc.${NEXT_N}" + echo "Computed rc: $RC_VERSION" + + # 5. Defensive: if the exact version already exists on the registry + # (race with another run), abort before re-publishing. + NPM_STDERR_EXISTS="$(mktemp)" + if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then + rm -f "$NPM_STDERR_EXISTS" + echo "::error::Version $RC_VERSION already exists on npm — aborting." + exit 1 + else + if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then + rm -f "$NPM_STDERR_EXISTS" + # Version doesn't exist — safe to proceed. + else + echo "::error::npm registry unreachable for existence check:" >&2 + cat "$NPM_STDERR_EXISTS" >&2 + rm -f "$NPM_STDERR_EXISTS" + exit 1 + fi + fi + + { + echo "base=$BASE" + echo "rc_n=$NEXT_N" + echo "rc_version=$RC_VERSION" + } >> "$GITHUB_OUTPUT" + + - name: Apply rc version in-CI + if: needs.route.outputs.mode == 'rc' + shell: bash + working-directory: gitnexus + run: | + set -euo pipefail + npm version "${{ steps.rc-version.outputs.rc_version }}" \ + --no-git-tag-version --allow-same-version + + - name: Build gitnexus run: npm run build working-directory: gitnexus - name: Dry-run publish - run: npm publish --dry-run - working-directory: gitnexus - - - name: Publish to npm - run: npm publish --provenance --access public + # Cheap verification that the tarball assembles before the real publish. + shell: bash working-directory: gitnexus env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TAG: ${{ needs.route.outputs.mode == 'rc' && 'rc' || 'latest' }} + run: npm publish --dry-run --tag "$NPM_TAG" - - name: Extract release notes from CHANGELOG + # ── Acquire the "rc lock" BEFORE publishing (idempotency anchor) ───── + # We create two refs and push atomically: + # v → annotated tag on a detached release commit whose + # tree contains the rewritten package.json, so the + # tag's source matches the npm tarball. + # rc/ → lightweight tag on HEAD; the guard's dedup key. + # Push fails → nothing published. Push succeeds, npm fails → marker + # blocks retries until manual cleanup (see Rollback Runbook in plan). + - name: Create and push rc tags + id: rc-tags + if: needs.route.outputs.mode == 'rc' + shell: bash + working-directory: gitnexus + env: + RC_VERSION: ${{ steps.rc-version.outputs.rc_version }} + HEAD_SHA: ${{ needs.rc-guard.outputs.head_sha }} + # Short-lived GitHub App token. Auth is supplied inline at push + # time via `http.extraheader` (per GitHub's documented + # x-access-token Basic pattern). It is NOT persisted in + # .git/config (artipacked audit) — checkout above ran with + # `persist-credentials: false`. + PUSH_TOKEN: ${{ steps.app-token.outputs.token }} + # App's slug from create-github-app-token (e.g. `gitnexus-release-bot`). + # Used to attribute the release commit to the App identity rather + # than the generic github-actions[bot]. The bot's numeric user-id + # is resolved at runtime via the GitHub API (the action does not + # expose it directly as of v3.2.0). + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + VTAG="v${RC_VERSION}" + MARKER="rc/${HEAD_SHA}" + + # Resolve the App's bot user-id and construct the noreply email + # in the GitHub-canonical `+[bot]@users.noreply.github.com` + # shape. `[bot]` is part of the actual login on GitHub. + # + # The lookup is wrapped in a bounded retry because the first RC + # after App installation may hit propagation delay (404), and + # transient api.github.com 5xx during heavy org activity is a real + # failure class. Without retry, every transient blip aborts the + # entire release after CI has already succeeded. + BOT_LOGIN="${APP_SLUG}[bot]" + BOT_USER_ID="" + api_stderr="$(mktemp)" + for attempt in 1 2 3; do + if BOT_USER_ID="$(gh api "/users/${BOT_LOGIN}" --jq .id 2>"$api_stderr")" \ + && [[ "${BOT_USER_ID}" =~ ^[0-9]+$ ]]; then + break + fi + BOT_USER_ID="" + if [ "$attempt" -lt 3 ]; then + echo "::warning::bot user-id lookup attempt ${attempt} failed; retrying in $((attempt * 5))s" + sleep $((attempt * 5)) + fi + done + if ! [[ "${BOT_USER_ID}" =~ ^[0-9]+$ ]]; then + echo "::error::Could not resolve bot user-id for ${BOT_LOGIN} after 3 attempts." + echo "::error::gh api stderr:" + cat "$api_stderr" >&2 || true + echo "::error::Common causes: (a) newly-installed App — user record still propagating to /users/ (wait ~5min, redispatch with force=true); (b) App lacks Metadata: read permission; (c) transient api.github.com 5xx (redispatch)." + rm -f "$api_stderr" + exit 1 + fi + rm -f "$api_stderr" + git config user.name "${BOT_LOGIN}" + git config user.email "${BOT_USER_ID}+${BOT_LOGIN}@users.noreply.github.com" + + # Detached release commit with the version bump — main stays + # pristine, but the v-tag's tree matches the published package + # exactly (release-integrity). + git add package.json package-lock.json 2>/dev/null || git add package.json + git commit -m "release: ${VTAG}" --allow-empty + RELEASE_SHA="$(git rev-parse HEAD)" + echo "Detached release commit: $RELEASE_SHA" + + git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG" + git tag "$MARKER" "$HEAD_SHA" + + # Inline auth header. The base64-encoded form is masked as well + # as the raw token, because GitHub's secret-masker only masks the + # raw value — any subsequent `set -x` / GIT_TRACE line would + # otherwise expose the encoded credential. + # + # `set +x` wraps the compute+mask pair so that if an operator + # enables ACTIONS_STEP_DEBUG=true for triage (which turns on + # `set -x` globally), the assignment is NOT traced for the one + # line between compute and mask-registration. Without this wrap, + # debug mode would log `+ auth_header='Authorization: Basic '` + # exposing a still-valid (~1h) App token. + { set +x; } 2>/dev/null + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${PUSH_TOKEN}" | base64 -w0)" + echo "::add-mask::${auth_header}" + # Re-enable tracing only when explicitly requested via step-debug. + if [ "${ACTIONS_STEP_DEBUG:-false}" = "true" ]; then set -x; fi + + # Atomic push of both refs. If either would clobber an existing + # remote ref, the push fails and we stop before npm publish. + git -c http.extraheader="${auth_header}" \ + push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" + + { + echo "vtag=$VTAG" + echo "marker=$MARKER" + echo "release_sha=$RELEASE_SHA" + } >> "$GITHUB_OUTPUT" + + - name: Set vtag (stable) + id: stable-vtag + if: needs.route.outputs.mode == 'stable' + shell: bash + # github.ref_name flows in via env to avoid templating into the + # shell source (template-injection audit). Even though refs are + # constrained by git naming rules, the env-passthrough pattern + # makes injection structurally impossible. + env: + REF_NAME: ${{ github.ref_name }} + run: | + echo "vtag=${REF_NAME}" >> "$GITHUB_OUTPUT" + + # ── vtag integrity gate ────────────────────────────────────────────── + # Fail closed before any artifact-producing step (npm publish, Release, + # Docker) runs against an empty or mode-mismatched vtag. Prevents the + # silent "Release named main" / "Docker tagged from ref fallback" + # failure modes that the previous draft was vulnerable to. + - name: vtag integrity gate + id: vtag-gate + shell: bash + env: + MODE: ${{ needs.route.outputs.mode }} + VTAG: ${{ steps.rc-tags.outputs.vtag || steps.stable-vtag.outputs.vtag }} + run: | + set -euo pipefail + + if [ -z "$VTAG" ]; then + echo "::error::vtag is empty — refusing to create GitHub Release or trigger Docker." + exit 1 + fi + + case "$MODE" in + rc) + if ! [[ "$VTAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then + echo "::error::vtag '${VTAG}' does not match rc shape ^v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+$" + exit 1 + fi + ;; + stable) + if ! [[ "$VTAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::vtag '${VTAG}' does not match stable shape ^v[0-9]+.[0-9]+.[0-9]+$" + exit 1 + fi + ;; + *) + echo "::error::unknown mode '${MODE}' at vtag integrity gate." + exit 1 + ;; + esac + + echo "vtag verified: ${VTAG} (mode=${MODE})" + echo "vtag=${VTAG}" >> "$GITHUB_OUTPUT" + + # npm Trusted Publishing (GA'd 2025-07-31). With the package registered + # as a trusted publisher on npmjs.com bound to this repo + this + # workflow file, npm authenticates via OIDC at publish time — + # NODE_AUTH_TOKEN is intentionally NOT set (an empty string would + # short-circuit the OIDC fallback; the env var must be unset, not + # blanked). Provenance is auto-attached by the registry on + # trusted-publisher publishes, so the explicit --provenance flag is + # dropped. + # + # Prerequisite: configure the package as a trusted publisher at + # https://www.npmjs.com/package/gitnexus/access (Publishing access → + # Trusted Publishers → GitHub Actions) bound to: + # Owner: + # Repository: GitNexus + # Workflow: publish.yml + # Environment: (none) + - name: Publish to npm + shell: bash + working-directory: gitnexus + env: + NPM_TAG: ${{ needs.route.outputs.mode == 'rc' && 'rc' || 'latest' }} + run: npm publish --access public --tag "$NPM_TAG" + + # ── Stable-only: pull CHANGELOG body if present ────────────────────── + - name: Extract release notes from CHANGELOG (stable) id: changelog + if: needs.route.outputs.mode == 'stable' shell: bash run: | VERSION="${GITHUB_REF#refs/tags/v}" @@ -98,5 +783,90 @@ jobs: - name: Create GitHub Release uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 with: - body_path: ${{ steps.changelog.outputs.fallback == 'false' && '/tmp/release-notes.md' || '' }} - generate_release_notes: ${{ steps.changelog.outputs.fallback == 'true' }} + tag_name: ${{ steps.vtag-gate.outputs.vtag }} + name: >- + ${{ needs.route.outputs.mode == 'rc' + && format('Release Candidate {0}', steps.vtag-gate.outputs.vtag) + || steps.vtag-gate.outputs.vtag }} + prerelease: ${{ needs.route.outputs.mode == 'rc' }} + make_latest: ${{ needs.route.outputs.mode == 'stable' && 'true' || 'false' }} + # Stable: prefer CHANGELOG body, fall back to auto-generated. + # RC: always auto-generated + the prerelease body block below. + body_path: >- + ${{ needs.route.outputs.mode == 'stable' && steps.changelog.outputs.fallback == 'false' + && '/tmp/release-notes.md' || '' }} + generate_release_notes: >- + ${{ needs.route.outputs.mode == 'rc' + || steps.changelog.outputs.fallback == 'true' }} + body: >- + ${{ needs.route.outputs.mode == 'rc' && format( + 'Automated release candidate build from `main`.{0}{0}**npm:** `npm install gitnexus@rc`{0}**Version:** `{1}`{0}**Target base:** `{2}` (rc #{3}){0}**Source commit (main):** {4}{0}**Release commit (versioned tree):** {5}{0}{0}Release candidates are pre-stable builds intended for early testing. Stable releases remain on the `latest` dist-tag.', + '\n', + steps.rc-version.outputs.rc_version, + steps.rc-version.outputs.base, + steps.rc-version.outputs.rc_n, + needs.rc-guard.outputs.head_sha, + steps.rc-tags.outputs.release_sha + ) || '' }} + + # ── RC partial-failure cleanup ─────────────────────────────────────── + # If anything after the atomic tag-push step failed (npm publish + # blew up, GitHub Release call timed out, etc.), the v-tag and + # rc/ marker are already on origin. External consumers + # (Renovate, Dependabot, Releases RSS) can ingest a phantom tag for + # a version that was never published to npm. This step deletes them + # automatically so the operator's recovery is just "redispatch with + # force=true on the next commit", not a manual ref cleanup. + # + # Scoped strictly to RC + real (non-dry-run) + the rc-tags step + # actually produced a vtag (otherwise nothing to clean up). The + # App token is still valid (~1h TTL, job timeout 20min). + - name: Cleanup pushed tags on partial failure + if: ${{ failure() && needs.route.outputs.mode == 'rc' && steps.rc-tags.outputs.vtag != '' }} + shell: bash + working-directory: gitnexus + env: + VTAG: ${{ steps.rc-tags.outputs.vtag }} + MARKER: ${{ steps.rc-tags.outputs.marker }} + PUSH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -uo pipefail + echo "::warning::Publish step failed after tag push. Cleaning up remote refs to prevent phantom-version ingestion by downstream consumers." + + { set +x; } 2>/dev/null + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${PUSH_TOKEN}" | base64 -w0)" + echo "::add-mask::${auth_header}" + if [ "${ACTIONS_STEP_DEBUG:-false}" = "true" ]; then set -x; fi + + # Delete v-tag and marker. Each delete is best-effort — if one + # is already absent (atomic push partially rejected, or earlier + # cleanup ran), the other still gets attempted. + for ref in "refs/tags/${VTAG}" "refs/tags/${MARKER}"; do + if git -c http.extraheader="${auth_header}" push origin --delete "${ref}" 2>&1; then + echo "deleted origin ${ref}" + else + echo "::warning::could not delete origin ${ref} — may already be absent or protected. Manual cleanup may be required." + fi + done + + echo "::notice::Cleanup complete. To retry the release, redispatch the workflow with force=true on the same SHA, or push a new commit to main." + + # ── Phase 5 (RC only): Docker images ─────────────────────────────────────── + # R6: Docker remains RC-only. Stable Docker builds are explicitly deferred. + # Secrets are passed explicitly (not via `secrets: inherit`) so the + # callee's secret surface is auditable from the caller's source. + docker: + name: Build & Push RC Docker images + needs: [route, publish] + if: ${{ needs.route.outputs.mode == 'rc' && needs.publish.outputs.vtag != '' }} + uses: ./.github/workflows/docker.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + permissions: + contents: read + packages: write + id-token: write + attestations: write + with: + tag: ${{ needs.publish.outputs.vtag }} diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml deleted file mode 100644 index 2129e765a..000000000 --- a/.github/workflows/release-candidate.yml +++ /dev/null @@ -1,459 +0,0 @@ -name: Release Candidate - -on: - # Publish a release-candidate build whenever a merge/commit lands on main. - # Docs/README-only changes are filtered out so prose updates don't - # cut a release. - push: - branches: [main] - paths-ignore: - - '**.md' - - 'docs/**' - - 'LICENSE' - workflow_dispatch: - inputs: - bump: - description: >- - Cycle policy. 'auto' (default) continues the active rc cycle on - this branch if there is one, otherwise bumps patch from latest. - Choose 'patch' / 'minor' / 'major' to explicitly start or reset - an rc cycle. - required: false - default: 'auto' - type: choice - options: - - auto - - patch - - minor - - major - force: - description: 'Publish even when HEAD already has an rc marker' - required: false - default: 'false' - type: choice - options: - - 'false' - - 'true' - -# No workflow-level permissions — scoped per job below. -permissions: {} - -# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". -# Serialize all runs on the same ref (push + workflow_dispatch) to prevent two publishes -# racing on the rc counter. cancel-in-progress: false — the earlier merge publishes first. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -jobs: - # ── Skip when HEAD already has an rc marker (retry / duplicate dispatch) ── - # The marker is a lightweight tag `rc/` pushed *before* `npm - # publish`, so a failed publish leaves the marker in place and the guard - # refuses to re-publish. Recovery path after a partial failure: - # git push --delete origin rc/ v - # then redispatch with force=true. - guard: - name: Check if release candidate should run - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: read # read PR labels on the merge commit - outputs: - should_run: ${{ steps.decide.outputs.should_run }} - head_sha: ${{ steps.decide.outputs.head_sha }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Decide - id: decide - shell: bash - env: - FORCE: ${{ inputs.force }} - BUMP_INPUT: ${{ inputs.bump }} - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - HEAD_SHA=$(git rev-parse HEAD) - echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" - - if [ "$FORCE" = "true" ]; then - echo "Force flag set — running regardless of marker tag." - echo "should_run=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # An explicit cycle reset on dispatch (bump != auto) also bypasses - # the dedup guard — the maintainer is deliberately asking for a - # new rc from the same commit. - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && [ -n "${BUMP_INPUT:-}" ] \ - && [ "${BUMP_INPUT:-auto}" != "auto" ]; then - echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup." - echo "should_run=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # ── Skip when the merge commit corresponds to a release ───────── - # Two complementary checks (belt-and-suspenders): - # 1. The HEAD commit subject matches `chore: release vX.Y.Z` - # (the canonical release-PR title in this repo). Anchored - # at both ends to require the bare title or the squash-merge - # `(#NNNN)` suffix exactly — rejects noisy variants like - # `chore: release v1.0.0 (something unrelated)`. - # 2. The squash-merged PR carries the `release` label. - # Either match suppresses the rc build — stable releases publish - # via publish.yml on the v-tag, so the rc cycle should pause for - # them rather than racing the npm publish. - HEAD_SUBJECT="$(git log -1 --pretty=%s HEAD)" - # Sanitise GitHub-Actions annotation prefixes before logging the - # raw subject — defence-in-depth so a hypothetical commit subject - # containing `::error::` or `::set-output::` cannot forge log - # annotations even though %s strips newlines. - HEAD_SUBJECT_SAFE="${HEAD_SUBJECT//::/__}" - RELEASE_SUBJECT_RE='^chore:[[:space:]]*release[[:space:]]+v[0-9]+\.[0-9]+\.[0-9]+([[:space:]]+\(#[0-9]+\))?$' - if [[ "$HEAD_SUBJECT" =~ $RELEASE_SUBJECT_RE ]]; then - echo "HEAD commit subject matches a release commit — skipping rc." - echo " subject (sanitised): $HEAD_SUBJECT_SAFE" - echo "should_run=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Squash-merge commits include `(#NNNN)` at the end of the subject. - if [[ "$HEAD_SUBJECT" =~ \(#([0-9]+)\)[[:space:]]*$ ]]; then - PR_NUM="${BASH_REMATCH[1]}" - echo "Detected squash-merge of PR #$PR_NUM — checking labels." - if LABELS_JSON="$(gh pr view "$PR_NUM" --repo "$REPO" --json labels 2>/dev/null)"; then - if printf '%s' "$LABELS_JSON" | jq -e '.labels[] | select(.name == "release")' >/dev/null; then - echo "PR #$PR_NUM has the 'release' label — skipping rc." - echo "should_run=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "PR #$PR_NUM has no 'release' label — proceeding." - else - # Lookup failure is not fatal — fall through to the dedup check - # so a transient GH API hiccup doesn't silently suppress rc builds. - echo "::warning::Could not read labels for PR #${PR_NUM} — falling through." - fi - fi - - # Dedup: is there already an rc/ marker pointing at HEAD? - MARKER="rc/${HEAD_SHA}" - if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then - echo "HEAD already has marker $MARKER — skipping." - echo "should_run=false" >> "$GITHUB_OUTPUT" - else - echo "No marker on HEAD — proceeding." - echo "should_run=true" >> "$GITHUB_OUTPUT" - fi - - # ── Reuse the stable CI workflow ───────────────────────────────────── - ci: - needs: guard - if: needs.guard.outputs.should_run == 'true' - uses: ./.github/workflows/ci.yml - permissions: - contents: read - secrets: inherit - - # ── Publish the rc build to npm + create GitHub prerelease ─────────── - publish: - name: Publish release candidate to npm - needs: [guard, ci] - if: needs.guard.outputs.should_run == 'true' - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - # The default GITHUB_TOKEN cannot be granted `workflows: write`, so - # tag pushes that reach a commit which modified `.github/workflows/**` - # are rejected with: "refusing to allow a GitHub App to create or - # update workflow ... without `workflows` permission". We pass a - # fine-grained PAT (RELEASE_PUSH_TOKEN, scoped to this repo with - # Contents: write + Workflows: write) to `actions/checkout` so that - # the subsequent `git push --atomic` of the v-tag and rc marker - # carries the PAT's identity. Job-level GITHUB_TOKEN keeps its - # scoped permissions for everything else (npm provenance, etc.). - contents: write # push rc tag + marker (via PAT) - id-token: write # npm provenance - outputs: - vtag: ${{ steps.reltag.outputs.vtag }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - fetch-tags: true - # Use the PAT so `origin` is preauthed for `git push`. Without - # this the default GITHUB_TOKEN is wired into the remote, and a - # workflows-touching tag push is rejected — see the permissions - # block above. - token: ${{ secrets.RELEASE_PUSH_TOKEN }} - - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 22 - registry-url: https://registry.npmjs.org - # Hermetic install — release-candidate produces shipped artifacts. - # setup-node v5+ caches by default when a packageManager field is - # present in package.json; explicit opt-out is required to clear - # the zizmor cache-poisoning audit. See cache-poisoning audit. - package-manager-cache: false - - - name: Build gitnexus-shared - run: npm install && npm run build - working-directory: gitnexus-shared - - - name: Install gitnexus dependencies - run: npm ci - working-directory: gitnexus - - - name: Resolve rc version - id: version - shell: bash - working-directory: gitnexus - env: - BUMP_INPUT: ${{ inputs.bump }} - EVENT_NAME: ${{ github.event_name }} - PKG_NAME: gitnexus - run: | - set -euo pipefail - - # 1. Current published `latest` — the floor for any new rc base. - # Only E404 ("never published") falls back to package.json; any - # other error (network, auth, malformed response) fails fast. - NPM_STDERR_LATEST="$(mktemp)" - if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then - : - else - if grep -q 'E404' "$NPM_STDERR_LATEST"; then - CURRENT_LATEST="$(node -p "require('./package.json').version")" - echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST" - else - echo "::error::npm registry unreachable for 'view version':" >&2 - cat "$NPM_STDERR_LATEST" >&2 - rm -f "$NPM_STDERR_LATEST" - exit 1 - fi - fi - rm -f "$NPM_STDERR_LATEST" - CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}" - - # 2. Full version list — needed for the counter and for active-cycle - # inference. Same E404-only fallback. - NPM_STDERR_VERSIONS="$(mktemp)" - if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then - : - else - if grep -q 'E404' "$NPM_STDERR_VERSIONS"; then - VERSIONS_JSON='[]' - echo "No published versions for $PKG_NAME yet (E404)." - else - echo "::error::npm registry unreachable for 'view versions':" >&2 - cat "$NPM_STDERR_VERSIONS" >&2 - rm -f "$NPM_STDERR_VERSIONS" - exit 1 - fi - fi - rm -f "$NPM_STDERR_VERSIONS" - - # 3. Base selection. - # - workflow_dispatch + bump ∈ {patch,minor,major} → explicit cycle - # reset from latest. - # - Everything else (push, or dispatch with bump=auto) → continue - # the highest active rc base > latest if one exists; else - # default to patch from latest. - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && [ -n "${BUMP_INPUT:-}" ] \ - && [ "${BUMP_INPUT:-auto}" != "auto" ]; then - BASE="$(npx --yes -p semver@7 semver -i "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")" - echo "Explicit bump=$BUMP_INPUT → BASE=$BASE" - else - cat > /tmp/active_base.mjs <<'NODESCRIPT' - const latest = process.env.LATEST; - let v; - try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } - if (!Array.isArray(v)) v = [v]; - const parse = s => s.split(".").map(n => parseInt(n, 10)); - const gt = (a, b) => { - const [A, B] = [parse(a), parse(b)]; - for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i]; - return false; - }; - const bases = new Set(); - for (const s of v) { - const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s); - if (m && gt(m[1], latest)) bases.add(m[1]); - } - if (!bases.size) { process.stdout.write(""); process.exit(0); } - const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1); - process.stdout.write(sorted[sorted.length - 1]); - NODESCRIPT - ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)" - if [ -n "$ACTIVE_BASE" ]; then - BASE="$ACTIVE_BASE" - echo "Continuing active rc cycle → BASE=$BASE" - else - BASE="$(npx --yes -p semver@7 semver -i patch "$CURRENT_LATEST_CLEAN")" - echo "No active rc cycle → patch bump from latest → BASE=$BASE" - fi - fi - - # 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1. - cat > /tmp/next_rc.mjs <<'NODESCRIPT' - const base = process.env.BASE; - const prefix = base + "-rc."; - let v; - try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } - if (!Array.isArray(v)) v = [v]; - const ns = v - .filter(s => typeof s === "string" && s.startsWith(prefix)) - .map(s => parseInt(s.slice(prefix.length), 10)) - .filter(n => Number.isInteger(n) && n >= 0); - process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1)); - NODESCRIPT - NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)" - RC_VERSION="${BASE}-rc.${NEXT_N}" - echo "Computed rc: $RC_VERSION" - - # 5. Defensive: if the exact version already exists on the registry - # (e.g., race with another run), abort before re-publishing. - # Same E404-only pattern used above — a transient network - # failure must fail loudly, not pretend the version is missing. - NPM_STDERR_EXISTS="$(mktemp)" - if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then - rm -f "$NPM_STDERR_EXISTS" - echo "::error::Version $RC_VERSION already exists on npm — aborting." - exit 1 - else - if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then - rm -f "$NPM_STDERR_EXISTS" - # Version doesn't exist — safe to proceed. - else - echo "::error::npm registry unreachable for existence check:" >&2 - cat "$NPM_STDERR_EXISTS" >&2 - rm -f "$NPM_STDERR_EXISTS" - exit 1 - fi - fi - - { - echo "base=$BASE" - echo "rc_n=$NEXT_N" - echo "rc_version=$RC_VERSION" - } >> "$GITHUB_OUTPUT" - - - name: Apply rc version in-CI - shell: bash - working-directory: gitnexus - run: | - set -euo pipefail - npm version "${{ steps.version.outputs.rc_version }}" \ - --no-git-tag-version --allow-same-version - - - name: Build gitnexus - run: npm run build - working-directory: gitnexus - - - name: Dry-run publish - run: npm publish --dry-run --tag rc - working-directory: gitnexus - - # ── Acquire the "rc lock" BEFORE publishing (fixes idempotency) ───── - # We create two tags and push them atomically: - # v → annotated tag on a detached release commit - # whose tree contains the rewritten package.json - # (so the tag's source matches the npm tarball) - # rc/ → lightweight tag on HEAD; the guard's dedup key - # If this push fails, nothing is published — safe. - # If this push succeeds but npm publish fails, the marker stays on - # the remote and blocks retries until an operator manually cleans up. - - name: Create and push rc tags - id: reltag - shell: bash - working-directory: gitnexus - env: - RC_VERSION: ${{ steps.version.outputs.rc_version }} - HEAD_SHA: ${{ needs.guard.outputs.head_sha }} - run: | - set -euo pipefail - VTAG="v${RC_VERSION}" - MARKER="rc/${HEAD_SHA}" - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - - # Detached release commit with the version bump — keeps `main` - # pristine but gives the v-tag a tree that matches the published - # package contents exactly (fixes release-integrity gap). - git add package.json package-lock.json 2>/dev/null || git add package.json - git commit -m "release: ${VTAG}" --allow-empty - RELEASE_SHA="$(git rev-parse HEAD)" - echo "Detached release commit: $RELEASE_SHA" - - # Annotated release tag on the release commit. - git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG" - # Lightweight marker on the user-visible HEAD for the guard. - git tag "$MARKER" "$HEAD_SHA" - - # Atomic push of both refs. If either would clobber an existing - # remote ref, the push fails and we stop before npm publish. - git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" - - { - echo "vtag=$VTAG" - echo "marker=$MARKER" - echo "release_sha=$RELEASE_SHA" - } >> "$GITHUB_OUTPUT" - - - name: Publish to npm (rc dist-tag) - run: npm publish --provenance --access public --tag rc - working-directory: gitnexus - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Create GitHub prerelease - uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 - with: - tag_name: ${{ steps.reltag.outputs.vtag }} - name: Release Candidate ${{ steps.reltag.outputs.vtag }} - prerelease: true - make_latest: 'false' - generate_release_notes: true - body: | - Automated release candidate build from `main`. - - **npm:** `npm install gitnexus@rc` - **Version:** `${{ steps.version.outputs.rc_version }}` - **Target base:** `${{ steps.version.outputs.base }}` (rc #${{ steps.version.outputs.rc_n }}) - **Source commit (main):** ${{ needs.guard.outputs.head_sha }} - **Release commit (versioned tree):** ${{ steps.reltag.outputs.release_sha }} - - Release candidates are pre-stable builds intended for early testing. - Stable releases remain on the `latest` dist-tag. - - # ── Build & push RC Docker images ──────────────────────────────────── - # Calls docker.yml as a reusable workflow so that the build, signing, and - # attestation logic stays in one place. The publish job exposes `vtag` - # (e.g. `v1.2.3-rc.1`) as an output so we can pass it as the tag input. - # RC images are signed with Cosign keyless signing; the OIDC identity - # will be `docker.yml@refs/heads/main` (the caller's ref) rather than a - # tag ref — see README.md § Docker for the correct verify command for RCs. - docker: - name: Build & Push RC Docker images - needs: [guard, publish] - if: needs.guard.outputs.should_run == 'true' && needs.publish.outputs.vtag != '' - uses: ./.github/workflows/docker.yml - # Reusable workflows do not receive caller secrets unless inherited; without - # this, DOCKERHUB_* / GITHUB_TOKEN are empty in docker.yml → "Username and - # password required" on Docker Hub login (see same pattern on `ci:` above). - secrets: inherit - permissions: - contents: read - packages: write - id-token: write - attestations: write - with: - tag: ${{ needs.publish.outputs.vtag }} diff --git a/.github/zizmor.yml b/.github/zizmor.yml index b2f89e3ba..5d93b4cce 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -37,7 +37,8 @@ rules: - pr-labeler.yml # Note: cache-poisoning is NOT exempted. The two prior findings in - # publish.yml and release-candidate.yml were fixed structurally by - # dropping `cache: npm` from those workflows (matches the pattern used - # by PyO3/maturin for the same audit). See the commit that added this - # file for the rationale. + # publish.yml and the former release-candidate.yml were fixed structurally + # by dropping `cache: npm` from those workflows (matches the pattern used + # by PyO3/maturin for the same audit). After the publish-workflow + # unification (issue #1609), only publish.yml remains; the same + # cache-poisoning hardening applies there. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4f6b12b2..e048d4f2f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -144,16 +144,18 @@ If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUD ## Releases -Two publish workflows ship `gitnexus` to npm: +One workflow ships `gitnexus` to npm — `.github/workflows/publish.yml`. It +routes between two modes based on the triggering event: -- **Stable** (`.github/workflows/publish.yml`) — triggered by pushing any `v*` - tag. Publishes to the `latest` dist-tag with a changelog-backed GitHub - release. Maintainers are expected to tag from `main` as a convention; the - workflow itself does not enforce branch reachability. -- **Release Candidate** (`.github/workflows/release-candidate.yml`) — runs on - every push to `main` (typically a merged PR) plus manual dispatch. Docs-only - changes are skipped via `paths-ignore`. Publishes to the `rc` dist-tag with - version `X.Y.Z-rc.N` and a GitHub prerelease, where: +- **Stable mode** — triggered by pushing any `v` tag (no `-rc.*` + suffix; RC tags are excluded at trigger via a negative glob). Publishes to + the `latest` dist-tag with a changelog-backed GitHub release. Maintainers + are expected to tag from `main` as a convention; the workflow itself does + not enforce branch reachability. No Docker build (RC-only). +- **Release-candidate mode** — runs on every push to `main` (typically a + merged PR) plus manual `workflow_dispatch`. Docs-only changes are skipped + via `paths-ignore`. Publishes to the `rc` dist-tag with version + `X.Y.Z-rc.N` and a GitHub prerelease, where: - `X.Y.Z` is selected automatically. On push (and on dispatch with `bump: auto`, the default) the workflow **continues the active rc cycle**: if the registry already has `X.Y.Z-rc.*` versions with `X.Y.Z` > current @@ -170,36 +172,64 @@ Two publish workflows ship `gitnexus` to npm: caller's ref — see README.md § Docker for the verify command). Idempotency: the workflow pushes an `rc/` marker tag and a - `v` release tag **atomically, before** calling `npm publish`. The guard - refuses to re-run once the marker exists, so a post-publish failure will - not mint a duplicate rc for the same commit. The `v` tag points at a - detached release commit whose `package.json` matches the npm tarball - exactly (traceable releases). Recovery after a partial failure: + `v` release tag **atomically, before** calling `npm publish`. The + RC guard refuses to re-run once the marker exists, so a post-publish + failure will not mint a duplicate rc for the same commit. The `v` + tag points at a detached release commit whose `package.json` matches + the npm tarball exactly (traceable releases). The RC tag is excluded + from this workflow's `push: tags:` filter, so it does **not** re-trigger + publishing — preventing the double-publish failure mode tracked in #1609. + Recovery after a partial failure: the workflow's `if: failure()` cleanup + step in the `publish` job auto-deletes the v-tag and marker on most + post-publish failures, so the typical retry is just: + + ```bash + gh workflow run publish.yml --ref main -f force=true + # or push a new commit to main, which will cut a fresh RC + ``` + + If auto-cleanup didn't run (e.g. the cleanup step itself failed, or the + failure happened in the route/rc-guard phase before the marker was + pushed), manual cleanup is: ```bash git push --delete origin rc/ v - # then redispatch the workflow with force: true + # then redispatch with force: true ``` + **Release-PR-skip subject pattern.** The rc-guard job recognizes a + squash-merged release commit by matching the commit subject against + `^chore: release vX.Y.Z` (optionally followed by ` (#NNNN)` for the + squash-merge PR-number suffix). Match is case-insensitive — `Chore: Release v1.2.3` + works too. PRs that should suppress the RC build must either use this + subject shape, or carry the `release` label so the label-based fallback + fires. Other release-style subjects (`chore(release): v1.2.3`, + `release: v1.2.3`) will NOT trigger the skip — please name the release + PR exactly `chore: release vX.Y.Z` to keep the dedup deterministic. + **Docker-only partial failure:** if `publish` succeeds (npm tarball + tags are live) but the `docker` job subsequently fails (e.g. GHCR flakiness), the npm RC is already published and the `rc/` marker is in place. - Re-running `release-candidate.yml` with `force: true` will abort at the - "Version already exists on npm" guard. To recover without cutting a new RC: + Recovery without cutting a new RC: ```bash - # 1. Manually trigger only the docker workflow, passing the existing RC tag: - gh workflow run docker.yml --ref main -f tag=v - # (requires a workflow_dispatch trigger on docker.yml — see note below) + # Re-run only the failed docker job from the original workflow run: + gh run rerun --failed ``` - Because `docker.yml` intentionally has no `workflow_dispatch` (images are - tag-driven by design), the practical recovery options are: - - Wait for the next commit on `main`, which will cut a new RC that includes - the Docker build. - - Manually run `docker build` + `docker push` locally and sign with Cosign - against the same digest. - - Delete `rc/` and `v` tags, then redispatch with `force: true` to re-run the full RC pipeline (cuts a new RC number). + Find the run ID via `gh run list --workflow=publish.yml --branch main`. + `docker.yml` intentionally has no `workflow_dispatch` trigger (images are + tag-driven by design), so the gh-run-rerun path is the supported recovery. + + **GitHub Release transient failure** (npm publish succeeded, Release step + failed): the npm artifact is live but no GitHub Release page exists. + Recover by either re-running the failed job (`gh run rerun --failed`), + or creating the Release manually: + + ```bash + gh release create v --prerelease --generate-notes # RC + gh release create v --notes-file gitnexus/CHANGELOG.md # stable + ``` The rc workflow never moves `latest`. To verify after a change, inspect dist-tags: diff --git a/README.md b/README.md index e08c0eb7d..5909554e9 100644 --- a/README.md +++ b/README.md @@ -429,7 +429,7 @@ The Docker images are version-locked to the npm package: Both registries receive the same digest from a single build step, so you can pull from either and the signature verifies identically. - Release-candidate images (e.g. `:1.7.0-rc.1`) are published alongside each - RC npm release. They are built by `release-candidate.yml` calling `docker.yml` + RC npm release. They are built by `publish.yml` calling `docker.yml` as a reusable workflow after the RC tag is created and pushed. - `:latest` is auto-promoted only from non-prerelease tags by the Docker metadata action, so it always points at a real, npm-published version. @@ -462,7 +462,7 @@ registries because both sets of tags were signed at the same digest in one workflow run. **Release candidates** — signed from `refs/heads/main` (the caller's ref when -`release-candidate.yml` invokes `docker.yml` as a reusable workflow): +`publish.yml` invokes `docker.yml` as a reusable workflow): ```bash cosign verify ghcr.io/abhigyanpatwari/gitnexus:1.7.0-rc.1 \ From f69c382bcb91373d12526c2c4d79281603ea2c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 16 May 2026 08:36:49 +0100 Subject: [PATCH 3/5] fix(ci): engage npm Trusted Publishing OIDC properly (#1627) First live-fire RC publish after #1610 failed at npm publish with E404. The if: failure() cleanup correctly auto-deleted the partial v-tag and rc-marker, but OIDC never engaged. Root cause: two coordinated upstream bugs. 1. actions/setup-node@v6 with registry-url: writes _authToken into the runner .npmrc AND exports NODE_AUTH_TOKEN from its token: input (defaulting to github.token). npm publish sends GITHUB_TOKEN as the bearer and the registry returns 404. OIDC never tried because npm thinks it already has a credential. See actions/setup-node#1440. 2. The Node 22 runner ships with npm 10.9.x. npm Trusted Publishing OIDC support requires npm >= 11.5.1. Fix: omit registry-url: from the setup-node step (per the consensus workaround in community discussion #176761), and add npm install -g npm@latest before publish. --provenance flag is NOT added; npm auto-attaches provenance under Trusted Publishing. Sources: - https://github.com/actions/setup-node/issues/1440 - https://github.com/orgs/community/discussions/176761 - https://docs.npmjs.com/trusted-publishers/ --- .github/workflows/publish.yml | 58 ++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7ce377ca3..b62c39406 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -372,13 +372,34 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 - registry-url: https://registry.npmjs.org + # `registry-url:` is intentionally OMITTED. Under npm Trusted + # Publishing, OIDC only engages when no credential is configured. + # Setting `registry-url:` would make setup-node write + # `//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}` into the + # runner's .npmrc AND export NODE_AUTH_TOKEN from its `token:` + # input (default github.token). `npm publish` would then attempt + # GITHUB_TOKEN as the npm token, get rejected with 404, and OIDC + # would never be tried. See actions/setup-node#1440 and the GitHub + # Community discussion #176761 for the upstream bug and consensus + # workaround. + # # Hermetic install for published artifacts — opt out of the v5+ # default packageManager-based caching (clears the zizmor - # zizmor cache-poisoning audit). ~30s slower per - # release; runs rarely. + # cache-poisoning audit). ~30s slower per release; runs rarely. package-manager-cache: false + # npm Trusted Publishing requires npm >= 11.5.1. The Node 22 runner + # currently ships with npm 10.9.x which has no OIDC support — without + # this upgrade, `npm publish` falls back to classic auth and the + # registry returns 404 because no token is configured. Upgrade + # globally so subsequent `npm` invocations in this job use the new + # binary. + - name: Upgrade npm for Trusted Publishing + shell: bash + run: | + npm install -g npm@latest + npm --version + - name: Build gitnexus-shared run: npm install && npm run build working-directory: gitnexus-shared @@ -741,19 +762,28 @@ jobs: echo "vtag verified: ${VTAG} (mode=${MODE})" echo "vtag=${VTAG}" >> "$GITHUB_OUTPUT" - # npm Trusted Publishing (GA'd 2025-07-31). With the package registered - # as a trusted publisher on npmjs.com bound to this repo + this - # workflow file, npm authenticates via OIDC at publish time — - # NODE_AUTH_TOKEN is intentionally NOT set (an empty string would - # short-circuit the OIDC fallback; the env var must be unset, not - # blanked). Provenance is auto-attached by the registry on - # trusted-publisher publishes, so the explicit --provenance flag is - # dropped. + # npm Trusted Publishing (GA'd 2025-07-31). OIDC authentication only + # engages when no npm credential is configured anywhere — the absence + # is the signal. Two upstream behaviors had to be neutralized for + # this to work: # - # Prerequisite: configure the package as a trusted publisher at + # 1. setup-node's `registry-url:` is omitted (see the setup-node + # step above). With it, setup-node writes + # `//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}` into + # .npmrc and exports NODE_AUTH_TOKEN from `token:` (defaulting + # to github.token). npm publish then sends GITHUB_TOKEN as the + # bearer credential and the registry returns 404. OIDC is never + # tried because npm thinks it already has a credential. + # 2. The runner's bundled npm (10.9.x on Node 22) has no OIDC + # support; the upgrade step above pins it to >= 11.5.1. + # + # Provenance is auto-attached by the registry on trusted-publisher + # publishes — no --provenance flag needed. + # + # Prerequisite: register the package as a trusted publisher at # https://www.npmjs.com/package/gitnexus/access (Publishing access → - # Trusted Publishers → GitHub Actions) bound to: - # Owner: + # Trusted Publishers → GitHub Actions): + # Owner: abhigyanpatwari # Repository: GitNexus # Workflow: publish.yml # Environment: (none) From f28185d67eb212a404efeed884737566ee64bc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 16 May 2026 09:18:08 +0100 Subject: [PATCH 4/5] fix(ci): bump publish job to Node 24 for npm OIDC support (#1628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1627's npm install -g npm@latest step crashed mid-install with MODULE_NOT_FOUND: promise-retry — a known fragility when npm self-upgrades. Node 22's bundled npm is 10.9.x (no OIDC). Fix: bump publish job's node-version to 24, which ships with npm 11.x natively. Package consumers unaffected (this Node version is only used during publish; engines.node is >=22.0.0; ci-tests.yml continues testing on Node 22). --- .github/workflows/publish.yml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b62c39406..122130310 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -371,7 +371,15 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 22 + # Node 24 ships with npm >= 11.5.x, which is the minimum that + # supports npm Trusted Publishing OIDC. Node 22 ships with npm + # 10.9.x (no OIDC) and `npm install -g npm@latest` to self-upgrade + # is fragile — it can crash the in-flight reify with + # `MODULE_NOT_FOUND` on `promise-retry` etc. Bumping the Node + # version is the clean fix; the package's `engines` field is + # `>=22.0.0` so consumer-side compatibility is unaffected (this + # Node version is only used during publish, not by package users). + node-version: 24 # `registry-url:` is intentionally OMITTED. Under npm Trusted # Publishing, OIDC only engages when no credential is configured. # Setting `registry-url:` would make setup-node write @@ -388,18 +396,6 @@ jobs: # cache-poisoning audit). ~30s slower per release; runs rarely. package-manager-cache: false - # npm Trusted Publishing requires npm >= 11.5.1. The Node 22 runner - # currently ships with npm 10.9.x which has no OIDC support — without - # this upgrade, `npm publish` falls back to classic auth and the - # registry returns 404 because no token is configured. Upgrade - # globally so subsequent `npm` invocations in this job use the new - # binary. - - name: Upgrade npm for Trusted Publishing - shell: bash - run: | - npm install -g npm@latest - npm --version - - name: Build gitnexus-shared run: npm install && npm run build working-directory: gitnexus-shared From fa06c5610bd91c835b30a0f7def52ae05fda363e Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 10:02:40 +0100 Subject: [PATCH 5/5] fix: resolve cross-file type propagation stall on large repos (#1626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * fix: add time-based deadline to cross-file type propagation to prevent stalling on large repos Adds a 2-minute wall-clock time limit (DEFAULT_CROSS_FILE_ELAPSED_MS) to runCrossFileBindingPropagation. When exceeded, the phase gracefully stops and logs a warning. Users can override via GITNEXUS_CROSS_FILE_TIMEOUT_MS env var. This prevents the analyze command from stalling for hours on very large repositories where per-file re-resolution is expensive. Fixes the reported issue where gitnexus analyze stalls at "Cross-file type propagation" for several hours on repos with 15000+ files. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/b8341947-557c-4111-a3a8-991ba455ab01 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: root cause - cache tree-sitter queries across files, add live progress reporting Root cause: cross-file propagation called processCalls() with 1 file at a time, causing Parser.Query to be recompiled from the query string for every single file (O(N) compilations vs O(1) for the whole phase). Additionally, progress was only reported once at the start, making the phase appear completely frozen. Fixes: - Add optional `compiledQueryCache` parameter to `processCalls` so callers that invoke it with single-file batches can share compiled query objects across calls. The cross-file phase now compiles each language's query string exactly once and reuses it for all files of that language (e.g. 1 TypeScript compile for 595+ files). - Pre-count candidate files and emit onProgress every 25 files showing "Cross-file type propagation (N/M files)..." so the UI shows real movement instead of a frozen bar. - Keep the wall-clock deadline (GITNEXUS_CROSS_FILE_TIMEOUT_MS) as a safety net for pathological inputs. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/f5028cc8-4bc9-4309-8ffb-798fe2bd7a0a Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix: address code review - use SupportedLanguages key type, rename queryCache to compiledQueryCache Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/f5028cc8-4bc9-4309-8ffb-798fe2bd7a0a Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * chore(autofix): apply prettier + eslint fixes via /autofix command * fix(cross-file): remove wall-clock timeout from type propagation The query compilation cache and live progress reporting address the original stall; the 2-minute deadline could truncate cross-file work on large repos. MAX_CROSS_FILE_REPROCESS (2000) remains as the only cap. * test(cross-file): verify compiledQueryCache is shared across all processCalls invocations Finding 1: O(N) query recompilation was fixed by sharing a compiledQueryCache Map across all processCalls invocations in runCrossFileBindingPropagation. This test verifies the fix is correctly wired: the same Map instance is passed as the 12th argument to every call, proving queries are compiled once per language, not once per file. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/3ab768d9-3993-4882-9d8f-17f7fcbd086e Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * test(cross-file): verify live progress events are emitted with N/M format Finding 2: frozen progress display was fixed by emitting onProgress every 25 files with "Cross-file type propagation (N/M files)..." messages instead of calling it once at phase start. This test verifies the fix with 50 candidate files: expects onProgress called 3 times (1 initial + at 25 + at 50) with correct N/M counters. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/3ab768d9-3993-4882-9d8f-17f7fcbd086e Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix(cross-file): skip registry-primary language files before readFileContents Finding 3 (from comment 4466231612): cross-file-impl was calling processCalls for every candidate file even when that file's language is registry-primary (TypeScript, C++, Python, Go, C#, PHP, C — since AGENTS.md v1.7.0). processCalls would immediately skip those files via its own isRegistryPrimary guard, but cross-file-impl still paid the full cost: readFileContents I/O, buildImportedReturnTypes, buildImportedRawReturnTypes, and Map allocation — all discarded. Fix: check isRegistryPrimary(lang) in both the totalCandidates pre-count loop and the levelCandidates builder, before any file I/O or map building. This eliminates 595+ no-op processCalls invocations on large TypeScript repos. Test: mocks isRegistryPrimary to always return true and verifies that processCalls is never invoked and result is 0. The mock also defaults to false in beforeEach so existing tests using .ts files are unaffected. Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/3ab768d9-3993-4882-9d8f-17f7fcbd086e Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * refactor(test): address code review - simplify mock factory, name the arg index constant Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/3ab768d9-3993-4882-9d8f-17f7fcbd086e Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- gitnexus/src/core/ingestion/call-processor.ts | 15 +- .../pipeline-phases/cross-file-impl.ts | 58 ++++++- gitnexus/test/unit/cross-file-impl.test.ts | 159 ++++++++++++++++++ 3 files changed, 230 insertions(+), 2 deletions(-) diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 35a59dab4..1b5a234b4 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -766,6 +766,15 @@ export const processCalls = async ( importedRawReturnTypesMap?: ReadonlyMap>, heritageMap?: HeritageMap, bindingAccumulator?: BindingAccumulator, + /** + * Optional cache for compiled `Parser.Query` objects keyed by language name. + * When provided, compiled queries are reused across calls instead of being + * re-compiled from the query string for every file. Callers that invoke + * `processCalls` many times with single-file batches (e.g. the cross-file + * propagation phase) should pass a long-lived map here to avoid O(N) + * query recompilation overhead. + */ + compiledQueryCache?: Map, ): Promise => { const parser = await loadParser(); const collectedHeritage: ExtractedHeritage[] = []; @@ -843,7 +852,11 @@ export const processCalls = async ( let matches; try { const lang = parser.getLanguage(); - const query = new Parser.Query(lang, queryStr); + let query = compiledQueryCache?.get(language); + if (!query) { + query = new Parser.Query(lang, queryStr); + compiledQueryCache?.set(language, query); + } matches = query.matches(tree.rootNode); } catch (queryError) { logger.warn({ queryError }, `Query error for ${file.path}:`); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts index 5c014ed73..6c10ccb11 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts @@ -16,12 +16,18 @@ import { } from '../call-processor.js'; import type { createResolutionContext } from '../model/resolution-context.js'; import { createASTCache } from '../ast-cache.js'; -import { type PipelineProgress, getLanguageFromFilename } from 'gitnexus-shared'; +import { + type PipelineProgress, + getLanguageFromFilename, + type SupportedLanguages, +} from 'gitnexus-shared'; import { readFileContents } from '../filesystem-walker.js'; import { isLanguageAvailable } from '../../tree-sitter/parser-loader.js'; +import { isRegistryPrimary } from '../registry-primary-flag.js'; import { topologicalLevelSort } from '../utils/graph-sort.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import { isDev } from '../utils/env.js'; +import type Parser from 'tree-sitter'; import { logger } from '../../logger.js'; /** Max AST trees to keep in LRU cache for cross-file binding propagation. */ @@ -114,6 +120,36 @@ export async function runCrossFileBindingPropagation( let crossFileResolved = 0; const crossFileStart = Date.now(); const astCache = createASTCache(AST_CACHE_CAP); + // Compiled query objects keyed by language name. Shared across all processCalls + // invocations in this phase so the same tree-sitter query string is only + // compiled once per language instead of once per file (O(1) vs O(N)). + const compiledQueryCache = new Map(); + + // Snapshot total topological candidates for progress math. We walk the + // levels once more here (fast — no I/O) so we can report meaningful + // percentages rather than a frozen display. + let totalCandidates = 0; + for (const level of levels) { + for (const filePath of level) { + if (totalCandidates >= MAX_CROSS_FILE_REPROCESS) break; + const imports = ctx.namedImportMap.get(filePath); + if (!imports) continue; + if (!allPathSet.has(filePath)) continue; + const lang = getLanguageFromFilename(filePath); + if (!lang || !isLanguageAvailable(lang)) continue; + // Registry-primary languages have their call resolution handled by the + // scope-resolution pipeline — processCalls skips them immediately. Skip + // here too so we avoid the I/O cost (readFileContents) and map-building + // overhead for files that would be no-ops anyway. + if (isRegistryPrimary(lang)) continue; + totalCandidates++; + } + if (totalCandidates >= MAX_CROSS_FILE_REPROCESS) break; + } + const cappedTotal = Math.min(totalCandidates, MAX_CROSS_FILE_REPROCESS); + + /** Emit a progress event every PROGRESS_INTERVAL files so the UI stays alive. */ + const PROGRESS_INTERVAL = 25; for (const level of levels) { const levelCandidates: { @@ -151,6 +187,10 @@ export async function runCrossFileBindingPropagation( const lang = getLanguageFromFilename(filePath); if (!lang || !isLanguageAvailable(lang)) continue; + // Registry-primary languages have their call resolution handled by the + // scope-resolution pipeline — processCalls skips them immediately. Skip + // here to avoid readFileContents I/O and map-building for no-op files. + if (isRegistryPrimary(lang)) continue; levelCandidates.push({ filePath, seeded, importedReturns, importedRawReturns }); } @@ -188,8 +228,24 @@ export async function runCrossFileBindingPropagation( bindings.size > 0 ? bindings : undefined, importedReturnTypesMap.size > 0 ? importedReturnTypesMap : undefined, importedRawReturnTypesMap.size > 0 ? importedRawReturnTypesMap : undefined, + undefined, + undefined, + compiledQueryCache, ); crossFileResolved++; + + // Emit progress every PROGRESS_INTERVAL files so the UI shows real + // movement instead of a frozen display (cross-file can take minutes + // on large repos with many cross-file imports). + if (crossFileResolved % PROGRESS_INTERVAL === 0 || crossFileResolved === cappedTotal) { + const pct = cappedTotal > 0 ? Math.round((crossFileResolved / cappedTotal) * 8) : 0; + onProgress({ + phase: 'parsing', + percent: 82 + pct, + message: `Cross-file type propagation (${crossFileResolved}/${cappedTotal} files)...`, + stats: { filesProcessed: crossFileResolved, totalFiles, nodesCreated: graph.nodeCount }, + }); + } } if (crossFileResolved >= MAX_CROSS_FILE_REPROCESS) { diff --git a/gitnexus/test/unit/cross-file-impl.test.ts b/gitnexus/test/unit/cross-file-impl.test.ts index 55a47a5b5..bf5241d1d 100644 --- a/gitnexus/test/unit/cross-file-impl.test.ts +++ b/gitnexus/test/unit/cross-file-impl.test.ts @@ -49,17 +49,36 @@ vi.mock('../../src/core/tree-sitter/parser-loader.js', async (importOriginal) => }; }); +// Default to non-registry-primary so existing tests (which use .ts files) are +// not affected by the isRegistryPrimary guard added in cross-file-impl. Tests +// that verify the skip behavior can override this with mockReturnValue(true). +vi.mock('../../src/core/ingestion/registry-primary-flag.js', () => ({ + isRegistryPrimary: vi.fn(() => false), +})); + import { runCrossFileBindingPropagation } from '../../src/core/ingestion/pipeline-phases/cross-file-impl.js'; import { processCalls } from '../../src/core/ingestion/call-processor.js'; +import { isRegistryPrimary } from '../../src/core/ingestion/registry-primary-flag.js'; import { createResolutionContext } from '../../src/core/ingestion/model/resolution-context.js'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import type { ExportedTypeMap } from '../../src/core/ingestion/call-processor.js'; const processCallsMock = vi.mocked(processCalls); +const isRegistryPrimaryMock = vi.mocked(isRegistryPrimary); + +/** + * Index of the `compiledQueryCache` parameter in the `processCalls` signature. + * graph(0), files(1), astCache(2), ctx(3), onProgress?(4), exportedTypeMap?(5), + * importedBindingsMap?(6), importedReturnTypesMap?(7), + * importedRawReturnTypesMap?(8), heritageMap?(9), bindingAccumulator?(10), + * compiledQueryCache?(11). + */ +const COMPILED_QUERY_CACHE_ARG_INDEX = 11; describe('runCrossFileBindingPropagation', () => { beforeEach(() => { processCallsMock.mockClear(); + isRegistryPrimaryMock.mockReturnValue(false); // reset to non-primary before each test }); it('returns 0 immediately when namedImportMap is empty', async () => { @@ -162,6 +181,103 @@ describe('runCrossFileBindingPropagation', () => { } }); + it('passes the same compiledQueryCache Map instance to every processCalls call', async () => { + // Verifies that the O(N)→O(1) query-cache fix is correctly wired: the + // `compiledQueryCache` created in runCrossFileBindingPropagation is shared + // across all processCalls invocations so each language's Parser.Query is + // compiled exactly once, not once per file. + const graph = createKnowledgeGraph(); + const ctx = createResolutionContext(); + + const exportedTypeMap: ExportedTypeMap = new Map([ + ['upstream.ts', new Map([['User', 'User']])], + ]); + ctx.importMap.set('upstream.ts', new Set()); + + const allPaths = ['upstream.ts']; + for (let i = 0; i < 3; i++) { + const file = `downstream${i}.ts`; + allPaths.push(file); + const bindings = new Map(); + bindings.set('User', { sourcePath: 'upstream.ts', exportedName: 'User' }); + ctx.namedImportMap.set(file, bindings); + ctx.importMap.set(file, new Set(['upstream.ts'])); + } + + await runCrossFileBindingPropagation( + graph, + ctx, + exportedTypeMap, + new Set(allPaths), + allPaths.length, + '/repo', + Date.now(), + () => {}, + ); + + expect(processCallsMock).toHaveBeenCalledTimes(3); + + // Argument index 11 is compiledQueryCache — see COMPILED_QUERY_CACHE_ARG_INDEX. + const caches = processCallsMock.mock.calls.map((call) => call[COMPILED_QUERY_CACHE_ARG_INDEX]); + // Every call must receive a non-null Map (not undefined). + for (const cache of caches) { + expect(cache).toBeDefined(); + expect(cache).toBeInstanceOf(Map); + } + // All calls share the SAME instance — the whole point of the cache. + expect(caches[1]).toBe(caches[0]); + expect(caches[2]).toBe(caches[0]); + }); + + it('emits live onProgress events every 25 files with N/M format', async () => { + // Verifies that the frozen-progress-display fix is correctly wired: + // onProgress must be called multiple times from the processing loop, + // not just once at phase start, so large repos show real movement in + // the UI instead of a frozen percentage bar. + const graph = createKnowledgeGraph(); + const ctx = createResolutionContext(); + + const exportedTypeMap: ExportedTypeMap = new Map([ + ['upstream.ts', new Map([['User', 'User']])], + ]); + ctx.importMap.set('upstream.ts', new Set()); + + const allPaths = ['upstream.ts']; + for (let i = 0; i < 50; i++) { + const file = `downstream${i}.ts`; + allPaths.push(file); + const bindings = new Map(); + bindings.set('User', { sourcePath: 'upstream.ts', exportedName: 'User' }); + ctx.namedImportMap.set(file, bindings); + ctx.importMap.set(file, new Set(['upstream.ts'])); + } + + const progressMessages: string[] = []; + const onProgress = vi.fn((p: { phase: string; percent: number; message: string }) => { + progressMessages.push(p.message); + }); + + await runCrossFileBindingPropagation( + graph, + ctx, + exportedTypeMap, + new Set(allPaths), + allPaths.length, + '/repo', + Date.now(), + onProgress, + ); + + // 1 initial call at phase start + 2 loop calls (at 25 and 50 files). + expect(onProgress).toHaveBeenCalledTimes(3); + + // Loop messages must carry the "N/M files" format so the UI is informative. + const loopMessages = progressMessages.filter((m) => m.match(/\(\d+\/\d+ files\)/)); + expect(loopMessages).toHaveLength(2); + expect(loopMessages[0]).toContain('(25/50 files)'); + expect(loopMessages[1]).toContain('(50/50 files)'); + }); + it('caps processing at MAX_CROSS_FILE_REPROCESS (2000)', async () => { const graph = createKnowledgeGraph(); const ctx = createResolutionContext(); @@ -203,4 +319,47 @@ describe('runCrossFileBindingPropagation', () => { expect(result).toBe(2000); expect(processCallsMock).toHaveBeenCalledTimes(2000); }); + + it('skips registry-primary language files without calling processCalls', async () => { + // Finding 3: on large TypeScript/C++ repos (registry-primary since v1.6.4+) + // cross-file-impl was calling processCalls 595× per candidate only for + // processCalls to immediately return (isRegistryPrimary guard inside). + // Now cross-file-impl filters them out BEFORE readFileContents so we avoid + // the I/O cost and map-building overhead entirely. + const graph = createKnowledgeGraph(); + const ctx = createResolutionContext(); + + const exportedTypeMap: ExportedTypeMap = new Map([ + ['upstream.ts', new Map([['User', 'User']])], + ]); + ctx.importMap.set('upstream.ts', new Set()); + + const allPaths = ['upstream.ts']; + for (let i = 0; i < 5; i++) { + const file = `downstream${i}.ts`; + allPaths.push(file); + const bindings = new Map(); + bindings.set('User', { sourcePath: 'upstream.ts', exportedName: 'User' }); + ctx.namedImportMap.set(file, bindings); + ctx.importMap.set(file, new Set(['upstream.ts'])); + } + + // Simulate all files being registry-primary (e.g. TypeScript on main branch). + isRegistryPrimaryMock.mockReturnValue(true); + + const result = await runCrossFileBindingPropagation( + graph, + ctx, + exportedTypeMap, + new Set(allPaths), + allPaths.length, + '/repo', + Date.now(), + () => {}, + ); + + // No files are candidates; no processCalls invocations. + expect(result).toBe(0); + expect(processCallsMock).not.toHaveBeenCalled(); + }); });