From ade206963355203db3f5ff83dfe7b0179f671eed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 05:35:27 +0100 Subject: [PATCH 01/51] chore(deps)(deps): bump brace-expansion from 5.0.5 to 5.0.6 in /gitnexus (#1689) --- gitnexus/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 383354253..fe2115d07 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -2461,9 +2461,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" From b37974fdac0bbbbcc3f1f9057af8c1fc213d2f37 Mon Sep 17 00:00:00 2001 From: DuduPhudu <34869259+ReidenXerx@users.noreply.github.com> Date: Tue, 19 May 2026 08:23:13 +0300 Subject: [PATCH 02/51] feat(javascript): migrate JavaScript to scope-based resolution (RFC #909 Ring 3, issue #928) (#1640) --- .claude/skills/gitnexus/gitnexus-cli/SKILL.md | 10 +- AGENTS.md | 149 +--- CLAUDE.md | 64 ++ .../ingestion/languages/javascript/arity.ts | 12 + .../languages/javascript/captures.ts | 722 ++++++++++++++++++ .../languages/javascript/import-target.ts | 72 ++ .../ingestion/languages/javascript/index.ts | 49 ++ .../languages/javascript/interpret.ts | 45 ++ .../languages/javascript/merge-bindings.ts | 21 + .../ingestion/languages/javascript/query.ts | 421 ++++++++++ .../languages/javascript/scope-resolver.ts | 89 +++ .../languages/javascript/simple-hooks.ts | 48 ++ .../core/ingestion/languages/typescript.ts | 25 + .../languages/typescript/simple-hooks.ts | 5 +- .../core/ingestion/registry-primary-flag.ts | 1 + .../scope-resolution/pipeline/registry.ts | 2 + .../js-consumer-before-provider/b-provider.js | 1 + .../js-cross-file/src/models.js | 1 + 18 files changed, 1623 insertions(+), 114 deletions(-) create mode 100644 gitnexus/src/core/ingestion/languages/javascript/arity.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/captures.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/import-target.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/index.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/interpret.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/merge-bindings.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/query.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts create mode 100644 gitnexus/src/core/ingestion/languages/javascript/simple-hooks.ts diff --git a/.claude/skills/gitnexus/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus/gitnexus-cli/SKILL.md index 24586205e..cd9a83be0 100644 --- a/.claude/skills/gitnexus/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-cli/SKILL.md @@ -17,11 +17,11 @@ npx gitnexus analyze Run from the project root. This parses all source files, builds the knowledge graph, writes it to `.gitnexus/`, and generates CLAUDE.md / AGENTS.md context files. -| Flag | Effect | -| ------------------- | ------------------------------------------------------------------------------------------------------- | -| `--force` | Force full re-index even if up to date | -| `--embeddings` | Enable embedding generation for semantic search (off by default) | -| `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | +| Flag | Effect | +| -------------- | ---------------------------------------------------------------- | +| `--force` | Force full re-index even if up to date | +| `--embeddings` | Enable embedding generation for semantic search (off by default) | +| `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. diff --git a/AGENTS.md b/AGENTS.md index 99fc68bd0..46960b967 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,131 +62,64 @@ Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING. # GitNexus — Code Intelligence -Indexed as **GitNexus** (4325 symbols, 10556 relationships, 300 execution flows). Use MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **GitNexus** (26675 symbols, 35395 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. -> If any tool warns the index is stale, run `npx gitnexus analyze` first. +> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. ## Always Do -- **MUST run impact analysis before editing any symbol.** `gitnexus_impact({target: "symbolName", direction: "upstream"})` — report blast radius to the user. -- **MUST run `gitnexus_detect_changes()` before committing** — verify only expected symbols and flows are affected. -- **MUST warn the user** if impact returns HIGH or CRITICAL risk. -- Explore unfamiliar code with `gitnexus_query({query: "concept"})` (process-grouped, ranked) instead of grepping. -- Full context on a symbol: `gitnexus_context({name: "symbolName"})`. - -## When Debugging - -1. `gitnexus_query({query: ""})` — find related execution flows -2. `gitnexus_context({name: ""})` — callers, callees, process participation -3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace flow step by step -4. Regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` - -## When Refactoring - -- **Rename:** `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Graph edits are safe; text_search edits need manual review. -- **Extract/Split:** `gitnexus_context` (incoming/outgoing refs) then `gitnexus_impact` (upstream callers) before moving code. -- **After any refactor:** `gitnexus_detect_changes({scope: "all"})` to verify scope. +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. +- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. ## Never Do -- Edit a symbol without running `gitnexus_impact` first. -- Ignore HIGH/CRITICAL risk warnings. -- Rename with find-and-replace — use `gitnexus_rename`. -- Commit without `gitnexus_detect_changes()`. -- Add language-specific behavior to shared ingestion code (`gitnexus/src/core/ingestion/`) — use a `LanguageProvider` hook. Seeing `provider.mroStrategy === 'xxx'` or an import from `languages/xxx.ts` in shared code means stop and add a hook. - -## Tools Quick Reference - -| Tool | When to use | Example | -|------|-------------|---------| -| `list_repos` | Discover indexed repos | `gitnexus_list_repos({})` | -| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | -| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | -| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | -| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | -| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | -| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | -| `api_impact` | Pre-change API route impact | `gitnexus_api_impact({route: "/api/users", method: "GET"})` | -| `route_map` | Route → handler → consumer map | `gitnexus_route_map({})` | -| `tool_map` | MCP/RPC tool definitions | `gitnexus_tool_map({})` | -| `shape_check` | Response shape vs consumer access | `gitnexus_shape_check({route: "/api/users"})` | -| `group_list` | List repo groups | `gitnexus_group_list({})` | -| `group_sync` | Rebuild group Contract Registry | `gitnexus_group_sync({name: "myGroup"})` | -| `query` (group mode) | Cross-repo search in a group (RRF-merged) | `gitnexus_query({repo: "@myGroup", query: "auth"})` | -| `context` (group mode) | 360° view across all member repos | `gitnexus_context({repo: "@myGroup", name: "validateUser"})` | -| `impact` (group mode) | Cross-repo blast radius via Contract Bridge | `gitnexus_impact({repo: "@myGroup", target: "X", direction: "upstream"})` | - -> Group mode: pass `repo: "@"` to fan out across all member repos, or `repo: "@/"` to target a single member (path keys from `group.yaml`). Optional `service: ""` filters by service root. Group-level state (contracts, staleness) lives in the resources table below — there are **no** `group_query` / `group_context` / `group_impact` / `group_contracts` / `group_status` MCP tools. -> -> For a full walkthrough of setting up a group across multiple repos that communicate over gRPC, see [docs/guides/microservices-grpc.md](docs/guides/microservices-grpc.md). - -## Impact Risk Levels - -| Depth | Meaning | Action | -|-------|---------|--------| -| d=1 | WILL BREAK — direct callers/importers | MUST update | -| d=2 | LIKELY AFFECTED — indirect deps | Should test | -| d=3 | MAY NEED TESTING — transitive | Test if critical path | +- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. +- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. +- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. +- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. ## Resources | Resource | Use for | |----------|---------| -| `gitnexus://repo/GitNexus/context` | Codebase overview, index freshness | +| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | | `gitnexus://repo/GitNexus/clusters` | All functional areas | | `gitnexus://repo/GitNexus/processes` | All execution flows | | `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace | -| `gitnexus://group/{name}/contracts` | Group Contract Registry (provider/consumer rows + cross-links) | -| `gitnexus://group/{name}/status` | Per-member index + Contract Registry staleness report | -## Self-Check Before Finishing +## CLI -1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL warnings were ignored -3. `gitnexus_detect_changes()` confirms expected scope -4. All d=1 dependents were updated - -## Keeping the Index Fresh - -```bash -npx gitnexus analyze # incremental by default; preserves embeddings -npx gitnexus analyze --force # full rebuild from scratch (opt out of incremental) -npx gitnexus analyze --embeddings # also generate embeddings for new/changed nodes -npx gitnexus analyze --drop-embeddings # explicit opt-in to wipe existing embeddings -``` - -`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** under `.gitnexus/parse-cache/` (per-chunk JSON shards plus `index.json`) for chunks whose file contents haven't changed since the last run. Older installs may still have a legacy single file `.gitnexus/parse-cache.json`, which is read for backward compatibility but no longer written. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). - -The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete the whole `.gitnexus/parse-cache/` directory (and remove any legacy `.gitnexus/parse-cache.json` if present) at any time — it'll be rebuilt on the next analyze. - -Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). A plain `analyze` no longer drops existing vectors — pass `--drop-embeddings` to wipe. - -> Claude Code: PostToolUse hook detects a stale index after `git commit` and `git merge` and prompts the agent to run `analyze`. The hook does not invoke `analyze` itself. - -## CLI Skills - -| Task | Skill file | -|------|-----------| -| Architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Debugging / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Refactoring | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools/resources/schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| CLI commands (index, status, clean, wiki) | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | - -## Hook env knobs - -The Claude Code hook (`gitnexus/hooks/claude/gitnexus-hook.cjs` and the mirrored plugin copy under `gitnexus-claude-plugin/hooks/`) honours these env vars. Defaults work for normal installations; set them only to override resolution. All path overrides ignore values that do not exist on disk and fall through to the standard resolution chain. - -| Env var | Type | Default | Purpose | -|---------|------|---------|---------| -| `GITNEXUS_HOOK_CLI_PATH` | path | resolved via package layout / `require.resolve` | Override path to the `gitnexus` CLI entry the hook spawns for `augment`. | -| `GITNEXUS_HOOK_LSOF_PATH` | path | `lsof` on `PATH` (with `/usr/bin/lsof`, `/usr/sbin/lsof`, `/sbin/lsof` fallbacks) | Override POSIX `lsof` location for the DB-lock probe. | -| `GITNEXUS_HOOK_PS_PATH` | path | `ps` on `PATH` (with `/bin/ps`, `/usr/bin/ps` fallbacks) | Override POSIX `ps` location. | -| `GITNEXUS_HOOK_POWERSHELL_PATH` | path | `%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe` (then `SysWOW64`, then `powershell.exe` on `PATH`) | Override Windows PowerShell location used by the Restart-Manager probe. | -| `GITNEXUS_HOOK_LINUX_PROC_BUDGET_MS` | integer ms | `1200` | Max wall-clock for the Linux `/proc` fd scan before bailing out to the `lsof` fallback. | -| `GITNEXUS_HOOK_RM_TARGET` | path | derived | Restart-Manager target file (the LadybugDB path under `.gitnexus/`). Set internally by the hook; rarely overridden manually. | -| `GITNEXUS_DEBUG` | boolean (`1`/`true`) | unset | Verbose stderr from the hook: prints discarded augment-stderr prefixes and one-shot `.ps1` load-failure warnings. | +| Task | Read this skill file | +|------|---------------------| +| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | +| Work in the Ingestion area (239 symbols) | `.claude/skills/generated/ingestion/SKILL.md` | +| Work in the Extractors area (135 symbols) | `.claude/skills/generated/extractors/SKILL.md` | +| Work in the Components area (112 symbols) | `.claude/skills/generated/components/SKILL.md` | +| Work in the Lbug area (96 symbols) | `.claude/skills/generated/lbug/SKILL.md` | +| Work in the Group area (94 symbols) | `.claude/skills/generated/group/SKILL.md` | +| Work in the Cli area (92 symbols) | `.claude/skills/generated/cli/SKILL.md` | +| Work in the Configs area (92 symbols) | `.claude/skills/generated/configs/SKILL.md` | +| Work in the Type-extractors area (90 symbols) | `.claude/skills/generated/type-extractors/SKILL.md` | +| Work in the Hooks area (88 symbols) | `.claude/skills/generated/hooks/SKILL.md` | +| Work in the Unit area (80 symbols) | `.claude/skills/generated/unit/SKILL.md` | +| Work in the Cpp area (73 symbols) | `.claude/skills/generated/cpp/SKILL.md` | +| Work in the Scope-resolution area (72 symbols) | `.claude/skills/generated/scope-resolution/SKILL.md` | +| Work in the Server area (66 symbols) | `.claude/skills/generated/server/SKILL.md` | +| Work in the Local area (61 symbols) | `.claude/skills/generated/local/SKILL.md` | +| Work in the Wiki area (60 symbols) | `.claude/skills/generated/wiki/SKILL.md` | +| Work in the Workers area (57 symbols) | `.claude/skills/generated/workers/SKILL.md` | +| Work in the Embeddings area (56 symbols) | `.claude/skills/generated/embeddings/SKILL.md` | +| Work in the Typescript area (53 symbols) | `.claude/skills/generated/typescript/SKILL.md` | +| Work in the Storage area (51 symbols) | `.claude/skills/generated/storage/SKILL.md` | +| Work in the Php area (48 symbols) | `.claude/skills/generated/php/SKILL.md` | diff --git a/CLAUDE.md b/CLAUDE.md index af4069fcd..70b3f1a36 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,3 +52,67 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g ## GitNexus rules See the ` … ` block in **[AGENTS.md](AGENTS.md)** for the canonical MCP tools, impact analysis rules, and index instructions. + + +# GitNexus — Code Intelligence + +This project is indexed by GitNexus as **GitNexus** (26675 symbols, 35395 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. + +> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. + +## Always Do + +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. +- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. + +## Never Do + +- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. +- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. +- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. +- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. + +## Resources + +| Resource | Use for | +|----------|---------| +| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | +| `gitnexus://repo/GitNexus/clusters` | All functional areas | +| `gitnexus://repo/GitNexus/processes` | All execution flows | +| `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace | + +## CLI + +| Task | Read this skill file | +|------|---------------------| +| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | +| Work in the Ingestion area (239 symbols) | `.claude/skills/generated/ingestion/SKILL.md` | +| Work in the Extractors area (135 symbols) | `.claude/skills/generated/extractors/SKILL.md` | +| Work in the Components area (112 symbols) | `.claude/skills/generated/components/SKILL.md` | +| Work in the Lbug area (96 symbols) | `.claude/skills/generated/lbug/SKILL.md` | +| Work in the Group area (94 symbols) | `.claude/skills/generated/group/SKILL.md` | +| Work in the Cli area (92 symbols) | `.claude/skills/generated/cli/SKILL.md` | +| Work in the Configs area (92 symbols) | `.claude/skills/generated/configs/SKILL.md` | +| Work in the Type-extractors area (90 symbols) | `.claude/skills/generated/type-extractors/SKILL.md` | +| Work in the Hooks area (88 symbols) | `.claude/skills/generated/hooks/SKILL.md` | +| Work in the Unit area (80 symbols) | `.claude/skills/generated/unit/SKILL.md` | +| Work in the Cpp area (73 symbols) | `.claude/skills/generated/cpp/SKILL.md` | +| Work in the Scope-resolution area (72 symbols) | `.claude/skills/generated/scope-resolution/SKILL.md` | +| Work in the Server area (66 symbols) | `.claude/skills/generated/server/SKILL.md` | +| Work in the Local area (61 symbols) | `.claude/skills/generated/local/SKILL.md` | +| Work in the Wiki area (60 symbols) | `.claude/skills/generated/wiki/SKILL.md` | +| Work in the Workers area (57 symbols) | `.claude/skills/generated/workers/SKILL.md` | +| Work in the Embeddings area (56 symbols) | `.claude/skills/generated/embeddings/SKILL.md` | +| Work in the Typescript area (53 symbols) | `.claude/skills/generated/typescript/SKILL.md` | +| Work in the Storage area (51 symbols) | `.claude/skills/generated/storage/SKILL.md` | +| Work in the Php area (48 symbols) | `.claude/skills/generated/php/SKILL.md` | + + diff --git a/gitnexus/src/core/ingestion/languages/javascript/arity.ts b/gitnexus/src/core/ingestion/languages/javascript/arity.ts new file mode 100644 index 000000000..f3702a4b2 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/arity.ts @@ -0,0 +1,12 @@ +/** + * Arity compatibility for JavaScript. + * + * Delegates to `typescriptArityCompatibility` unchanged — JavaScript + * supports the same arity constructs (rest parameters `...args`, default + * parameters `p = v`) and the metadata shape (`parameterCount`, + * `requiredParameterCount`, `parameterTypes`) is synthesized by the same + * `computeTsArityMetadata` function (which understands both TS and JS + * parameter node types via `extractTsJsParameters`). + */ + +export { typescriptArityCompatibility as jsArityCompatibility } from '../typescript/arity.js'; diff --git a/gitnexus/src/core/ingestion/languages/javascript/captures.ts b/gitnexus/src/core/ingestion/languages/javascript/captures.ts new file mode 100644 index 000000000..a04844059 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/captures.ts @@ -0,0 +1,722 @@ +/** + * `emitScopeCaptures` for JavaScript. + * + * Adapts `emitTsScopeCaptures` for the JavaScript grammar: + * + * 1. **JS grammar** — uses `tree-sitter-javascript` instead of + * `tree-sitter-typescript`. The JS scope query is a subset of the + * TypeScript one (TypeScript-only node types dropped). + * + * 2. **CJS `require()` decomposition** — `const { X } = require('./m')` + * and `const X = require('./m')` are walked in a post-query pass and + * synthesized as `@import.kind/name/alias/source` markers so that + * `interpretJsImport` can recover a `ParsedImport` using the same + * shape as the TypeScript ESM decomposer. + * + * 3. **JSDoc type bindings** — JavaScript has no static type annotations + * so `@type-binding.parameter` / `@type-binding.return` must be + * inferred from leading JSDoc comments. A lightweight regex scanner + * (`parseJsDocParams` / `parseJsDocReturn`) extracts `@param {T} n` + * and `@returns {T}` tags and emits synthetic captures positioned on + * the annotated function node. + * + * 4. **Shared synthesis passes** — destructuring, for-of map-tuple, and + * instanceof narrowing passes are duplicated from `typescript/captures.ts` + * (they are pure AST operations with no grammar-specific logic). + * + * Pure given the input source text. No I/O, no globals consulted. + */ + +import type { Capture, CaptureMatch } from 'gitnexus-shared'; +import { + findNodeAtRange, + nodeToCapture, + syntheticCapture, + type SyntaxNode, +} from '../../utils/ast-helpers.js'; +import { splitImportStatement } from '../typescript/import-decomposer.js'; +import { getJsParser, getJsScopeQuery, jsCachedTreeMatchesGrammar } from './query.js'; +import { computeTsArityMetadata } from '../typescript/arity-metadata.js'; +import { synthesizeTsReceiverBinding } from '../typescript/receiver-binding.js'; +import { getTreeSitterBufferSize } from '../../constants.js'; +import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; + +/** JS function-like node types that may carry a synthesized `this` binding. + * Kept in sync with the `@scope.function` patterns in `query.ts`. */ +const FUNCTION_NODE_TYPES = [ + 'method_definition', + 'arrow_function', + 'function_expression', + 'function_declaration', + 'generator_function_declaration', +] as const; + +/** Declaration anchors that carry function-like arity metadata. */ +const FUNCTION_DECL_TAGS = ['@declaration.method', '@declaration.function'] as const; + +/** Callsite anchors that should carry `@reference.arity` + param types. */ +const CALL_TAGS = [ + '@reference.call.free', + '@reference.call.member', + '@reference.call.constructor', +] as const; + +function pickFirstDefined(grouped: CaptureMatch, tags: readonly string[]): Capture | undefined { + for (const tag of tags) { + const cap = grouped[tag]; + if (cap !== undefined) return cap; + } + return undefined; +} + +/** Filter `@reference.read.member` in non-read contexts (same logic as TS). */ +function shouldEmitReadMember(memberNode: SyntaxNode): boolean { + const parent = memberNode.parent; + if (parent === null) return true; + switch (parent.type) { + case 'call_expression': + return parent.childForFieldName('function')?.id !== memberNode.id; + case 'new_expression': + return parent.childForFieldName('constructor')?.id !== memberNode.id; + case 'assignment_expression': + case 'augmented_assignment_expression': + return parent.childForFieldName('left')?.id !== memberNode.id; + case 'jsx_self_closing_element': + case 'jsx_opening_element': + return parent.childForFieldName('name')?.id !== memberNode.id; + default: + return true; + } +} + +/** Find the first JS function-like node at the given range. */ +function findFunctionNode(rootNode: SyntaxNode, range: Capture['range']): SyntaxNode | null { + for (const nodeType of FUNCTION_NODE_TYPES) { + const n = findNodeAtRange(rootNode, range, nodeType); + if (n !== null) return n; + } + return null; +} + +/** Infer a callsite argument's static type from literal shapes. */ +function inferArgType(argNode: SyntaxNode): string { + switch (argNode.type) { + case 'number': + return 'number'; + case 'string': + case 'template_string': + return 'string'; + case 'true': + case 'false': + return 'boolean'; + case 'null': + return 'null'; + case 'undefined': + return 'undefined'; + case 'array': + return 'Array'; + case 'object': + return 'object'; + case 'regex': + return 'RegExp'; + case 'new_expression': { + const ctor = argNode.childForFieldName('constructor'); + return ctor?.text ?? ''; + } + default: + return ''; + } +} + +// ─── CJS require() decomposition ───────────────────────────────────────── + +/** + * Walk the AST and synthesize `@import.*` captures for CJS `require()` calls: + * + * - `const { X, Y } = require('./m')` → one match per destructured name, + * `@import.kind = 'named'`, `@import.name = X / Y`. + * - `const X = require('./m')` → `@import.kind = 'namespace'`, + * `@import.alias = X` (the whole module is bound to X). + * - `require('./m')` as a bare expression-statement → side-effect. + * + * CJS named-alias form (`const { X: alias } = require('./m')`) emits + * `@import.kind = 'named-alias'` with `@import.name = X` and + * `@import.alias = alias`. + * + * The synthesized markers are identical to those produced by + * `splitImportStatement` for ESM, so `interpretJsImport` can delegate + * unchanged to `interpretTsImport` for all cases. + */ +function synthesizeCjsImports(root: SyntaxNode, out: CaptureMatch[]): void { + const stack: SyntaxNode[] = [root]; + for (;;) { + const node = stack.pop(); + if (node === undefined) break; + for (const child of node.namedChildren) { + if (child !== null) stack.push(child); + } + + if (node.type !== 'call_expression') continue; + + // Require call: function must be bare identifier "require". + const fn = node.childForFieldName('function'); + if (fn === null || fn.type !== 'identifier' || fn.text !== 'require') continue; + + const argsNode = node.childForFieldName('arguments'); + if (argsNode === null) continue; + + // Source must be a string literal. + const firstArg = argsNode.namedChild(0); + if (firstArg === null || firstArg.type !== 'string') continue; + const rawSource = firstArg.text; // includes surrounding quotes + const source = firstArg.namedChild(0)?.text ?? rawSource.slice(1, -1); + + const parent = node.parent; + + // Case 1: const { X } = require('./m') OR const X = require('./m') + if (parent?.type === 'variable_declarator') { + const nameNode = parent.childForFieldName('name'); + if (nameNode === null) continue; + + if (nameNode.type === 'object_pattern') { + // Destructured: emit one match per specifier. + for (const field of nameNode.namedChildren) { + if (field === null) continue; + if (field.type === 'shorthand_property_identifier_pattern') { + const name = field.text; + out.push({ + '@import.statement': syntheticCapture('@import.statement', node, rawSource), + '@import.kind': syntheticCapture('@import.kind', node, 'named'), + '@import.name': syntheticCapture('@import.name', field, name), + '@import.source': syntheticCapture('@import.source', firstArg, source), + }); + } else if (field.type === 'pair_pattern') { + const key = field.childForFieldName('key'); + const value = field.childForFieldName('value'); + if (key === null || value === null || value.type !== 'identifier') continue; + out.push({ + '@import.statement': syntheticCapture('@import.statement', node, rawSource), + '@import.kind': syntheticCapture('@import.kind', node, 'named-alias'), + '@import.name': syntheticCapture('@import.name', key, key.text), + '@import.alias': syntheticCapture('@import.alias', value, value.text), + '@import.source': syntheticCapture('@import.source', firstArg, source), + }); + } + } + } else if (nameNode.type === 'identifier') { + // Namespace-style: const X = require('./m') → bind whole module to X. + out.push({ + '@import.statement': syntheticCapture('@import.statement', node, rawSource), + '@import.kind': syntheticCapture('@import.kind', node, 'namespace'), + '@import.alias': syntheticCapture('@import.alias', nameNode, nameNode.text), + '@import.source': syntheticCapture('@import.source', firstArg, source), + }); + } + continue; + } + + // Case 2: bare require('./m') — side-effect import. + if (parent?.type === 'expression_statement') { + out.push({ + '@import.statement': syntheticCapture('@import.statement', node, rawSource), + '@import.kind': syntheticCapture('@import.kind', node, 'side-effect'), + '@import.source': syntheticCapture('@import.source', firstArg, source), + }); + } + } +} + +// ─── JSDoc type binding synthesis ──────────────────────────────────────── + +interface JsDocParam { + readonly name: string; + readonly type: string; +} + +/** Extract `@param {Type} name` entries from a JSDoc comment block. */ +function parseJsDocParams(text: string): readonly JsDocParam[] { + const results: JsDocParam[] = []; + // Match @param {Type} name or @param {Type} [name] (optional) + const re = /@param\s+\{([^}]+)\}\s+\[?(\w+)\]?/g; + let m: RegExpExecArray | null; + while ((m = re.exec(text)) !== null) { + results.push({ type: m[1].trim(), name: m[2].trim() }); + } + return results; +} + +/** Extract `@returns {Type}` or `@return {Type}` from a JSDoc comment. */ +function parseJsDocReturn(text: string): string | null { + const m = /@returns?\s+\{([^}]+)\}/.exec(text); + return m ? m[1].trim() : null; +} + +/** Extract `@type {Type}` from a JSDoc comment (variable-level annotation). */ +function parseJsDocType(text: string): string | null { + const m = /@type\s+\{([^}]+)\}/.exec(text); + return m ? m[1].trim() : null; +} + +/** + * Walk the AST and synthesize `@type-binding.*` captures from JSDoc + * comments immediately preceding function declarations / expressions. + * + * Only `/** … *​/` block comments are scanned. Line comments (`//`) are + * intentionally excluded — JSDoc lives in block comments. + * + * Emits: + * - `@type-binding.parameter` for each `@param {T} n` tag. + * - `@type-binding.return` for `@returns {T}` / `@return {T}`. + * - `@type-binding.annotation` for `@type {T}` on `let`/`const`/`var` + * declarations — covers the common `/** @type {User} *​/ const u = …` + * pattern (ECMA-262 §14.3.1/§14.3.2 variable declarations). + * + * The binding is anchored on the function node so `tsBindingScopeFor` + * can hoist method return-type bindings to Module scope (matching the + * TypeScript path where `hoistTypeBindingsToModule: true`). + */ +function synthesizeJsDocBindings(root: SyntaxNode, out: CaptureMatch[]): void { + const stack: SyntaxNode[] = [root]; + for (;;) { + const node = stack.pop(); + if (node === undefined) break; + for (const child of node.namedChildren) { + if (child !== null) stack.push(child); + } + + const isFnDecl = + node.type === 'function_declaration' || node.type === 'generator_function_declaration'; + const isMethodDef = node.type === 'method_definition'; + // Also check lexical_declaration containing an arrow/fn-expression + const isLexDecl = node.type === 'lexical_declaration' || node.type === 'variable_declaration'; + + if (!isFnDecl && !isMethodDef && !isLexDecl) continue; + + // For `export function foo() { ... }`, the JSDoc comment precedes the + // wrapping export_statement, not the inner function_declaration. + // Walk up to the export_statement so the preceding-sibling search finds it. + const lookupNode = + (isFnDecl || isLexDecl) && node.parent?.type === 'export_statement' ? node.parent : node; + + // Find the preceding sibling comment. + let sibling = lookupNode.previousNamedSibling; + while (sibling !== null && sibling.type === 'comment') { + const text = sibling.text; + if (text.startsWith('/**')) { + // Found a JSDoc block. + const params = parseJsDocParams(text); + const retType = parseJsDocReturn(text); + const varType = isLexDecl ? parseJsDocType(text) : null; + + // Determine the anchor node (the function-like node, for hoisting). + const anchor = node; + + for (const p of params) { + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', anchor, p.name), + '@type-binding.type': syntheticCapture('@type-binding.type', anchor, p.type), + '@type-binding.parameter': syntheticCapture('@type-binding.parameter', anchor, '1'), + }); + } + + if (retType !== null) { + // For named functions, use the function name as the binding name so + // `hoistTypeBindingsToModule` knows which function's return type this is. + let fnName: string | null = null; + if (isFnDecl) { + fnName = node.childForFieldName('name')?.text ?? null; + } else if (isMethodDef) { + // method_definition uses `name:` field for the method name + const nameNode = node.childForFieldName('name'); + if (nameNode?.type === 'property_identifier') fnName = nameNode.text; + } else if (isLexDecl) { + const declarator = node.namedChild(0); + const nameNode = declarator?.childForFieldName('name'); + if (nameNode?.type === 'identifier') fnName = nameNode.text; + } + if (fnName !== null) { + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', anchor, fnName), + '@type-binding.type': syntheticCapture('@type-binding.type', anchor, retType), + '@type-binding.return': syntheticCapture('@type-binding.return', anchor, '1'), + }); + } + } + + // @type {T} on let/const/var: `/** @type {User} */ const u = getUser()`. + // Emits annotation-strength binding (source = 'annotation') so it + // overrides any weaker constructor/alias inference on the same name. + if (varType !== null) { + for (const declarator of node.namedChildren) { + if (declarator === null || declarator.type !== 'variable_declarator') continue; + const nameNode = declarator.childForFieldName('name'); + if (nameNode === null || nameNode.type !== 'identifier') continue; + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', nameNode, nameNode.text), + '@type-binding.type': syntheticCapture('@type-binding.type', nameNode, varType), + '@type-binding.annotation': syntheticCapture( + '@type-binding.annotation', + nameNode, + '1', + ), + }); + } + } + + break; + } + sibling = sibling.previousNamedSibling; + } + } +} + +// ─── Destructuring / for-of / instanceof (shared with TS captures) ─────── + +function synthesizeDestructuringBindings(root: SyntaxNode, out: CaptureMatch[]): void { + const stack: SyntaxNode[] = [root]; + for (;;) { + const node = stack.pop(); + if (node === undefined) break; + for (const child of node.namedChildren) { + if (child !== null) stack.push(child); + } + if (node.type !== 'variable_declarator') continue; + const nameNode = node.childForFieldName('name'); + const valueNode = node.childForFieldName('value'); + if (nameNode === null || valueNode === null) continue; + if (nameNode.type !== 'object_pattern') continue; + if (valueNode.type !== 'identifier') continue; + const rhsName = valueNode.text; + for (const fieldNode of nameNode.namedChildren) { + if (fieldNode === null) continue; + if (fieldNode.type === 'shorthand_property_identifier_pattern') { + const localName = fieldNode.text; + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', fieldNode, localName), + '@type-binding.type': syntheticCapture( + '@type-binding.type', + fieldNode, + `${rhsName}.${localName}`, + ), + '@type-binding.destructured': syntheticCapture( + '@type-binding.destructured', + fieldNode, + fieldNode.text, + ), + }); + } else if (fieldNode.type === 'pair_pattern') { + const key = fieldNode.childForFieldName('key'); + const value = fieldNode.childForFieldName('value'); + if (key === null || value === null || value.type !== 'identifier') continue; + const fieldName = key.text; + const localName = value.text; + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', value, localName), + '@type-binding.type': syntheticCapture( + '@type-binding.type', + fieldNode, + `${rhsName}.${fieldName}`, + ), + '@type-binding.destructured': syntheticCapture( + '@type-binding.destructured', + fieldNode, + fieldNode.text, + ), + }); + } + } + } +} + +function synthesizeForOfMapTupleBindings(root: SyntaxNode, out: CaptureMatch[]): void { + const stack: SyntaxNode[] = [root]; + for (;;) { + const node = stack.pop(); + if (node === undefined) break; + for (const child of node.namedChildren) { + if (child !== null) stack.push(child); + } + if (node.type !== 'for_in_statement') continue; + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + if (left === null || right === null) continue; + if (left.type !== 'array_pattern' || right.type !== 'identifier') continue; + const rhs = right.text; + let slot = 0; + for (const child of left.namedChildren) { + if (child === null || child.type !== 'identifier') continue; + const localName = child.text; + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', child, localName), + '@type-binding.type': syntheticCapture( + '@type-binding.type', + child, + `__MAP_TUPLE_${slot}__:${rhs}`, + ), + '@type-binding.map-tuple-entry': syntheticCapture( + '@type-binding.map-tuple-entry', + child, + String(slot), + ), + }); + slot++; + } + } +} + +function synthesizeInstanceofNarrowings(root: SyntaxNode, out: CaptureMatch[]): void { + const stack: SyntaxNode[] = [root]; + for (;;) { + const node = stack.pop(); + if (node === undefined) break; + for (const child of node.namedChildren) { + if (child !== null) stack.push(child); + } + if (node.type !== 'if_statement') continue; + const cond = node.childForFieldName('condition'); + if (cond === null) continue; + const inner = cond.type === 'parenthesized_expression' ? cond.namedChildren[0] : cond; + if (inner === null || inner.type !== 'binary_expression') continue; + const op = inner.childForFieldName('operator'); + const left = inner.childForFieldName('left'); + const right = inner.childForFieldName('right'); + if (op === null || left === null || right === null) continue; + if (op.type !== 'instanceof') continue; + if (left.type !== 'identifier') continue; + if (right.type !== 'identifier') continue; + const varName = left.text; + const typeName = right.text; + const cons = node.childForFieldName('consequence'); + if (cons === null) continue; + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', cons, varName), + '@type-binding.type': syntheticCapture('@type-binding.type', right, typeName), + '@type-binding.instanceof-narrow': syntheticCapture( + '@type-binding.instanceof-narrow', + cons, + '1', + ), + }); + } +} + +// ─── Constructor field type bindings ───────────────────────────────────── + +/** + * Synthesize class-scope type bindings from `this.X = new Y()` assignments + * inside constructor method bodies. Covers the traditional ES5+ OOP pattern: + * + * class User { + * constructor() { + * /** @type {Address} *\/ + * this.address = new Address(); + * } + * } + * + * The emitted `@type-binding.class-field` is hoisted to the Class scope by + * `tsBindingScopeFor` so that compound-receiver resolution can look up + * `User.address → Address` when resolving `user.address.save()`. + * + * Type source priority: + * 1. JSDoc `@type {T}` comment immediately preceding the statement + * 2. `new Y()` constructor inference + */ +function synthesizeConstructorFieldBindings(root: SyntaxNode, out: CaptureMatch[]): void { + const stack: SyntaxNode[] = [root]; + for (;;) { + const node = stack.pop(); + if (node === undefined) break; + for (const child of node.namedChildren) { + if (child !== null) stack.push(child); + } + // Only process constructor method definitions + if (node.type !== 'method_definition') continue; + const nameNode = node.childForFieldName('name'); + if (nameNode?.text !== 'constructor') continue; + + const body = node.childForFieldName('body'); + if (body === null) continue; + + for (const stmt of body.namedChildren) { + if (stmt === null || stmt.type !== 'expression_statement') continue; + const expr = stmt.namedChild(0); + if (expr === null || expr.type !== 'assignment_expression') continue; + + const left = expr.childForFieldName('left'); + const right = expr.childForFieldName('right'); + if (left === null || right === null) continue; + if (left.type !== 'member_expression') continue; + + const obj = left.childForFieldName('object'); + const prop = left.childForFieldName('property'); + if (obj === null || prop === null) continue; + if (obj.text !== 'this' || prop.type !== 'property_identifier') continue; + + const fieldName = prop.text; + + // Prefer JSDoc @type annotation on the preceding sibling comment. + let typeName: string | null = null; + const prevSib: SyntaxNode | null = stmt.previousNamedSibling; + if (prevSib !== null && prevSib.type === 'comment') { + const m = /@type\s*\{([^}]+)\}/.exec(prevSib.text); + if (m?.[1]) typeName = m[1].trim(); + } + // Fall back to constructor inference from `new Y()`. + if (typeName === null && right.type === 'new_expression') { + const ctor = right.childForFieldName('constructor'); + if (ctor !== null && ctor.type === 'identifier') typeName = ctor.text; + } + if (typeName === null) continue; + + out.push({ + '@type-binding.name': syntheticCapture('@type-binding.name', prop, fieldName), + '@type-binding.type': syntheticCapture('@type-binding.type', prop, typeName), + // Anchor: positioned inside the constructor body so tsBindingScopeFor + // can walk up from the Function (constructor) scope to the Class scope. + '@type-binding.class-field': syntheticCapture('@type-binding.class-field', stmt, '1'), + }); + } + } +} + +// ─── Main emitter ────────────────────────────────────────────────────────── + +export function emitJsScopeCaptures( + sourceText: string, + filePath: string, + cachedTree?: unknown, +): readonly CaptureMatch[] { + let tree = cachedTree as ReturnType['parse']> | undefined; + if (tree !== undefined && !jsCachedTreeMatchesGrammar(tree)) { + tree = undefined; + } + if (tree === undefined) { + tree = parseSourceSafe(getJsParser(filePath), sourceText, undefined, { + bufferSize: getTreeSitterBufferSize(sourceText), + }); + } + + const rawMatches = getJsScopeQuery(filePath).matches(tree.rootNode); + const out: CaptureMatch[] = []; + + for (const m of rawMatches) { + const grouped: Record = {}; + for (const c of m.captures) { + const tag = '@' + c.name; + grouped[tag] = nodeToCapture(tag, c.node); + } + if (Object.keys(grouped).length === 0) continue; + + // Decompose ESM import_statement / re-export export_statement. + if (grouped['@import.statement'] !== undefined) { + const stmtCapture = grouped['@import.statement']; + const stmtNode = + findNodeAtRange(tree.rootNode, stmtCapture.range, 'import_statement') ?? + findNodeAtRange(tree.rootNode, stmtCapture.range, 'export_statement'); + if (stmtNode !== null) { + const decomposed = splitImportStatement(stmtNode); + for (const d of decomposed) out.push(d); + } + continue; + } + + // Decompose dynamic import() calls. + if (grouped['@import.dynamic'] !== undefined) { + const dynCapture = grouped['@import.dynamic']; + const callNode = findNodeAtRange(tree.rootNode, dynCapture.range, 'call_expression'); + if (callNode !== null) { + const decomposed = splitImportStatement(callNode); + for (const d of decomposed) out.push(d); + } + continue; + } + + // Filter @reference.read.member false-positives. + if (grouped['@reference.read.member'] !== undefined) { + const anchor = grouped['@reference.read.member']; + const memberNode = findNodeAtRange(tree.rootNode, anchor.range, 'member_expression'); + if (memberNode === null || !shouldEmitReadMember(memberNode)) { + continue; + } + } + + // Synthesize arity metadata on function-like declarations. + const declAnchor = pickFirstDefined(grouped, FUNCTION_DECL_TAGS); + if (declAnchor !== undefined) { + const fnNode = findFunctionNode(tree.rootNode, declAnchor.range); + if (fnNode !== null) { + const arity = computeTsArityMetadata(fnNode); + if (arity.parameterCount !== undefined) { + grouped['@declaration.parameter-count'] = syntheticCapture( + '@declaration.parameter-count', + fnNode, + String(arity.parameterCount), + ); + } + if (arity.requiredParameterCount !== undefined) { + grouped['@declaration.required-parameter-count'] = syntheticCapture( + '@declaration.required-parameter-count', + fnNode, + String(arity.requiredParameterCount), + ); + } + if (arity.parameterTypes !== undefined) { + grouped['@declaration.parameter-types'] = syntheticCapture( + '@declaration.parameter-types', + fnNode, + JSON.stringify(arity.parameterTypes), + ); + } + } + } + + // Synthesize @reference.arity on callsites. + const callAnchor = pickFirstDefined(grouped, CALL_TAGS); + if (callAnchor !== undefined && grouped['@reference.arity'] === undefined) { + const callNode = + findNodeAtRange(tree.rootNode, callAnchor.range, 'call_expression') ?? + findNodeAtRange(tree.rootNode, callAnchor.range, 'new_expression'); + if (callNode !== null) { + const argList = callNode.childForFieldName('arguments'); + const args: SyntaxNode[] = + argList === null + ? [] + : argList.namedChildren.filter( + (c): c is SyntaxNode => c !== null && c.type !== 'comment', + ); + grouped['@reference.arity'] = syntheticCapture( + '@reference.arity', + callNode, + String(args.length), + ); + grouped['@reference.parameter-types'] = syntheticCapture( + '@reference.parameter-types', + callNode, + JSON.stringify(args.map(inferArgType)), + ); + } + } + + out.push(grouped); + + // Synthesize `this` receiver type-bindings on class member functions. + const scopeFnAnchor = grouped['@scope.function']; + if (scopeFnAnchor !== undefined) { + const fnNode = findFunctionNode(tree.rootNode, scopeFnAnchor.range); + if (fnNode !== null) { + const synth = synthesizeTsReceiverBinding(fnNode); + if (synth !== null) out.push(synth); + } + } + } + + // Post-query synthesis passes. + synthesizeCjsImports(tree.rootNode, out); + synthesizeJsDocBindings(tree.rootNode, out); + synthesizeConstructorFieldBindings(tree.rootNode, out); + synthesizeDestructuringBindings(tree.rootNode, out); + synthesizeForOfMapTupleBindings(tree.rootNode, out); + synthesizeInstanceofNarrowings(tree.rootNode, out); + + return out; +} diff --git a/gitnexus/src/core/ingestion/languages/javascript/import-target.ts b/gitnexus/src/core/ingestion/languages/javascript/import-target.ts new file mode 100644 index 000000000..bfdfe9951 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/import-target.ts @@ -0,0 +1,72 @@ +/** + * Import-target resolver for JavaScript. + * + * Delegates to the TypeScript `resolveTsTarget` standard-strategy resolver + * with `language: SupportedLanguages.JavaScript` so the resolver tries + * `.js` / `.jsx` extensions in addition to (or instead of) `.ts` / `.tsx`. + * + * The `TsResolveContext.language` flag already exists in `import-target.ts` + * and the resolver (`resolveImportPath`) already branches on it — this + * adapter just wires the right value in. + * + * CJS `require()` calls reference the same module-path strings as ESM + * `import` statements, so the resolver handles them uniformly without any + * CJS-specific logic here. + * + * No `tsconfig.json` path-alias support (JavaScript projects don't use + * `tsconfig.json` compilerOptions.paths in general). Projects that DO use + * tsconfig-based aliases alongside JavaScript can still resolve via the + * standard extension-suffix fallback; the alias branch is a no-op when + * `tsconfigPaths` is null. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import { resolveTsTarget, type TsResolveContext } from '../typescript/import-target.js'; + +export type JsResolveContext = TsResolveContext; + +type PassCache = { + readonly key: ReadonlySet; + readonly allFilePaths: Set; + readonly allFileList: readonly string[]; + readonly normalizedFileList: readonly string[]; + readonly resolveCache: Map; +}; + +/** + * Build a memoized `resolveImportTarget` adapter for JavaScript. + * Caches the derived arrays and per-pass resolve cache across + * `resolveImportTarget` calls within a single workspace pass. + */ +export function makeJsResolveImportTarget(): ( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, + resolutionConfig?: unknown, +) => string | readonly string[] | null { + let cached: PassCache | null = null; + + return (targetRaw, fromFile, allFilePaths) => { + if (cached === null || cached.key !== allFilePaths) { + const allFileList = Array.from(allFilePaths); + cached = { + key: allFilePaths, + allFilePaths: new Set(allFilePaths), + allFileList, + normalizedFileList: allFileList.map((f) => f.toLowerCase()), + resolveCache: new Map(), + }; + } + + const ws: JsResolveContext = { + fromFile, + language: SupportedLanguages.JavaScript, + allFilePaths: cached.allFilePaths, + allFileList: cached.allFileList, + normalizedFileList: cached.normalizedFileList, + resolveCache: cached.resolveCache, + tsconfigPaths: null, + }; + return resolveTsTarget(targetRaw, ws); + }; +} diff --git a/gitnexus/src/core/ingestion/languages/javascript/index.ts b/gitnexus/src/core/ingestion/languages/javascript/index.ts new file mode 100644 index 000000000..e887bab51 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/index.ts @@ -0,0 +1,49 @@ +/** + * JavaScript scope-resolution hooks (RFC #909 Ring 3, issue #928). + * + * Public API barrel. Consumers should import from this file rather + * than the individual modules. + * + * Module layout (each file is a single concern): + * + * - `query.ts` — JS scope query string + lazy parser/query + * singletons (`getJsParser`, `getJsScopeQuery`) + * - `captures.ts` — `emitJsScopeCaptures` — runs the JS scope query, + * synthesizes CJS require() imports and JSDoc- + * derived type bindings, delegates arity synthesis + * and destructuring/instanceof passes to shared + * or TypeScript utilities + * - `interpret.ts` — `interpretJsImport` / `interpretJsTypeBinding` + * (delegate to TypeScript interpreters — same + * capture-marker vocabulary) + * - `simple-hooks.ts` — `jsBindingScopeFor` (var hoisting), + * `jsImportOwningScope`, `jsReceiverBinding` + * (all delegate to TypeScript counterparts) + * - `merge-bindings.ts` — `jsMergeBindings` (LEGB via typescriptMergeBindings) + * - `arity.ts` — `jsArityCompatibility` (delegates to TS function) + * - `import-target.ts` — `makeJsResolveImportTarget` (memoized adapter) + * - `scope-resolver.ts` — `javascriptScopeResolver` wiring object + * + * ## Known limitations + * + * 1. **JSDoc coverage** — `@param {T} name`, `@returns {T}` / `@return {T}`, + * and `@type {T}` on variable declarations are synthesized. `@typedef` + * is not yet synthesized (tracked in #1646). + * 2. **CJS chained destructuring** — `const { X: { Y } } = require(...)` + * (nested destructuring) emits only the outer `X` binding; `Y` is not + * resolved. + * 3. **Dynamic require** — `require(computedPath)` is skipped (non-literal + * argument — cannot statically resolve the target). + * 4. **`module.exports` / `exports.X`** — CJS export forms are not yet + * modeled as re-exports. The finalize algorithm treats the exporting + * module as a namespace; importers that do `const X = require('./m')` + * bind the module namespace, and member-call resolution walks the + * class graph from there. + */ + +export { emitJsScopeCaptures } from './captures.js'; +export { interpretJsImport, interpretJsTypeBinding } from './interpret.js'; +export { jsMergeBindings } from './merge-bindings.js'; +export { jsArityCompatibility } from './arity.js'; +export { makeJsResolveImportTarget } from './import-target.js'; +export { jsBindingScopeFor, jsImportOwningScope, jsReceiverBinding } from './simple-hooks.js'; diff --git a/gitnexus/src/core/ingestion/languages/javascript/interpret.ts b/gitnexus/src/core/ingestion/languages/javascript/interpret.ts new file mode 100644 index 000000000..7d3688fa9 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/interpret.ts @@ -0,0 +1,45 @@ +/** + * Capture-match → semantic-shape interpreters for JavaScript. + * + * `interpretJsImport` delegates to `interpretTsImport` for all cases + * because `emitJsScopeCaptures` synthesizes the same + * `@import.kind/name/alias/source` markers for both ESM and CJS imports. + * + * The `@import.kind` values emitted for CJS by `captures.ts`: + * + * - `'named'` : `const { X } = require('./m')` → named import + * - `'named-alias'` : `const { X: Y } = require('./m')` → aliased import + * - `'namespace'` : `const X = require('./m')` → namespace import + * - `'side-effect'` : `require('./m')` bare expression → side-effect + * + * These match the kinds `interpretTsImport` already handles for ESM + * (`import { X }`, `import { X as Y }`, `import * as X`, `import './m'`), + * so no new branch is needed here. + * + * `interpretJsTypeBinding` handles the JS-only `@type-binding.class-field` + * tag before delegating to `interpretTsTypeBinding`. The class-field tag + * is emitted by `synthesizeConstructorFieldBindings` and should produce + * `source = 'annotation'` — the same strength as an explicit type + * annotation. Remapping it to `@type-binding.annotation` achieves this + * without adding a JS-specific branch to the shared TS interpreter + * (DoD.md §2.2). + */ + +import type { CaptureMatch, ParsedImport, ParsedTypeBinding } from 'gitnexus-shared'; +import { interpretTsImport, interpretTsTypeBinding } from '../typescript/interpret.js'; + +export function interpretJsImport(captures: CaptureMatch): ParsedImport | null { + return interpretTsImport(captures); +} + +export function interpretJsTypeBinding(captures: CaptureMatch): ParsedTypeBinding | null { + // @type-binding.class-field is a JS-only tag emitted by + // synthesizeConstructorFieldBindings. Remap it to the standard + // @type-binding.annotation tag so interpretTsTypeBinding assigns + // source = 'annotation' without a JS-specific branch in shared code. + if (captures['@type-binding.class-field'] !== undefined) { + const { '@type-binding.class-field': classField, ...rest } = captures; + return interpretTsTypeBinding({ ...rest, '@type-binding.annotation': classField }); + } + return interpretTsTypeBinding(captures); +} diff --git a/gitnexus/src/core/ingestion/languages/javascript/merge-bindings.ts b/gitnexus/src/core/ingestion/languages/javascript/merge-bindings.ts new file mode 100644 index 000000000..271c38684 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/merge-bindings.ts @@ -0,0 +1,21 @@ +/** + * Binding-merge precedence for JavaScript. + * + * JavaScript has no TypeScript declaration-merging (no `interface + class` + * coexisting in the same scope, no `namespace + class` dual-space declarations). + * However, `typescriptMergeBindings` handles these by falling back to + * `['value']` for any `NodeLabel` not explicitly mapped to multiple spaces — + * which is what every JavaScript declaration produces. The result is pure + * LEGB precedence without any cross-space logic, which is exactly what + * JavaScript needs. + * + * Reuse rather than reimplementing to keep the single source of truth for + * the tier (local 0 / import-namespace-reexport 1 / wildcard 2) ordering. + */ + +import type { BindingRef } from 'gitnexus-shared'; +import { typescriptMergeBindings } from '../typescript/merge-bindings.js'; + +export function jsMergeBindings(bindings: readonly BindingRef[]): readonly BindingRef[] { + return typescriptMergeBindings(bindings); +} diff --git a/gitnexus/src/core/ingestion/languages/javascript/query.ts b/gitnexus/src/core/ingestion/languages/javascript/query.ts new file mode 100644 index 000000000..20cfc4f2e --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/query.ts @@ -0,0 +1,421 @@ +/** + * Tree-sitter query for JavaScript scope captures (RFC §5.1, Ring 3). + * + * Subset of the TypeScript scope query (`languages/typescript/query.ts`) + * compiled against `tree-sitter-javascript`. TypeScript-only node types + * (`interface_declaration`, `type_alias_declaration`, `enum_declaration`, + * `internal_module`, `abstract_class_declaration`, `function_signature`, + * `method_signature`, `abstract_method_signature`, `type_annotation`, + * `public_field_definition`) are dropped because: + * + * 1. The JS grammar doesn't define them — the query compiler would + * throw `InvalidNodeType` if they were included. + * 2. JavaScript has no static type annotations, so the `@type-binding.*` + * patterns derived from TS annotation nodes don't apply. + * + * What IS shared with the TypeScript query: + * + * - Scope patterns: `program`, `class_declaration`, `(class)` (the JS + * grammar node for class expressions — NOT `class_expression`, which + * does not exist in `tree-sitter-javascript`), `function_declaration`, + * `generator_function_declaration`, `function_expression`, + * `arrow_function`, `method_definition`. + * - Declaration patterns for functions, classes, const/let/var, + * object-property arrows (Zustand, TanStack, etc.), and HOC-wrapped + * variable declarations (forwardRef / memo / useCallback / useMemo). + * - Import patterns: `import_statement`, `export_statement` re-exports, + * and dynamic `import()` (represented as `call_expression(import)` in + * both grammars — the `import` leaf node exists in tree-sitter-javascript + * as well as tree-sitter-typescript). + * - Type-binding patterns that work without static annotations: + * constructor inference (`new User()`), call-result alias + * (`const u = getUser()`), member-access alias (`const a = u.addr`), + * identifier alias, assignment rebind, and for-of element bindings. + * JSDoc-derived type bindings (`@param {User} u`, `@returns {User}`) + * are handled separately in `captures.ts` via comment-node scanning. + * - Reference patterns: free calls, member calls, constructor calls, + * write-access, read-access, and dynamic import. + * + * CJS `require()` is NOT captured here; it is handled in `captures.ts` + * by scanning parent context (destructured vs. namespace) of `call_expression` + * nodes whose callee is the identifier `require`. + * + * Grammar version: `tree-sitter-javascript` pinned in gitnexus/package.json. + * + * Exposes lazy `Parser` and `Query` singletons so callers don't pay + * tree-sitter init cost per file. + */ + +import Parser from 'tree-sitter'; +import JS from 'tree-sitter-javascript'; + +const JS_GRAMMAR = JS as Parameters[0]; + +/** True when the file should be parsed with the JSX-extended query. */ +function isJsxFile(filePath: string): boolean { + return filePath.endsWith('.jsx'); +} + +const JAVASCRIPT_SCOPE_QUERY = ` +;; Scopes — module / class-likes / function-likes +(program) @scope.module + +(class_declaration) @scope.class +(class) @scope.class + +(function_declaration) @scope.function +(generator_function_declaration) @scope.function +(function_expression) @scope.function +(arrow_function) @scope.function +(method_definition) @scope.function + +;; Declarations — classes +(class_declaration + name: (identifier) @declaration.name) @declaration.class + +;; Declarations — methods (inside class bodies) +(method_definition + name: (property_identifier) @declaration.name) @declaration.method + +;; Declarations — class fields (JS uses field_definition, not public_field_definition) +(field_definition + property: (property_identifier) @declaration.name) @declaration.property + +;; Declarations — free functions +(function_declaration + name: (identifier) @declaration.name) @declaration.function + +(generator_function_declaration + name: (identifier) @declaration.name) @declaration.function + +;; Arrow / function-expression assigned to a const/let/var. +;; Anchor discipline: @declaration.function sits on the INNER arrow or +;; function_expression, NOT on the lexical_declaration wrapper. This +;; aligns anchor.range with the @scope.function range so +;; pass2AttachDeclarations resolves the innermost scope correctly and +;; resolveCallerGraphId walks up to the right caller anchor. +(lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (arrow_function) @declaration.function)) + +(lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (function_expression) @declaration.function)) + +(export_statement + declaration: (lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (arrow_function) @declaration.function))) + +(export_statement + declaration: (lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (function_expression) @declaration.function))) + +(variable_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (arrow_function) @declaration.function)) + +(variable_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (function_expression) @declaration.function)) + +;; Object-property arrows / function expressions named by their pair key. +;; Same anchor discipline as the lexical_declaration block above: the +;; @declaration.function capture must sit on the INNER arrow/fn-expression. +(pair + key: (property_identifier) @declaration.name + value: (arrow_function) @declaration.function) + +(pair + key: (property_identifier) @declaration.name + value: (function_expression) @declaration.function) + +(pair + key: (string (string_fragment) @declaration.name) + value: (arrow_function) @declaration.function) + +(pair + key: (string (string_fragment) @declaration.name) + value: (function_expression) @declaration.function) + +;; HOC-wrapped variable declarations: const X = HOC((args) => { ... }). +;; Covers React.forwardRef, memo, useCallback, useMemo, observer, +;; debounce, and any user-defined HOC factory. +(lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (call_expression + arguments: (arguments + (arrow_function) @declaration.function)))) + +(lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (call_expression + arguments: (arguments + (function_expression) @declaration.function)))) + +(export_statement + declaration: (lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (call_expression + arguments: (arguments + (arrow_function) @declaration.function))))) + +(export_statement + declaration: (lexical_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (call_expression + arguments: (arguments + (function_expression) @declaration.function))))) + +(variable_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (call_expression + arguments: (arguments + (arrow_function) @declaration.function)))) + +(variable_declaration + (variable_declarator + name: (identifier) @declaration.name + value: (call_expression + arguments: (arguments + (function_expression) @declaration.function)))) + +;; Variable / constant declarations (non-function values). +(lexical_declaration + (variable_declarator + name: (identifier) @declaration.name)) @declaration.const + +(export_statement + declaration: (lexical_declaration + (variable_declarator + name: (identifier) @declaration.name))) @declaration.const + +(variable_declaration + (variable_declarator + name: (identifier) @declaration.name)) @declaration.variable + +;; Imports (ESM) — single anchor per statement; decomposer emits per-specifier markers. +(import_statement) @import.statement + +;; Re-exports with a source clause. +(export_statement + source: (string)) @import.statement + +;; Dynamic imports: import('./m') — tree-sitter-javascript represents this +;; as call_expression with a named import leaf as the function field, +;; identical to tree-sitter-typescript. +(call_expression + function: (import)) @import.dynamic + +;; ── Type bindings (no static annotations in JS; inferred from AST shape) ── + +;; Constructor-inferred: const u = new User() +(variable_declarator + name: (identifier) @type-binding.name + value: (new_expression + constructor: (identifier) @type-binding.type)) @type-binding.constructor + +;; Qualified constructor: const u = new models.User() +(variable_declarator + name: (identifier) @type-binding.name + value: (new_expression + constructor: (member_expression) @type-binding.type)) @type-binding.constructor + +;; Call-result alias: const u = getUser() +(variable_declarator + name: (identifier) @type-binding.name + value: (call_expression + function: (identifier) @type-binding.type)) @type-binding.alias + +;; Member-call alias: const u = svc.getUser() +(variable_declarator + name: (identifier) @type-binding.name + value: (call_expression + function: (member_expression) @type-binding.type)) @type-binding.alias + +;; Await chain: const u = await getUser() / await svc.getUser() +(variable_declarator + name: (identifier) @type-binding.name + value: (await_expression + (call_expression + function: (identifier) @type-binding.type))) @type-binding.alias + +(variable_declarator + name: (identifier) @type-binding.name + value: (await_expression + (call_expression + function: (member_expression) @type-binding.type))) @type-binding.alias + +;; Member-access alias: const addr = user.address +(variable_declarator + name: (identifier) @type-binding.name + value: (member_expression) @type-binding.type) @type-binding.member-alias + +;; Identifier alias: const alias = user +(variable_declarator + name: (identifier) @type-binding.name + value: (identifier) @type-binding.type) @type-binding.alias + +;; Assignment rebind: u = new User() / u = getUser() +(assignment_expression + left: (identifier) @type-binding.name + right: (new_expression + constructor: (identifier) @type-binding.type)) @type-binding.constructor + +(assignment_expression + left: (identifier) @type-binding.name + right: (call_expression + function: (identifier) @type-binding.type)) @type-binding.alias + +(assignment_expression + left: (identifier) @type-binding.name + right: (identifier) @type-binding.type) @type-binding.alias + +;; For-of element: for (const u of users) / for (const u of getUsers()) +(for_in_statement + left: (identifier) @type-binding.name + right: (identifier) @type-binding.type) @type-binding.alias + +(for_in_statement + left: (identifier) @type-binding.name + right: (call_expression + function: (identifier) @type-binding.type)) @type-binding.alias + +(for_in_statement + left: (identifier) @type-binding.name + right: (call_expression + function: (member_expression) @type-binding.type)) @type-binding.alias + +(for_in_statement + left: (identifier) @type-binding.name + right: (member_expression + property: (property_identifier) @type-binding.type)) @type-binding.alias + +;; ── References ──────────────────────────────────────────────────────────── + +;; Free calls: fn(args). The dynamic-import filter runs in captures.ts. +(call_expression + function: (identifier) @reference.name) @reference.call.free + +;; Awaited free call: await fn(...) re-associated by tree-sitter. +(call_expression + function: (await_expression + (identifier) @reference.name)) @reference.call.free + +;; Member calls: obj.method() (includes optional chain). +(call_expression + function: (member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name)) @reference.call.member + +;; Awaited member call: await svc.m(...) +(call_expression + function: (await_expression + (member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name))) @reference.call.member + +;; Constructor calls: new User() / new ns.User() +(new_expression + constructor: (identifier) @reference.name) @reference.call.constructor + +(new_expression + constructor: (member_expression) @reference.call.constructor.qualified) @reference.call.constructor + +;; Write access: obj.field = value +(assignment_expression + left: (member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name)) @reference.write.member + +(augmented_assignment_expression + left: (member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name)) @reference.write.member + +;; Read access: obj.field (in read context; captures.ts filters non-reads). +(member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name) @reference.read.member +`; + +/** JSX-only suffix — appended when compiling against the JSX grammar for .jsx files. */ +const JSX_QUERY_SUFFIX = ` +;; +((jsx_self_closing_element + name: (identifier) @reference.name) @reference.call.free + (#match? @reference.name "^[A-Z]")) + +;; ... +((jsx_opening_element + name: (identifier) @reference.name) @reference.call.free + (#match? @reference.name "^[A-Z]")) + +;; +(jsx_self_closing_element + name: (member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name)) @reference.call.member + +(jsx_opening_element + name: (member_expression + object: (_) @reference.receiver + property: (property_identifier) @reference.name)) @reference.call.member +`; + +let _jsParser: Parser | null = null; +let _jsQuery: Parser.Query | null = null; +let _jsxParser: Parser | null = null; +let _jsxQuery: Parser.Query | null = null; + +export function getJsParser(filePath?: string): Parser { + // JSX files use the same JavaScript grammar in tree-sitter-javascript; + // both .js and .jsx parse with the same grammar object. We keep separate + // singletons only to mirror the TypeScript pattern and in case a future + // version of the grammar diverges. + if (filePath !== undefined && isJsxFile(filePath)) { + if (_jsxParser === null) { + _jsxParser = new Parser(); + _jsxParser.setLanguage(JS_GRAMMAR); + } + return _jsxParser; + } + if (_jsParser === null) { + _jsParser = new Parser(); + _jsParser.setLanguage(JS_GRAMMAR); + } + return _jsParser; +} + +export function getJsScopeQuery(filePath?: string): Parser.Query { + if (filePath !== undefined && isJsxFile(filePath)) { + if (_jsxQuery === null) { + _jsxQuery = new Parser.Query(JS_GRAMMAR, JAVASCRIPT_SCOPE_QUERY + JSX_QUERY_SUFFIX); + } + return _jsxQuery; + } + if (_jsQuery === null) { + _jsQuery = new Parser.Query(JS_GRAMMAR, JAVASCRIPT_SCOPE_QUERY); + } + return _jsQuery; +} + +/** Validate that a cached Tree was produced by the JS grammar. */ +export function jsCachedTreeMatchesGrammar(tree: unknown): boolean { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const lang = (tree as any)?.getLanguage?.(); + if (lang === undefined || lang === null) return true; + return lang === JS_GRAMMAR; +} diff --git a/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts new file mode 100644 index 000000000..1536e793e --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts @@ -0,0 +1,89 @@ +/** + * JavaScript `ScopeResolver` registered in `SCOPE_RESOLVERS` and + * consumed by the generic `runScopeResolution` orchestrator + * (RFC #909 Ring 3, issue #928). + * + * Follows the same minimal wiring-only pattern as TypeScript (the third + * migration). Per-hook logic lives in sibling modules: + * + * - `query.ts` — JS scope query + parser/query singletons + * - `captures.ts` — `emitJsScopeCaptures` (JS grammar, CJS, JSDoc) + * - `interpret.ts` — `interpretJsImport` (delegates to TS interpreter) + * - `simple-hooks.ts` — `jsBindingScopeFor`, `jsImportOwningScope`, + * `jsReceiverBinding` (all delegate to TS hooks) + * - `merge-bindings.ts` — `jsMergeBindings` (delegates to TS function) + * - `arity.ts` — `jsArityCompatibility` (delegates to TS function) + * - `import-target.ts` — `makeJsResolveImportTarget` (TS resolver, JS extensions) + * + * See `./index.ts` for the full per-module rationale. + * + * ## Key differences from TypeScript resolver + * + * - `fieldFallbackOnMethodLookup: true` — JavaScript is dynamically typed; + * the field-fallback heuristic is ENABLED (unlike TypeScript, which + * disables it because the type-binding layer is precise). + * - `allowGlobalFreeCallFallback: true` — CJS `require` patterns and + * global helpers (e.g. `process`, `console`) benefit from workspace- + * wide unique-name fallback. TypeScript uses explicit imports. + * - `loadResolutionConfig` is omitted — JavaScript projects don't use + * `tsconfig.json` path aliases in general. `tsconfigPaths: null` is + * threaded through the resolver adapter. + * - `hoistTypeBindingsToModule: true` — JSDoc `@returns {T}` bindings are + * synthesized on the function scope and hoisted, matching TypeScript's + * method return-type hoisting strategy for cross-file chain resolution. + */ + +import type { ParsedFile } from 'gitnexus-shared'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; +import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; +import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import { javascriptProvider } from '../typescript.js'; +import { jsMergeBindings } from './merge-bindings.js'; +import { jsArityCompatibility } from './arity.js'; +import { makeJsResolveImportTarget } from './import-target.js'; + +const javascriptScopeResolver: ScopeResolver = { + language: SupportedLanguages.JavaScript, + languageProvider: javascriptProvider, + importEdgeReason: 'javascript-scope: import', + + resolveImportTarget: makeJsResolveImportTarget(), + + // JavaScript LEGB — same tier ordering as TypeScript; no declaration- + // merging across type/value/namespace spaces. + mergeBindings: (existing, incoming) => [...jsMergeBindings([...existing, ...incoming])], + + // Adapter: jsArityCompatibility uses (def, callsite); contract is (callsite, def). + arityCompatibility: (callsite, def) => jsArityCompatibility(def, callsite), + + buildMro: (graph, parsedFiles, nodeLookup) => + buildMro(graph, parsedFiles, nodeLookup, defaultLinearize), + + populateOwners: (parsed: ParsedFile) => populateClassOwnedMembers(parsed), + + // JavaScript `super` keyword: same pattern as TypeScript. + isSuperReceiver: (text) => /^super(\s*\(|\s*\.|\s*\[|\s*$)/.test(text.trim()), + + // JavaScript is dynamically typed — enable the field-fallback heuristic + // so member-call receivers without type annotations can still resolve + // through declared class fields (e.g. JSDoc-typed fields). + fieldFallbackOnMethodLookup: true, + + // Return-type propagation (across ESM imports) mirrors TypeScript's + // default behavior. JSDoc @returns bindings are hoisted to Module scope + // and propagated to importers via the standard mechanism. + propagatesReturnTypesAcrossImports: true, + + // JSDoc @returns bindings are synthesized on the function/method node + // and hoisted to Module scope by `jsBindingScopeFor` (identical to the + // TypeScript `tsBindingScopeFor` `@type-binding.return` branch). + hoistTypeBindingsToModule: true, + + // CJS-heavy codebases often have utility functions exported without + // explicit imports at the call site. Workspace-wide unique-name fallback + // recovers these edges. + allowGlobalFreeCallFallback: true, +}; + +export { javascriptScopeResolver }; diff --git a/gitnexus/src/core/ingestion/languages/javascript/simple-hooks.ts b/gitnexus/src/core/ingestion/languages/javascript/simple-hooks.ts new file mode 100644 index 000000000..c0947ba5a --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/javascript/simple-hooks.ts @@ -0,0 +1,48 @@ +/** + * Simple hooks for the JavaScript scope-resolution provider. + * + * `jsBindingScopeFor` wraps `tsBindingScopeFor` and adds the JS-only + * `@type-binding.class-field` hoisting rule. The other two hooks + * (`jsImportOwningScope`, `jsReceiverBinding`) are identical to their + * TypeScript counterparts and are re-exported directly. + * + * ## Why class-field hoisting lives here (not in `tsBindingScopeFor`) + * + * `@type-binding.class-field` is emitted exclusively by + * `synthesizeConstructorFieldBindings` in `captures.ts`, which is a + * JavaScript-only synthesis pass. TypeScript uses + * `@type-binding.parameter-property` for constructor parameter + * properties instead. Keeping the JS-only rule in the JS hook file + * prevents language-specific logic from leaking into shared TypeScript + * infrastructure (DoD.md §2.2). + */ + +import type { CaptureMatch, Scope, ScopeId, ScopeTree } from 'gitnexus-shared'; +import { tsBindingScopeFor, walkToScope } from '../typescript/simple-hooks.js'; + +export { + tsImportOwningScope as jsImportOwningScope, + tsReceiverBinding as jsReceiverBinding, +} from '../typescript/simple-hooks.js'; + +/** + * Like `tsBindingScopeFor` but additionally hoists + * `@type-binding.class-field` captures to the enclosing Class scope. + * + * `@type-binding.class-field` is anchored inside the constructor body + * (by `synthesizeConstructorFieldBindings`) so that `walkToScope` can + * walk up from the Function (constructor) scope to the Class scope. + * This puts `User.address → Address` in the class's typeBindings so + * compound-receiver resolution finds it when resolving + * `user.address.save()`. + */ +export function jsBindingScopeFor( + decl: CaptureMatch, + innermost: Scope, + tree: ScopeTree, +): ScopeId | null { + if (decl['@type-binding.class-field'] !== undefined) { + return walkToScope(innermost, tree, 'Class'); + } + return tsBindingScopeFor(decl, innermost, tree); +} diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index c3fad3518..f597396b6 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -56,6 +56,16 @@ import { typescriptArityCompatibility, resolveTsImportTarget, } from './typescript/index.js'; +import { + emitJsScopeCaptures, + interpretJsImport, + interpretJsTypeBinding, + jsBindingScopeFor, + jsImportOwningScope, + jsReceiverBinding, + jsMergeBindings, + jsArityCompatibility, +} from './javascript/index.js'; /** * TypeScript/JavaScript: arrow_function and function_expression are @@ -359,4 +369,19 @@ export const javascriptProvider = defineLanguage({ classExtractor: createClassExtractor(javascriptClassConfig), heritageExtractor: createHeritageExtractor(SupportedLanguages.JavaScript), builtInNames: BUILT_INS, + + // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── + // JavaScript is the fourth migration after Python, C#, and TypeScript. + // Hooks are thin wrappers over the TypeScript implementations where + // semantics are identical; JS-specific additions (CJS require(), + // JSDoc type bindings) live in ./javascript/captures.ts. + // See ./javascript/index.ts for the full per-module rationale. + emitScopeCaptures: emitJsScopeCaptures, + interpretImport: interpretJsImport, + interpretTypeBinding: interpretJsTypeBinding, + bindingScopeFor: jsBindingScopeFor, + importOwningScope: jsImportOwningScope, + mergeBindings: (_scope, bindings) => jsMergeBindings(bindings), + receiverBinding: jsReceiverBinding, + arityCompatibility: jsArityCompatibility, }); diff --git a/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts b/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts index 1dbb03dc7..f8f7302fa 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/simple-hooks.ts @@ -75,8 +75,11 @@ export function tsBindingScopeFor( * any of `kinds`. Returns the matching scope's id or `null` when no * ancestor matches (e.g., a return type binding emitted outside any * Module scope — shouldn't happen in well-formed input). + * + * Exported so language-specific hook wrappers (e.g. `jsBindingScopeFor`) + * can reuse it without duplicating the traversal logic. */ -function walkToScope( +export function walkToScope( from: Scope, tree: ScopeTree, ...kinds: readonly Scope['kind'][] diff --git a/gitnexus/src/core/ingestion/registry-primary-flag.ts b/gitnexus/src/core/ingestion/registry-primary-flag.ts index 869157adb..e050c6553 100644 --- a/gitnexus/src/core/ingestion/registry-primary-flag.ts +++ b/gitnexus/src/core/ingestion/registry-primary-flag.ts @@ -74,6 +74,7 @@ export const MIGRATED_LANGUAGES: ReadonlySet = new Set = n [SupportedLanguages.C, cScopeResolver], [SupportedLanguages.CPlusPlus, cppScopeResolver], [SupportedLanguages.PHP, phpScopeResolver], + [SupportedLanguages.JavaScript, javascriptScopeResolver], ]); diff --git a/gitnexus/test/fixtures/cross-file-binding/js-consumer-before-provider/b-provider.js b/gitnexus/test/fixtures/cross-file-binding/js-consumer-before-provider/b-provider.js index 99c85948a..b04e7c84a 100644 --- a/gitnexus/test/fixtures/cross-file-binding/js-consumer-before-provider/b-provider.js +++ b/gitnexus/test/fixtures/cross-file-binding/js-consumer-before-provider/b-provider.js @@ -2,6 +2,7 @@ export class User { save() {} } +/** @returns {User} */ export function getUser() { return new User(); } diff --git a/gitnexus/test/fixtures/cross-file-binding/js-cross-file/src/models.js b/gitnexus/test/fixtures/cross-file-binding/js-cross-file/src/models.js index 5d33e9be4..59bb80058 100644 --- a/gitnexus/test/fixtures/cross-file-binding/js-cross-file/src/models.js +++ b/gitnexus/test/fixtures/cross-file-binding/js-cross-file/src/models.js @@ -3,6 +3,7 @@ export class User { getName() { return ''; } } +/** @returns {User} */ export function getUser() { return new User(); } From 73543a471449f468bb9b831c053677b9651087fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 06:54:14 +0100 Subject: [PATCH 03/51] chore(deps)(deps-dev): bump @types/node in /gitnexus (#1696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 25.6.2 to 25.7.0. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node) --- updated-dependencies: - dependency-name: "@types/node" dependency-version: 25.7.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar --- gitnexus/package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index fe2115d07..1e5bb9a39 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -2065,12 +2065,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.6.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz", - "integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==", + "version": "25.7.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.7.0.tgz", + "integrity": "sha512-z+pdZyxE+RTQE9AcboAZCb4otwcrvgHD+GlBpPgn0emDVt0ohrTMhAwlr2Wd9nZ+nihhYFxO2pThz3C5qSu2Eg==", "license": "MIT", "dependencies": { - "undici-types": "~7.19.0" + "undici-types": "~7.21.0" } }, "node_modules/@types/qs": { @@ -5551,9 +5551,9 @@ } }, "node_modules/undici-types": { - "version": "7.19.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", - "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.21.0.tgz", + "integrity": "sha512-w9IMgQrz4O0YN1LtB7K5P63vhlIOvC7opSmouCJ+ZywlPAlO9gIkJ+otk6LvGpAs2wg4econaCz3TvQ9xPoyuQ==", "license": "MIT" }, "node_modules/universalify": { From 637cfca39ce2054956fe7a10a7d546e6a98c06d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 06:54:33 +0100 Subject: [PATCH 04/51] chore(deps)(deps): bump express-rate-limit in /gitnexus (#1697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [express-rate-limit](https://github.com/express-rate-limit/express-rate-limit) from 8.5.1 to 8.5.2. - [Release notes](https://github.com/express-rate-limit/express-rate-limit/releases) - [Commits](https://github.com/express-rate-limit/express-rate-limit/compare/v8.5.1...v8.5.2) --- updated-dependencies: - dependency-name: express-rate-limit dependency-version: 8.5.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> --- gitnexus/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 1e5bb9a39..1881b8ab0 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -3059,9 +3059,9 @@ } }, "node_modules/express-rate-limit": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.1.tgz", - "integrity": "sha512-5O6KYmyJEpuPJV5hNTXKbAHWRqrzyu+OI3vUnSd2kXFubIVpG7ezpgxQy76Zo5GQZtrQBg86hF+CM/NX+cioiQ==", + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", "license": "MIT", "dependencies": { "ip-address": "^10.2.0" From 6210d80f1e790640848baaefc6144a9d99096c6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 06:55:04 +0100 Subject: [PATCH 05/51] chore(deps)(deps-dev): bump tsx from 4.21.0 to 4.21.1 in /gitnexus (#1698) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [tsx](https://github.com/privatenumber/tsx) from 4.21.0 to 4.21.1. - [Release notes](https://github.com/privatenumber/tsx/releases) - [Changelog](https://github.com/privatenumber/tsx/blob/master/release.config.cjs) - [Commits](https://github.com/privatenumber/tsx/compare/v4.21.0...v4.21.1) --- updated-dependencies: - dependency-name: tsx dependency-version: 4.21.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> --- gitnexus/package-lock.json | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 1881b8ab0..253b770d5 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -3284,19 +3284,6 @@ "node": ">= 0.4" } }, - "node_modules/get-tsconfig": { - "version": "4.13.7", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.7.tgz", - "integrity": "sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, "node_modules/gitnexus-shared": { "resolved": "../gitnexus-shared", "link": true @@ -4692,16 +4679,6 @@ "node": ">=0.10.0" } }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, "node_modules/rolldown": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.1.tgz", @@ -5492,14 +5469,13 @@ "optional": true }, "node_modules/tsx": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", - "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.1.tgz", + "integrity": "sha512-5QE2Q04cN1u0993w0LT5rPw3faZqZU1fFn1mGE0pV53N1Dn7c+QFFxQu1mBeSgeOXwFyTicZw02wVgp3Tb5cAQ==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "~0.27.0", - "get-tsconfig": "^4.7.5" + "esbuild": "~0.27.0" }, "bin": { "tsx": "dist/cli.mjs" From 15a667ae5ec77ca261902561e8574664c119a73a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 06:55:24 +0100 Subject: [PATCH 06/51] chore(deps)(deps-dev): bump @vitest/coverage-v8 in /gitnexus (#1604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@vitest/coverage-v8](https://github.com/vitest-dev/vitest/tree/HEAD/packages/coverage-v8) from 4.1.5 to 4.1.6. - [Release notes](https://github.com/vitest-dev/vitest/releases) - [Commits](https://github.com/vitest-dev/vitest/commits/v4.1.6/packages/coverage-v8) --- updated-dependencies: - dependency-name: "@vitest/coverage-v8" dependency-version: 4.1.6 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> From c746f30c909b9f6c3ab9ebdcc2949ddec7fe618b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 06:55:44 +0100 Subject: [PATCH 07/51] chore(deps)(deps): bump langsmith (#1552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the npm_and_yarn group with 1 update in the /gitnexus-web directory: [langsmith](https://github.com/langchain-ai/langsmith-sdk). Updates `langsmith` from 0.5.23 to 0.6.3 - [Release notes](https://github.com/langchain-ai/langsmith-sdk/releases) - [Commits](https://github.com/langchain-ai/langsmith-sdk/commits) --- updated-dependencies: - dependency-name: langsmith dependency-version: 0.6.3 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> --- gitnexus-web/package-lock.json | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index cf69e1083..6cf5bcf2d 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -5599,13 +5599,12 @@ } }, "node_modules/langsmith": { - "version": "0.5.23", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.5.23.tgz", - "integrity": "sha512-dE/M/2Gg2S2R8ygDdkWGJVO3JstijvsNvPXsy9V8WGbpb88Zn8xF/aTjPx4mIy5gIoo02T6FssOgYyLf51Dv1Q==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.6.3.tgz", + "integrity": "sha512-pXrQ4/4myQvjFFOAUmt5pWRrLEZR20gzIJD7MNdUH+5/S5nLI4ZRBo/SYKC6coaYj9pYTfQdBIzcs+3kfJ5uDA==", "license": "MIT", "dependencies": { - "p-queue": "6.6.2", - "uuid": "10.0.0" + "p-queue": "6.6.2" }, "peerDependencies": { "@opentelemetry/api": "*", @@ -5632,19 +5631,6 @@ } } }, - "node_modules/langsmith/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/layout-base": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", From dad1ca7ab55ee3dae13729e4cbbce54b8c070f15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 06:56:04 +0100 Subject: [PATCH 08/51] chore(deps)(deps): bump zod from 3.25.76 to 4.3.6 in /gitnexus-web (#1464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [zod](https://github.com/colinhacks/zod) from 3.25.76 to 4.3.6. - [Release notes](https://github.com/colinhacks/zod/releases) - [Commits](https://github.com/colinhacks/zod/compare/v3.25.76...v4.3.6) --- updated-dependencies: - dependency-name: zod dependency-version: 4.3.6 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> --- gitnexus-web/package-lock.json | 8 ++++---- gitnexus-web/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 6cf5bcf2d..6297ec5c5 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -41,7 +41,7 @@ "sigma": "^3.0.2", "tailwindcss": "^4.2.4", "uuid": "^14.0.0", - "zod": "^3.25.76" + "zod": "^4.3.6" }, "devDependencies": { "@babel/types": "^7.29.0", @@ -8889,9 +8889,9 @@ } }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 211bb65c7..e93dc82c2 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -51,7 +51,7 @@ "sigma": "^3.0.2", "tailwindcss": "^4.2.4", "uuid": "^14.0.0", - "zod": "^3.25.76" + "zod": "^4.3.6" }, "devDependencies": { "@babel/types": "^7.29.0", From 18167400c45cea270d544a9b98350ed26cdd90bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 07:43:07 +0100 Subject: [PATCH 09/51] chore(deps)(deps): bump express and @types/express in /gitnexus (#872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps)(deps): bump express and @types/express in /gitnexus Bumps [express](https://github.com/expressjs/express) and [@types/express](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/express). These dependencies needed to be updated together. Updates `express` from 4.22.1 to 5.2.1 - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/v4.22.1...v5.2.1) Updates `@types/express` from 4.17.25 to 5.0.6 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/express) --- updated-dependencies: - dependency-name: "@types/express" dependency-version: 5.0.6 dependency-type: direct:development update-type: version-update:semver-major - dependency-name: express dependency-version: 5.2.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * fix(server): normalize jobId param for Express 5 SSE routes Express 5 types req.params values as string | string[]. mountSSEProgress uses a dynamic route path so TypeScript cannot narrow jobId; assert it once with assertString and reuse in the SSE progress callback. Co-authored-by: Cursor * chore(ci): retrigger CI Co-authored-by: Cursor --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: Cursor --- gitnexus/package-lock.json | 786 +++++++++---------------------------- gitnexus/package.json | 4 +- gitnexus/src/server/api.ts | 11 +- 3 files changed, 201 insertions(+), 600 deletions(-) diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 253b770d5..adfac8ee8 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -17,7 +17,7 @@ "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", - "express": "^4.19.2", + "express": "^5.2.1", "express-rate-limit": "^8.4.1", "glob": "^13.0.6", "graphology": "^0.26.0", @@ -52,7 +52,7 @@ "devDependencies": { "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", - "@types/express": "^4.17.21", + "@types/express": "^5.0.6", "@types/js-yaml": "^4.0.9", "@types/node": "^25.6.0", "@types/uuid": "^11.0.0", @@ -1290,268 +1290,6 @@ } } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/accepts": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", - "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", - "license": "MIT", - "dependencies": { - "mime-types": "^3.0.0", - "negotiator": "^1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", - "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.3", - "http-errors": "^2.0.0", - "iconv-lite": "^0.7.0", - "on-finished": "^2.4.1", - "qs": "^6.14.1", - "raw-body": "^3.0.1", - "type-is": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", - "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", - "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", - "license": "MIT", - "engines": { - "node": ">=6.6.0" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", - "license": "MIT", - "dependencies": { - "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", - "cookie": "^0.7.1", - "cookie-signature": "^1.2.1", - "debug": "^4.4.0", - "depd": "^2.0.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "finalhandler": "^2.1.0", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "merge-descriptors": "^2.0.0", - "mime-types": "^3.0.0", - "on-finished": "^2.4.1", - "once": "^1.4.0", - "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", - "range-parser": "^1.2.1", - "router": "^2.2.0", - "send": "^1.1.0", - "serve-static": "^2.2.0", - "statuses": "^2.0.1", - "type-is": "^2.0.1", - "vary": "^1.1.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", - "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "on-finished": "^2.4.1", - "parseurl": "^1.3.3", - "statuses": "^2.0.1" - }, - "engines": { - "node": ">= 18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", - "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", - "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", - "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "fresh": "^2.0.0", - "http-errors": "^2.0.1", - "mime-types": "^3.0.2", - "ms": "^2.1.3", - "on-finished": "^2.4.1", - "range-parser": "^1.2.1", - "statuses": "^2.0.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", - "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", - "license": "MIT", - "dependencies": { - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "parseurl": "^1.3.3", - "send": "^1.2.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", - "license": "MIT", - "dependencies": { - "content-type": "^1.0.5", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -2018,22 +1756,21 @@ "license": "MIT" }, "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "dev": true, "license": "MIT", "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.19.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", - "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz", + "integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==", "dev": true, "license": "MIT", "dependencies": { @@ -2057,13 +1794,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/node": { "version": "25.7.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.7.0.tgz", @@ -2074,9 +1804,9 @@ } }, "node_modules/@types/qs": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz", - "integrity": "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow==", + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", "dev": true, "license": "MIT" }, @@ -2098,25 +1828,13 @@ } }, "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", "dev": true, "license": "MIT", "dependencies": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mime": "^1", "@types/node": "*" } }, @@ -2276,13 +1994,13 @@ } }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" @@ -2360,12 +2078,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -2407,57 +2119,27 @@ } }, "node_modules/body-parser": { - "version": "1.20.4", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", - "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", "license": "MIT", "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.14.0", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/body-parser/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/body-parser/node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "unpipe": "~1.0.0" + "node": ">=18" }, - "engines": { - "node": ">= 0.8" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/brace-expansion": { @@ -2612,15 +2294,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -2649,10 +2332,13 @@ } }, "node_modules/cookie-signature": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", - "license": "MIT" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/cors": { "version": "2.8.6", @@ -2784,16 +2470,6 @@ "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -3013,45 +2689,42 @@ } }, "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.3", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.14.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" }, "engines": { - "node": ">= 0.10.0" + "node": ">= 18" }, "funding": { "type": "opencollective", @@ -3076,21 +2749,6 @@ "express": ">= 4.11" } }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/fast-copy": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.3.tgz", @@ -3144,38 +2802,26 @@ } }, "node_modules/finalhandler": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/finalhandler/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/finalhandler/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/flatbuffers": { "version": "25.9.23", "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", @@ -3192,12 +2838,12 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/fs-extra": { @@ -3499,15 +3145,19 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/ignore": { @@ -4018,63 +3668,49 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", + "engines": { + "node": ">=18" + }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", "dependencies": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/minimatch": { @@ -4157,9 +3793,9 @@ } }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -4363,12 +3999,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" - }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -4621,22 +4251,6 @@ "node": ">= 0.10" } }, - "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -4739,26 +4353,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/safe-stable-stringify": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", @@ -4803,44 +4397,31 @@ } }, "node_modules/send": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", - "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.1", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "~2.4.1", - "range-parser": "~1.2.1", - "statuses": "~2.0.2" + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/serialize-error": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", @@ -4857,18 +4438,22 @@ } }, "node_modules/serve-static": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", - "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "~0.19.1" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/setprototypeof": { @@ -5500,16 +5085,34 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { - "node": ">= 0.6" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/typescript": { @@ -5556,15 +5159,6 @@ "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", "license": "MIT" }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.0.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 7447961ba..3863a0457 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -60,7 +60,7 @@ "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", - "express": "^4.19.2", + "express": "^5.2.1", "express-rate-limit": "^8.4.1", "glob": "^13.0.6", "graphology": "^0.26.0", @@ -100,7 +100,7 @@ "devDependencies": { "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", - "@types/express": "^4.17.21", + "@types/express": "^5.0.6", "@types/js-yaml": "^4.0.9", "@types/node": "^25.6.0", "@types/uuid": "^11.0.0", diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 4c08857ba..7fc892574 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -448,7 +448,14 @@ export const streamGraphNdjson = async ( */ const mountSSEProgress = (app: express.Express, routePath: string, jm: JobManager) => { app.get(routePath, (req, res) => { - const job = jm.getJob(req.params.jobId); + let jobId: string; + try { + jobId = assertString(req.params.jobId, 'jobId'); + } catch (err: any) { + res.status(err.status ?? 400).json({ error: err.message }); + return; + } + const job = jm.getJob(jobId); if (!job) { res.status(404).json({ error: 'Job not found' }); return; @@ -494,7 +501,7 @@ const mountSSEProgress = (app: express.Express, routePath: string, jm: JobManage try { eventId++; if (progress.phase === 'complete' || progress.phase === 'failed') { - const eventJob = jm.getJob(req.params.jobId); + const eventJob = jm.getJob(jobId); res.write( `id: ${eventId}\nevent: ${progress.phase}\ndata: ${JSON.stringify({ repoName: eventJob?.repoName, From 55f8d442f6f44b3611e3eba9c5d1d09f11e36663 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 19 May 2026 08:17:26 +0100 Subject: [PATCH 10/51] fix(mcp): setup fallback on Windows when global `gitnexus` resolves to a non-spawnable shim (#1694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * fix: avoid invalid Windows MCP shim paths Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/a052306e-483a-42d0-b65a-2646906457c7 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * test: cover .ps1 windows mcp fallback Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/5aaed570-2a0b-4ed9-a0ac-ca099ce5675e Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * test: assert windows fallback for cursor and codex Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/5aaed570-2a0b-4ed9-a0ac-ca099ce5675e Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * chore(autofix): apply prettier + eslint fixes via /autofix command --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> Co-authored-by: Gergő Magyar Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- gitnexus/src/cli/setup.ts | 10 +++-- gitnexus/test/unit/setup-codex.test.ts | 20 +++++++++ gitnexus/test/unit/setup-jsonc.test.ts | 60 ++++++++++++++++++++++++++ gitnexus/test/unit/setup.test.ts | 24 +++++++++-- 4 files changed, 106 insertions(+), 8 deletions(-) diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index 3e7cbad8f..fe9d86f52 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -61,11 +61,13 @@ function resolveGitnexusBin(): string | null { .filter(Boolean); if (isWin) { - // On Windows, `where` returns multiple entries (e.g. the POSIX shell - // script AND the .cmd/.bat wrapper). Prefer the wrapper because - // child_process.spawn() cannot execute a shell script directly. + // On Windows, npm global installs can surface multiple launchers for the + // same package (e.g. a POSIX shell shim plus .cmd/.bat wrappers). Claude + // and the other MCP hosts need a directly spawnable command path, so only + // accept the Windows wrapper. If it is missing, fall back to the slower + // npx entry instead of persisting a non-spawnable shim path. const cmdLine = lines.find((l) => /\.(cmd|bat)$/i.test(l)); - return cmdLine || lines[0] || null; + return cmdLine || null; } return lines[0] || null; diff --git a/gitnexus/test/unit/setup-codex.test.ts b/gitnexus/test/unit/setup-codex.test.ts index 95761bed0..5951c9325 100644 --- a/gitnexus/test/unit/setup-codex.test.ts +++ b/gitnexus/test/unit/setup-codex.test.ts @@ -15,8 +15,13 @@ const execFileMock = vi.fn((...args: any[]) => { } }); +const execFileSyncMock = vi.fn(() => { + throw new Error('not found'); +}); + vi.mock('child_process', () => ({ execFile: execFileMock, + execFileSync: execFileSyncMock, })); describe('setupCommand codex execution', () => { @@ -74,6 +79,21 @@ describe('setupCommand codex execution', () => { ); }); + it('uses Windows npx fallback arguments when where returns only a non-wrapper shim', async () => { + execFileSyncMock.mockReturnValueOnce('C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus\n'); + + const { setupCommand } = await import('../../src/cli/setup.js'); + + await setupCommand(); + + expect(execFileMock).toHaveBeenCalledWith( + 'codex', + ['mcp', 'add', 'gitnexus', '--', 'cmd', '/c', 'npx', '-y', NPX_REF, 'mcp'], + { shell: true }, + expect.any(Function), + ); + }); + it('invokes codex mcp add without shell on non-Windows and does not write fallback config', async () => { setPlatform('darwin'); diff --git a/gitnexus/test/unit/setup-jsonc.test.ts b/gitnexus/test/unit/setup-jsonc.test.ts index e31955950..f4f44c618 100644 --- a/gitnexus/test/unit/setup-jsonc.test.ts +++ b/gitnexus/test/unit/setup-jsonc.test.ts @@ -241,6 +241,50 @@ describe('setupOpenCode — JSONC preservation', () => { }); }); + it('uses Windows npx fallback when where returns only a non-wrapper shim', async () => { + setPlatform('win32'); + execFileSyncMock.mockReturnValueOnce('C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus\n'); + + const jsonc = `{ + "model": "test", + "mcp": {} +}`; + await fs.writeFile(opencodeJsonPath(), jsonc, 'utf-8'); + + const { setupCommand } = await import('../../src/cli/setup.js'); + await setupCommand(); + + const raw = await fs.readFile(opencodeJsonPath(), 'utf-8'); + const config = parseJsonc(raw); + + expect(config.mcp.gitnexus).toEqual({ + type: 'local', + command: ['cmd', '/c', 'npx', '-y', NPX_REF, 'mcp'], + }); + }); + + it('uses Windows npx fallback when where returns only a .ps1 path', async () => { + setPlatform('win32'); + execFileSyncMock.mockReturnValueOnce('C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus.ps1\n'); + + const jsonc = `{ + "model": "test", + "mcp": {} +}`; + await fs.writeFile(opencodeJsonPath(), jsonc, 'utf-8'); + + const { setupCommand } = await import('../../src/cli/setup.js'); + await setupCommand(); + + const raw = await fs.readFile(opencodeJsonPath(), 'utf-8'); + const config = parseJsonc(raw); + + expect(config.mcp.gitnexus).toEqual({ + type: 'local', + command: ['cmd', '/c', 'npx', '-y', NPX_REF, 'mcp'], + }); + }); + it('preserves tab indentation in existing file', async () => { const tabbed = `{\n\t"model": "test"\n}`; await fs.writeFile(opencodeJsonPath(), tabbed, 'utf-8'); @@ -360,6 +404,22 @@ describe('setupCursor — JSONC preservation', () => { const raw = await fs.readFile(mcpPath(), 'utf-8'); expect(raw).toBe(corrupt); }); + + it('uses Windows npx fallback when where returns only a non-wrapper shim', async () => { + setPlatform('win32'); + execFileSyncMock.mockReturnValueOnce('C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus\n'); + + const { setupCommand } = await import('../../src/cli/setup.js'); + await setupCommand(); + + const raw = await fs.readFile(mcpPath(), 'utf-8'); + const config = parseJsonc(raw); + + expect(config.mcpServers.gitnexus).toEqual({ + command: 'cmd', + args: ['/c', 'npx', '-y', NPX_REF, 'mcp'], + }); + }); }); describe('setupClaudeCode — JSONC preservation', () => { diff --git a/gitnexus/test/unit/setup.test.ts b/gitnexus/test/unit/setup.test.ts index bdf1cd1fd..ee43d97d7 100644 --- a/gitnexus/test/unit/setup.test.ts +++ b/gitnexus/test/unit/setup.test.ts @@ -282,9 +282,9 @@ describe('setupClaudeCode', () => { ).resolves.toBeUndefined(); }); - it('falls back to first line on Windows when no .cmd/.bat wrapper found', async () => { + it('falls back to npx on Windows when no .cmd/.bat wrapper is found', async () => { setPlatform('win32'); - // Edge case: where returns only the POSIX script (no .cmd wrapper) + // Edge case: where returns only a non-spawnable shim (no .cmd wrapper) execFileSyncMock.mockReturnValueOnce('C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus\n'); const { setupCommand } = await import('../../src/cli/setup.js'); @@ -294,8 +294,24 @@ describe('setupClaudeCode', () => { const config = JSON.parse(raw); expect(config.mcpServers.gitnexus).toEqual({ - command: 'C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus', - args: ['mcp'], + command: 'cmd', + args: ['/c', 'npx', '-y', NPX_REF, 'mcp'], + }); + }); + + it('falls back to npx on Windows when where returns only a .ps1 path', async () => { + setPlatform('win32'); + execFileSyncMock.mockReturnValueOnce('C:\\Users\\dev\\AppData\\Roaming\\npm\\gitnexus.ps1\n'); + + const { setupCommand } = await import('../../src/cli/setup.js'); + await setupCommand(); + + const raw = await fs.readFile(path.join(tempHome, '.claude.json'), 'utf-8'); + const config = JSON.parse(raw); + + expect(config.mcpServers.gitnexus).toEqual({ + command: 'cmd', + args: ['/c', 'npx', '-y', NPX_REF, 'mcp'], }); }); }); From 803f0bed5f7d714d3ee8577e1e42f17bbf371dca Mon Sep 17 00:00:00 2001 From: LocallyInsaneDB Date: Tue, 19 May 2026 13:09:21 +0200 Subject: [PATCH 11/51] fix(lbug): probe-then-load FTS extension on Windows (#1690) (#1692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(lbug): probe-then-load FTS extension on Windows (#1690) The Windows skip-on-process.platform==='win32' guard in pool-adapter.ts hard-skipped loadFTSExtension() for every Windows host, even when the FTS extension binary was already present locally at ~/.lbdb/extension//win_amd64/fts/libfts.lbug_extension. That left BM25 silently degraded on Windows hosts that had a working extension on disk, with no error path — `gitnexus doctor` still reported FTS as available, but query returned 0 BM25 hits. This patch adds hasLocalWinFtsExtension() which probes ~/.lbdb/extension/*/win_amd64/fts/ before the Windows skip. When a binary is on disk we call loadFTSExtension(..., { policy: 'load-only' }); the crashing install path documented in #1199 / #1217 is never exercised at query time, and LadybugDB's version-specific resolution combined with the ExtensionManager's tryLoad try/catch handles stale or zero-byte sibling version dirs cleanly (no dlopen attempted on a stale binary). When no binary is on disk at all, we fall back to the upstream skip so install-time SIGSEGV continues to be avoided. Verified on Windows 10 + Node 22.19.0 + gitnexus 1.6.5 + @ladybugdb/core 0.16.1 with the FTS extension cached at 0.16.0: * BM25 timing goes from 0 → ~250-326ms on previously-zero queries * gitnexus context / impact / cypher unaffected * Adversarial-mixed-state run (real 0.16.0 binary + zero-byte stubs at 0.15.0, 0.16.1, 0.17.0): exits 0, no SIGSEGV, FTS resolves to the real 0.16.0 binary, BM25 returns real hits * Stub-only state at the resolution path (0.16.0, zero-byte): exits 0, emits "FTS extension unavailable; load-only policy: extension not pre-installed", FTS marked unavailable cleanly via markUnavailable in extension-loader.ts — no silent greenlight Closes #1690 * test(lbug): cover hasLocalWinFtsExtension probe + format pool-adapter - Export hasLocalWinFtsExtension and add lbug-pool-win-fts-probe.test.ts with 7 cases against a real tmpdir + os.homedir spy: * missing ~/.lbdb/extension dir -> false * extension root present but no version dirs -> false * one version dir with binary present -> true * zero-byte stub at probe path -> true (LOAD failure handled downstream) * multi-version with binary only in a non-first dir -> true * multi-version with no binary anywhere (Nix/Bazel/MDM tree) -> false * fs.readdir throws (EACCES) -> false The Windows conditional in doInitLbug / initLbugWithDb is intentionally not unit-isolated: it reduces to `probe ? load : true` over a fully constructed lbug.Database + Connection pool, which the test/integration/lbug-pool*.test.ts suites already exercise on the windows-latest CI matrix. - Apply prettier format to the fs.stat() call in pool-adapter.ts, resolving the quality/format CI failure surfaced by gitnexus/autofix. Addresses DoD §2.7 test-coverage blocker raised in the production- readiness review on #1692, and the dir-exists-no-file regression case raised on #1690. Refs #1690. --------- Co-authored-by: Gergő Magyar --- gitnexus/src/core/lbug/pool-adapter.ts | 68 +++++++-- .../test/unit/lbug-pool-win-fts-probe.test.ts | 132 ++++++++++++++++++ 2 files changed, 191 insertions(+), 9 deletions(-) create mode 100644 gitnexus/test/unit/lbug-pool-win-fts-probe.test.ts diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index d373d13c4..f551c65f8 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -16,6 +16,8 @@ */ import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; import lbug from '@ladybugdb/core'; import { isReadOnlyDbError, loadFTSExtension } from './lbug-adapter.js'; import { @@ -24,6 +26,42 @@ import { WAL_RECOVERY_SUGGESTION, } from './lbug-config.js'; +/** + * Probe whether a Windows FTS extension binary is locally installed under + * ~/.lbdb/extension//win_amd64/fts/. Returns true on the first + * version dir whose libfts.lbug_extension exists on disk; false if the + * extension root is missing or contains no FTS binary. + * + * Gates the Windows skip-FTS-load guard below so we only skip the load + * when no extension binary is present. When at least one binary exists, + * loadFTSExtension is called with policy: 'load-only' — LadybugDB resolves + * LOAD EXTENSION fts to its version-specific path internally, and the + * ExtensionManager's tryLoad try/catch handles version-mismatch errors + * cleanly without ever attempting dlopen of a stale binary. The install + * path that the #1199/#1217 SIGSEGV documented is never exercised at + * query time. + * + * Exported so unit tests can exercise the probe directly against a + * temp-dir plus spied `os.homedir()` — see lbug-pool-win-fts-probe.test.ts. + */ +export async function hasLocalWinFtsExtension(): Promise { + try { + const extRoot = path.join(os.homedir(), '.lbdb', 'extension'); + const versions = await fs.readdir(extRoot); + for (const v of versions) { + try { + await fs.stat(path.join(extRoot, v, 'win_amd64', 'fts', 'libfts.lbug_extension')); + return true; + } catch { + /* missing for this version, keep looking */ + } + } + } catch { + /* no .lbdb/extension dir */ + } + return false; +} + /** Per-repo pool: one Database, many Connections */ interface PoolEntry { db: lbug.Database; @@ -423,14 +461,24 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { // install; analyze owns extension installation. If LOAD fails, search // features degrade gracefully and the user-facing query path proceeds. if (!shared.ftsLoaded) { - // Windows guard: LOAD EXTENSION fts crashes with SIGSEGV on Windows when - // the FTS extension binary is not installed locally (@ladybugdb/core native - // bug — the extension loader hits an unhandled error path that signals SIGSEGV - // rather than throwing a JS exception, so try/catch cannot protect here). - // Skip the load on Windows; bm25-index.js catches the resulting Kuzu catalog - // errors and returns empty BM25 results gracefully. Graph queries are unaffected. + // Windows guard: LOAD EXTENSION fts crashes with SIGSEGV on Windows during + // *install* — the @ladybugdb/core out-of-process installer hits an unhandled + // error path that signals SIGSEGV instead of throwing (see #1199, #1217). + // The previous unconditional skip was over-broad: it also disabled FTS on + // hosts where the binary was already on disk and only needed LOAD, leaving + // BM25 silently degraded with no error path (see #1690). + // + // Probe ~/.lbdb/extension/*/win_amd64/fts/ first. If any binary is on disk + // we run loadFTSExtension(..., 'load-only'); the install path is never + // exercised, and LadybugDB's version-specific resolution + ExtensionManager + // try/catch handle stale/zero-byte siblings cleanly (verified empirically + // on Win10 + Node 22.19 + gitnexus 1.6.5 + @ladybugdb/core 0.16.1). With + // no binary at all, we fall back to the upstream skip so install-time + // SIGSEGV continues to be avoided. if (process.platform === 'win32') { - shared.ftsLoaded = true; + shared.ftsLoaded = (await hasLocalWinFtsExtension()) + ? await loadFTSExtension(available[0], { policy: 'load-only' }) + : true; } else { shared.ftsLoaded = await loadFTSExtension(available[0], { policy: 'load-only' }); } @@ -497,10 +545,12 @@ export async function initLbugWithDb( // Load FTS extension if not already loaded on this Database. // policy: 'load-only' — same contract as initLbug above; the read pool // must not block on a network install during query execution. - // Windows guard: same SIGSEGV risk as doInitLbug above — skip on Windows. + // Windows guard: same probe-then-load policy as doInitLbug above. if (!shared.ftsLoaded) { if (process.platform === 'win32') { - shared.ftsLoaded = true; + shared.ftsLoaded = (await hasLocalWinFtsExtension()) + ? await loadFTSExtension(available[0], { policy: 'load-only' }) + : true; } else { shared.ftsLoaded = await loadFTSExtension(available[0], { policy: 'load-only' }); } diff --git a/gitnexus/test/unit/lbug-pool-win-fts-probe.test.ts b/gitnexus/test/unit/lbug-pool-win-fts-probe.test.ts new file mode 100644 index 000000000..fec1fe471 --- /dev/null +++ b/gitnexus/test/unit/lbug-pool-win-fts-probe.test.ts @@ -0,0 +1,132 @@ +/** + * Unit tests for the Windows FTS probe in pool-adapter.ts. + * + * Covers `hasLocalWinFtsExtension()` — the helper that gates the + * Windows-only skip of `loadFTSExtension` in `doInitLbug` and + * `initLbugWithDb`. Issue #1690 / PR #1692. + * + * The probe is exercised against a real temp filesystem with + * `os.homedir()` spied to point at the tempdir. This tests the + * actual fs surface (readdir/stat semantics, missing-dir behavior, + * zero-byte file handling) rather than mocking fs internals. + * + * The Windows-branch conditional in `doInitLbug` / `initLbugWithDb` + * is intentionally not unit-tested in isolation: those functions + * require a fully constructed `lbug.Database` + `Connection` pool + * and are exercised end-to-end by `test/integration/lbug-pool*.test.ts` + * on the `windows-latest` matrix. The conditional itself is a single + * expression — `(await hasLocalWinFtsExtension()) ? load : true` — + * whose correctness reduces to the probe being correctly tested here. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import os from 'os'; +import path from 'path'; +import fs from 'fs/promises'; + +// Stub out the LadybugDB native loader and its transitive importers so that +// importing pool-adapter.ts in this unit test does not pull in the .node binary +// (which is built by the postinstall script and is not always present in the +// dev install used for unit tests). +vi.mock('@ladybugdb/core', () => ({ + default: { Database: vi.fn(), Connection: vi.fn() }, +})); +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + isReadOnlyDbError: vi.fn(() => false), + loadFTSExtension: vi.fn(), +})); +vi.mock('../../src/core/lbug/lbug-config.js', () => ({ + createLbugDatabase: vi.fn(), + isWalCorruptionError: vi.fn(() => false), + WAL_RECOVERY_SUGGESTION: '', +})); + +import { hasLocalWinFtsExtension } from '../../src/core/lbug/pool-adapter.js'; + +describe('hasLocalWinFtsExtension', () => { + let tmpHome: string; + + beforeEach(async () => { + tmpHome = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-fts-probe-')); + vi.spyOn(os, 'homedir').mockReturnValue(tmpHome); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(tmpHome, { recursive: true, force: true }); + }); + + it('returns false when ~/.lbdb/extension does not exist', async () => { + // tmpHome is empty; the probe should swallow the readdir ENOENT and return false. + await expect(hasLocalWinFtsExtension()).resolves.toBe(false); + }); + + it('returns false when ~/.lbdb/extension exists but has no version dirs', async () => { + await fs.mkdir(path.join(tmpHome, '.lbdb', 'extension'), { recursive: true }); + await expect(hasLocalWinFtsExtension()).resolves.toBe(false); + }); + + it('returns true when a single version dir contains the FTS binary', async () => { + const ftsDir = path.join(tmpHome, '.lbdb', 'extension', '0.16.0', 'win_amd64', 'fts'); + await fs.mkdir(ftsDir, { recursive: true }); + await fs.writeFile(path.join(ftsDir, 'libfts.lbug_extension'), Buffer.from('mock-binary')); + await expect(hasLocalWinFtsExtension()).resolves.toBe(true); + }); + + it('returns true when the binary is a zero-byte stub (LOAD failure handled downstream)', async () => { + // Empirically verified in #1690 thread: LadybugDB resolves LOAD EXTENSION fts to a + // version-specific path internally and the ExtensionManager's tryLoad try/catch + // catches the resulting load error cleanly. Probe is intentionally generous here; + // safety lives in the loader, not the probe. + const ftsDir = path.join(tmpHome, '.lbdb', 'extension', '0.16.0', 'win_amd64', 'fts'); + await fs.mkdir(ftsDir, { recursive: true }); + await fs.writeFile(path.join(ftsDir, 'libfts.lbug_extension'), ''); + await expect(hasLocalWinFtsExtension()).resolves.toBe(true); + }); + + it('returns true when multiple version dirs exist and only one carries the binary', async () => { + const versions = ['0.15.0', '0.16.0', '0.17.0']; + for (const v of versions) { + await fs.mkdir(path.join(tmpHome, '.lbdb', 'extension', v, 'win_amd64', 'fts'), { + recursive: true, + }); + } + // Only 0.16.0 has the binary; the probe should keep iterating past empty siblings. + await fs.writeFile( + path.join( + tmpHome, + '.lbdb', + 'extension', + '0.16.0', + 'win_amd64', + 'fts', + 'libfts.lbug_extension', + ), + Buffer.from('mock-binary'), + ); + await expect(hasLocalWinFtsExtension()).resolves.toBe(true); + }); + + it('returns false when version dirs exist but none contain the binary', async () => { + // Adversarial topology raised by #1690 review: tree exists (Nix store, Bazel + // sandbox seeding, corporate MDM-prepopulated user dirs) but the actual + // libfts.lbug_extension file is absent. Probe must distinguish file from dir. + const versions = ['0.15.0', '0.16.0', '0.17.0']; + for (const v of versions) { + await fs.mkdir(path.join(tmpHome, '.lbdb', 'extension', v, 'win_amd64', 'fts'), { + recursive: true, + }); + } + await expect(hasLocalWinFtsExtension()).resolves.toBe(false); + }); + + it('returns false when fs.readdir throws (e.g. permission denied on the extension root)', async () => { + // Cover the outer try/catch — any fs error walking the extension root is + // treated as "no binary present", matching the upstream skip-guard intent. + const eaccess = Object.assign(new Error('EACCES: permission denied'), { + code: 'EACCES', + }) as NodeJS.ErrnoException; + vi.spyOn(fs, 'readdir').mockRejectedValue(eaccess); + await expect(hasLocalWinFtsExtension()).resolves.toBe(false); + }); +}); From 92ad0f5491d13a89c7689ca54f0f4ad52d884820 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 05:38:26 +0100 Subject: [PATCH 12/51] chore(deps): bump idna in /eval in the uv group across 1 directory (#1713) --- eval/uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eval/uv.lock b/eval/uv.lock index 04b89f336..a78eadf06 100644 --- a/eval/uv.lock +++ b/eval/uv.lock @@ -760,11 +760,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] [[package]] From d7e1815aa310ff497178f240cde77819d1cf750f Mon Sep 17 00:00:00 2001 From: Nilotpal Kashyap <87768618+NilotpalK@users.noreply.github.com> Date: Wed, 20 May 2026 11:16:16 +0530 Subject: [PATCH 13/51] fix(detect-changes): guard resolveWorktreeCwd against overriding a separately-indexed worktree (#1691) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(detect-changes): guard resolveWorktreeCwd against overriding a separately-indexed worktree When the repo registry entry points to a linked worktree (both main checkout and worktree indexed separately), resolveWorktreeCwd was incorrectly replacing the correct worktree repoPath with the server's main-checkout launch directory. Both share the same canonical root so the existing same-repo check passed, causing git diff to run from the wrong directory and return 0 changes (issue #1659). Fix: early-exit guard — if tryRealpath(repoPath) differs from tryRealpath(getCanonicalRepoRoot(repoPath)), repoPath is itself a linked worktree and is returned unchanged. Auto-detection only fires when repoPath equals the canonical main-checkout root. Also normalises the launchCanonical comparison in the auto-detect path to use tryRealpath for cross-platform consistency. Regression test: 'returns worktreeDir unchanged when repoPath IS a linked worktree and launchCwd is the main checkout'. * test(detect-changes): add worktreeA→worktreeB case and assumption comment Cover the missing case from the production-readiness review: repoPath = wt-A (indexed), launchCwd = wt-B (server on a different linked worktree). The guard fires on repoPath being a worktree regardless of launchCwd, so wt-A is returned unchanged. Also add an inline comment documenting the assumption that repoPath is a git root or linked-worktree root (not an arbitrary subdirectory), as noted in Finding 2 of the review. * refactor(detect-changes): validate repoPath is a git root before canonical comparison Instead of relying on a comment asserting repoPath is always a git root, call getGitRoot(repoPath) first. Only if the result matches repoPath itself do we call getCanonicalRepoRoot and apply the guard. This eliminates the over-classification risk for subdirectory repoPath values and makes the assumption explicit in code. repoCanonical is shared across both the guard and the auto-detect block. --------- Co-authored-by: Gergő Magyar --- gitnexus/src/mcp/local/local-backend.ts | 31 +++++++- .../test/unit/detect-changes-worktree.test.ts | 75 +++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 720e73eaa..4cff47a28 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -248,6 +248,29 @@ function tryRealpath(p: string): string { */ export function resolveWorktreeCwd(repoPath: string, launchCwd: string): string { try { + // Verify repoPath is a git root before comparing against its canonical + // root. If getGitRoot returns a different path, repoPath is an arbitrary + // subdirectory — skip both the linked-worktree guard and auto-detection + // and fall through to the repoPath fallback. + const repoGitRoot = getGitRoot(repoPath); + const repoCanonical = + repoGitRoot && tryRealpath(repoGitRoot) === tryRealpath(repoPath) + ? getCanonicalRepoRoot(repoPath) + : null; + + // Early exit: if repoPath is a linked worktree (differs from its canonical + // main-checkout root), return it unchanged. Do NOT override it with the + // server's launch directory — that would silently replace the explicitly- + // resolved worktree index with the main checkout. + // + // getCanonicalRepoRoot returns the main-checkout path for both the checkout + // and all linked worktrees: + // repoPath === canonical → main checkout (auto-detect may fire below) + // repoPath !== canonical → linked worktree (return as-is) + if (repoCanonical && tryRealpath(repoPath) !== tryRealpath(repoCanonical)) { + return repoPath; + } + const launchGitRoot = getGitRoot(launchCwd); if (launchGitRoot) { // Normalise via realpathSync before comparing so macOS /var → /private/var @@ -256,8 +279,12 @@ export function resolveWorktreeCwd(repoPath: string, launchCwd: string): string const realRepo = tryRealpath(repoPath); if (realLaunch !== realRepo) { const launchCanonical = getCanonicalRepoRoot(launchCwd); - const repoCanonical = getCanonicalRepoRoot(repoPath); - if (launchCanonical && repoCanonical && launchCanonical === repoCanonical) { + // Use tryRealpath on both canonical values for cross-platform safety. + if ( + launchCanonical && + repoCanonical && + tryRealpath(launchCanonical) === tryRealpath(repoCanonical) + ) { return launchGitRoot; } } diff --git a/gitnexus/test/unit/detect-changes-worktree.test.ts b/gitnexus/test/unit/detect-changes-worktree.test.ts index 02e440100..925731118 100644 --- a/gitnexus/test/unit/detect-changes-worktree.test.ts +++ b/gitnexus/test/unit/detect-changes-worktree.test.ts @@ -189,6 +189,81 @@ describe('resolveWorktreeCwd — auto-detection helper', () => { rmSync(repoB, { recursive: true, force: true }); } }); + + it('returns worktreeDir unchanged when repoPath IS a linked worktree and launchCwd is the main checkout', () => { + // Regression for: detect_changes returns no changes when the MCP server + // runs from the main checkout but the resolved repo index is a separately- + // indexed linked worktree (issue #1659 / dpearson2699 report). + // + // Before the fix, resolveWorktreeCwd would detect that launchCwd (main + // checkout) and repoPath (worktree) share the same canonical root and + // wrongly override repoPath with the main-checkout path, causing git diff + // to run from the wrong directory and return 0 changes. + const repoDir = mkdtempSync(path.join(os.tmpdir(), 'gitnexus-rwc-idx-wt-')); + try { + execSync('git init -q', { cwd: repoDir, stdio: 'ignore' }); + execSync('git config user.email "test@example.com"', { cwd: repoDir, stdio: 'ignore' }); + execSync('git config user.name "Test"', { cwd: repoDir, stdio: 'ignore' }); + writeFileSync(path.join(repoDir, 'x.ts'), 'export const x = 1;\n'); + execSync('git add x.ts', { cwd: repoDir, stdio: 'ignore' }); + execSync('git commit -q -m "initial"', { cwd: repoDir, stdio: 'ignore' }); + + const worktreeDir = path.join(repoDir, 'wt-indexed'); + execSync(`git worktree add -q -b indexed "${worktreeDir}"`, { + cwd: repoDir, + stdio: 'ignore', + }); + + // Simulate: repo registry entry points to the worktree (repoPath = worktreeDir) + // but the MCP server was launched from the main checkout (launchCwd = repoDir). + // resolveWorktreeCwd must NOT override the correct worktree path with repoDir. + const result = resolveWorktreeCwd(worktreeDir, repoDir); + expect(realpathSync.native(result)).toBe(realpathSync.native(worktreeDir)); + expect(realpathSync.native(result)).not.toBe(realpathSync.native(repoDir)); + } finally { + try { + execSync('git worktree remove -f wt-indexed', { cwd: repoDir, stdio: 'ignore' }); + } catch { + // ignore + } + rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('returns worktreeA unchanged when both repoPath and launchCwd are different linked worktrees of the same repo', () => { + // Covers: repoPath = wt-A (indexed), launchCwd = wt-B (server launched from another worktree). + // The guard fires on repoPath being a linked worktree regardless of what launchCwd is, + // so wt-A must be returned unchanged — not wt-B, not the main checkout. + const repoDir = mkdtempSync(path.join(os.tmpdir(), 'gitnexus-rwc-two-wt-')); + try { + execSync('git init -q', { cwd: repoDir, stdio: 'ignore' }); + execSync('git config user.email "test@example.com"', { cwd: repoDir, stdio: 'ignore' }); + execSync('git config user.name "Test"', { cwd: repoDir, stdio: 'ignore' }); + writeFileSync(path.join(repoDir, 'x.ts'), 'export const x = 1;\n'); + execSync('git add x.ts', { cwd: repoDir, stdio: 'ignore' }); + execSync('git commit -q -m "initial"', { cwd: repoDir, stdio: 'ignore' }); + + const worktreeA = path.join(repoDir, 'wt-a'); + const worktreeB = path.join(repoDir, 'wt-b'); + execSync(`git worktree add -q -b branch-a "${worktreeA}"`, { cwd: repoDir, stdio: 'ignore' }); + execSync(`git worktree add -q -b branch-b "${worktreeB}"`, { cwd: repoDir, stdio: 'ignore' }); + + // repoPath = wt-A (the indexed worktree), launchCwd = wt-B (where the server runs). + // resolveWorktreeCwd must return wt-A — the indexed path — unchanged. + const result = resolveWorktreeCwd(worktreeA, worktreeB); + expect(realpathSync.native(result)).toBe(realpathSync.native(worktreeA)); + expect(realpathSync.native(result)).not.toBe(realpathSync.native(worktreeB)); + expect(realpathSync.native(result)).not.toBe(realpathSync.native(repoDir)); + } finally { + try { + execSync('git worktree remove -f wt-a', { cwd: repoDir, stdio: 'ignore' }); + execSync('git worktree remove -f wt-b', { cwd: repoDir, stdio: 'ignore' }); + } catch { + // ignore + } + rmSync(repoDir, { recursive: true, force: true }); + } + }); }); // ── Guard logic via real path arithmetic ───────────────────────────────────── From b4a2a4b91eaa744357f0d5ad4ae9265617d7f805 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 08:00:03 +0100 Subject: [PATCH 14/51] fix(ingestion): Prioritize same-module Java type resolution for duplicate FQNs across modules (#1712) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * Fix Java same-name type resolution with same-module priority Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/df0843e3-e244-4e0f-a94a-311df3899bd0 * Refine Java ambiguity fallback safety check Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/df0843e3-e244-4e0f-a94a-311df3899bd0 * Remove Java-specific fallback from shared scope walkers Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/e882906a-2c96-411e-94a5-123a345421a9 * Harden Java module key and ambiguous owner fallback Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/e882906a-2c96-411e-94a5-123a345421a9 * Add negative assertions for duplicate-FQN module edges Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/e882906a-2c96-411e-94a5-123a345421a9 * Make Java same-module ordering path-agnostic Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/b317a759-f6bc-4590-bd2a-628f0ee9c477 * Refine generic Java path-affinity ordering safeguards Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/b317a759-f6bc-4590-bd2a-628f0ee9c477 * Polish Java path-affinity ordering clarity Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/b317a759-f6bc-4590-bd2a-628f0ee9c477 * Simplify Java path-affinity ordering logic Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/b317a759-f6bc-4590-bd2a-628f0ee9c477 * Revert legacy DAG Java ambiguity ordering changes Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/94e50cf2-9733-4e69-a0eb-9fd38cbdb589 * Skip duplicate-FQN Java assertions in legacy parity mode Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/1b560efa-1b3b-4697-b590-c6ef447f431e * Tighten duplicate-FQN Java CALLS edge cardinality assertions Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/67c18f93-5e56-4b15-8404-cdf1be9b4485 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Gergő Magyar --- gitnexus/src/core/ingestion/languages/java.ts | 44 +++++++++++++ .../src/main/java/com/example/Module1App.java | 8 +++ .../main/java/com/example/UserService.java | 6 ++ .../src/main/java/com/example/Module2App.java | 8 +++ .../main/java/com/example/UserService.java | 6 ++ .../test/integration/resolvers/helpers.ts | 7 ++ .../test/integration/resolvers/java.test.ts | 66 +++++++++++++++++++ 7 files changed, 145 insertions(+) create mode 100644 gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/Module1App.java create mode 100644 gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/UserService.java create mode 100644 gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/Module2App.java create mode 100644 gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/UserService.java diff --git a/gitnexus/src/core/ingestion/languages/java.ts b/gitnexus/src/core/ingestion/languages/java.ts index c70eacb10..20755d7cd 100644 --- a/gitnexus/src/core/ingestion/languages/java.ts +++ b/gitnexus/src/core/ingestion/languages/java.ts @@ -27,6 +27,7 @@ import { javaMethodConfig } from '../method-extractors/configs/jvm.js'; import { createVariableExtractor } from '../variable-extractors/generic.js'; import { javaVariableConfig } from '../variable-extractors/configs/jvm.js'; import { createHeritageExtractor } from '../heritage-extractors/generic.js'; +import type { SymbolDefinition } from 'gitnexus-shared'; import { emitJavaScopeCaptures, interpretJavaImport, @@ -39,6 +40,48 @@ import { resolveJavaImportTarget, } from './java/index.js'; +const orderJavaSameNameTypeCandidates = ({ + callSiteFilePath, + candidates, +}: { + readonly typeName: string; + readonly callSiteFilePath: string; + readonly candidates: readonly SymbolDefinition[]; +}): readonly SymbolDefinition[] | null => { + if (!callSiteFilePath.endsWith('.java')) return null; + if (candidates.length <= 1) return null; + const callerDir = splitDirectorySegments(callSiteFilePath); + + const scored = candidates.map((candidate, index) => ({ + candidate, + index, + score: sharedPrefixLength(callerDir, splitDirectorySegments(candidate.filePath)), + })); + const bestScore = Math.max(...scored.map((entry) => entry.score)); + // When all candidates tie, we have no structural signal to prefer one path. + // Returning null keeps downstream ambiguity handling conservative. + if (scored.every((entry) => entry.score === bestScore)) return null; + + const ordered = [...scored] + .sort((a, b) => b.score - a.score || a.index - b.index) + .map((entry) => entry.candidate); + return ordered; +}; + +const splitDirectorySegments = (filePath: string): string[] => { + const normalized = filePath.replace(/\\/g, '/'); + // Remove empty segments from leading/trailing/multiple slashes, then drop filename. + const segments = normalized.split('/').filter(Boolean); + return segments.slice(0, -1); +}; + +const sharedPrefixLength = (left: readonly string[], right: readonly string[]): number => { + const max = Math.min(left.length, right.length); + let idx = 0; + while (idx < max && left[idx] === right[idx]) idx += 1; + return idx; +}; + export const javaProvider = defineLanguage({ id: SupportedLanguages.Java, extensions: ['.java'], @@ -87,4 +130,5 @@ export const javaProvider = defineLanguage({ receiverBinding: javaReceiverBinding, arityCompatibility: javaArityCompatibility, resolveImportTarget: resolveJavaImportTarget, + orderSameNameTypeCandidates: orderJavaSameNameTypeCandidates, }); diff --git a/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/Module1App.java b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/Module1App.java new file mode 100644 index 000000000..242956497 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/Module1App.java @@ -0,0 +1,8 @@ +package com.example; + +public class Module1App { + public void run() { + UserService service = new UserService(); + service.ping(); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/UserService.java b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/UserService.java new file mode 100644 index 000000000..9daea3d45 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module1/src/main/java/com/example/UserService.java @@ -0,0 +1,6 @@ +package com.example; + +public class UserService { + public void ping() { + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/Module2App.java b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/Module2App.java new file mode 100644 index 000000000..7d24142b8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/Module2App.java @@ -0,0 +1,8 @@ +package com.example; + +public class Module2App { + public void run() { + UserService service = new UserService(); + service.ping(); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/UserService.java b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/UserService.java new file mode 100644 index 000000000..9daea3d45 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-duplicate-fqn-modules/module2/src/main/java/com/example/UserService.java @@ -0,0 +1,6 @@ +package com.example; + +public class UserService { + public void ping() { + } +} diff --git a/gitnexus/test/integration/resolvers/helpers.ts b/gitnexus/test/integration/resolvers/helpers.ts index cc2697849..bd5eaa474 100644 --- a/gitnexus/test/integration/resolvers/helpers.ts +++ b/gitnexus/test/integration/resolvers/helpers.ts @@ -34,6 +34,13 @@ const LEGACY_RESOLVER_PARITY_EXPECTED_FAILURES: Readonly { }); }); +describe('Java same-module priority for duplicate FQNs', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'java-duplicate-fqn-modules'), () => {}); + }, 60000); + + it('resolves Module1App.run calls to module1 UserService, not module2', () => { + const calls = getRelationships(result, 'CALLS'); + const module1ToModule1 = calls.filter( + (c) => + c.source === 'run' && + c.target === 'UserService' && + c.sourceFilePath === 'module1/src/main/java/com/example/Module1App.java' && + c.targetFilePath === 'module1/src/main/java/com/example/UserService.java', + ); + const module1ToModule2 = calls.filter( + (c) => + c.source === 'run' && + c.target === 'UserService' && + c.sourceFilePath === 'module1/src/main/java/com/example/Module1App.java' && + c.targetFilePath === 'module2/src/main/java/com/example/UserService.java', + ); + const module1ToAnyUserService = calls.filter( + (c) => + c.source === 'run' && + c.target === 'UserService' && + c.sourceFilePath === 'module1/src/main/java/com/example/Module1App.java' && + /module[12]\/src\/main\/java\/com\/example\/UserService\.java/.test(c.targetFilePath), + ); + + expect(module1ToModule1.length).toBe(1); + expect(module1ToModule2.length).toBe(0); + expect(module1ToAnyUserService.length).toBe(1); + }); + + it('resolves Module2App.run calls to module2 UserService, not module1', () => { + const calls = getRelationships(result, 'CALLS'); + const module2ToModule2 = calls.filter( + (c) => + c.source === 'run' && + c.target === 'UserService' && + c.sourceFilePath === 'module2/src/main/java/com/example/Module2App.java' && + c.targetFilePath === 'module2/src/main/java/com/example/UserService.java', + ); + const module2ToModule1 = calls.filter( + (c) => + c.source === 'run' && + c.target === 'UserService' && + c.sourceFilePath === 'module2/src/main/java/com/example/Module2App.java' && + c.targetFilePath === 'module1/src/main/java/com/example/UserService.java', + ); + const module2ToAnyUserService = calls.filter( + (c) => + c.source === 'run' && + c.target === 'UserService' && + c.sourceFilePath === 'module2/src/main/java/com/example/Module2App.java' && + /module[12]\/src\/main\/java\/com\/example\/UserService\.java/.test(c.targetFilePath), + ); + + expect(module2ToModule2.length).toBe(1); + expect(module2ToModule1.length).toBe(0); + expect(module2ToAnyUserService.length).toBe(1); + }); +}); + // --------------------------------------------------------------------------- // Member-call resolution: obj.method() resolves through pipeline // --------------------------------------------------------------------------- From dae70a26ea1b71ad40d1ef58a4ff18db031a8992 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Wed, 20 May 2026 12:06:51 +0100 Subject: [PATCH 15/51] feat(cpp): Add pointer nullptr ellipsis conversion ranks (#1708) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add C++ pointer null ellipsis ranks * test(cpp): Strengthen pointer overload assertions --------- Co-authored-by: Gergő Magyar --- .../languages/cpp/conversion-rank.ts | 73 ++++++++---- .../passes/free-call-fallback.ts | 12 +- .../passes/overload-narrowing.ts | 83 ++++++++++++- .../lib.cpp | 12 ++ .../cpp-overload-pointer-null-ellipsis/lib.h | 38 ++++++ .../test/integration/resolvers/cpp.test.ts | 58 +++++++++ .../test/integration/resolvers/helpers.ts | 6 + .../cpp/cpp-overload-ranking.test.ts | 111 ++++++++++++++++++ 8 files changed, 362 insertions(+), 31 deletions(-) create mode 100644 gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.cpp create mode 100644 gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.h create mode 100644 gitnexus/test/unit/scope-resolution/cpp/cpp-overload-ranking.test.ts diff --git a/gitnexus/src/core/ingestion/languages/cpp/conversion-rank.ts b/gitnexus/src/core/ingestion/languages/cpp/conversion-rank.ts index 2a9e3bc01..bea3600a7 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/conversion-rank.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/conversion-rank.ts @@ -1,32 +1,30 @@ /** - * C++ conversion-rank scoring for overload resolution (#1578). + * C++ conversion-rank scoring for overload resolution (#1578, #1637). * - * Operates on **normalized** type strings (output of - * `normalizeCppParamType` in `arity-metadata.ts`). After normalization: - * - int/long/short/unsigned → 'int' - * - float/double → 'double' - * - char → 'char', bool → 'bool' - * - * Because the normalizer collapses promotion pairs (int↔long, - * float↔double) to the same string, those promotions are invisible at - * this layer — they appear as exact matches (rank 0). + * Operates on normalized type strings (output of `normalizeCppParamType` + * in `arity-metadata.ts`) plus optional shape sidecars from #1630. + * Normalization intentionally collapses cv/ref/pointer spelling for stable + * graph IDs, so pointer/nullptr rules must consult `ParameterTypeClass`. * * Post-normalization ranking: - * - rank 0 — exact (same normalized type) - * - rank 1 — integral promotion (char→int, bool→int) - * - rank 2 — standard arithmetic conversion (int↔double, char→double, - * bool→double) - * - Infinity — mismatch (string↔int, user types, pointers, etc.) + * - rank 0: exact (same normalized type) + * - rank 1: integral promotion (char -> int, bool -> int) + * - rank 2: standard conversion (arithmetic, nullptr -> T*, T* -> bool, + * T* -> void*) + * - rank 3: nullptr -> bool (kept worse than nullptr -> T*) + * - rank 4: ellipsis conversion (worst viable) + * - Infinity: mismatch (string -> int, user types, unsupported shapes) * - * This function is intentionally C++-specific (issue #1578 pitfall: - * keep conversion-rank tables out of shared overload-narrowing). Other - * languages may define their own `ConversionRankFn` in the future. + * This function is intentionally C++-specific. Other languages may define + * their own `ConversionRankFn` in the future. */ +import type { ParameterTypeClass } from 'gitnexus-shared'; + /** Set of normalized arithmetic types that support implicit conversion. */ const ARITHMETIC = new Set(['int', 'double', 'char', 'bool']); -/** Integral promotion targets: char→int and bool→int are rank 1. */ +/** Integral promotion targets: char -> int and bool -> int are rank 1. */ const INTEGRAL_PROMOTION = new Map([ ['char', 'int'], ['bool', 'int'], @@ -35,13 +33,40 @@ const INTEGRAL_PROMOTION = new Map([ /** * Return the conversion rank from `argType` to `paramType`. * - * @returns 0 for exact match, 1 for integral promotion (char/bool→int), - * 2 for standard arithmetic conversion, Infinity for mismatch. + * @returns 0 for exact match, 1 for integral promotion, 2 for standard + * conversion, 3 for nullptr -> bool, 4 for ellipsis, Infinity + * for mismatch. */ -export function cppConversionRank(argType: string, paramType: string): number { - if (argType === paramType) return 0; - // Integral promotions: char→int, bool→int (ISO C++ [conv.prom]) +export function cppConversionRank( + argType: string, + paramType: string, + argTypeClass?: ParameterTypeClass, + paramTypeClass?: ParameterTypeClass, +): number { + if (argType === paramType) { + return exactShapeCompatible(argTypeClass, paramTypeClass) ? 0 : Infinity; + } + if (paramType === '...') return 4; if (INTEGRAL_PROMOTION.get(argType) === paramType) return 1; if (ARITHMETIC.has(argType) && ARITHMETIC.has(paramType)) return 2; + if (argType === 'null' && isPointer(paramTypeClass)) return 2; + if (argType === 'null' && paramType === 'bool') return 3; + if (isPointer(argTypeClass) && paramType === 'bool') return 2; + if (isPointer(argTypeClass) && isPointer(paramTypeClass) && paramType === 'void') return 2; return Infinity; } + +function isPointer(typeClass: ParameterTypeClass | undefined): boolean { + return typeClass?.indirection === 'pointer' && typeClass.pointerDepth > 0; +} + +function exactShapeCompatible( + argTypeClass: ParameterTypeClass | undefined, + paramTypeClass: ParameterTypeClass | undefined, +): boolean { + if (argTypeClass === undefined || paramTypeClass === undefined) return true; + if (argTypeClass.indirection === 'unknown' || paramTypeClass.indirection === 'unknown') { + return true; + } + return isPointer(argTypeClass) === isPointer(paramTypeClass); +} diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index f3010abf4..0d537db07 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -17,7 +17,13 @@ * generalization plan. */ -import type { ParsedFile, Reference, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import type { + ParameterTypeClass, + ParsedFile, + Reference, + ScopeId, + SymbolDefinition, +} from 'gitnexus-shared'; import type { KnowledgeGraph } from '../../../graph/types.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; import type { SemanticModel } from '../../model/semantic-model.js'; @@ -277,6 +283,7 @@ export function emitFreeCallFallback( }) : undefined, site.argumentTypes, + site.argumentTypeClasses, options.conversionRankFn, ); } @@ -342,6 +349,7 @@ function pickUniqueGlobalCallable( callArity?: number, isCallerVisible?: (candidate: SymbolDefinition) => boolean, callArgTypes?: readonly string[], + callArgTypeClasses?: readonly ParameterTypeClass[], conversionRankFn?: ConversionRankFn, ): SymbolDefinition | undefined { const scopeDefs: SymbolDefinition[] = []; @@ -380,6 +388,7 @@ function pickUniqueGlobalCallable( // disambiguate (e.g., `f(int)` vs `f(double)` called with `f(2.5)`). if (scopeDefs.length > 1) { const narrowed = narrowOverloadCandidates(scopeDefs, callArity, callArgTypes, { + argumentTypeClasses: callArgTypeClasses, conversionRankFn, }); if (narrowed.length === 1) return narrowed[0]; @@ -420,6 +429,7 @@ function pickUniqueGlobalCallable( // Same argument-type + conversion-rank narrowing for the model pool. if (defs.length > 1) { const narrowed = narrowOverloadCandidates(defs, callArity, callArgTypes, { + argumentTypeClasses: callArgTypeClasses, conversionRankFn, }); if (narrowed.length === 1) return narrowed[0]; diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/overload-narrowing.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/overload-narrowing.ts index b9b452eb7..f28cd3185 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/overload-narrowing.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/overload-narrowing.ts @@ -38,7 +38,13 @@ * 5. Empty input returns empty output. */ -import type { ArityVerdict, Callsite, ConstraintContext, SymbolDefinition } from 'gitnexus-shared'; +import type { + ArityVerdict, + Callsite, + ConstraintContext, + ParameterTypeClass, + SymbolDefinition, +} from 'gitnexus-shared'; /** * Per-slot conversion-rank function. Returns a numeric cost for @@ -51,7 +57,12 @@ import type { ArityVerdict, Callsite, ConstraintContext, SymbolDefinition } from * Each language provides its own implementation. The function operates * on normalized type strings (output of the language's type normalizer). */ -export type ConversionRankFn = (argType: string, paramType: string) => number; +export type ConversionRankFn = ( + argType: string, + paramType: string, + argTypeClass?: ParameterTypeClass, + paramTypeClass?: ParameterTypeClass, +) => number; /** * Optional hook bundle for narrowing extension points. Threaded in @@ -130,7 +141,16 @@ export function narrowOverloadCandidates( if (params === undefined) return false; for (let i = 0; i < argTypes.length && i < params.length; i++) { if (argTypes[i] === '') continue; - if (argTypes[i] !== params[i]) return false; + if ( + !exactTypeSlotMatches( + argTypes[i], + params[i], + hookCtx?.argumentTypeClasses?.[i], + d.parameterTypeClasses?.[i], + ) + ) { + return false; + } } return true; }); @@ -144,7 +164,12 @@ export function narrowOverloadCandidates( // are returned; multiple survivors are genuinely ambiguous. When // ranking also yields empty, fall through to the arity-filtered // `candidates` set — matches pre-#1606 behavior. - const ranked = rankByConversion(candidates, argTypes, hookCtx.conversionRankFn); + const ranked = rankByConversion( + candidates, + argTypes, + hookCtx.conversionRankFn, + hookCtx.argumentTypeClasses, + ); if (ranked.length > 0) result = ranked; } } @@ -183,6 +208,27 @@ export function narrowOverloadCandidates( return result; } +function exactTypeSlotMatches( + argType: string, + paramType: string, + argTypeClass?: ParameterTypeClass, + paramTypeClass?: ParameterTypeClass, +): boolean { + if (argType !== paramType) return false; + // C++ normalizes away pointer markers (`int*` -> `int`). When both sides + // provide shape sidecars, do not let that collapse make `int` exactly match + // `int*`. Unknown sidecar evidence preserves the previous string-only path. + if (argTypeClass === undefined || paramTypeClass === undefined) return true; + if (argTypeClass.indirection === 'unknown' || paramTypeClass.indirection === 'unknown') { + return true; + } + return isPointerShape(argTypeClass) === isPointerShape(paramTypeClass); +} + +function isPointerShape(typeClass: ParameterTypeClass): boolean { + return typeClass.indirection === 'pointer' && typeClass.pointerDepth > 0; +} + /** * Pairwise dominance comparison (ISO C++ [over.ics.rank]). * @@ -199,6 +245,7 @@ function rankByConversion( candidates: readonly SymbolDefinition[], argTypes: readonly string[], rankFn: ConversionRankFn, + argTypeClasses?: readonly ParameterTypeClass[], ): readonly SymbolDefinition[] { // Step 1: compute per-slot ranks and exclude non-viable candidates. const viable: Array<{ def: SymbolDefinition; ranks: number[] }> = []; @@ -207,12 +254,22 @@ function rankByConversion( if (params === undefined) continue; const ranks: number[] = []; let ok = true; - for (let i = 0; i < argTypes.length && i < params.length; i++) { + for (let i = 0; i < argTypes.length; i++) { + const paramType = parameterTypeAt(params, i); + if (paramType === undefined) { + ok = false; + break; + } if (argTypes[i] === '') { ranks.push(0); // unknown arg → any-match (rank 0) continue; } - const r = rankFn(argTypes[i], params[i]); + const r = rankFn( + argTypes[i], + paramType, + argTypeClasses?.[i], + parameterTypeClassAt(d.parameterTypeClasses, i), + ); if (!isFinite(r)) { ok = false; break; @@ -239,6 +296,20 @@ function rankByConversion( return viable.filter((_, idx) => !dominated.has(idx)).map((v) => v.def); } +function parameterTypeAt(params: readonly string[], argIndex: number): string | undefined { + if (argIndex < params.length) return params[argIndex]; + return params[params.length - 1] === '...' ? '...' : undefined; +} + +function parameterTypeClassAt( + params: readonly ParameterTypeClass[] | undefined, + argIndex: number, +): ParameterTypeClass | undefined { + if (params === undefined) return undefined; + if (argIndex < params.length) return params[argIndex]; + return params[params.length - 1]?.base === '...' ? params[params.length - 1] : undefined; +} + /** * Compare two per-slot rank vectors. * Returns -1 if `a` dominates `b` (not worse everywhere, better somewhere), diff --git a/gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.cpp b/gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.cpp new file mode 100644 index 000000000..c6b0bb5a2 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.cpp @@ -0,0 +1,12 @@ +#include "lib.h" + +void Service::f(int* p) {} +void Service::f(bool flag) {} + +void Service::g(int a, int b) {} +void Service::g(int a, ...) {} + +void Service::h(int a, double b) {} +void Service::h(int a, ...) {} + +void Service::k(int a, ...) {} diff --git a/gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.h b/gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.h new file mode 100644 index 000000000..c85f258a0 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/cpp-overload-pointer-null-ellipsis/lib.h @@ -0,0 +1,38 @@ +#pragma once + +class Service { +public: + void f(int* p); + void f(bool flag); + + void g(int a, int b); + void g(int a, ...); + + void h(int a, double b); + void h(int a, ...); + + void k(int a, ...); + + void runNullptr() { + f(nullptr); + } + + void runPointer() { + int* p = nullptr; + f(p); + } + + void runBoolConversion() { + f(42); + } + + void run() { + int* p = nullptr; + f(nullptr); + f(p); + f(42); + g(1, 2); + h(1, 'a'); + k(1, 2, 3); + } +}; diff --git a/gitnexus/test/integration/resolvers/cpp.test.ts b/gitnexus/test/integration/resolvers/cpp.test.ts index 1537c3884..e1376938e 100644 --- a/gitnexus/test/integration/resolvers/cpp.test.ts +++ b/gitnexus/test/integration/resolvers/cpp.test.ts @@ -1836,6 +1836,64 @@ describe('C++ overload resolution — conversion-rank disambiguation (#1578)', ( }); }); +// C++ overload resolution: pointer/nullptr/ellipsis conversion ranks (#1637) +describe('C++ overload resolution — pointer/nullptr/ellipsis ranks (#1637)', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'cpp-overload-pointer-null-ellipsis'), + () => {}, + ); + }, 60000); + + it('f(nullptr) and f(p) resolve to f(int*) while f(42) resolves to f(bool)', () => { + const calls = getRelationships(result, 'CALLS'); + + const nullptrCall = calls.find((c) => c.source === 'runNullptr' && c.target === 'f'); + const pointerCall = calls.find((c) => c.source === 'runPointer' && c.target === 'f'); + const boolCall = calls.find((c) => c.source === 'runBoolConversion' && c.target === 'f'); + + expect( + result.graph.getNode(nullptrCall?.rel.targetId ?? '')?.properties.parameterTypes, + ).toEqual(['int']); + expect( + result.graph.getNode(pointerCall?.rel.targetId ?? '')?.properties.parameterTypes, + ).toEqual(['int']); + expect(result.graph.getNode(boolCall?.rel.targetId ?? '')?.properties.parameterTypes).toEqual([ + 'bool', + ]); + }); + + it('g(1, 2) resolves to fixed-arity g(int, int), not g(int, ...)', () => { + const calls = getRelationships(result, 'CALLS'); + const gCalls = calls.filter((c) => c.source === 'run' && c.target === 'g'); + + expect(gCalls.length).toBe(1); + const tgt = result.graph.getNode(gCalls[0].rel.targetId); + expect(tgt?.properties.parameterTypes).toEqual(['int', 'int']); + }); + + it("h(1, 'a') resolves to h(int, double), not h(int, ...)", () => { + const calls = getRelationships(result, 'CALLS'); + const hCalls = calls.filter((c) => c.source === 'run' && c.target === 'h'); + + expect(hCalls.length).toBe(1); + const tgt = result.graph.getNode(hCalls[0].rel.targetId); + expect(tgt?.properties.parameterTypes).toEqual(['int', 'double']); + }); + + it('k(1, 2, 3) keeps the ellipsis overload viable when it is the only match', () => { + const calls = getRelationships(result, 'CALLS'); + const kCalls = calls.filter((c) => c.source === 'run' && c.target === 'k'); + + expect(kCalls.length).toBe(1); + const tgt = result.graph.getNode(kCalls[0].rel.targetId); + expect(tgt?.properties.parameterCount).toBeUndefined(); + expect(tgt?.properties.parameterTypes).toEqual(['int']); + }); +}); + // --------------------------------------------------------------------------- // U3: anonymous-namespace symbols MUST NOT leak across translation units // (full-pipeline integration test; unit-level coverage exists separately) diff --git a/gitnexus/test/integration/resolvers/helpers.ts b/gitnexus/test/integration/resolvers/helpers.ts index bd5eaa474..296f3e458 100644 --- a/gitnexus/test/integration/resolvers/helpers.ts +++ b/gitnexus/test/integration/resolvers/helpers.ts @@ -196,6 +196,12 @@ const LEGACY_RESOLVER_PARITY_EXPECTED_FAILURES: Readonly` overloads // guarded by mutually-exclusive `enable_if_t` predicates collapse diff --git a/gitnexus/test/unit/scope-resolution/cpp/cpp-overload-ranking.test.ts b/gitnexus/test/unit/scope-resolution/cpp/cpp-overload-ranking.test.ts new file mode 100644 index 000000000..5cb679a37 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/cpp/cpp-overload-ranking.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest'; +import type { ParameterTypeClass, SymbolDefinition } from 'gitnexus-shared'; +import { cppConversionRank } from '../../../../src/core/ingestion/languages/cpp/conversion-rank.js'; +import { narrowOverloadCandidates } from '../../../../src/core/ingestion/scope-resolution/passes/overload-narrowing.js'; + +const value = (base: string): ParameterTypeClass => ({ + base, + cv: 'none', + indirection: 'value', + pointerDepth: 0, +}); + +const pointer = (base: string): ParameterTypeClass => ({ + base, + cv: 'none', + indirection: 'pointer', + pointerDepth: 1, +}); + +const ellipsis = (): ParameterTypeClass => ({ + base: '...', + cv: 'unknown', + indirection: 'unknown', + pointerDepth: 0, +}); + +const mkDef = ( + nodeId: string, + parameterTypes: readonly string[], + parameterTypeClasses: readonly ParameterTypeClass[], +): SymbolDefinition => ({ + nodeId, + filePath: 'service.cpp', + type: 'Method', + parameterCount: parameterTypes.includes('...') ? undefined : parameterTypes.length, + requiredParameterCount: parameterTypes.includes('...') + ? parameterTypes.indexOf('...') + : parameterTypes.length, + parameterTypes: [...parameterTypes], + parameterTypeClasses: [...parameterTypeClasses], +}); + +describe('cppConversionRank pointer/nullptr/ellipsis ranks (#1637)', () => { + it('ranks nullptr -> T* ahead of nullptr -> bool', () => { + expect(cppConversionRank('null', 'int', value('null'), pointer('int'))).toBe(2); + expect(cppConversionRank('null', 'bool', value('null'), value('bool'))).toBe(3); + }); + + it('ranks pointer -> bool and pointer -> void* as standard conversions', () => { + expect(cppConversionRank('int', 'bool', pointer('int'), value('bool'))).toBe(2); + expect(cppConversionRank('int', 'void', pointer('int'), pointer('void'))).toBe(2); + }); + + it('keeps pointer exact matches shape-aware', () => { + expect(cppConversionRank('int', 'int', pointer('int'), pointer('int'))).toBe(0); + expect(cppConversionRank('int', 'int', value('int'), pointer('int'))).toBe(Infinity); + }); + + it('ranks ellipsis as the worst viable conversion', () => { + expect(cppConversionRank('int', '...', value('int'), ellipsis())).toBe(4); + }); +}); + +describe('narrowOverloadCandidates with C++ pointer-rank sidecars (#1637)', () => { + it('selects pointer overload for nullptr over bool overload', () => { + const byPointer = mkDef('f:intptr', ['int'], [pointer('int')]); + const byBool = mkDef('f:bool', ['bool'], [value('bool')]); + + const result = narrowOverloadCandidates([byPointer, byBool], 1, ['null'], { + argumentTypeClasses: [value('null')], + conversionRankFn: cppConversionRank, + }); + + expect(result.map((d) => d.nodeId)).toEqual(['f:intptr']); + }); + + it('does not treat normalized value and pointer types as exact matches', () => { + const byPointer = mkDef('f:intptr', ['int'], [pointer('int')]); + const byBool = mkDef('f:bool', ['bool'], [value('bool')]); + + const result = narrowOverloadCandidates([byPointer, byBool], 1, ['int'], { + argumentTypeClasses: [value('int')], + conversionRankFn: cppConversionRank, + }); + + expect(result.map((d) => d.nodeId)).toEqual(['f:bool']); + }); + + it('selects fixed-arity overload over ellipsis', () => { + const exact = mkDef('g:int-int', ['int', 'int'], [value('int'), value('int')]); + const variadic = mkDef('g:ellipsis', ['int', '...'], [value('int'), ellipsis()]); + + const result = narrowOverloadCandidates([exact, variadic], 2, ['int', 'int'], { + argumentTypeClasses: [value('int'), value('int')], + conversionRankFn: cppConversionRank, + }); + + expect(result.map((d) => d.nodeId)).toEqual(['g:int-int']); + }); + + it('keeps an ellipsis overload viable when it is the only match', () => { + const variadic = mkDef('log:ellipsis', ['int', '...'], [value('int'), ellipsis()]); + + const result = narrowOverloadCandidates([variadic], 3, ['int', 'int', 'double'], { + argumentTypeClasses: [value('int'), value('int'), value('double')], + conversionRankFn: cppConversionRank, + }); + + expect(result.map((d) => d.nodeId)).toEqual(['log:ellipsis']); + }); +}); From f350ae278aaa02362bea31b6a257b7779c5f2f1d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 13:37:04 +0100 Subject: [PATCH 16/51] feat: Add `analyze --repair-fts`, enforce FTS verification, and harden repair safeguards (#1720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * feat(analyze): add --repair-fts and verify FTS index rebuilds Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/dccb3673-af86-43aa-aede-2e1449399775 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * refactor(fts): tighten repair/verify messaging and option naming Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/dccb3673-af86-43aa-aede-2e1449399775 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * docs: highlight analyze --repair-fts vs --force in READMEs Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/61edc967-debc-419f-9f51-aebf2ef08d22 Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> * fix(analyze): guard repair mode against missing graph store * fix(cli): reject --repair-fts with --force * test(analyze): document repair-store fixture intent * test(analyze): tidy repair failure fixtures and constants * test(analyze): clarify mock constants in repair tests * test(analyze): rename simulated missing-index constant * test(analyze): clarify mocked graph shape in full-verify test * refactor(analyze): finalize flag validation and test clarity * test(skip-git): avoid hard failing when FTS extension is unavailable * test(skip-git): log visible FTS-unavailable test skips * test(skip-git): tighten FTS-unavailable error detection * test(skip-git): simplify FTS-unavailable message checks * test(skip-git): avoid HOME pointing at parent repo in fixture env * fix(analyze): address Claude follow-up findings for repair guardrails Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/2f7243d3-ba16-4d83-86e5-17e6c58a3b0d * fix(repair-fts): clarify invalid graph-store preflight errors Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/2f7243d3-ba16-4d83-86e5-17e6c58a3b0d * test(analyze): strengthen assertions for conflict and missing-store errors Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/2f7243d3-ba16-4d83-86e5-17e6c58a3b0d * fix(repair-fts): make invalid graph-store type errors explicit Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/2f7243d3-ba16-4d83-86e5-17e6c58a3b0d * fix(repair-fts): improve graph-store type diagnostics Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/2f7243d3-ba16-4d83-86e5-17e6c58a3b0d --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: magyargergo <11230420+magyargergo@users.noreply.github.com> Co-authored-by: Gergő Magyar --- README.md | 3 +- gitnexus/README.md | 3 +- gitnexus/src/cli/analyze.ts | 25 ++ gitnexus/src/cli/index.ts | 1 + gitnexus/src/core/lbug/lbug-adapter.ts | 5 +- gitnexus/src/core/run-analyze.ts | 96 ++++++- gitnexus/src/core/search/fts-indexes.ts | 39 ++- gitnexus/src/mcp/local/local-backend.ts | 2 +- gitnexus/src/server/api.ts | 2 +- .../test/unit/analyze-no-stats-bridge.test.ts | 41 ++- gitnexus/test/unit/bm25-search.test.ts | 25 ++ gitnexus/test/unit/calltool-dispatch.test.ts | 2 +- gitnexus/test/unit/cli-index-help.test.ts | 7 + .../test/unit/run-analyze-fts-repair.test.ts | 243 ++++++++++++++++++ gitnexus/test/unit/skip-git-cli.test.ts | 141 +++++++--- 15 files changed, 590 insertions(+), 45 deletions(-) create mode 100644 gitnexus/test/unit/run-analyze-fts-repair.test.ts diff --git a/README.md b/README.md index 714d2cbfd..6e6c7193d 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,8 @@ args = ["-y", "gitnexus@latest", "mcp"] ```bash gitnexus setup # Configure MCP for your editors (one-time) gitnexus analyze [path] # Index a repository (or update stale index) -gitnexus analyze --force # Force full re-index +gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data +gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild gitnexus analyze --skills # Generate repo-specific skill files from detected communities gitnexus analyze --skip-embeddings # Skip embedding generation (faster) gitnexus analyze --skip-agents-md # Preserve custom AGENTS.md/CLAUDE.md gitnexus section edits diff --git a/gitnexus/README.md b/gitnexus/README.md index 55cdc12f7..640053013 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -151,7 +151,8 @@ Your AI agent gets these tools automatically: ```bash gitnexus setup # Configure MCP for your editors (one-time) gitnexus analyze [path] # Index a repository (or update stale index) -gitnexus analyze --force # Force full re-index +gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data +gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild gitnexus analyze --embeddings # Enable embedding generation (slower, better search) gitnexus analyze --skip-agents-md # Preserve custom AGENTS.md/CLAUDE.md gitnexus section edits gitnexus analyze --verbose # Log skipped files when parsers are unavailable diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index e24b8c894..70183ecf9 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -166,6 +166,7 @@ function ensureHeap(): boolean { export interface AnalyzeOptions { force?: boolean; + repairFts?: boolean; /** * Embedding generation toggle. Commander parses `--embeddings [limit]` as: * - `undefined` when the flag is omitted @@ -343,6 +344,15 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption process.env.GITNEXUS_EMBEDDING_DEVICE = options.embeddingDevice; } + if (options?.repairFts && options?.force) { + cliError( + ' Cannot combine `--repair-fts` with `--force`. ' + + 'Use `--repair-fts` for fast FTS-only repair, or `--force` for a full rebuild.\n', + ); + process.exitCode = 1; + return; + } + console.log('\n GitNexus Analyzer\n'); // `--index-only` is the stronger contract — it suppresses every form of file @@ -521,9 +531,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption // needs a fresh pipelineResult. Has no bearing on the registry // collision guard (see allowDuplicateName below). force: options?.force || options?.skills, + repairFts: options?.repairFts, embeddings: embeddingsEnabled, embeddingsNodeLimit, dropEmbeddings: options?.dropEmbeddings, + verbose: options?.verbose, skipGit: options?.skipGit, skipAgentsMd, skipSkills, @@ -568,6 +580,19 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption return; } + if (result.ftsRepairedOnly) { + clearInterval(elapsedTimer); + process.removeListener('SIGINT', sigintHandler); + console.log = origLog; + // eslint-disable-next-line no-console -- restoring after intentional progress-bar routing + console.warn = origWarn; + // eslint-disable-next-line no-console -- restoring after intentional progress-bar routing + console.error = origError; + bar.stop(); + console.log(' FTS indexes repaired successfully\n'); + return; + } + // Post-finalize invariant (#1169): runFullAnalysis nominally writes // meta.json and registers the repo, but on Windows it has been // observed to return successfully with neither artifact present diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 5b317e5c6..9450698d3 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -23,6 +23,7 @@ program .command('analyze [path]') .description('Index a repository (full analysis)') .option('-f, --force', 'Force full re-index even if up to date') + .option('--repair-fts', 'Repair/rebuild search FTS indexes without full re-analysis') .option( '--embeddings [limit]', 'Enable embedding generation for semantic search (off by default). ' + diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 8330614b2..77e9f65f1 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -1651,7 +1651,10 @@ export const createFTSIndex = async ( if (ensuredFTSIndexes.has(key)) return; if (!(await loadFTSExtension())) { - return; + throw new Error( + `FTS extension unavailable - cannot create FTS index ${tableName}.${indexName}. ` + + 'Run `gitnexus doctor` and ensure the LadybugDB FTS extension is installed and loadable on this machine.', + ); } const propList = properties.map((p) => `'${p}'`).join(', '); diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 425f18f9a..e11d0833b 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -25,7 +25,7 @@ import { deleteAllCommunitiesAndProcesses, queryImporters, } from './lbug/lbug-adapter.js'; -import { createSearchFTSIndexes } from './search/fts-indexes.js'; +import { createSearchFTSIndexes, verifySearchFTSIndexes } from './search/fts-indexes.js'; import { getStoragePaths, saveMeta, @@ -71,6 +71,10 @@ export interface AnalyzeOptions { * bypass. See `allowDuplicateName` below. */ force?: boolean; + /** Repair only search indexes without re-running full parsing/indexing. */ + repairFts?: boolean; + /** Emit per-index FTS create logs. */ + verbose?: boolean; embeddings?: boolean; /** * Override the auto-skip node-count cap for embedding generation. @@ -126,6 +130,8 @@ export interface AnalyzeResult { alreadyUpToDate?: boolean; /** The raw pipeline result — only populated when needed by callers (e.g. skill generation). */ pipelineResult?: any; + /** True when analyze only repaired FTS indexes and skipped pipeline re-analysis. */ + ftsRepairedOnly?: boolean; } // Re-export the pure flag-derivation helper so external callers (and tests) @@ -190,6 +196,78 @@ export async function runFullAnalysis( const currentCommit = repoHasGit ? getCurrentCommit(repoPath) : ''; const existingMeta = await loadMeta(storagePath); + // ── FTS-only repair path ──────────────────────────────────────────── + if (options.repairFts) { + if (!existingMeta) { + throw new Error( + 'Cannot repair FTS indexes because this repository has not been analyzed yet. ' + + 'Run `gitnexus analyze` first to create the initial index, then retry `--repair-fts`.', + ); + } + let lbugStat; + try { + lbugStat = await fs.lstat(lbugPath); + } catch { + throw new Error( + `Cannot repair FTS indexes: graph store at ${lbugPath} is missing. ` + + 'Run `gitnexus analyze` (full) to rebuild from scratch.', + ); + } + if (!lbugStat.isFile()) { + const foundType = lbugStat.isDirectory() + ? 'a directory' + : lbugStat.isSymbolicLink() + ? 'a symbolic link' + : lbugStat.isSocket() + ? 'a socket' + : lbugStat.isBlockDevice() + ? 'a block device' + : lbugStat.isCharacterDevice() + ? 'a character device' + : lbugStat.isFIFO() + ? 'a FIFO' + : 'not a regular file'; + throw new Error( + `Cannot repair FTS indexes: graph store at ${lbugPath} is ${foundType} (expected a file). ` + + 'Run `gitnexus analyze` (full) to rebuild from scratch.', + ); + } + try { + await initLbug(lbugPath); + progress('fts', 85, 'Repairing search indexes...'); + await createSearchFTSIndexes({ + onIndexStart: options.verbose + ? (table, indexName) => log(`FTS: creating ${table}.${indexName}`) + : undefined, + onIndexReady: options.verbose + ? (table, indexName) => log(`FTS: ready ${table}.${indexName}`) + : undefined, + }); + const missing = await verifySearchFTSIndexes(executeQuery); + if (missing.length > 0) { + throw new Error( + `FTS repair failed - missing indexes after rebuild: ${missing.join(', ')}. ` + + 'Run `gitnexus analyze --force` to perform a full graph+FTS rebuild; ' + + 'if that also fails, verify FTS extension availability via `gitnexus doctor`.', + ); + } + await ensureGitNexusIgnored(repoPath); + progress('fts', 90, 'Search indexes ready'); + progress('done', 100, 'Done'); + return { + repoName: + options.registryName ?? + getInferredRepoName(repoPath) ?? + path.basename(resolveRepoIdentityRoot(repoPath)), + repoPath, + stats: existingMeta.stats ?? {}, + ftsRepairedOnly: true, + }; + } finally { + await closeLbug().catch(() => {}); + } + } + // ── Crash recovery: dirty flag forces full rebuild ──────────────── // If the previous incremental run set incrementalInProgress and didn't // clear it, the on-disk index may be in a half-state. Cheapest path @@ -583,7 +661,21 @@ export async function runFullAnalysis( // ── Phase 3: FTS (85–90%) ───────────────────────────────────────── progress('fts', 85, 'Creating search indexes...'); - await createSearchFTSIndexes(); + await createSearchFTSIndexes({ + onIndexStart: options.verbose + ? (table, indexName) => log(`FTS: creating ${table}.${indexName}`) + : undefined, + onIndexReady: options.verbose + ? (table, indexName) => log(`FTS: ready ${table}.${indexName}`) + : undefined, + }); + const missingIndexNames = await verifySearchFTSIndexes(executeQuery); + if (missingIndexNames.length > 0) { + throw new Error( + `FTS verification failed - missing indexes after analyze: ${missingIndexNames.join(', ')}. ` + + 'Check FTS extension availability, then retry `gitnexus analyze --force` for a full rebuild.', + ); + } progress('fts', 90, 'Search indexes ready'); // ── Phase 3.5: Re-insert cached embeddings ──────────────────────── diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index 5dfbde647..01147beed 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -1,8 +1,45 @@ import { createFTSIndex } from '../lbug/lbug-adapter.js'; import { FTS_INDEXES } from './fts-schema.js'; -export async function createSearchFTSIndexes(): Promise { +export interface CreateSearchFTSIndexesOptions { + onIndexStart?: (table: string, indexName: string) => void; + onIndexReady?: (table: string, indexName: string) => void; +} + +export async function createSearchFTSIndexes( + options?: CreateSearchFTSIndexesOptions, +): Promise { for (const { table, indexName, properties } of FTS_INDEXES) { + options?.onIndexStart?.(table, indexName); await createFTSIndex(table, indexName, [...properties]); + options?.onIndexReady?.(table, indexName); } } + +export async function verifySearchFTSIndexes( + executeQuery: (cypher: string) => Promise, +): Promise { + const safeIdentifier = (value: string): string => { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) { + throw new Error(`Invalid FTS identifier: ${value}`); + } + return value; + }; + + const missing: string[] = []; + for (const { table, indexName } of FTS_INDEXES) { + const safeTable = safeIdentifier(table); + const safeIndex = safeIdentifier(indexName); + const probe = ` + CALL QUERY_FTS_INDEX('${safeTable}', '${safeIndex}', '__gitnexus_fts_probe__', conjunctive := false) + RETURN score + LIMIT 1 + `; + try { + await executeQuery(probe); + } catch { + missing.push(`${table}.${indexName}`); + } + } + return missing; +} diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 4cff47a28..d1297ea17 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -1066,7 +1066,7 @@ export class LocalBackend { timing, ...(!ftsUsed && { warning: - 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.', + 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.', }), }; } diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 7fc892574..796fa04e1 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -1233,7 +1233,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const response: any = { results: results.searchResults ?? results }; if (results.ftsAvailable === false) { response.warning = - 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.'; + 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.'; } res.json(response); } catch (err: any) { diff --git a/gitnexus/test/unit/analyze-no-stats-bridge.test.ts b/gitnexus/test/unit/analyze-no-stats-bridge.test.ts index f941141ef..b2d368c2f 100644 --- a/gitnexus/test/unit/analyze-no-stats-bridge.test.ts +++ b/gitnexus/test/unit/analyze-no-stats-bridge.test.ts @@ -1,16 +1,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -const { runFullAnalysisMock, generateAIContextFilesMock, generateSkillFilesMock } = vi.hoisted( - () => { +const { runFullAnalysisMock, generateAIContextFilesMock, generateSkillFilesMock, cliErrorMock } = + vi.hoisted(() => { const runFullAnalysisMock = vi.fn(); const generateAIContextFilesMock = vi.fn(async () => ({ files: [] as string[] })); const generateSkillFilesMock = vi.fn(async () => ({ skills: [{ name: 'c', label: 'Community', symbolCount: 1, fileCount: 1 }], outputPath: '/repo/.claude/skills/generated', })); - return { runFullAnalysisMock, generateAIContextFilesMock, generateSkillFilesMock }; - }, -); + const cliErrorMock = vi.fn(); + return { + runFullAnalysisMock, + generateAIContextFilesMock, + generateSkillFilesMock, + cliErrorMock, + }; + }); vi.mock('../../src/core/run-analyze.js', () => ({ runFullAnalysis: runFullAnalysisMock, @@ -24,6 +29,10 @@ vi.mock('../../src/cli/skill-gen.js', () => ({ generateSkillFiles: generateSkillFilesMock, })); +vi.mock('../../src/cli/cli-message.js', () => ({ + cliError: cliErrorMock, +})); + vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ closeLbug: vi.fn(async () => undefined), })); @@ -62,6 +71,7 @@ describe('analyzeCommand commander → runFullAnalysis noStats bridge (#1477)', skills: [{ name: 'c', label: 'Community', symbolCount: 1, fileCount: 1 }], outputPath: '/repo/.claude/skills/generated', }); + cliErrorMock.mockReset(); process.exitCode = undefined; process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=8192`.trim(); }); @@ -104,6 +114,27 @@ describe('analyzeCommand commander → runFullAnalysis noStats bridge (#1477)', expect(opts.skipAgentsMd).toBe(true); }); + it('passes --repair-fts through to runFullAnalysis', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, { repairFts: true }); + + const opts = runFullAnalysisMock.mock.calls[0][1]; + expect(opts.repairFts).toBe(true); + }); + + it('rejects combining --repair-fts with --force', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, { repairFts: true, force: true }); + + expect(process.exitCode).toBe(1); + expect(cliErrorMock).toHaveBeenCalledWith( + expect.stringMatching(/cannot combine `--repair-fts` with `--force`/i), + ); + expect(runFullAnalysisMock).not.toHaveBeenCalled(); + }); + it('passes stats:false as noStats to generateAIContextFiles on the --skills regeneration path (#1477)', async () => { runFullAnalysisMock.mockResolvedValueOnce({ repoName: 'repo', diff --git a/gitnexus/test/unit/bm25-search.test.ts b/gitnexus/test/unit/bm25-search.test.ts index 6b878ef1b..984e6364a 100644 --- a/gitnexus/test/unit/bm25-search.test.ts +++ b/gitnexus/test/unit/bm25-search.test.ts @@ -40,6 +40,31 @@ describe('BM25 search', () => { ['Interface', 'interface_fts', ['name', 'content']], ]); }); + + it('verifies all configured FTS indexes are queryable', async () => { + const executeQuery = vi.fn().mockResolvedValue([]); + const { verifySearchFTSIndexes } = await import('../../src/core/search/fts-indexes.js'); + + const missing = await verifySearchFTSIndexes(executeQuery); + + expect(missing).toEqual([]); + expect(executeQuery).toHaveBeenCalledTimes(5); + }); + + it('reports missing indexes when an FTS probe fails', async () => { + const executeQuery = vi + .fn() + .mockResolvedValueOnce([]) + .mockRejectedValueOnce(new Error('index does not exist')) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]); + const { verifySearchFTSIndexes } = await import('../../src/core/search/fts-indexes.js'); + + const missing = await verifySearchFTSIndexes(executeQuery); + + expect(missing).toEqual(['Function.function_fts']); + }); }); describe('searchFTSFromLbug', () => { diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index 1ad46ba72..8c69a23d5 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -203,7 +203,7 @@ describe('LocalBackend.callTool', () => { const result = await backend.callTool('query', { query: 'ProcessActivity' }); expect(result).toHaveProperty('warning'); - expect((result as any).warning).toMatch(/gitnexus analyze --force/); + expect((result as any).warning).toMatch(/gitnexus analyze --repair-fts/); }); it('does not include warning when ftsAvailable is true with zero results', async () => { diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index 889a7473f..00f5c0572 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -76,4 +76,11 @@ describe('CLI help surface', () => { expect(result.stdout).toContain('understand-quickly'); expect(result.stdout).toContain('UNDERSTAND_QUICKLY_TOKEN'); }); + + it('analyze help includes the FTS repair option', () => { + const result = runHelp('analyze'); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('--repair-fts'); + }); }); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts new file mode 100644 index 000000000..c35aae6e8 --- /dev/null +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -0,0 +1,243 @@ +import fs from 'fs/promises'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { getStoragePaths, saveMeta } from '../../src/storage/repo-manager.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const SIMULATED_MISSING_FTS_INDEX_NAME = 'File.file_fts'; +const PLACEHOLDER_GRAPH_STORE_CONTENT = 'fixture'; + +const createPlaceholderGraphStore = async (lbugPath: string): Promise => { + // Repair mode gates on existence before `initLbug` takes over open/validate. + // A placeholder file is enough to exercise this preflight branch. + await fs.writeFile(lbugPath, PLACEHOLDER_GRAPH_STORE_CONTENT); +}; + +const escapeForRegex = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +describe('runFullAnalysis FTS repair and verification failure paths', () => { + afterEach(() => { + vi.doUnmock('../../src/core/lbug/lbug-adapter.js'); + vi.doUnmock('../../src/core/search/fts-indexes.js'); + vi.doUnmock('../../src/core/ingestion/pipeline.js'); + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('fails repair mode when no base meta exists', async () => { + const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-no-meta-'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow(/has not been analyzed yet/i); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('fails repair mode when graph store is missing', async () => { + const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-missing-store-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow(new RegExp(`graph store at ${escapeForRegex(lbugPath)} is missing`)); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('fails repair mode when graph store path is not a file', async () => { + const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-store-not-file-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + }); + await fs.mkdir(lbugPath, { recursive: true }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow( + new RegExp( + `graph store at ${escapeForRegex(lbugPath)} is a directory \\(expected a file\\)`, + ), + ); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('fails repair mode when FTS verify still reports missing indexes', async () => { + const closeLbugMock = vi.fn(async () => undefined); + vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ + initLbug: vi.fn(async () => undefined), + loadGraphToLbug: vi.fn(async () => undefined), + getLbugStats: vi.fn(async () => ({})), + executeQuery: vi.fn(async () => []), + executeWithReusedStatement: vi.fn(async () => []), + closeLbug: closeLbugMock, + loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), + deleteNodesForFile: vi.fn(async () => undefined), + deleteAllCommunitiesAndProcesses: vi.fn(async () => undefined), + queryImporters: vi.fn(async () => []), + })); + vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + createSearchFTSIndexes: vi.fn(async () => undefined), + verifySearchFTSIndexes: vi.fn(async () => [SIMULATED_MISSING_FTS_INDEX_NAME]), + })); + + const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-verify-fail-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow(/FTS repair failed - missing indexes after rebuild/i); + expect(closeLbugMock).toHaveBeenCalled(); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('surfaces extension-unavailable errors from FTS index creation in repair mode', async () => { + vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ + initLbug: vi.fn(async () => undefined), + loadGraphToLbug: vi.fn(async () => undefined), + getLbugStats: vi.fn(async () => ({})), + executeQuery: vi.fn(async () => []), + executeWithReusedStatement: vi.fn(async () => []), + closeLbug: vi.fn(async () => undefined), + loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), + deleteNodesForFile: vi.fn(async () => undefined), + deleteAllCommunitiesAndProcesses: vi.fn(async () => undefined), + queryImporters: vi.fn(async () => []), + })); + vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + createSearchFTSIndexes: vi.fn(async () => { + throw new Error('FTS extension unavailable'); + }), + verifySearchFTSIndexes: vi.fn(async () => []), + })); + + const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-extension-fail-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow(/FTS extension unavailable/i); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('fails full analyze when FTS verification reports missing indexes after creation', async () => { + vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ + initLbug: vi.fn(async () => undefined), + loadGraphToLbug: vi.fn(async () => undefined), + getLbugStats: vi.fn(async () => ({ nodes: 0, edges: 0, communities: 0, processes: 0 })), + executeQuery: vi.fn(async () => []), + executeWithReusedStatement: vi.fn(async () => []), + closeLbug: vi.fn(async () => undefined), + loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), + deleteNodesForFile: vi.fn(async () => undefined), + deleteAllCommunitiesAndProcesses: vi.fn(async () => undefined), + queryImporters: vi.fn(async () => []), + })); + vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + createSearchFTSIndexes: vi.fn(async () => undefined), + verifySearchFTSIndexes: vi.fn(async () => ['Function.function_fts']), + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + // Full-analyze path only needs `forEachNode` before the FTS verify guard. + graph: { forEachNode: () => undefined }, + })), + })); + + const tmpRepo = await createTempDir('gitnexus-run-analyze-full-verify-fail-'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { force: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow(/FTS verification failed - missing indexes after analyze/i); + } finally { + await tmpRepo.cleanup(); + } + }); +}); diff --git a/gitnexus/test/unit/skip-git-cli.test.ts b/gitnexus/test/unit/skip-git-cli.test.ts index 80c07ed17..3b9b7b7e8 100644 --- a/gitnexus/test/unit/skip-git-cli.test.ts +++ b/gitnexus/test/unit/skip-git-cli.test.ts @@ -6,6 +6,32 @@ import fs from 'fs'; describe('--skip-git CLI flag', () => { const cliPath = path.resolve(__dirname, '../../dist/cli/index.js'); + const ftsUnavailableMessage = 'FTS extension unavailable - cannot create FTS index'; + + interface ExecSyncLikeError { + message?: string; + stdout?: string | Buffer; + stderr?: string | Buffer; + } + + const isFtsUnavailableError = (err: unknown): boolean => { + if (!err || typeof err !== 'object') return false; + const e = err as ExecSyncLikeError; + return ( + e.message?.includes(ftsUnavailableMessage) || + e.stdout?.toString().includes(ftsUnavailableMessage) || + e.stderr?.toString().includes(ftsUnavailableMessage) + ); + }; + + const shouldSkipForFtsUnavailable = (err: unknown, testName: string): boolean => { + if (!isFtsUnavailableError(err)) return false; + + console.warn( + `[skip-git-cli.test] Skipping "${testName}" because FTS extension is unavailable.`, + ); + return true; + }; it('Commander maps --skip-git to options.skipGit (not --no-git inversion)', () => { // Verify the CLI defines --skip-git and --skip-agents-md in analyze help. @@ -37,14 +63,26 @@ describe('--skip-git CLI flag', () => { ...process.env, HOME: gitnexusHome, GITNEXUS_HOME: gitnexusHome, - GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', }; try { - const output = execSync( - `node "${cliPath}" analyze "${tmpDir}" --index-only --skills --skip-agents-md`, - { encoding: 'utf8', timeout: 60000, env }, - ); + let output: string; + try { + output = execSync( + `node "${cliPath}" analyze "${tmpDir}" --index-only --skills --skip-agents-md`, + { + encoding: 'utf8', + timeout: 60000, + env, + }, + ); + } catch (err: unknown) { + if ( + shouldSkipForFtsUnavailable(err, 'warns when --index-only overrides --skills (PR 1485)') + ) + return; + throw err; + } expect(output).toContain('--index-only overrides --skills'); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -87,15 +125,25 @@ describe('--skip-git CLI flag', () => { ...process.env, HOME: gitnexusHome, GITNEXUS_HOME: gitnexusHome, - GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', }; try { - execSync(`node "${cliPath}" analyze "${tmpDir}" --skip-git --skip-agents-md`, { - encoding: 'utf8', - timeout: 60000, - env, - }); + try { + execSync(`node "${cliPath}" analyze "${tmpDir}" --skip-git --skip-agents-md`, { + encoding: 'utf8', + timeout: 60000, + env, + }); + } catch (err: unknown) { + if ( + shouldSkipForFtsUnavailable( + err, + 'still respects .gitnexusignore when run with --skip-git', + ) + ) + return; + throw err; + } const keepContext = execSync( `node "${cliPath}" context keep --repo "${path.basename(tmpDir)}"`, @@ -130,9 +178,8 @@ describe('--skip-git CLI flag', () => { function testEnv() { return { ...process.env, - HOME: parentDir, + HOME: gitnexusHome, GITNEXUS_HOME: gitnexusHome, - GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', }; } @@ -221,12 +268,24 @@ describe('--skip-git CLI flag', () => { createTestStructure(); try { // Run analyze from COOLIO with --skip-git - const output = execSync(`node "${cliPath}" analyze --skip-git --skip-agents-md`, { - cwd: path.join(parentDir, 'COOLIO'), - encoding: 'utf8', - timeout: 60000, - env: testEnv(), - }); + let output: string; + try { + output = execSync(`node "${cliPath}" analyze --skip-git --skip-agents-md`, { + cwd: path.join(parentDir, 'COOLIO'), + encoding: 'utf8', + timeout: 60000, + env: testEnv(), + }); + } catch (err: unknown) { + if ( + shouldSkipForFtsUnavailable( + err, + 'from subdir inside parent git repo, indexes subdir not parent', + ) + ) + return; + throw err; + } // Should mention COOLIO not the parent dir name expect(output).toContain('COOLIO'); @@ -255,12 +314,23 @@ describe('--skip-git CLI flag', () => { stdio: 'ignore', }); - execSync(`node "${cliPath}" analyze --skip-git --skip-agents-md`, { - cwd: path.join(parentDir, 'COOLIO'), - encoding: 'utf8', - timeout: 60000, - env: testEnv(), - }); + try { + execSync(`node "${cliPath}" analyze --skip-git --skip-agents-md`, { + cwd: path.join(parentDir, 'COOLIO'), + encoding: 'utf8', + timeout: 60000, + env: testEnv(), + }); + } catch (err: unknown) { + if ( + shouldSkipForFtsUnavailable( + err, + 'keeps parent git status clean for --skip-git subdir analyze (#1233)', + ) + ) + return; + throw err; + } expect( fs.readFileSync(path.join(parentDir, 'COOLIO', '.gitnexus', '.gitignore'), 'utf8'), @@ -278,12 +348,21 @@ describe('--skip-git CLI flag', () => { it('explicit input path with --skip-git indexes subdir', () => { createTestStructure(); try { - const output = execSync(`node "${cliPath}" analyze ./COOLIO --skip-git --skip-agents-md`, { - cwd: parentDir, - encoding: 'utf8', - timeout: 60000, - env: testEnv(), - }); + let output: string; + try { + output = execSync(`node "${cliPath}" analyze ./COOLIO --skip-git --skip-agents-md`, { + cwd: parentDir, + encoding: 'utf8', + timeout: 60000, + env: testEnv(), + }); + } catch (err: unknown) { + if ( + shouldSkipForFtsUnavailable(err, 'explicit input path with --skip-git indexes subdir') + ) + return; + throw err; + } expect(output).toContain('COOLIO'); expectCoolioRegistryEntry(); From df1882d36b45dc2cd9b83d1ab3ae648a92db9a01 Mon Sep 17 00:00:00 2001 From: jelsco <58397194+jelsco@users.noreply.github.com> Date: Wed, 20 May 2026 07:37:58 -0600 Subject: [PATCH 17/51] fix(ingestion): surface skipped large-file paths by default (#1659) (#1661) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ingestion): surface skipped large-file paths by default (#1659) The 512 KB skip threshold in filesystem-walker is necessary, but the existing warning only said "Skipped N large files" with no paths unless GITNEXUS_VERBOSE=1 was set. In a repo with one or two oversized first- party source files (e.g. a 17K-line cron handler), every IMPORTS/CALLS edge from that file silently disappeared and the surface looked like a Python resolver bug. Issue #1659 was filed against the resolver for exactly that reason, but the resolver was fine; the file was being dropped before parse. Changes: * Always print up to 5 skipped paths after the count line. * If more than 5 were skipped, append "...and N more" with a hint to set GITNEXUS_VERBOSE=1 for the full list. * When running at the default threshold, emit a one-line hint about GITNEXUS_MAX_FILE_SIZE= so operators know how to widen it. * Cover the new behavior with three additional tests in the existing filesystem-walker integration suite, plus a new describe block for the >5 preview-cap case. Verified end-to-end on a 680-file Python repo that hit #1659: before the patch, "Skipped 3 large files (>512KB, ...)" was the only signal and impact upstream of a function called from cron.py returned 1 of 5 real callers; after the patch the cron file is listed by name with the hint, and running with GITNEXUS_MAX_FILE_SIZE=1024 brings the missing callers back (impactedCount 1 -> 9). * fix(ingestion): address #1661 adversarial review follow-ups (F1/F2/F3) Three non-blocking nits flagged by the adversarial review on #1661: F1 (output stability) — skippedLargePaths was populated by concurrent fs.stat callbacks in batches of 32, so push order within a batch was completion-order rather than input-order. The default preview's "first 5" could vary across runs on the same repo. Fix: sort the array before slicing. New test asserts the verbose output is in sorted order. F2 (boundary coverage) — the preview-cap describe block created 8 large files, so the SKIPPED_PREVIEW_CAP = 5 comparison was never exercised at the exact <= boundary. A future off-by-one (<= → <) would not fail the suite. Fix: add two tests, one with exactly 5 files (all listed, no truncation) and one with exactly 6 files (5 listed plus "...and 1 more"). F3 (hint accuracy) — isDefault compared effective bytes, so an operator who explicitly set GITNEXUS_MAX_FILE_SIZE=512 (the same KB as the default) would still see the "Set GITNEXUS_MAX_FILE_SIZE=..." hint. Fix: gate the hint on whether the env var is unset, not on the resulting byte value. New test pins the explicit-default-value case. All 34 filesystem-walker tests pass (was 30; +4 new). Prettier clean, typecheck clean for the changed files. --------- Co-authored-by: scotjelinski <58397194+scotjelinski@users.noreply.github.com> Co-authored-by: Gergő Magyar --- .../src/core/ingestion/filesystem-walker.ts | 27 ++- .../integration/filesystem-walker.test.ts | 167 ++++++++++++++++++ 2 files changed, 190 insertions(+), 4 deletions(-) diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 4d6725e24..c30ea4321 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -74,12 +74,31 @@ export const walkRepositoryPaths = async ( if (skippedLarge > 0) { const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; + const isOverrideUnset = !process.env.GITNEXUS_MAX_FILE_SIZE; const suffix = isDefault ? ', likely generated/vendored' : ''; logger.warn(` Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`); - if (isVerboseIngestionEnabled()) { - for (const p of skippedLargePaths) { - logger.warn(` - ${p}`); - } + + // Always show at least the first few paths so users can diagnose why + // edges are missing from a specific file (issue #1659). The full list is + // gated behind GITNEXUS_VERBOSE=1 to avoid flooding output on repos with + // many generated/vendored blobs. Sort before slicing so the preview is + // stable across runs (fs.stat callbacks race within each batch). + skippedLargePaths.sort(); + const SKIPPED_PREVIEW_CAP = 5; + const showAll = isVerboseIngestionEnabled() || skippedLargePaths.length <= SKIPPED_PREVIEW_CAP; + const preview = showAll ? skippedLargePaths : skippedLargePaths.slice(0, SKIPPED_PREVIEW_CAP); + for (const p of preview) { + logger.warn(` - ${p}`); + } + if (!showAll) { + const remaining = skippedLargePaths.length - SKIPPED_PREVIEW_CAP; + logger.warn(` ...and ${remaining} more (set GITNEXUS_VERBOSE=1 to list them all)`); + } + // Only hint about the env var when the user has not set it at all. An + // explicit GITNEXUS_MAX_FILE_SIZE=512 happens to resolve to the same + // bytes as the default but the operator clearly already knows the knob. + if (isDefault && isOverrideUnset) { + logger.warn(` Set GITNEXUS_MAX_FILE_SIZE= to include files above the default cap.`); } } diff --git a/gitnexus/test/integration/filesystem-walker.test.ts b/gitnexus/test/integration/filesystem-walker.test.ts index 2a1257ccc..1d743a898 100644 --- a/gitnexus/test/integration/filesystem-walker.test.ts +++ b/gitnexus/test/integration/filesystem-walker.test.ts @@ -398,5 +398,172 @@ describe('filesystem-walker', () => { expect(skipWarnings.length).toBeGreaterThan(0); expect(String(skipWarnings[0].msg ?? '')).toContain('generated/vendored'); }); + + // Regression: issue #1659. The skipped-paths list and the + // GITNEXUS_MAX_FILE_SIZE hint must appear by default, otherwise users + // see "Skipped N large files" with no actionable detail and misdiagnose + // missing IMPORTS/CALLS edges as a resolver bug. + it('lists the skipped path by default (not gated behind GITNEXUS_VERBOSE)', async () => { + await walkRepositoryPaths(sizeDir); + const pathWarnings = cap.records().filter((r) => String(r.msg ?? '').includes(BIG_FILE)); + expect(pathWarnings.length).toBeGreaterThan(0); + }); + + it('emits a GITNEXUS_MAX_FILE_SIZE hint when running with the default cap', async () => { + await walkRepositoryPaths(sizeDir); + const hint = cap + .records() + .filter((r) => String(r.msg ?? '').includes('GITNEXUS_MAX_FILE_SIZE=')); + expect(hint.length).toBe(1); + }); + + it('omits the GITNEXUS_MAX_FILE_SIZE hint when an override is active', async () => { + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; + await walkRepositoryPaths(sizeDir); + const hint = cap + .records() + .filter((r) => String(r.msg ?? '').includes('GITNEXUS_MAX_FILE_SIZE=')); + expect(hint.length).toBe(0); + }); + + // Edge case from the #1661 adversarial review: setting GITNEXUS_MAX_FILE_SIZE + // to the same value as the default (512KB) used to still print the hint + // because the byte comparison resolved to equal. The hint should care + // about whether the operator set the env var, not what value they chose. + it('omits the GITNEXUS_MAX_FILE_SIZE hint when the override equals the default value', async () => { + process.env.GITNEXUS_MAX_FILE_SIZE = '512'; + await walkRepositoryPaths(sizeDir); + const hint = cap + .records() + .filter((r) => String(r.msg ?? '').includes('GITNEXUS_MAX_FILE_SIZE=')); + expect(hint.length).toBe(0); + }); + }); + + describe('large file skip preview cap (#1659)', () => { + let manyDir: string; + const ORIGINAL_ENV = process.env.GITNEXUS_MAX_FILE_SIZE; + const ORIGINAL_VERBOSE = process.env.GITNEXUS_VERBOSE; + let cap: ReturnType; + + beforeAll(async () => { + manyDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-size-many-')); + await fs.mkdir(path.join(manyDir, 'src'), { recursive: true }); + // 8 files >512KB so the preview-cap path (5) is exercised. + for (let i = 0; i < 8; i++) { + await fs.writeFile(path.join(manyDir, 'src', `big${i}.ts`), 'x'.repeat(600 * 1024)); + } + }); + + afterAll(async () => { + await fs.rm(manyDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + delete process.env.GITNEXUS_VERBOSE; + _resetMaxFileSizeWarnings(); + cap = _captureLogger(); + }); + + afterEach(() => { + if (ORIGINAL_ENV === undefined) { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + } else { + process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL_ENV; + } + if (ORIGINAL_VERBOSE === undefined) { + delete process.env.GITNEXUS_VERBOSE; + } else { + process.env.GITNEXUS_VERBOSE = ORIGINAL_VERBOSE; + } + cap.restore(); + }); + + it('truncates the path list to 5 and mentions GITNEXUS_VERBOSE when over the cap', async () => { + await walkRepositoryPaths(manyDir); + const pathLines = cap.records().filter((r) => /^\s*-\s/.test(String(r.msg ?? ''))); + expect(pathLines.length).toBe(5); + const more = cap + .records() + .filter((r) => String(r.msg ?? '').includes('and 3 more (set GITNEXUS_VERBOSE=1')); + expect(more.length).toBe(1); + }); + + // Boundary check from the #1661 adversarial review: the SKIPPED_PREVIEW_CAP + // comparison is `<=`, so 5 paths should list all five without a truncation + // line and 6 paths should list exactly five plus "...and 1 more". Tested + // explicitly so a future off-by-one refactor (`<=` → `<`) fails fast. + it('lists all paths and omits the truncation line at exactly 5 skipped files', async () => { + const fiveDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-size-five-')); + try { + await fs.mkdir(path.join(fiveDir, 'src'), { recursive: true }); + for (let i = 0; i < 5; i++) { + await fs.writeFile(path.join(fiveDir, 'src', `big${i}.ts`), 'x'.repeat(600 * 1024)); + } + await walkRepositoryPaths(fiveDir); + const pathLines = cap.records().filter((r) => /^\s*-\s/.test(String(r.msg ?? ''))); + expect(pathLines.length).toBe(5); + const more = cap.records().filter((r) => String(r.msg ?? '').includes('...and ')); + expect(more.length).toBe(0); + } finally { + await fs.rm(fiveDir, { recursive: true, force: true }); + } + }); + + it('lists exactly 5 paths plus "...and 1 more" at exactly 6 skipped files', async () => { + const sixDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-size-six-')); + try { + await fs.mkdir(path.join(sixDir, 'src'), { recursive: true }); + for (let i = 0; i < 6; i++) { + await fs.writeFile(path.join(sixDir, 'src', `big${i}.ts`), 'x'.repeat(600 * 1024)); + } + await walkRepositoryPaths(sixDir); + const pathLines = cap.records().filter((r) => /^\s*-\s/.test(String(r.msg ?? ''))); + expect(pathLines.length).toBe(5); + const more = cap + .records() + .filter((r) => String(r.msg ?? '').includes('and 1 more (set GITNEXUS_VERBOSE=1')); + expect(more.length).toBe(1); + } finally { + await fs.rm(sixDir, { recursive: true, force: true }); + } + }); + + it('lists every skipped path when GITNEXUS_VERBOSE=1', async () => { + process.env.GITNEXUS_VERBOSE = '1'; + await walkRepositoryPaths(manyDir); + const pathLines = cap.records().filter((r) => /^\s*-\s/.test(String(r.msg ?? ''))); + expect(pathLines.length).toBe(8); + const more = cap.records().filter((r) => String(r.msg ?? '').includes('and ')); + expect(more.length).toBe(0); + }); + + // Issue #1659 follow-up (PR #1661 review): paths were pushed in fs.stat + // completion order, so the default preview could vary between runs on + // the same repo. The implementation sorts skippedLargePaths before + // slicing, so the listed paths come out in sorted order, which is the + // stable contract operators can rely on. + it('lists skipped paths in sorted order (deterministic preview)', async () => { + process.env.GITNEXUS_VERBOSE = '1'; + await walkRepositoryPaths(manyDir); + const pathLines = cap + .records() + .map((r) => String(r.msg ?? '')) + .filter((m) => /^\s*-\s/.test(m)) + .map((m) => m.replace(/^\s*-\s*/, '')); + expect(pathLines).toEqual([...pathLines].sort()); + // sanity-check we actually saw all 8 of the manyDir fixture + expect(pathLines).toEqual([ + 'src/big0.ts', + 'src/big1.ts', + 'src/big2.ts', + 'src/big3.ts', + 'src/big4.ts', + 'src/big5.ts', + 'src/big6.ts', + 'src/big7.ts', + ]); + }); }); }); From 4d2ed0e52501104450a2c9200a543d1633690875 Mon Sep 17 00:00:00 2001 From: Shane Thurston Wijaya <129602553+sanguine59@users.noreply.github.com> Date: Wed, 20 May 2026 22:14:13 +0700 Subject: [PATCH 18/51] fix(eval-server): localhost now doesn't normalize into IPv4 instead lets OS decide which to bind (#1722) * fix(eval-server): localhost now doesn't normalize into IPv4 instead lets OS decide which to bind * fix(eval-server): EADDRNOTAVAIL now treats as potential IPv6 * test(eval-server): new integration test for --host localhost * docs(eval-server): updated eval/README.md based on latest update * fix(eval-server): clarify EADDRNOTAVAIL diagnostic, guard server.address(), and soften localhost docs --- eval/README.md | 2 + gitnexus/src/cli/eval-server.ts | 32 +++++-- gitnexus/test/integration/cli-e2e.test.ts | 97 ++++++++++++++++++++++ gitnexus/test/unit/eval-formatters.test.ts | 4 +- 4 files changed, 124 insertions(+), 11 deletions(-) diff --git a/eval/README.md b/eval/README.md index e3539cf20..1b01bce59 100644 --- a/eval/README.md +++ b/eval/README.md @@ -211,6 +211,8 @@ environment: Defaults are `port: 4848` and `host: 127.0.0.1` (loopback only). Use `0.0.0.0` only when the agent container needs to reach the eval-server from a separate network namespace. The health probe and tool scripts connect via the configured bind host (defaulting to `127.0.0.1`), which is reachable for both loopback and all-interface binds. +`"localhost"` is also a valid `eval_server_host` value. The OS resolves it at bind time — typically `127.0.0.1` on dual-stack or IPv4-only systems, and `::1` on IPv6-only systems. The exact result depends on your `/etc/hosts` and `gai.conf`. The READY signal will reflect the actual bound address (e.g. `GITNEXUS_EVAL_SERVER_READY:127.0.0.1:4848` or `GITNEXUS_EVAL_SERVER_READY:[::1]:4848`), not the literal string `localhost`. Use this when you want the server to bind to whichever loopback address the OS prefers rather than forcing IPv4. + **Running eval-server directly in Docker / Docker Compose:** ```bash diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index 8735abbef..88ad10592 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -44,10 +44,12 @@ export interface EvalServerOptions { /** * Validate the --host value. Accepts IPv4, IPv6, or "localhost". - * Returns the normalised host string, or null if invalid. + * Returns the host string unchanged, or null if invalid. + * "localhost" is passed through so the OS resolves it to the correct loopback + * address (127.0.0.1 or ::1) at bind time rather than forcing IPv4. */ export function validateHost(raw: string): string | null { - if (raw === 'localhost') return '127.0.0.1'; + if (raw === 'localhost') return raw; if (isIPv4(raw) || isIPv6(raw)) return raw; return null; } @@ -470,12 +472,14 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise { }, 30000); }); }, 35000); + + it('emits READY signal with bound IP (not literal "localhost") when --host localhost is used', () => { + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + '--import', + tsxImportUrl, + cliEntry, + 'eval-server', + '--port', + '0', + '--host', + 'localhost', + '--idle-timeout', + '3', + ], + { + cwd: MINI_REPO, + stdio: ['ignore', 'pipe', 'pipe'], + env: cliEnv(), + }, + ); + + let stdoutBuffer = ''; + let settled = false; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.kill('SIGTERM'); + fn(); + }; + + child.stdout.on('data', async (chunk: Buffer) => { + stdoutBuffer += chunk.toString(); + const readyLine = stdoutBuffer + .split('\n') + .find((l) => l.startsWith('GITNEXUS_EVAL_SERVER_READY:')); + if (!readyLine || settled) return; + + // The signal must contain a real bound IP, not the literal input string + if (readyLine.includes(':localhost:')) { + settle(() => + reject( + new Error( + `READY signal contained literal "localhost" instead of a bound IP:\n${readyLine}`, + ), + ), + ); + return; + } + + // Parse host and port: everything after the prefix up to the last colon + const withoutPrefix = readyLine.slice('GITNEXUS_EVAL_SERVER_READY:'.length); + const lastColon = withoutPrefix.lastIndexOf(':'); + const signalHost = withoutPrefix.slice(0, lastColon); // "127.0.0.1" or "[::1]" + const boundPort = withoutPrefix.slice(lastColon + 1).trim(); + if (!boundPort || isNaN(Number(boundPort))) { + settle(() => reject(new Error(`Could not parse port from READY signal: ${readyLine}`))); + return; + } + + // Probe /health at the bound address to confirm the server is reachable + try { + const res = await fetch(`http://${signalHost}:${boundPort}/health`); + if (res.status === 200) { + settle(resolve); + } else { + settle(() => reject(new Error(`/health returned ${res.status}, expected 200`))); + } + } catch (err) { + settle(() => + reject( + new Error( + `eval-server bound to localhost but /health unreachable at ${signalHost}:${boundPort}: ${err}`, + ), + ), + ); + } + }); + + child.stderr.on('data', (chunk: Buffer) => { + const text = chunk.toString(); + if (text.includes('unknown option') || text.includes('error: unknown')) { + settle(() => reject(new Error(`eval-server rejected --host flag:\n${text}`))); + } + }); + + const timer = setTimeout(() => { + settle(() => + reject(new Error('eval-server --host localhost did not emit READY signal within 30s')), + ); + }, 30000); + }); + }, 35000); }); }); diff --git a/gitnexus/test/unit/eval-formatters.test.ts b/gitnexus/test/unit/eval-formatters.test.ts index 6cefbcdca..cfc09acd1 100644 --- a/gitnexus/test/unit/eval-formatters.test.ts +++ b/gitnexus/test/unit/eval-formatters.test.ts @@ -19,8 +19,8 @@ import { // ─── validateHost ──────────────────────────────────────────────────── describe('validateHost', () => { - it('normalizes "localhost" to "127.0.0.1"', () => { - expect(validateHost('localhost')).toBe('127.0.0.1'); + it('passes "localhost" through unchanged', () => { + expect(validateHost('localhost')).toBe('localhost'); }); it('accepts valid IPv4 addresses', () => { From aa8f4d6efe928e9fadd1f16208810fe3ea870d2d Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Wed, 20 May 2026 17:44:07 +0100 Subject: [PATCH 19/51] fix(group): Union HTTP graph and source contracts (#1709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Union HTTP graph and source contracts * test(group): Document HTTP source union follow-ups --------- Co-authored-by: Gergő Magyar --- .../group/extractors/http-route-extractor.ts | 46 ++++-- .../unit/group/http-route-extractor.test.ts | 148 ++++++++++++++++++ 2 files changed, 180 insertions(+), 14 deletions(-) diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index 898a22c38..54aeb9150 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -18,12 +18,14 @@ import { getPluginForFile, HTTP_SCAN_GLOB, type HttpDetection } from './http-pat * the preferred path because the graph has richer symbol metadata * (real uids, class/method structure, etc.). * - * 2. **Source-scan fallback (Strategy B)** — parse files directly with - * the per-language plugin registry in `./http-patterns/`. Used when - * the graph has no routes/fetches for this repo (e.g. a repo that - * hasn't been indexed yet, or whose indexer doesn't know the - * framework). Each plugin owns its tree-sitter grammar and query - * sources — this orchestrator imports NO grammars or query strings. + * 2. **Source-scan supplement (Strategy B)** — parse files directly with + * the per-language plugin registry in `./http-patterns/`. Used to + * fill gaps when graph extraction only covers part of a polyglot repo + * (e.g. Java graph routes plus Go source-scan routes). Graph entries + * remain authoritative for duplicate contract IDs because they carry + * richer symbol metadata. Each plugin owns its tree-sitter grammar + * and query sources — this orchestrator imports NO grammars or query + * strings. * * Adding a new language for Strategy B is a one-file edit in * `http-patterns/index.ts`: register a new `HttpLanguagePlugin` and @@ -194,17 +196,19 @@ export class HttpRouteExtractor implements ContractExtractor { const graphProviders = dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, getDetections) : []; - const providers = - graphProviders.length > 0 - ? graphProviders - : this.extractProvidersSourceScan(await getScannedFiles(), getDetections); + // Source scan always runs to capture routes in languages/files not covered + // by graph edges; the glob and per-file parse results are cached above. + const providers = this.mergeGraphAndSourceContracts( + graphProviders, + this.extractProvidersSourceScan(await getScannedFiles(), getDetections), + ); const graphConsumers = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : []; - const consumers = - graphConsumers.length > 0 - ? graphConsumers - : this.extractConsumersSourceScan(await getScannedFiles(), getDetections); + const consumers = this.mergeGraphAndSourceContracts( + graphConsumers, + this.extractConsumersSourceScan(await getScannedFiles(), getDetections), + ); return [...providers, ...consumers]; } @@ -473,4 +477,18 @@ export class HttpRouteExtractor implements ContractExtractor { } return out; } + + private mergeGraphAndSourceContracts( + graphContracts: ExtractedContract[], + sourceContracts: ExtractedContract[], + ): ExtractedContract[] { + const seenContractIds = new Set(graphContracts.map((c) => c.contractId)); + const out = [...graphContracts]; + for (const contract of sourceContracts) { + if (seenContractIds.has(contract.contractId)) continue; + seenContractIds.add(contract.contractId); + out.push(contract); + } + return out; + } } diff --git a/gitnexus/test/unit/group/http-route-extractor.test.ts b/gitnexus/test/unit/group/http-route-extractor.test.ts index aa648a71d..d2c1b3fa4 100644 --- a/gitnexus/test/unit/group/http-route-extractor.test.ts +++ b/gitnexus/test/unit/group/http-route-extractor.test.ts @@ -92,6 +92,77 @@ public class UserController { expect(getRoute!.confidence).toBe(0.9); expect(getRoute!.symbolUid).not.toBe('file-uid-ctrl'); }); + + it('supplements graph providers with source-scan providers from other files', async () => { + const dir = path.join(tmpDir, 'graph-source-provider-union'); + fs.mkdirSync(path.join(dir, 'src/controller'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'cmd'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src/controller/UserController.java'), + ` +@RestController +@RequestMapping("/api/v2") +public class UserController { + @GetMapping("/users") + public List list() { return service.findAll(); } +} +`, + ); + fs.writeFileSync( + path.join(dir, 'cmd/server.go'), + ` +package main + +func healthHandler(w http.ResponseWriter, r *http.Request) {} + +func main() { + http.HandleFunc("/api/health", healthHandler) +} +`, + ); + + const mockDbExecutor = async (query: string) => { + if (query.includes('HANDLES_ROUTE')) { + return [ + { + fileId: 'file-uid-ctrl', + filePath: 'src/controller/UserController.java', + routePath: '/api/v2/users', + routeId: 'route-uid-users', + responseKeys: null, + routeSource: 'decorator-GetMapping', + }, + ]; + } + if (query.includes('FETCHES')) return []; + if (query.includes('CONTAINS')) { + return [ + { + uid: 'uid-ctrl-list', + name: 'list', + filePath: 'src/controller/UserController.java', + labels: ['Method'], + }, + ]; + } + return []; + }; + + const contracts = await extractor.extract(mockDbExecutor, dir, makeRepo(dir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + const graphRouteMatches = providers.filter( + (c) => c.contractId === 'http::GET::/api/v2/users', + ); + expect(graphRouteMatches).toHaveLength(1); + expect(graphRouteMatches[0].symbolUid).toBe('uid-ctrl-list'); + expect(graphRouteMatches[0].meta.extractionStrategy).toBe('graph_assisted'); + + const sourceRoute = providers.find((c) => c.contractId === 'http::GET::/api/health'); + expect(sourceRoute).toBeDefined(); + expect(sourceRoute?.symbolName).toBe('healthHandler'); + expect(sourceRoute?.meta.extractionStrategy).toBe('source_scan'); + }); }); describe('provider extraction — source-scan fallback (Strategy B)', () => { @@ -166,6 +237,30 @@ export default router; ).toBeDefined(); }); + it('dedupes source-only providers by contract id', async () => { + const dir = path.join(tmpDir, 'source-only-same-contract-id'); + fs.mkdirSync(path.join(dir, 'src/routes'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src/routes/health-a.ts'), + ` +router.get('/api/health', healthA); +`, + ); + fs.writeFileSync( + path.join(dir, 'src/routes/health-b.ts'), + ` +router.get('/api/health', healthB); +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const providers = contracts.filter((c) => c.contractId === 'http::GET::/api/health'); + + expect(providers).toHaveLength(1); + expect(providers[0].role).toBe('provider'); + expect(providers[0].meta.extractionStrategy).toBe('source_scan'); + }); + it('extracts Go Gin and Echo route registrations', async () => { const dir = path.join(tmpDir, 'go-frameworks'); fs.mkdirSync(path.join(dir, 'cmd'), { recursive: true }); @@ -740,6 +835,59 @@ async def create_user(user: UserCreate): expect(consumers[0].confidence).toBe(0.9); expect(consumers[0].symbolName).toBe('fetchUsers'); }); + + it('supplements graph consumers with source-scan consumers from other files', async () => { + const dir = path.join(tmpDir, 'graph-source-consumer-union'); + fs.mkdirSync(path.join(dir, 'src/api'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'src/api/graph.ts'), 'export const api = {};'); + fs.writeFileSync( + path.join(dir, 'src/api/health.ts'), + ` +export async function fetchHealth() { + const res = await fetch('/api/health'); + return res.json(); +} +`, + ); + + const mockDbExecutor = async (query: string) => { + if (query.includes('HANDLES_ROUTE')) return []; + if (query.includes('FETCHES')) { + return [ + { + fileId: 'file-uid-api', + filePath: 'src/api/graph.ts', + routePath: '/api/users', + routeId: 'route-uid-users', + fetchReason: 'fetch-url-match', + }, + ]; + } + if (query.includes('CONTAINS')) { + return [ + { + uid: 'uid-fn-fetch', + name: 'fetchUsers', + filePath: 'src/api/graph.ts', + labels: ['Function'], + }, + ]; + } + return []; + }; + + const contracts = await extractor.extract(mockDbExecutor, dir, makeRepo(dir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + const graphConsumer = consumers.find((c) => c.contractId === 'http::GET::/api/users'); + expect(graphConsumer).toBeDefined(); + expect(graphConsumer?.symbolUid).toBe('uid-fn-fetch'); + expect(graphConsumer?.meta.extractionStrategy).toBe('graph_assisted'); + + const sourceConsumer = consumers.find((c) => c.contractId === 'http::GET::/api/health'); + expect(sourceConsumer).toBeDefined(); + expect(sourceConsumer?.meta.extractionStrategy).toBe('source_scan'); + }); }); describe('edge cases', () => { From c34c36036f2db44abf8858782faed665fc5b3f49 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 20:39:35 +0100 Subject: [PATCH 20/51] =?UTF-8?q?fix(workers):=20resilient=20+=20zero-copy?= =?UTF-8?q?=20ingestion=20worker=20pool=20=E2=80=94=20prevent=20analyze=20?= =?UTF-8?q?hangs=20on=20TS-root-scale=20loads=20(#1693)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * fix: skip worker-timeout files in sequential fallback and optimize TS capture node lookup Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/0e53743e-0600-4690-bd0d-198894daef58 * refactor: clarify TS capture helpers after validation feedback Agent-Logs-Url: https://github.com/abhigyanpatwari/GitNexus/sessions/0e53743e-0600-4690-bd0d-198894daef58 * fix(workers): exclude in-flight file on worker error/exit, not just singleton timeout WorkerPoolDispatchError previously surfaced the stalled path only for the singleton-timeout final-fail branch. Worker `error` and `exit` events (and the msg-channel `error` reply) fell back to plain `Error`, so the sequential fallback re-attempted every file in the active job — re-hanging on the same pathological file when the worker crashed mid-parse. Lift the in-flight-file inference into `inFlightExcludePath(job, lastProgress)` and wire it into the three remaining in-pool failure sites. `lastProgress` is already in `runWorker` scope, so `items[lastProgress]` (the next file the worker was about to acknowledge) is the best single guess at the culprit; earlier files are still re-tried sequentially. Returns `[]` when no path is determinable (`lastProgress >= items.length`, or path missing/non-string) so sequential retries the whole job. Replacement-worker startup failures stay plain `Error` (no job context); the result-before-flush protocol bug stays plain `Error` (code fault, not file). Tests cover the three new exclusion paths plus a negative test confirming non-WorkerPoolDispatchError throws fall through to full sequential retry. * fix(review): apply autofix feedback - Use cause-neutral "worker-excluded" label in skip messages and tests now that worker error/exit paths share the same exclusion contract as singleton-timeout (correctness + maintainability reviewers). - Add JSDoc to findSelfOrAncestorOfType{s} explaining the parent-walk short-circuit vs root-DFS fallback (maintainability reviewer). * feat(workers): resilient + scalable worker pool Restructures `createWorkerPool` so a single bad file no longer kills the pool for the rest of an analyze run. Five interlocking layers: 1. **Auto-respawn on error/exit** — worker death triggers `replaceWorker` on the same slot, bounded by `maxRespawnsPerSlot` (default 3). The slot is dropped from rotation when the budget is exhausted; other slots keep running. 2. **Circuit breaker** — replaces the permanent `poolBroken=true` with a consecutive-failure counter. The pool only trips after `consecutiveFailureThreshold` deaths (default `max(3, poolSize)`) with no successful job in between. A successful job resets the counter so transient bursts of bad files don't escalate. 3. **Session-scoped file quarantine** — paths identified as the in-flight file at the moment of a worker death are added to a `Set` on the pool. `dispatch()` filters quarantined items up front (they never reach a worker again this pool lifetime). Exposed via the new `WorkerPool.getQuarantinedPaths()` so callers can log/route them. `processParsing` surfaces the per-chunk quarantine summary alongside the existing fallback-exclusion log. 4. **Authoritative in-flight tracking** — `parse-worker.ts` emits `{type:'starting-file', path}` before each file. The pool tracks this per slot and uses it for crash attribution, falling back to the `items[lastProgress]` heuristic only when no starting-file has been observed (very-early crash, older worker build). Closes the reorder/race concerns raised by reviewers C1 and R3 in the earlier review run. 5. **Per-job cumulative timeout budget** — each `WorkerJob` tracks the total wall time spent across attempts/splits/retries. When the budget is exhausted (default 5x `subBatchIdleTimeoutMs`), the pool surfaces the in-flight path instead of letting exponential backoff balloon into multi-hour stalls. Cross-layer wiring: a new `wakeIdleSlots` helper kicks any non-busy live slot when items are requeued (after a death or split-retry), so a dropped slot doesn't strand work in the queue. `recoverAndResume` consolidates the per-job teardown shared by the three in-pool death sites (`error`, `exit`, msg-channel `error`). New env knobs: `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT`, `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS`, `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD`. New `WorkerPoolOptions.workerFactory` injection point for unit tests. Tests: 12 new unit tests using a FakeWorker mock cover quarantine seeding, slot-respawn, slot-drop after budget, breaker trip + reset, and quarantine filtering. Plus option-resolution tests for the three new env vars. All 19 worker-pool/-fallback/-options tests pass; full unit suite 6040 passed / 30 skipped / 0 failed. * fix(workers): apply code-review fixes (12 findings) Walks through every finding from ce-code-review run 20260519-094648-3549cf5e. All 12 picked Apply. Critical: - F1 — Layer 5 cumulative-timeout exhaustion no longer silently drops the rest of the job. `requeueRemainder` is now invoked before `handleWorkerDeath` in both Layer 5 and singleton-final-fail give-up paths so non-quarantined items get re-tried by another worker. - F2 — idle-timer recovery overhaul. `!shouldContinue` branch no longer calls `replaceWorker` (double-spawn race with the `handleWorkerDeath` inside `requeueAfterTimeout`). `shouldContinue` branch now enforces `maxRespawnsPerSlot` before respawning, closing the budget-bypass for the timeout-retry path. Also fixes premature `maybeDone` by simplifying the bookkeeping. - F3 — `requeueRemainder` no longer pre-charges `cumulativeTimeoutMs` by `job.timeoutMs`. The death itself consumed no budget, so the next `requeueAfterTimeout` was double-billing the first attempt. - F4 — `WorkerPool.getQuarantinedPaths` is now optional on the interface, matching the defensive `?.()` call site and the existing mocks. Removes the contract-vs-callsite contradiction. - F5 — per-job unattributed-death tracking. When a worker dies with no exclusion attribution, `requeueRemainder` tracks death count per `startIndex`. First time: re-queue intact. Second time: quarantine items[0] as best guess, or drop the job entirely when items lack paths. Bounds the death loop the original design admitted to. - F6 — per-slot consecutive-failure counter. Replaces the pool-wide scalar so a chronically-failing slot trips the breaker on its own streak instead of being masked by another slot's successes. Smaller: - F7 — exhaustiveness `never` check on `WorkerOutgoingMessage` union. - F8 — recursive `runWorker` on fully-quarantined jobs converted to a while-loop. - F9 — `tripBreaker` calls `reject(err)` BEFORE awaiting `worker.terminate()`. A stuck terminate no longer blocks the caller. - F10 — `parsing-processor.ts` quarantine log de-duplicates per pool instance via a `WeakMap`. Only newly-quarantined paths are logged in each chunk; the per-chunk count still surfaces via progress. - F11 — extract `firstPath` local in `requeueAfterTimeout`; eliminates double `itemPath` call and the `unknown as string` cast. Tests (F12, 6 new): - crash-error event path (errorHandler). - F5 drop-branch coverage via items without `.path`. - Common-case unattributable crash falling back to items[0] heuristic. - `replaceWorker` startup failure (workerFactory emits 'exit' before 'online'). - All-slots-dropped breaker trip. - `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` env override. Residual gap (deferred): no unit test exercises the Layer 5 cumulative-budget runtime path — requires fake-timer interleaving with FakeWorker that's too brittle for this iteration. Tracked. Unit suite: 257 files / 6056 passed / 30 skipped / 0 failed. * test(workers): integration tests for resilience layers + fix requeue-after-timeout flow Adds 6 new real-worker integration tests covering the PR #1693 resilience layers + fixes 3 follow-on bugs surfaced while writing them. New integration coverage (real worker threads + temp fixture scripts): - `respawns the slot after worker process.exit and finishes the work on the replacement` — exercises Layer 1 auto-respawn + Layer 3 quarantine through real IPC. - `attributes exactly via authoritative starting-file message on worker crash` — Layer 4 end-to-end: starting-file message → exact quarantine attribution (not the items[0] heuristic). - `quarantine filters subsequent dispatches without sending to a worker` — second dispatch's sub-batch payload audited via filesystem; the quarantined path is never sent across the message channel. - `drops a slot after maxRespawnsPerSlot and continues on the survivor` — 2-slot pool, slot dies twice past budget, survivor finishes re-queued remainder. - `trips the circuit breaker on cascading per-slot consecutive failures` — single-slot pool, dies on every job, breaker trips after consecutiveFailureThreshold with WorkerPoolDispatchError carrying the cumulative quarantine. - `survives a worker error event (uncaught throw) the same as a process.exit` — validates recoverAndResume on the errorHandler path via a real worker `throw` (not just process.exit). Bug fixes uncovered while writing these tests: 1. **Stack-overflow recursion in runWorker's no-worker branch** — `if (!worker) { ...; wakeIdleSlots(); maybeDone(); }` recursed indefinitely when multiple slots were mid-respawn simultaneously (wakeIdleSlots → runWorker → no worker → wakeIdleSlots → …). Removed the wakeIdleSlots call: the slot's own respawn IIFE owns runWorker post-respawn, and other slots will pick up work via finishJob's runWorker. 2. **requeueAfterTimeout dispatched work before respawn completed** — the F2 fix had `requeueAfterTimeout` `void`-discarding `handleWorkerDeath`, so the `!shouldContinue` IIFE had no way to know when the respawn finished. New design: `requeueAfterTimeout` returns a `TimeoutDecision` discriminated union; the IIFE owns the death-and-respawn-and-dispatch orchestration in an async closure so it can `await handleWorkerDeath` and then call `runWorker` deterministically. 3. **Stalled-singleton + protocol-error + replacement-startup-crash tests** had stale contracts predating the resilience refactor. The stalled-singleton no longer rejects (it quarantines + resolves `[]`); the protocol-error rejection message now mentions "circuit breaker tripped"; the replacement-startup-crash test documents the known `waitForWorkerOnline` race (online fires before the worker's main script runs, so a top-level throw looks like a successful spawn) — the test asserts the file is quarantined via the second-idle-timeout give-up path. Full suite: 334 files / 8982 passed / 43 skipped / 0 failed (second run; first run had a Vitest-reported flake from an uncaught worker exception bleeding into the test report — repeated runs are clean). * perf(workers): raise pool cap to cores-1 + defer per-chunk extraction to keep workers busy User reported 4-5% CPU utilization on a multi-core machine during ingestion. Two structural reasons: 1. **Pool cap.** `createWorkerPool` resolved size as `Math.min(8, max(1, os.cpus().length - 1))` — a 16-core box got 8 workers (50% theoretical max). U1 lifts the default to `min(16, max(1, cores - 1))`, exposes `GITNEXUS_WORKER_POOL_SIZE` env override, and adds `--workers ` CLI flag (`0` disables the pool for sequential fallback). 2. **Per-chunk extraction serialized the loop.** Per chunk: dispatch → await workers → main-thread `processImportsFromExtracted` + `processHeritageFromExtracted` + `processRoutesFromExtracted` + `synthesizeWildcardImportBindings` + `seedCrossFileReceiverTypes` → next chunk dispatch. Workers sat idle through every extraction block. U2 (revised from the plan's pipelined-chunks design) defers these passes to a single end-of-loop batch. Chunk loop becomes parse + merge + accumulate. Resolution sees strictly-more-info (full repo graph) so cross-chunk import/heritage targets resolve at least as well as before. Memory cost: `deferredWorkerImports` accumulates across chunks; bounded by total file count, acceptable. Plan deviation note: the plan called for an in-flight chunk pipeline (N concurrent dispatches with bounded memory). That design needed either a `processParsing` API refactor or duplicating its catch-block fallback in `parse-impl`. The deferred-extraction approach delivers the same "workers stay busy" outcome with much smaller surface area and zero changes to `processParsing`. The `GITNEXUS_PARSE_CHUNK_CONCURRENCY` env var documented in U2 of the plan is therefore not implemented in this commit; if memory growth from `deferredWorkerImports` becomes a problem at very-large-repo scale, a bounded sliding-window variant can land as a follow-up. Tests: - New `test/unit/analyze-worker-pool-size.test.ts` covers --workers validation (5 invalid inputs rejected with exit code 1 + clear error; valid integers set the env var; `--workers 0` routes to sequential). - Extended `worker-pool-resilience.test.ts` with `resolveAutoPoolSize` scenarios: env override, env=0, env above cap, invalid env fallback, auto-formula match, integer return type. - Full unit suite: 6097 / 6127 passed / 30 skipped / 0 failed. - Full integration suite (second run): 77 / 78 passed / 1 skipped / 0 failed. First run had a known cosmetic flake from an uncaught worker exception bleeding into the test reporter. Resilience contract from PR #1693 preserved: per-slot respawn budget, circuit breaker, quarantine, authoritative in-flight tracking, cumulative timeout budget — all unchanged. New env vars surfaced in --help: GITNEXUS_WORKER_POOL_SIZE, GITNEXUS_PARSE_CHUNK_CONCURRENCY (reserved for future bounded pipelining). * docs(readme): document --workers CLI flag * feat(workers): add getStats() and per-chunk throughput logging * test(workers): cleanup leaked temp-dirs and drop duplicate option-resolution block - Add afterEach to worker-pool-resilience.test.ts cleaning up the per-test temp directory created by beforeEach (~25 stale dirs per CI run previously). - Delete the duplicated describe('worker pool option resolution', ...) block. Verified the first block (lines 490-532) is a strict superset (includes the GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS env test the second block omitted), so deletion loses no test coverage. Addresses PR #1693 review findings L2 (temp-dir leak) and L3 (duplicate block). * feat(cli): thread --workers via PipelineOptions + snapshot/restore CLI env Resolves PR #1693 review B2 (env-var leak in long-running hosts): - --workers is now threaded through AnalyzeOptions -> runFullAnalysis -> PipelineOptions.workerPoolSize -> createWorkerPool's explicit poolSize arg, bypassing the GITNEXUS_WORKER_POOL_SIZE env channel. The env var remains as a back-compat fallback inside resolveAutoPoolSize for operators who set it directly. - analyzeCommand and wikiCommand snapshot the GITNEXUS_* env vars they mutate at function entry and restore them in finally. Inner *Impl extraction keeps the diff surgical (no body re-indent). process.exit(0) on the CLI success path still terminates the process; restoration matters for programmatic callers (tests, long-running hosts) reaching early-return paths or the alreadyUpToDate fast path. - Tests updated to assert the new behavior: analyze-worker-pool-size.test.ts: workerPoolSize flows through runFullAnalysis options; env is not mutated; back-to-back calls see their own values, not the previous call's leak. analyze-worker-timeout.test.ts: env IS set during the runFullAnalysis call (captured via mockImplementation) and restored after, proving the timeout reaches downstream while the leak fix holds. - Also addresses L4: afterEach NODE_OPTIONS restore so back-to-back test runs don't accumulate --max-old-space-size=8192 tokens. Addresses PR #1693 review B2 (blocker) and L4 (test polish). * feat(workers): harden worker lifecycle (messageerror + availableParallelism + ready handshake) Resolves PR #1693 review H1, H2, M4: H1 - messageerror handler at every dispatch site V8 deserialization failure on postMessage previously left the message silently lost; the pool would wait out the idle timeout (default 30s) instead of treating it as worker death. The dispatch loop now wires worker.once('messageerror', ...) alongside error/exit and routes through recoverAndResume so the existing per-slot respawn budget, in-flight file attribution, and circuit-breaker layers fire as designed. H2 - resolveAutoPoolSize uses os.availableParallelism() Mirrors the pattern at capabilities.ts:85 (defaultEmbeddingThreads). os.cpus().length returns the host CPU count, which over-sizes the pool on cgroup-limited containers, taskset-restricted runtimes, and CI runners with explicit CPU quotas. Falls back to os.cpus().length on Node < 18.14. M4 - worker-side ready handshake replaces online-trust parse-worker.ts now emits {type: 'ready'} after all top-of-script initialization completes, BEFORE the message handler is attached. The pool's renamed waitForWorkerReady listens for this message under a bounded WORKER_READY_TIMEOUT_MS (5s) budget instead of trusting Node's online event - which fires when the worker thread starts, BEFORE the script body runs, letting init crashes slip past pool startup. ready is added to WorkerOutgoingMessage with an exhaustiveness-checked no-op branch in the dispatch handler (defensive: the message is consumed by waitForWorkerReady before dispatch handlers attach). messageerror is wired into waitForWorkerReady the same way. Test scaffolding: - FakeWorker emits {type: 'ready'} in addition to 'online' so replacement workers in unit tests don't hit the 5s budget. - Integration test ad-hoc worker scripts go through a writeReadyWorker helper that prepends the ready handshake. Tests intending to script "crash BEFORE ready" can bypass the helper. 61/61 worker-pool unit tests pass; 28/28 integration tests pass. * feat(parse-impl): monotonic progress + verbose-gated throughput log + seed-before-build Resolves PR #1693 review M2, M3, L1, L5 in a single parse-impl.ts pass: M2 - Monotonic progress through deferred phase (no more "stuck at 82%") Previously the deferred resolution stages (imports, heritage, routes, calls) all emitted percent: 82 — the UI looked frozen for the duration of the deferred work, which on large repos is several seconds to minutes and visually identical to the hang PR #1693 set out to fix. Redistributed: parse phase: 20-70 (was 20-82) imports: 70-75 heritage: 75-80 routes: 80-85 calls: 85-95 Each deferred stage now advances through its own band via the existing per-batch progress callback. Skipped stages (zero deferred input) leave their band as a no-op jump - the next stage still starts at its own band, preserving strict monotonicity. The "no parseable files" early return now jumps to 95 (was 82), and the duplicate "Parsing N files..." announcement is suppressed when totalParseable === 0 to avoid a non-monotonic 95 -> 20 regression that pre-existed (uncovered by the new monotonic test). M3 - Throughput log gated on `--verbose`, not just NODE_ENV=development The per-chunk files/s log was gated on `isDev`, so operators running `gitnexus analyze --verbose` in a production install never saw it. Now fires when (isDev || isVerboseIngestionEnabled()) — matches the documented promise that `--verbose` shows tuning observability. L1 - Typo rename: `chunkChunkStartMs` -> `chunkStartMs` L5 - `buildExportedTypeMapFromGraph` runs BEFORE `seedCrossFileReceiverTypes` Previously the seeding branch was reached with `exportedTypeMap.size === 0` in the worker path (the map was only built far below, AFTER the seeding branch), so the seed dead-coded itself silently and call resolution never got the cross-file receiver-type enrichment. Now the map is populated from the in-progress graph before the seed call; the post-parse builder remains as a defensive sequential-path fallback, guarded by `size === 0` so we don't pay the cost twice on the worker path. Net win: cross-file CALLS edges that previously had no receiver type now get enriched. New test: parse-impl-progress-monotonic.test.ts Asserts the emitted percent stream is strictly non-decreasing across the parse + deferred phases, and that the deferred band (>=70) is actually reached. Also pins the "no parseable files" path to exactly [95] so the 95 -> 20 regression we just fixed can't re-emerge. * feat(parse-impl): bounded chunk concurrency via file-pre-fetch pipeline Resolves PR #1693 review B1 (GITNEXUS_PARSE_CHUNK_CONCURRENCY documented in --help but unimplemented). The chunk loop now pre-fetches chunk file contents up to `parseChunkConcurrency` chunks ahead of the worker-dispatch cursor so disk I/O overlaps with worker compute. Worker dispatch itself stays serial because WorkerPool.dispatch is not reentrant — concurrent calls would race on the shared per-slot busy/in-flight state, regressing the hang/resilience work this PR is built on. The pre-fetch path is the honest interpretation of "concurrent in-flight parse chunks" that the help text advertises: I/O overlap, not parallel worker dispatch. Concurrency value resolution: 1. PipelineOptions.parseChunkConcurrency (threaded from CLI) 2. GITNEXUS_PARSE_CHUNK_CONCURRENCY env var 3. Default 2 (matches the help text) F4 (wildcard-synthesis ordering) is preserved: deferred-state aggregation runs in chunkIdx order because the for-loop iterates sequentially after awaiting each chunk's pre-fetched contents. Cross-chunk processors (processImportsFromExtracted, synthesizeWildcardImportBindings, etc.) still run only after all chunks complete — they see deterministic input regardless of file-read completion order. Concurrency=1 produces behavior identical to the pure-serial loop; that's the regression baseline. New test: parse-impl-chunk-concurrency.test.ts - Asserts graph output is identical (nodeCount + relationshipCount) between parseChunkConcurrency=1 and =2 — the critical correctness invariant. Exact .toBe(N) comparisons per DoD §2.7 (the second run's counts must equal the first run's exactly). - Pins specific fixture symbols (foo/bar/Baz) under both parseChunkConcurrency=1 and the env-fallback (3) path. - Env-fallback test confirms GITNEXUS_PARSE_CHUNK_CONCURRENCY is honored when the option is undefined. * test(workers): pin cumulative-timeout exhaustion behavior Resolves PR #1693 review M6: the existing resilience suite asserts only the *default value* of maxCumulativeTimeoutMs (5x subBatchIdleTimeoutMs), not that dispatch actually aborts the offending job when the cumulative wall-clock budget is exhausted. Without this test, a future refactor could remove the exhaustion branch in requeueAfterTimeout and the suite would stay green while the pool sat in retry loops for an hour on a real production stall. Scenario: subBatchIdleTimeoutMs = 100ms timeoutBackoffFactor = 10 maxCumulativeTimeoutMs = 300ms Single file, HangingWorker that never responds. First attempt times out at 100ms (cumulative=100). The next backoff (1000ms, cumulative 1100ms) exceeds the 300ms cap, so requeueAfterTimeout returns give-up on the first timeout retry and the file goes to the session quarantine. Asserts: - pool.getQuarantinedPaths() includes 'src/stuck.ts' after dispatch - if dispatch rejected, the error is a WorkerPoolDispatchError (the typed surface that routes to sequential fallback) Uses a local minimal HangingWorker double rather than the full action-scripted FakeWorker from worker-pool-resilience.test.ts — the inverse pattern (always hang) doesn't need the scripted-action machinery and keeps the test file focused on the one behavior. * docs(readme): add environment-variables reference table Resolves PR #1693 review L6: operator-facing env vars were either mentioned inline (GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS) or only documented via `gitnexus --help`, with no single place to look up the full set. The new "Environment variables" subsection under the Quick Start CLI block lists every operator-facing knob with default, effect, and tuning guidance, matching the names in cli/index.ts addHelpText post-U2 / U1. Covers: GITNEXUS_WORKER_POOL_SIZE (--workers) GITNEXUS_PARSE_CHUNK_CONCURRENCY (newly real per U1) GITNEXUS_VERBOSE (--verbose) GITNEXUS_MAX_FILE_SIZE (--max-file-size) GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS (--worker-timeout × 1000) GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES GITNEXUS_CHUNK_BYTE_BUDGET GITNEXUS_NO_GITIGNORE GITNEXUS_SKIP_OPTIONAL_GRAMMARS CLI flag vs env-var precedence is stated explicitly (CLI > env > default) so operators running long-lived hosts (MCP server, eval-server) know which channel wins. * test(workers): pin quarantine path round-trip and non-normalization contract Resolves PR #1693 review M5 (Windows quarantine path-normalization coverage). worker-pool.ts quarantines paths via a Set keyed by exact string equality. The existing suite never asserted this contract, which lets a future "helpfully normalizing" refactor on one side of the pipeline (caller, worker, or pool) silently break quarantine filtering on Windows. This file pins the contract from both directions: 1. Round-trip: a path the caller dispatches with backslashes (src\bad.ts) flows through starting-file -> death -> quarantine -> next-dispatch filter verbatim. The replacement worker never sees the re-dispatched bad path because the pool's pre-dispatch filter short-circuits it. 2. Non-normalization: quarantining src\poison.ts does NOT filter src/poison.ts. Whoever changes that contract has to update this test alongside (the load-bearing assertion catches accidental path.normalize() calls in the quarantine path). Runs on every platform — the path strings are test-injected, so the test exercises the same code path regardless of the host's path.sep. Used a self-contained FakeWorker that emits {type:'ready'} for U3's waitForWorkerReady handshake, so the test doesn't depend on the larger worker-pool-resilience.test.ts harness. * test(typescript): pin capture-anchor rewrite invariants (B5 regression) Resolves PR #1693 review B5: the captures.ts ancestor-walk rewrite (findSelfOrAncestorOfType[s] + pickFirstNode replacing the prior findNodeAtRange-from-root path) was semantically equivalent to its predecessor per Lane 4 of the production-readiness review, but the existing typescript-captures.test.ts didn't pin the specific sharp edges where an over-aggressive walk would silently break captures. This file does. Each test exercises a capture class whose anchor type is one the rewrite explicitly handles: - member call obj.foo() -> @reference.call.member (call_expression anchor walks to self) - dynamic import import("./helper") -> raw @import.dynamic gets decomposed by splitImportStatement into @import.statement with @import.kind=dynamic + @import.source stripped of quotes - JSX in .tsx -> @reference.call.free emitted (TSX query pattern, query.ts:899-905) but @declaration.parameter-count is NOT synthesized because findSelfOrAncestorOfType('call_expression') returns null on a jsx_self_closing_element anchor. Pre-rewrite the range lookup also returned null. Pinning this contract catches accidental "walk JSX -> outer call" refactors. - constructor `new Foo(1,2)` -> @reference.call.constructor (new_expression anchor walks to self) - named/namespace import + re-export -> @import.statement (one each) - class method override -> @declaration.method per class, no collapse - member read obj.foo (no call) -> @reference.read.member All assertions use exact .toBe(N) per DoD §2.7. * test(parse-impl): pin multi-chunk graph equivalence under deferred extraction Resolves PR #1693 review B4: the deferred-extraction reorder (moving processImportsFromExtracted / Heritage / Routes / Wildcard / ReceiverTypes from per-chunk to end-of-loop) was proven observably equivalent by Lane 4 of the production-readiness review. Until now, the existing suite never asserted cross-chunk graph equivalence, which lets a future refactor that accidentally tightens the per-chunk vs end-of-loop coupling silently break cross-chunk resolution. This test forces multi-chunk parsing on a small fixture by setting GITNEXUS_CHUNK_BYTE_BUDGET=64 BEFORE the parse-impl module loads (the budget is captured at module load via vi.resetModules — a future move to function-scope env reads is U14 in Phase 2). Then runs the same fixture under a 10MB budget (single chunk) and asserts the two graphs are byte-identical: same nodeCount, same relationshipCount, exact .toBe(N) per DoD §2.7. Fixture: 3-file class hierarchy with cross-file inheritance — Animal (a.ts) -> Dog extends Animal (b.ts) -> makeDog returns Dog (c.ts). Forces the resolver to chain imports + heritage across chunks. A second test pins specific symbol names (Animal, Dog, makeDog, speak, bark) in the multi-chunk graph so a regression in chunk-boundary resolution surfaces as a missing-symbol failure with a specific diagnostic instead of a bare count mismatch. * test(parse-impl): wall-clock integration pinning multi-chunk pipeline (B3) Resolves PR #1693 review B3 — the final P0/P1 merge blocker. With this test, all five doc-review blockers (B1-B5) are pinned by regression coverage. The PR's headline claim is "analyze no longer hangs on TS-root-shaped loads". The existing suite pins each resilience layer (worker-pool- resilience.test.ts), the deferred-extraction equivalence (U7), and the chunk-concurrency contract (U1). What was missing: a single end-to-end run that exercises the full chunked parse-and-resolve path on a multi-chunk fixture, BOUNDED by a wall-clock budget so a regression that re-introduces the hang fails this test loudly via timeout rather than slipping past as a count drift. Implementation: - 17-file synthetic fixture: 15 small modules (one function each), one "realistic dense" complex.ts (30 functions + class + interface), and an index.ts re-exporting them. Forces cross-chunk import chains. - GITNEXUS_CHUNK_BYTE_BUDGET=64 via vi.resetModules forces multi-chunk parsing on the small fixture. - Promise.race with 30s timeout: a hang fails as "exceeded WALL_CLOCK_BUDGET_MS — likely the hang B3 was meant to prevent", not as a bounds-only inequality (DoD §2.7 distinction — hang-detector via exception, not regression-mask via inequality). - Exact .toBe(true) assertions on specific expected symbols (fn0..fn14, Service, Config, configure, describe, complex0/15/29) so a silent mid-chunk crash that exits 0 without producing graph data also fails this test, not just the hang case. Scope: runs the sequential-fallback path (skipWorkers: true) because the full real-worker scenario requires a built dist/parse-worker.js and ~60s wall-clock per run — appropriate for a CI-integration job, not vitest. The load-bearing invariants pinned here catch the bulk of B3's concern; the dist-worker swap is a Phase 2 follow-up documented in the file header. * refactor(parse-impl): move chunk-byte-budget env read to function scope Resolves PR #1693 review F7 / U14: pre-U14, `CHUNK_BYTE_BUDGET` was a module-load IIFE constant that captured `GITNEXUS_CHUNK_BYTE_BUDGET` once and froze the value for the module's lifetime. That defeated per-call option threading (a future `PipelineOptions.chunkByteBudget` was silently no-op'd because the function body read the frozen module-level constant) AND forced tests to use `vi.resetModules` to vary chunk layout. The U7 deferred-extraction test and the U6 multi-chunk integration test both used the workaround. After this change: - `DEFAULT_CHUNK_BYTE_BUDGET = 2 * 1024 * 1024` stays as a module-level constant — purely a default, no env access. - `resolveChunkByteBudget(options)` runs per call: option wins, then env, then default. Same options-first/env-fallback/default pattern as resolveAutoPoolSize and the U1 parseChunkConcurrency resolver — keeps the ingestion code's configuration model uniform. - `PipelineOptions.chunkByteBudget?` added with documentation that threading through options lets long-running hosts (eval-server, MCP daemon) size per-call without leaking process.env state across analyze invocations. New test (parse-impl-env-reads.test.ts) pins all four behaviors: 1. option-first: option present + env present -> option wins 2. env-fallback: option absent + env present -> env wins 3. default-fallback: both absent -> 2 MB default 4. per-call: two back-to-back runs in the same vitest worker with different chunkByteBudget option values observe their OWN values, proving the module-load freeze is gone (no vi.resetModules in this test — that's the invariant being verified). All four assertions use exact `.toBe(N)` per DoD §2.7. The chunk count is observed by parsing the `Parsing chunk X/Y` progress message stream — a stable proxy that doesn't require exposing internal parse-impl counter state. Note: U7 and U6 tests still use `vi.resetModules` because they were written before this change. A follow-up cleanup could simplify those tests (drop the resetModules dance, pass chunkByteBudget via options), but they pass as-is so this commit doesn't touch them. * feat(workers): per-slot generation counter for late-event protection (U12) Adds a monotonic per-slot generation counter to createWorkerPool's state. Each successful worker replacement (replaceWorker) bumps the slot's counter exactly once — atomically with the workers[slotIndex] swap, so observers (getStats) see the new (worker, generation) pair consistently. Handler closures in the dispatch loop capture the slot's generation at attach time and short-circuit when they fire on a stale generation. In the current implementation, cleanup() synchronously removes listeners on a Worker instance the moment a death is observed, so no listener naturally fires on a stale generation — the guard is a defensive layer protecting against any future refactor that loosens cleanup() ordering or re-attaches handlers across the swap. The load-bearing observable is the slotGenerations[] array exposed via WorkerPoolStats so operators (and tests) can confirm a slot was actually replaced and not just the same worker recycled. Implementation: - const slotGenerations: number[] = new Array(size).fill(0) in createWorkerPool's per-pool state, alongside respawnCount and consecutiveFailuresPerSlot. - replaceWorker: slotGenerations[workerIndex]++ AFTER the workers[workerIndex] = replacement swap (only on the success branch — drop-slot paths leave the counter unchanged). - runWorker dispatch loop: const slotGen = slotGenerations[workerIndex] captured before handler attachment; every handler (handler / errorHandler / exitHandler / messageErrorHandler) starts with `if (slotGenerations[workerIndex] !== slotGen) return`. - WorkerPoolStats gains `readonly slotGenerations: readonly number[]`. - getStats() returns slotGenerations.slice() so callers can't mutate pool state by writing to the returned array. Two existing toEqual snapshots in worker-pool-resilience.test.ts extended with the new slotGenerations field (both expect all-zeros — neither test scenario triggers a respawn). New test file (worker-pool-slot-generation.test.ts, 4 tests): 1. Fresh pool: every slot at generation 0. 2. Successful crash + respawn: generation bumps to 1 exactly once. 3. Crash that drops the slot (maxRespawnsPerSlot:0): generation stays at 0 because no successful respawn happened. The dispatch rejection on breaker trip is the expected outcome here; the load-bearing assertion is the post-rejection stats. 4. Multi-slot independence: one slot crashing bumps only that slot's generation, not the other. Order-independent via sort() because the round-robin assignment isn't pinned by contract. All assertions exact .toEqual / .toBe per DoD §2.7. * docs(bench): add parse-throughput benchmark scaffold (R13) Resolves PR #1693 review R13 (benchmark artifact requirement). Creates `gitnexus/bench/parse-throughput.md` documenting: - Synthetic fixture spec (same shape as the U6 integration test, so CI smoke baseline and ad-hoc benchmark exercise the same paths). - What to measure (wall-clock, peak heap, chunk count, getStats snapshot) and the hardware-shape metadata to record alongside. - Harness recipe — vitest + env-var overrides to exercise sequential fallback vs worker-pool paths. - Latest-measurement table with placeholder rows for the three paths (sequential, workers+concurrency, workers single-threaded) and an explicit "Status: scaffold — fill in before merging" callout. The U6 test's observed ~6 s wall-clock is captured as a smoke-baseline. - Operator-tuning quick reference cross-linked to the README env-var section (U11) so the doc is actionable without re-reading the PR. - "What this benchmark does NOT measure" section explicitly scoping the artifact's limits (synthetic ≠ real-repo, throughput-only ≠ resilience-tested, Phase 3 IPC repack row reserved for U16-U17). Mitigates the doc-review SG5 "static doc drift" concern via: 1. Explicit "regenerate this file before merging" callout at the top. 2. Self-contained methodology so anyone can re-run the numbers. 3. Cross-links to the U6 integration test that already bounds the wall-clock as part of the CI suite — so "is it still completing?" is regression-tested even if the numbers in this doc drift. The standalone harness script (`bench/scripts/parse-throughput.ts`) remains a stretch goal per the original plan. The U6 vitest with verbose ingestion logs covers the primary observability gap until the standalone harness lands. * perf(parse-impl): free deferred-extraction arrays after consumption (U15 lightweight M1) PR #1693 review M1 noted that the deferred-extraction accumulator arrays (`deferredWorkerImports`, `deferredWorkerCalls`, `deferredWorkerHeritage`, `deferredConstructorBindings`, `deferredAssignments`) were retained until function return, making peak accumulator memory O(repo) instead of O(in-flight stage). This commit implements the LIGHTWEIGHT version: free each array immediately after its last consumer drains/reads it, dropping peak accumulator memory progressively through the deferred-extraction stages. The structural per-chunk streaming variant (the original U15 framing) is deliberately deferred — the doc-review's adversarial reviewer (A4) flagged it as defending unmeasured memory pressure, and the simpler array-clearing captures the bulk of the benefit without committing to a scheduling-strategy decision (microtask vs parallel extractor task vs worker-side) that profile data should inform. Clears added: 1. After `processImportsFromExtracted` (the sole consumer of `deferredWorkerImports`): clear the imports array before the heavier heritage/calls stages run. 2. After `buildHeritageMap` (the LAST consumer of the raw `deferredWorkerHeritage` records — processCallsFromExtracted reads from the derived `fullWorkerHeritageMap` instead): clear the heritage array before the call-resolution stage. 3. After `processAssignmentsFromExtracted` (the joint last consumer with processCallsFromExtracted for the calls/ bindings/assignments triple): clear all three before downstream graph-build / scope-resolution uses its own working memory. Arrays returned in the function result object (allFetchCalls, allExtractedRoutes, allDecoratorRoutes, allToolDefs, allORMQueries, allParsedFiles) intentionally stay live — downstream consumers need them. Graph-output equivalence is preserved (U7 multi-chunk equivalence test passes — the clears happen AFTER each array's last consumer has copied data into the graph or derived structures). * feat(workers): introduce protocol.ts wire-format module (U16, IPC scaffold) Defines the binary frame for worker-thread IPC as an isolated, fully-tested module. Production wiring is deferred to U17 — shipping the wire-format contract first de-risks the migration by establishing a single source of truth for the byte layout. Resolves the scaffold half of PR #1693 review R12. Wire layout (per message, single buffer): +---------+-----------+---------------------+ | tag | length | payload bytes … | | 1 byte | 4 bytes | | +---------+-----------+---------------------+ tag : MessageTag enum value (0x01 DispatchJob ... 0x08 Ready) length : little-endian uint32 byte count for the payload region payload: UTF-8 JSON-encoded value, possibly "null" Why JSON for the body (rather than per-shape binary encoders): the doc-review adversarial reviewer (A2) flagged that a true per-shape binary encoder for the result message — which carries nested heterogeneous extracted-call / import / heritage / route arrays — would be 500-1500 LOC and a substantial maintenance burden. The honest perf win the IPC repack targets is moving file CONTENTS via ArrayBuffer transferList (zero-copy ownership transfer for the largest single piece of state in any message). That win is captured by U17 layering transferList over the bulk file-content payload while keeping this module's framing for the surrounding metadata. If U18 benchmark data shows the JSON body is itself a bottleneck after U17 lands, a follow-up unit can swap to per-shape binary encoding behind the same encodeMessage / decodeMessage surface without changing the frame. API: - MessageTag (const object): stable byte tags 0x01..0x08 - PROTOCOL_HEADER_BYTES = 5 - ProtocolDecodeError extends Error: distinct class so U17's pool-side handler can route protocol violations through the existing messageerror recovery layer (U3 H1) distinctly from other failure classes - encodeMessage(tag, payload): Buffer - decodeMessage(buf): { tag, payload } - Uses Buffer#subarray instead of the deprecated Buffer#slice Tests (18, all exact-equality per DoD §2.7): - byte layout (tag at offset 0, length LE uint32 at offset 1) - empty/null payload encodes to 5-byte header + 4-byte "null" body - round-trip for every MessageTag with representative payloads - non-ASCII path string (UTF-8 byte-length boundary) - 9 MB payload (well past the existing 8 MB sub-batch budget) - decode errors surface as ProtocolDecodeError, not generic Error: * buffer < header size * tag outside valid range * declared length exceeds buffer * payload bytes are not valid JSON - error class name is preserved through prototype chain so callers can `err instanceof ProtocolDecodeError` reliably * refactor(workers): extract quarantine into its own module (U13 partial) Honest partial U13: extract the quarantine resilience layer (Layer 3 of the 5-layer model) into a dedicated module with a small explicit interface. The full 5-module split that the original plan named was flagged by doc-review A10 as abstraction-without-multi-consumer-demand ("Each has exactly one consumer: worker-pool.ts. None of these layers is imported elsewhere in the codebase pre-extraction, and the plan doesn't identify any future consumer.") This commit ships the smallest self-contained layer as a named module to validate the factory + interface pattern with minimal risk. The remaining four layers (respawn-budget, cumulative-timeout, circuit-breaker, slot-attribution) stay inline until a real second consumer emerges (e.g., a non-parse worker pool that reuses the same resilience layers). Module shape (`workers/quarantine.ts`, ~30 LOC): interface Quarantine { add(path: string): void; has(path: string): boolean; snapshot(): string[]; // defensive copy readonly size: number; // getter, reflects state at access time } function createQuarantine(): Quarantine Replaces in `worker-pool.ts`: - `const quarantined: Set = new Set()` -> `createQuarantine()` - `quarantined.has(p)` -> `quarantine.has(p)` (2 sites) - `quarantined.add(p)` -> `quarantine.add(p)` (2 sites) - `quarantined.size` -> `quarantine.size` (2 sites) - `Array.from(quarantined)` -> `quarantine.snapshot()` (6 sites) Public worker-pool.ts API is unchanged — `getQuarantinedPaths()` still returns the same defensive `string[]` copy. The behavioral contract is preserved: paths are quarantined as opaque strings (the U9 / M5 non-normalization contract still holds — see the new dedicated test). Tests: - 8 isolated unit tests for the quarantine module — pins the interface contract (empty start, add/has/size, dedup on repeated add, no separator normalization, snapshot defensive copy + freshness, size-getter live behavior). - All 86 existing worker-pool tests pass unchanged — they exercise the quarantine through the pool and act as the regression net for behavior preservation. Why not the full 5-module extraction in this commit: doc-review A10's concern is real — a single-consumer abstraction adds module-boundary overhead (5 sets of imports, 5 dedicated test files, 5 interfaces to keep in sync with worker-pool) without any structural benefit until a second consumer materializes. Extracting one validates the pattern; the remaining four can be moved on demand. * feat(workers): wire protocol.ts encoded IPC into parse-worker + pool (U17) Production worker IPC now uses the U16 binary wire format (1-byte tag + 4-byte LE length + UTF-8 JSON body) end-to-end. The pool encodes every outgoing `sub-batch` / `flush` dispatch via `encodeMessage`; the worker decodes incoming frames via `decodeMessage` and encodes its `ready`, `starting-file`, `progress`, `sub-batch-done`, `result`, `warning`, and `error` outputs the same way. The load-bearing correctness fix is making `decodeMessage` accept `Uint8Array` rather than only `Buffer`: Node's `worker_threads` `postMessage` structured-clones the payload, which strips the `Buffer` prototype on the receive side. A frame sent as `Buffer` arrives as a plain `Uint8Array`, and `Buffer.isBuffer(raw)` returns false — so the first attempt at U17 (gating decode on `Buffer.isBuffer`) silently treated every incoming frame as POJO and the worker never responded. The fix adopts the underlying memory zero-copy via `Buffer.from(view.buffer, view.byteOffset, view.byteLength)` and uses `raw instanceof Uint8Array` at every call site (parse-worker decode, pool dispatch handler, pool ready-handshake handler, FakeWorker test mocks, and the integration-test worker preamble). The pool stays tolerant of POJO incoming so unit-test FakeWorkers don't need rewriting — only the new outgoing encoded dispatches require the test scaffolding to decode on receive, which the test FakeWorkers and the integration test's inline `parentPort.on` wrapper now do. The slot-drop integration test was rewritten from a shared-counter-file race (which pre-U17 timing happened to land on the assertion-friendly counter==2 endpoint, but post-U17 protocol decoding latency shifted to counter==1 and produced 3 quarantines instead of 2) to a deterministic path-based crash trigger: slot 0 crashes on a.ts, respawns, crashes on the requeued b.ts, slot is dropped after budget exhausted; slot 1 handles [c.ts, d.ts] normally. Outcome no longer depends on inter-worker file-write ordering. Protocol coverage adds two regression tests pinning the Uint8Array decode path: structured-clone-stripped frames decode identically to their Buffer originals, and Uint8Array views with non-zero byteOffset into a wider ArrayBuffer also decode correctly (catches `Buffer.from(uint8)` copying semantics if a future refactor loses the zero-copy adoption). All 94 worker-pool tests (9 files, unit + integration) pass; the full unit suite (6128 tests across 268 files) passes unchanged. * perf(workers): zero-copy file content transfer via transferList (U19) Pool dispatch now hoists `{path, content: string}[]` file contents OUT of the U17 JSON envelope into separately-allocated `Uint8Array`s whose ArrayBuffers are passed to `worker.postMessage`'s `transferList` for zero-copy ownership transfer. The envelope itself carries only lightweight metadata (`{path, byteLength}` per file) and is structure- cloned the same as before. What this saves vs U17 baseline: - **JSON.stringify of file contents on main thread** drops to zero — the envelope is now O(paths + sizes), not O(total bytes). For a 200- file sub-batch of 10 KB TS files, that's ~2 MB of escape processing per dispatch that disappears. JSON.stringify's per-character branch on quotes/backslashes/control chars is roughly 2x slower than UTF-8 transcode in TextEncoder, so the replacement is a CPU win even though it adds a single TextEncoder.encode per file. - **Structured-clone memcpy of file contents** drops to zero — the contents' backing ArrayBuffers are ownership-transferred, not copied into the worker's heap. The envelope's struct-clone cost is now proportional to metadata size only. - **JSON.parse on worker thread** likewise no longer scales with content size. Worker decodes each `Uint8Array` to string via `TextDecoder` lazily at the parse boundary — runs on the worker thread, parallel with continued main-thread work, vs U17's sequential JSON.parse blocking the worker before processBatch can start. Pipelining: TextEncoder.encode (main) and TextDecoder.decode (worker) can both run while the OTHER side is doing useful work. Under U17, struct-clone was a synchronous main-thread blocker. The ArrayBuffer ownership contract is load-bearing: - File-content `Uint8Array`s are allocated via `TextEncoder.encode`, NOT `Buffer.from(str, 'utf8')`. TextEncoder produces a dedicated ArrayBuffer per call; `Buffer.from(str)` carves from Node's shared `Buffer.poolSize` slab for small strings, so transferring one pool-backed Buffer's ArrayBuffer would detach every other Buffer that shares that slab — silent data corruption. - The envelope itself is NOT transferred. It MAY be pool-backed by `encodeMessage`, and at ~30-80 bytes/file the struct-clone cost is negligible. Not transferring avoids the same detach-collateral risk the contents path is careful to dodge. Detection is strict: every input element must have both `path: string` and `content: string`. A single non-conforming element disqualifies the whole batch from the transfer path and falls back to the legacy single-Uint8Array `encodeMessage` envelope. Safer than partial transfer (which would split a sub-batch into mixed-shape messages the worker can't reassemble). `parse-worker.ts` `decodeIncomingMessage` recognizes the hybrid `{envelope, contents}` shape, decodes the envelope, zips metadata positionally with the contents array, decodes UTF-8 → string per file, and hands the reassembled `ParseWorkerInput[]` to the existing `processBatch`. Identical downstream behavior to U17 — the IPC optimization is invisible above this line. Test scaffolding (3 FakeWorkers + 1 integration-test preamble) gain a `decodeDispatchedMessage` helper that tolerates BOTH shapes (legacy single-frame Uint8Array AND the new hybrid envelope+contents) so the in-process unit mocks keep their existing action-scripting API and the 9 ad-hoc integration test workers keep their `msg.type === 'sub-batch'` handlers unchanged. `buildDispatchMessage` is now exported from worker-pool.ts so its contract can be tested in isolation. A new `test/unit/worker-pool-transferlist.test.ts` pins: - hybrid shape produced for parse-worker inputs - transferList carries one ArrayBuffer per file in input order - envelope decodes to metadata only (no `content` field) - content bytes round-trip byte-for-byte through UTF-8 (ASCII, multi-byte, surrogate-pair emoji) - each content's ArrayBuffer is independently allocated (no pool sharing) — the load-bearing transfer-safety invariant - non-parse shapes, empty arrays, and mixed-conformance arrays all fall back to the legacy single-frame path All 271 test files (6166 unit + integration tests) pass. * fix(workers,tests,docs): apply ce-code-review findings (16 items) Walks the full set of findings from a multi-agent code review (11 reviewers, 1 maintainability dispatch lost to tool-permission denial) of the PR #1693 branch. All 16 actionable findings — 4 P1, 4 P2, 8 P3 — applied in a single pass against a consistent tree. Tests pass (269/269 unit files, 29/29 integration). P1 — bounds-only / disguised-bounds assertions across 4 test files (per user-memory DoD §2.7): - worker-pool.test.ts: 5 sites — `nodes.length > 0` dropped (redundant after `.toContain('validateInput')`); `files.length >= 4` pinned to `.toBe(7)` (mini-repo/src has exactly 7 .ts files); `results.length > 0` pinned to `.toHaveLength(1)` (default sub-batch absorbs all 7); `result.fileCount >= 0` pinned to `.toBe(1)` (empty file is still "processed"); `warnRecords.length > 0` replaced with content- predicate `/respawn|dropping|replacement|did not report ready/` (catches silenced warnings); `fallbackExcludePaths.length > 0` pinned to exact `['one.ts', 'two.ts']` (deterministic given the single-slot pool + 2 items + per-item starting-file). - parse-impl-fallback.test.ts: 3 sites — `astCacheClearCalls >= 1` pinned to exact 4 (per-chunk × 2 + finally × 2); the two error-path delta checks pinned to exact +2 and +3 (verified empirically). - parse-impl-progress-monotonic.test.ts: `percents.length > 0` → `.not.toEqual([])`; per-element `Math.max(prev, cur)` tautology replaced with direct `if (cur < prev) throw`; final-percent `Math.min(last, 95)` tautology pinned to exact `.toBe(70)` (3-file skipWorkers fixture's deferred band lands at the band start). - parse-impl-large-fixture.test.ts: `Math.min(elapsedMs, BUDGET)` tautology removed; Promise.race rejection is the load-bearing wall-clock check. P1 — terminate() lacks `.catch` mask: - worker-pool.ts terminate() now matches the `.catch(() => undefined)` pattern used at every other internal terminate site. Prevents a hung/OOM worker's terminate rejection from masking the original pipeline error when called from parse-impl.ts's finally block, and guarantees `workers.length = 0` / `activeSlots.clear()` always run. P1 — hybrid envelope length-mismatch + null-payload silent data loss: - parse-worker.ts decodeIncomingMessage: explicit non-null-and-typed check before `.type` access (decodeMessage permits null payloads per encodeMessage contract); explicit length-equality assertion between `decoded.files` and `contents` before zipping. Without these, `TextDecoder.decode(undefined)` silently returns "" and produces empty-content graph nodes — a contract violation that used to be undetectable. Both throws route through the outer try/catch → worker `error` reply → pool's recoverAndResume. P1 — unsafe casts at the IPC boundary: - buildDispatchMessage now uses a properly-typed `isParseWorkerItemArray` type guard. The narrowed branch accesses `item.path` and `item.content` as statically-typed strings — a future rename of `ParseWorkerInput.content` would fail to compile inside the branch instead of silently mismatching at runtime. The remaining decodeMessage payload casts are bounded by the F3/F6 runtime guards. P2 — idle-timeout retry bypasses circuit breaker: - worker-pool.ts timeout-retry IIFE now increments `consecutiveFailuresPerSlot[workerIndex]` alongside `respawnCount`. A slot that consistently times out (vs crashes) now trips the per-slot breaker, instead of consuming its full respawn budget over potentially tens of minutes without the breaker firing. P2 — null/non-object worker message crashes pool handler: - Dispatch handler in worker-pool.ts now guards `null / non-object / no string type discriminant` before `msg.type` access and routes through recoverAndResume on violation. Previously a legitimate `null` payload would throw TypeError out of the EventEmitter listener → uncaughtException on main, crashing the analyze. P2 — workerPoolSize === 0 creates unusable pool: - parse-impl.ts now treats `workerPoolSize === 0` as `skipWorkers` at the gate. Matches the PipelineOptions docstring contract ("0 disables the pool entirely — equivalent to skipWorkers"); avoids constructing a pool that rejects every dispatch and logs "Worker pool parsing stopped" per chunk. P2 — encodeMessage 2-buffer allocation per frame: - protocol.ts encodeMessage coalesced to a single `Buffer.allocUnsafe + writeUInt8 + writeUInt32LE + buf.write (string, offset, 'utf8')`. Drops the intermediate `Buffer.from(JSON.stringify(...), 'utf8')` allocation + memcpy. Length pre-check via `Buffer.byteLength(string, 'utf8')` surfaces the uint32 cap before any allocation. P3 — slotGenerations made optional on WorkerPoolStats so external implementations of getStats() that predate U12 don't compile-break; in-repo callers already use optional chaining. P3 — buildDispatchMessage marked `@internal` so it isn't surfaced as public API by typedoc / api-extractor (it's a test-only export). P3 — verboseThroughputLog hoisted above the chunk loop (env vars can't change mid-run; one O(env-read) per analyze, not per chunk). P3 — corrected the messageerror routing comment in worker-pool.ts dispatch handler. `ProtocolDecodeError` is caught by the surrounding try/catch — distinct from `messageerror`, which fires for V8 structured-clone failures before the message body would reach the handler. P3 — initial pool spawn now uses a `Promise.allSettled` ready-handshake gate symmetric with `replaceWorker`. Dispatch awaits this gate before selecting slots, so an init-crashing initial worker is dropped from `activeSlots` and a downstream OOM/missing-native-binding failure surfaces in seconds (bounded by WORKER_READY_TIMEOUT_MS) rather than waiting for the first idle timeout (30s default). P3 — `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT`, `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS`, `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` added to: - CLI `--help` text in src/cli/index.ts - Root README env-var table - gitnexus/README troubleshooting section (new "Worker pool resilience tuning" subsection) P3 — CLI `catch (e: any)` / `catch (err: any)` in analyze.ts replaced with `catch (err: unknown)` + narrowed access; matches modern TS best practice and the codebase pattern at other catch sites. P3 — `WorkerPoolStats.terminated: boolean` field added (optional, for backward compatibility). `terminate()` sets it true; `getStats()` surfaces it. Distinguishes graceful shutdown from a circuit-breaker trip in observability surfaces. Coverage / advisory items not addressed in this commit (kept in the report only): - maintainability reviewer failed (Read/Bash denied) — god-module audit on worker-pool.ts (~1400 LOC) carried as residual risk - quarantine case-sensitivity contract unpinned (adversarial #8) - WORKER_READY_TIMEOUT_MS env-configurability (adversarial #2) - chunk-byte-budget × parseChunkConcurrency memory multiplier doc (adversarial #5) - MCP discoverability gaps for env vars / verbose (agent-native W1/W2) - bench/parse-throughput.md scaffold-with-TBD-rows (PS RR-003) * fix(parsing): sequential gap-fill for worker-quarantined chunk files (U20.U1) When the worker pool's Layer 3 quarantine filters one or more files out of a chunk's dispatch, the worker results returned to processParsing are silently narrower than the input chunk. Without this reparse, the graph for this run would be missing every quarantined file's symbols/imports/calls/heritage with no failure signal. After the existing per-chunk quarantine log emits in processParsing's worker-path try-block, run processParsingSequential on JUST the quarantined-in-chunk files. The sequential path writes directly to the graph, so symbols for those files land alongside worker output for the surviving files. Mirrors the WorkerPoolDispatchError catch-block's processParsingSequential call shape — same signature, same args, same scopeTreeCache wiring. Emits a structured warn naming `reparsedPaths` so operators can observe the sequential fall-through. This fixes the in-run side of the corruption Codex's adversarial review of PR #1693 flagged. The cross-run side (chunk-cache poisoning) is closed by U20.U2 in a follow-up commit. References plan: docs/plans/2026-05-20-002-fix-chunk-cache-corruption-on-worker-quarantine-plan.md * fix(parse-impl): suppress chunk-cache write when any chunk file was quarantined (U20.U2) The chunk hash at parse-impl.ts:424-428 is computed from every file in the chunk. The worker pool's Layer 3 quarantine (worker-pool.ts createQuarantine) filters quarantined files out of dispatch, so `rawResults` reflects only the surviving files. Before this commit, the write at line 500-507 stored that partial result under the full-coverage chunk hash — and on the next analyze with unchanged content, the cache HIT branch (line 439-464) silently replayed the incomplete result. Symbols from the quarantined file were missing from the graph for as long as the cache survived. Codex's adversarial review of PR #1693 flagged this as a silent- corruption class because there's no failure signal: no warn log during the replay, no graph-equivalence check, no exit code change. The corruption only surfaces if an operator notices a missing symbol in `gitnexus_query` output. Guard the write with `chunkFiles.some(f => quarantineSet.has(f.path))`. When any chunk file is in the worker pool's cumulative quarantine snapshot, skip the `parseCache.entries.set` call. Emits a verbose- only info log so operators investigating "why aren't my chunks caching" have a diagnostic trail. Skipping the write means the next analyze gets a cache miss for this chunk and re-dispatches it. Quarantine is session-scoped (a fresh createWorkerPool starts with an empty quarantine), so the new pool gives the quarantined file another chance. If quarantine fires again, U20.U1's sequential gap-fill still produces a complete graph for that run; the cache stays empty for the chunk until a fully-clean dispatch lands. The cache-hit replay branch at parse-impl.ts:439-464 is unchanged. Its contract strengthens: "cache entries are complete" becomes true post-fix, but the replay code doesn't need to know that. Closes the cross-run side of the Codex finding. U20.U3 adds the regression test. References plan: docs/plans/2026-05-20-002-fix-chunk-cache-corruption-on-worker-quarantine-plan.md * test(parse-impl): integration regression for quarantine + chunk-cache (U20.U3) Pins the U20 fix end-to-end via REAL `worker_threads` + `createWorkerPool`. Mirrors the writeReadyWorker pattern from `test/integration/worker-pool.test.ts` — inline READY_PREAMBLE + custom test worker script that: 1. Decodes the U17/U19 IPC protocol (Buffer frame OR hybrid envelope/ contents shape) the same way the production parse-worker does. 2. Emits a `{type:'ready'}` handshake so the pool's `waitForWorkerReady` resolves promptly. 3. On a sub-batch containing `poison.ts`, emits starting-file + `process.exit(134)`. The pool attributes the death to `poison.ts` via the in-flight signal and adds it to the session-scoped quarantine. 4. On a sub-batch without poison, synthesizes a minimal valid `ParseWorkerResult` with one `Function` node per file (no tree-sitter dep in the test worker — the synthesized nodes give `mergeChunkResults` deterministic content for the graph). Assertions exercise both fix layers: - U1 (sequential gap-fill in processParsing): the graph contains a `Function` node named `poison` AFTER the run. The custom worker never emits anything for `poison.ts`, so the only path for that symbol to reach the graph is `processParsing`'s sequential reparse of the quarantined-in-chunk file using the real tree-sitter parser against the actual source. - U2 (cache-write suppression in runChunkedParseAndResolve): `parseCache.entries` does NOT contain the chunk hash after the run; `parseCache.usedKeys` DOES contain it (chunk processed, cache write specifically skipped). - Cross-run: a second pass over the same fixture with the same parseCache and a fresh worker pool re-dispatches the chunk (cache empty), the worker crashes again, sequential gap-fill runs again, and the cache stays empty. Pins the round-trip contract. Adds `workerUrlForTest?: URL` to PipelineOptions — same `@internal` test-only injection precedent as `workerThresholdsForTest` (already in PipelineOptions for thresholds). When set, parse-impl uses the provided URL instead of the src/ → dist/ resolution dance. Production call sites never set this field; the only consumer today is this integration test. Why integration over unit: - The fix lives at the boundary between parsing-processor.ts and parse-impl.ts under a real WorkerPool. Unit-mocking the worker-pool module bypasses the structured-clone boundary, the dispatch lifecycle, and the actual quarantine flow — it verifies the test setup rather than the contract. The real worker thread executing through the U17/U19 IPC protocol IS the load-bearing surface. - User-explicit preference (saved as feedback_integration_over_vimock.md memory). For worker-pool / parse-impl / IPC-touching code: write integration tests under test/integration/ using writeReadyWorker patterns; avoid vi.mock on worker-pool.js. Test wall-clock: under 2s; both `it` blocks together complete in ~1.8s under the existing CI conditions. References plan: docs/plans/2026-05-20-002-fix-chunk-cache-corruption-on-worker-quarantine-plan.md * refactor(parsing): remove sequential-parser fallback (U20 design pivot) The worker pool's resilience layers — respawn budget, circuit breaker, quarantine, slot-attribution, cumulative timeout — are now the SOLE contract for handling worker failures. Two sequential-reparse paths are removed from processParsing: 1. **U20.U1 sequential gap-fill for quarantined chunk files** (just added in commit 7dd489e9, now reverted). The pre-emptive rescue would re-run processParsingSequential on the file that ALREADY killed a worker — which for the most common quarantine cause (tree-sitter native SIGSEGV on a pathological file) re-triggers the same native crash on the main thread, killing the entire analyze. The "rescue" turned silent missing-symbols into a louder analyze-wide crash. Drop the rescue; accept the per-run gap. 2. **Pre-existing WorkerPoolDispatchError catch-block sequential fallback** (in production since PR #1693's resilience layer landed). Same risk class — when the pool exhausts its respawn budget / trips the circuit breaker, the failing files are precisely the ones likely to crash a sequential parser too. The "graceful degradation" hid pool failures behind degraded-but- completing analyze runs, making operational issues harder to surface and diagnose. Drop the catch-block; WorkerPoolDispatchError propagates to the analyze entry point where the user sees a clear hard signal. What stays: - The `skipWorkers: true` / small-repo path that uses `processParsingSequential` as the EXPLICIT primary path (not a fallback). Caller-driven opt-out and tiny-repo perf optimization are different intents. - U2's chunk-cache write suppression in parse-impl.ts (commit 7c9c9556). When quarantine fires, the chunk stays uncached so the next analyze with a fresh pool retries the file cleanly. That's the cross-run correctness Codex's adversarial review actually asked for. - The per-chunk quarantine warn log (parsing-processor.ts) — operators see which files were skipped, both immediately and across runs. What changed: - `processParsing` worker-path try-block: unwrapped. The `processParsingWithWorkers` call is now direct (no try/catch wrapping); errors propagate to the chunk-loop caller. - `parsing-worker-fallback.test.ts` rewritten: the previous 5 tests asserted graceful sequential-fallback behavior. Replaced with 3 tests pinning the new contract — raw Error propagates, WorkerPool- DispatchError propagates with fallbackExcludePaths intact, normal quarantine signal does NOT throw and surfaces via progress detail. - `parse-impl-quarantine-cache-skip.test.ts` (U20 integration test) updated: poison.ts is NOT in the post-run graph; surviving files are; chunk-cache stays empty; second pass re-dispatches and leaves cache empty. - Plan doc updated to mark R1 as dropped and explain the U20 pivot in the Summary. User decision: explicit directive ("let's remove the sequential fallback entirely we must rely on entirely that the parallel process is resilient enough to work itself through the code base"). The pool's resilience layers are designed for this — respawn budget, circuit breaker, quarantine, slot-generation, cumulative-timeout cap — and adding a layer below them was redundant insurance with real downside. Tests: 269/269 unit files (6135 tests) green. 31/31 worker-pool + parse-impl integration tests green. The 2 reported "errors" in the integration run are the pre-existing intentional-process.exit unhandled- exception leaks from test workers — unchanged by U20. References plan: docs/plans/2026-05-20-002-fix-chunk-cache-corruption-on-worker-quarantine-plan.md * fix(workers,tests,docs): address ce-ultrareview findings F1/F2/F3/F4 Multi-lane review run on the PR #1693 branch surfaced four addressable items beyond the blocking three. F1 (minor, CodeQL): unused `findMatch` helper in test/unit/scope-resolution/typescript/typescript-captures-anchor.test.ts:28 removed. `countMatchesTsx` flagged by the same CodeQL pass is a false positive — it's called at line 88 by the JSX-anchor regression tests so the rewrite case actually fires under TSX, not just TS. F2 (medium, docs): bench/parse-throughput.md retitled as "(scaffold)" with an explicit "no measurement data has been collected yet" note above the table. The self-contradictory "Regenerate this file before merging any PR that touches the ingestion pipeline" instruction is dropped — the file ships intentionally without numbers; the load-bearing perf-regression protection lives in test/integration/parse-impl-large-fixture.test.ts (U6, 30s Promise.race wall-clock budget). The Latest measurement section now preserves the ~6s sequential observation as a smoke reference, not as a regression target. F3 (low, API hygiene): `WorkerPoolDispatchError.fallbackExcludePaths` renamed to `quarantinedPaths`. The "fallback" terminology was load-bearing under the pre-U20 design when `processParsing`'s sequential-fallback catch-block consumed it to filter the fallback file list. After commit be1f65c removed that catch-block, no production code reads the field — but it stays populated by the pool because the snapshot is genuinely useful operator diagnostics when the breaker trips. The rename clarifies the field's actual semantics (here are the files the pool quarantined before it tripped) without changing wire behavior. Definition + the lone surviving in-pool comment reference + both test assertions updated. F4 (low → real fix, reliability): timeout-retry IIFE in worker-pool.ts now consults `consecutiveFailureThreshold` and trips the circuit breaker when the per-slot consecutive-failure count crosses it. Closes a gap left by ce-code-review's REL-02 patch — that fix added the `consecutiveFailuresPerSlot[workerIndex]++` increment in the timeout-retry path but did NOT add the corresponding threshold-check + tripBreaker call. Result: chronic pure-timeout deaths accumulated counts that never tripped the breaker until the slot also hit `respawnCount > maxRespawnsPerSlot`. Now timeouts and crashes are structurally treated the same way by the breaker, which is what the REL-02 increment was meant to enable. Test coverage: worker-pool-resilience.test.ts already exercises the breaker via the shared handleWorkerDeath path; this new branch traces the same trip semantics with a different entry point, so the breaker-tripped state is observable via the same `getStats().poolBroken` and `WorkerPoolDispatchError.quarantinedPaths` surface. Out of scope here (caller actions or future PRs): - F5 (info): cumulative-quarantine cache check is safe in practice because chunks are alphabetically deterministic; no action. - F6 (low): exit-code-0 quarantine exemption — pre-existing P2 residual, bounded by quarantine + respawn budget; deferred. - F7 (info): dispatch non-reentrancy contract documented but not enforced; no production caller violates it; deferred. - PR title `[WIP]` removal — happens on GitHub side. Tests: 274/274 test files (6185 passing, 30 skipped). The single "error" in the integration runner is the pre-existing intentional- process.exit unhandled-exception leak from the deliberate startup- crash test worker, unchanged by these fixes. * fix(workers): swap protocol body from JSON to V8 serialize/deserialize CI scope-parity tests on Ubuntu surfaced silent data loss in the worker IPC: `Phase 'scopeResolution' failed: scope.typeBindings is not iterable` (Python, Go) and `importerModule.typeBindings.has is not a function` (Python). Plus three #1066 large-file regression tests (Python / C# / TypeScript) failed because call relationships weren't resolving from the worker output. **Root cause:** U17 introduced `JSON.stringify`/`JSON.parse` as the protocol body codec. JSON has no representation for `Map`, `Set`, `Date`, `RegExp`, `BigInt`, `TypedArray`, `undefined` values, or circular refs — `JSON.stringify(someMap)` returns `"{}"`. Production scope-resolution code keys data structures on Maps throughout (`ParsedFile.scopes[*].typeBindings: ReadonlyMap`, plus `bindings`, `bySourceScope`, `byTargetDef`, the finalize-algorithm edge indexes, etc.). The JSON round-trip silently turned every Map into an empty object, manifesting downstream as iteration / `.has` calls failing on the decoded payload. **Fix:** replace the JSON body with `node:v8`'s `serialize` / `deserialize`. That's the same structured-clone algorithm Node's `worker.postMessage` uses natively — bit-for-bit compatible with the pre-U17 implicit-clone path. Full type fidelity for Map, Set, Date, RegExp, BigInt, TypedArray, undefined values, and circular refs. No external dependency. A previous iteration of this fix attempted to bolt a Map/Set replacer+reviver onto the JSON path. Rejected in favor of V8 serialization because: - the JSON tag-marker approach requires per-type registration (Map, Set; then Date, RegExp, BigInt would each need their own sentinels); V8 handles them all uniformly - keys to JSON-encode would still need handling for nested types (and the marker approach doesn't survive nested Maps-in-Maps cleanly without recursive replacer logic) - V8 is faster than JSON for object-heavy payloads anyway (binary format, no string escaping pass) - the user-explicit ask was "a much more generic solution that will work for everything" — V8 serialization IS the generic solution Trade-offs documented in the module header: - body bytes are opaque (binary, not human-readable) — debugging requires `v8.deserialize` ad-hoc; protocol.test.ts exercises every supported MessageTag including the new type-fidelity cases as a regression net. - format is tied to the running Node major. Pool always spawns workers on the same Node instance the main thread runs, so this is moot in production. Would matter if frames ever persisted to disk (nothing does today). Protocol test file rewritten: - drops the JSON-specific byte-layout assertions (e.g. `body must equal "null" string`) — replaced with V8-derived expected lengths - adds a "structured-clone type fidelity" describe block that pins Map, nested Map, Set, Date, RegExp, BigInt, TypedArray, undefined values, and circular-ref round-trips. These are the load-bearing regression tests preventing a future "optimize" PR from quietly swapping V8 back to JSON. - the bad-body decode-error test now uses arbitrary non-V8 bytes instead of `{not-json}` — same intent. Integration test READY_PREAMBLEs (worker-pool.test.ts and parse-impl-quarantine-cache-skip.test.ts) update their inline decoders to use `v8.deserialize` matching the production codec. Both files have a standalone CJS worker preamble that can't import dist/protocol.js by relative path, so the V8 dependency is required via `node:v8` directly. Tests: 271/271 unit files (6163 tests + 30 skipped). 28/28 worker-pool integration. 3/3 parse-impl integration. 791/791 scope-parity tests (the four CI-failing files: python.test.ts, go.test.ts, typescript.test.ts, csharp.test.ts) all green again. References plan: docs/plans/2026-05-20-002-fix-chunk-cache-corruption-on-worker-quarantine-plan.md * refactor(workers): drop protocol.ts; use native postMessage + transferList The protocol.ts framing layer was redundant — Node's `worker.postMessage` already runs V8 structured-clone internally, the same algorithm that backed `v8.serialize`. Wrapping V8.serialize → Buffer → postMessage(struct-clone-Buffer) was a double-walk: one full structured-clone pass to produce the Buffer, then another pass when postMessage cloned that Buffer across threads. This commit cuts the wrapper layer; workers and pool exchange POJO directly via `worker.postMessage(value, transferList)`, with file-content `ArrayBuffer`s in `transferList` for zero-copy ownership transfer. What changes: - **Deleted** `src/core/ingestion/workers/protocol.ts` (~180 LOC) + `test/unit/workers/protocol.test.ts` (~250 LOC). The MessageTag enum / ProtocolDecodeError / encodeMessage / decodeMessage surface is gone. Tag-based routing is replaced by the `msg.type` discriminant that every receive site already checks. Protocol-decode errors map to Node's `messageerror` event (V8 deserialization failures during postMessage), which the pool already wires to `recoverAndResume`. - **`worker-pool.ts`**: `decodeIncomingWorkerMessage` removed; handlers receive POJO directly. `buildDispatchMessage` now returns `{message: {type:'sub-batch', files: [{path, content: Uint8Array}]}, transferList: ArrayBuffer[]}`. The Uint8Array-per-content allocation via `TextEncoder.encode` is preserved (it's the load-bearing transfer-safety contract that keeps content out of Node's shared `Buffer.poolSize` slab). Flush dispatch is now plain `worker.postMessage({type:'flush'})`. - **`parse-worker.ts`**: `decodeIncomingMessage` removed. The message handler receives POJO directly; the only conversion is `Uint8Array → string` for sub-batch file contents at the `decodeSubBatchFiles` boundary, before handing to `processBatch`. Outgoing messages are emitted as POJO via plain `parentPort.postMessage({type:'starting-file', ...})` etc. The `sharedHybridDecoder` is now `sharedContentDecoder` (same intent, clearer name for the simpler shape). - **Test scaffolding**: FakeWorkers in `worker-pool-resilience`, `worker-pool-windows-quarantine`, and `worker-pool-slot-generation` drop their `decodeMessage` import + `decodeDispatchedMessage` helper. The helpers stay (still convert `files[i].content` Uint8Array → string for test-action introspection) but no longer touch any protocol framing — just shape-check for sub-batch. - **Integration READY_PREAMBLEs** (worker-pool.test.ts and parse-impl-quarantine-cache-skip.test.ts): drop the inline v8.deserialize + envelope-unzip logic; the preamble is now just the ready handshake + a `parentPort.on` wrapper that converts `files[i].content` Uint8Array → string for the ad-hoc test worker scripts. - **`worker-pool-transferlist.test.ts`**: contract tests updated for the new buildDispatchMessage shape — no `envelope` field anymore; `message.files[i].content` is Uint8Array; transferList holds each content.buffer in input order. Pool-slab independence still pinned. What stays the same: - Zero-copy file-content transfer via transferList — every file's ArrayBuffer is ownership-transferred to the worker (no copy). - Full structured-clone type fidelity — Map / Set / Date / RegExp / BigInt / TypedArray / undefined / circular refs all preserved by Node's native postMessage. The V8 fix from commit 06f6957e is inherent in this path; there's no JSON layer to lose them. - TextEncoder-per-content allocation — keeps content buffers out of the shared `Buffer.poolSize` slab so transferring one cannot detach another. - The pool's resilience layers (respawn, breaker, quarantine, starting-file attribution, cumulative timeout, ready handshake, slot-generation guard) — unchanged. - U20 chunk-cache write suppression on quarantine — unchanged. Net: ~430 LOC removed (protocol.ts + tests + inline decoders + helpers), ~120 LOC simplified in worker-pool.ts and parse-worker.ts. One less serialization pass per message on the hot path. Tests: 270/270 unit files (6133 + 30 skipped). 822/822 integration tests including the four CI-failing scope-parity files (Python, Go, TypeScript, C#) — the V8-fidelity contract holds via native postMessage with no explicit serializer. The single "error" reported in worker-pool.test.ts is the pre-existing intentional process.exit unhandled-exception artifact from the deliberate startup-crash test, unchanged by this commit. * refactor(parse-worker): drop legacy single-message dispatch mode The `parentPort.on('message', ...)` handler had an `Array.isArray(msg)` branch left over from a pre-sub-batch dispatch shape — the pool used to send the items array directly, before the worker pool added sub-batching and the `{type:'sub-batch', files: ...}` envelope. No production caller has dispatched that shape since the sub-batching refactor landed; verified by grepping the repo for `postMessage([` patterns (zero matches). The `ParseWorkerInput[]` arm in the `WorkerIncomingMessage` discriminated union also blocked exhaustiveness narrowing — flagged by the kieran-typescript code review (RR-01) as "if a future unit removes the legacy array path, this arm should be dropped." Dropping it now. What changes: - Remove the `Array.isArray(msg)` branch from the message handler. - Drop `ParseWorkerInput[]` from the `WorkerIncomingMessage` union; it's now a clean `{type:'sub-batch'} | {type:'flush'}` discriminated union, so the dispatch switch is exhaustive over `msg.type`. Tests: 71/71 worker-pool unit + integration tests green (resilience, slot-generation, windows-quarantine, transferlist, parsing-worker- fallback, worker-pool integration, parse-impl-quarantine-cache-skip). --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Gergő Magyar --- README.md | 171 +-- gitnexus/README.md | 10 + gitnexus/bench/parse-throughput.md | 175 +++ gitnexus/src/cli/analyze.ts | 89 +- gitnexus/src/cli/index.ts | 9 + gitnexus/src/cli/wiki.ts | 18 + .../languages/typescript/captures.ts | 78 +- .../src/core/ingestion/parsing-processor.ts | 94 +- .../ingestion/pipeline-phases/parse-impl.ts | 422 ++++-- gitnexus/src/core/ingestion/pipeline.ts | 50 + .../core/ingestion/workers/parse-worker.ts | 80 +- .../src/core/ingestion/workers/quarantine.ts | 59 + .../src/core/ingestion/workers/worker-pool.ts | 1155 +++++++++++++++-- gitnexus/src/core/run-analyze.ts | 10 +- .../parse-impl-large-fixture.test.ts | 177 +++ .../parse-impl-quarantine-cache-skip.test.ts | 388 ++++++ gitnexus/test/integration/worker-pool.test.ts | 512 +++++++- .../unit/analyze-worker-pool-size.test.ts | 140 ++ .../test/unit/analyze-worker-timeout.test.ts | 42 +- .../unit/parse-impl-chunk-concurrency.test.ts | 131 ++ .../parse-impl-deferred-extraction.test.ts | 142 ++ .../test/unit/parse-impl-env-reads.test.ts | 147 +++ .../test/unit/parse-impl-fallback.test.ts | 22 +- .../parse-impl-progress-monotonic.test.ts | 132 ++ .../test/unit/parsing-worker-fallback.test.ts | 139 +- .../typescript-captures-anchor.test.ts | 147 +++ .../worker-pool-cumulative-timeout.test.ts | 129 ++ .../test/unit/worker-pool-resilience.test.ts | 646 +++++++++ .../unit/worker-pool-slot-generation.test.ts | 225 ++++ .../unit/worker-pool-transferlist.test.ts | 115 ++ .../worker-pool-windows-quarantine.test.ts | 209 +++ gitnexus/test/unit/workers/quarantine.test.ts | 96 ++ 32 files changed, 5606 insertions(+), 353 deletions(-) create mode 100644 gitnexus/bench/parse-throughput.md create mode 100644 gitnexus/src/core/ingestion/workers/quarantine.ts create mode 100644 gitnexus/test/integration/parse-impl-large-fixture.test.ts create mode 100644 gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts create mode 100644 gitnexus/test/unit/analyze-worker-pool-size.test.ts create mode 100644 gitnexus/test/unit/parse-impl-chunk-concurrency.test.ts create mode 100644 gitnexus/test/unit/parse-impl-deferred-extraction.test.ts create mode 100644 gitnexus/test/unit/parse-impl-env-reads.test.ts create mode 100644 gitnexus/test/unit/parse-impl-progress-monotonic.test.ts create mode 100644 gitnexus/test/unit/scope-resolution/typescript/typescript-captures-anchor.test.ts create mode 100644 gitnexus/test/unit/worker-pool-cumulative-timeout.test.ts create mode 100644 gitnexus/test/unit/worker-pool-resilience.test.ts create mode 100644 gitnexus/test/unit/worker-pool-slot-generation.test.ts create mode 100644 gitnexus/test/unit/worker-pool-transferlist.test.ts create mode 100644 gitnexus/test/unit/worker-pool-windows-quarantine.test.ts create mode 100644 gitnexus/test/unit/workers/quarantine.test.ts diff --git a/README.md b/README.md index 6e6c7193d..b3caf7f7a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # GitNexus + **⚠️ Important Notice:** GitNexus has NO official cryptocurrency, token, or coin. Any token/coin using the GitNexus name on Pump.fun or any other platform is **not affiliated with, endorsed by, or created by** this project or its maintainers. Do not purchase any cryptocurrency claiming association with GitNexus.
@@ -30,14 +31,9 @@ Indexes any codebase into a knowledge graph — every dependency, call chain, cluster, and execution flow — then exposes it through smart tools so AI agents never miss code. - - - https://github.com/user-attachments/assets/172685ba-8e54-4ea7-9ad1-e31a3398da72 - - -> *Like DeepWiki, but deeper.* DeepWiki helps you *understand* code. GitNexus lets you *analyze* it — because a knowledge graph tracks every relationship, not just descriptions. +> _Like DeepWiki, but deeper._ DeepWiki helps you _understand_ code. GitNexus lets you _analyze_ it — because a knowledge graph tracks every relationship, not just descriptions. **TL;DR:** The **Web UI** is a quick way to chat with any repo. The **CLI + MCP** is how you make your AI agent actually reliable — it gives Cursor, Claude Code, Codex, and friends a deep architectural view of your codebase so they stop missing dependencies, breaking call chains, and shipping blind edits. Even smaller models get full architectural clarity, making it compete with Goliath models. @@ -47,18 +43,17 @@ https://github.com/user-attachments/assets/172685ba-8e54-4ea7-9ad1-e31a3398da72 [![Star History Chart](https://api.star-history.com/svg?repos=abhigyanpatwari/GitNexus&type=date&legend=top-left)](https://www.star-history.com/#abhigyanpatwari/GitNexus&type=date&legend=top-left) - ## Two Ways to Use GitNexus -| | **CLI + MCP** | **Web UI** | -| ----------------- | -------------------------------------------------------------- | ------------------------------------------------------------ | -| **What** | Index repos locally, connect AI agents via MCP | Visual graph explorer + AI chat in browser | -| **For** | Daily development with Cursor, Claude Code, Codex, Windsurf, OpenCode | Quick exploration, demos, one-off analysis | -| **Scale** | Full repos, any size | Limited by browser memory (~5k files), or unlimited via backend mode | -| **Install** | `npm install -g gitnexus` | No install — [gitnexus.vercel.app](https://gitnexus.vercel.app) | -| **Storage** | LadybugDB native (fast, persistent) | LadybugDB WASM (in-memory, per session) | -| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | -| **Privacy** | Everything local, no network | Everything in-browser, no server | +| | **CLI + MCP** | **Web UI** | +| ----------- | --------------------------------------------------------------------- | -------------------------------------------------------------------- | +| **What** | Index repos locally, connect AI agents via MCP | Visual graph explorer + AI chat in browser | +| **For** | Daily development with Cursor, Claude Code, Codex, Windsurf, OpenCode | Quick exploration, demos, one-off analysis | +| **Scale** | Full repos, any size | Limited by browser memory (~5k files), or unlimited via backend mode | +| **Install** | `npm install -g gitnexus` | No install — [gitnexus.vercel.app](https://gitnexus.vercel.app) | +| **Storage** | LadybugDB native (fast, persistent) | LadybugDB WASM (in-memory, per session) | +| **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | +| **Privacy** | Everything local, no network | Everything in-browser, no server | > **Bridge mode:** `gitnexus serve` connects the two — the web UI auto-detects the local server and can browse all your CLI-indexed repos without re-uploading or re-indexing. @@ -69,6 +64,7 @@ https://github.com/user-attachments/assets/172685ba-8e54-4ea7-9ad1-e31a3398da72 GitNexus is available as an **enterprise offering** - either as a fully managed **SaaS** or a **self-hosted** deployment. Also available for **commercial use** of the OSS version with proper licensing. Enterprise includes: + - **PR Review** - automated blast radius analysis on pull requests - **Auto-updating Code Wiki** - always up-to-date documentation (Code Wiki is also available in OSS) - **Auto-reindexing** - knowledge graph stays fresh automatically @@ -77,6 +73,7 @@ Enterprise includes: - **Priority feature/language support** - request new languages or features **Upcoming:** + - Auto regression forensics - End-to-end test generation @@ -117,13 +114,13 @@ To configure MCP for your editor, run `npx gitnexus setup` once — or set it up ### Editor Support -| Editor | MCP | Skills | Hooks (auto-augment) | Support | -| --------------------- | --- | ------ | -------------------- | -------------- | -| **Claude Code** | Yes | Yes | Yes (PreToolUse + PostToolUse) | **Full** | -| **Cursor** | Yes | Yes | Yes (postToolUse, [manual install](gitnexus-cursor-integration/README.md#hook-install)) | **Full** | -| **Codex** | Yes | Yes | — | MCP + Skills | -| **Windsurf** | Yes | — | — | MCP | -| **OpenCode** | Yes | Yes | — | MCP + Skills | +| Editor | MCP | Skills | Hooks (auto-augment) | Support | +| --------------- | --- | ------ | --------------------------------------------------------------------------------------- | ------------ | +| **Claude Code** | Yes | Yes | Yes (PreToolUse + PostToolUse) | **Full** | +| **Cursor** | Yes | Yes | Yes (postToolUse, [manual install](gitnexus-cursor-integration/README.md#hook-install)) | **Full** | +| **Codex** | Yes | Yes | — | MCP + Skills | +| **Windsurf** | Yes | — | — | MCP | +| **OpenCode** | Yes | Yes | — | MCP + Skills | > **Claude Code** gets the deepest integration: MCP tools + agent skills + PreToolUse hooks that enrich searches with graph context + PostToolUse hooks that detect a stale index after commits and prompt the agent to reindex. @@ -131,10 +128,10 @@ To configure MCP for your editor, run `npx gitnexus setup` once — or set it up Built by the community — not officially maintained, but worth checking out. -| Project | Author | Description | -|---------|--------|-------------| -| [pi-gitnexus](https://github.com/tintinweb/pi-gitnexus) | [@tintinweb](https://github.com/tintinweb) | GitNexus plugin for [pi](https://pi.dev) — `pi install npm:pi-gitnexus` | -| [gitnexus-stable-ops](https://github.com/ShunsukeHayashi/gitnexus-stable-ops) | [@ShunsukeHayashi](https://github.com/ShunsukeHayashi) | Stable ops & deployment workflows (Miyabi ecosystem) | +| Project | Author | Description | +| ----------------------------------------------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------- | +| [pi-gitnexus](https://github.com/tintinweb/pi-gitnexus) | [@tintinweb](https://github.com/tintinweb) | GitNexus plugin for [pi](https://pi.dev) — `pi install npm:pi-gitnexus` | +| [gitnexus-stable-ops](https://github.com/ShunsukeHayashi/gitnexus-stable-ops) | [@ShunsukeHayashi](https://github.com/ShunsukeHayashi) | Stable ops & deployment workflows (Miyabi ecosystem) | > Have a project built on GitNexus? Open a PR to add it here! @@ -206,6 +203,7 @@ gitnexus analyze --skip-git # Index folders that are not Git repositories gitnexus analyze --embeddings # Enable embedding generation (slower, better search) gitnexus analyze --verbose # Log skipped files when parsers are unavailable gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses +gitnexus analyze --workers # Parse worker pool size (default: cores-1, capped at 16; 0 = sequential) gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus list # List all indexed repositories @@ -230,6 +228,25 @@ gitnexus group status # Check staleness of repos in a group If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. +#### Environment variables + +Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max-file-size`, `--verbose`). Use the env-var form when you'd otherwise repeat the same flag every run, or when invoking GitNexus from a long-running host (MCP server, eval-server, CI shell) that already manages its own environment. CLI flags take precedence over env vars; env vars take precedence over built-in defaults. + +| Variable | Default | Effect | Tune when… | +| -------------------------------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size. `0` disables the pool (sequential fallback). Equivalent to `--workers `. | Constrained containers (cgroup CPU limits), CI runners with explicit quotas, or debugging a worker-only crash via `0`. | +| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | +| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | +| `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | +| `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | +| `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | +| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | +| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Combined with `timeoutBackoffFactor`, prevents exponentially-growing retries from stalling for hours. | Slow files that legitimately need long total retry windows; lower to fail-fast on stalls. | +| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD`| `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | +| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | +| `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | +| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips native builds for `tree-sitter-dart` / `tree-sitter-proto` at install time. | Installing on a host without a C++ toolchain; you're willing to skip Dart/Proto parsing. | + #### Publishing to understand-quickly (opt-in) [`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) is a public registry of code-knowledge graphs that lists `gitnexus@1` as a first-class format. After registering your repo once (`npx @understand-quickly/cli add` or the [wizard](https://looptech-ai.github.io/understand-quickly/add.html)), `gitnexus publish` fires a single `repository_dispatch` event so the registry resyncs your entry on demand instead of waiting for the nightly job. @@ -240,27 +257,27 @@ It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained G **16 tools** exposed via MCP (11 per-repo + 5 group): -| Tool | What It Does | `repo` Param | -| ------------------ | ----------------------------------------------------------------- | -------------- | -| `list_repos` | Discover all indexed repositories | — | -| `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | -| `context` | 360-degree symbol view — categorized refs, process participation | Optional | -| `impact` | Blast radius analysis with depth grouping and confidence | Optional | -| `detect_changes` | Git-diff impact — maps changed lines to affected processes | Optional | -| `rename` | Multi-file coordinated rename with graph + text search | Optional | -| `cypher` | Raw Cypher graph queries | Optional | -| `group_list` | List configured repository groups | — | -| `group_sync` | Extract contracts and match across repos/services | — | -| `group_contracts`| Inspect extracted contracts and cross-links | — | -| `group_query` | Search execution flows across all repos in a group | — | -| `group_status` | Check staleness of repos in a group | — | +| Tool | What It Does | `repo` Param | +| ----------------- | ---------------------------------------------------------------- | ------------ | +| `list_repos` | Discover all indexed repositories | — | +| `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | +| `context` | 360-degree symbol view — categorized refs, process participation | Optional | +| `impact` | Blast radius analysis with depth grouping and confidence | Optional | +| `detect_changes` | Git-diff impact — maps changed lines to affected processes | Optional | +| `rename` | Multi-file coordinated rename with graph + text search | Optional | +| `cypher` | Raw Cypher graph queries | Optional | +| `group_list` | List configured repository groups | — | +| `group_sync` | Extract contracts and match across repos/services | — | +| `group_contracts` | Inspect extracted contracts and cross-links | — | +| `group_query` | Search execution flows across all repos in a group | — | +| `group_status` | Check staleness of repos in a group | — | > When only one repo is indexed, the `repo` parameter is optional. With multiple repos, specify which one: `query({query: "auth", repo: "my-app"})`. **Resources** for instant context: -| Resource | Purpose | -| ----------------------------------------- | ---------------------------------------------------- | +| Resource | Purpose | +| --------------------------------------- | ---------------------------------------------------- | | `gitnexus://repos` | List all indexed repositories (read this first) | | `gitnexus://repo/{name}/context` | Codebase stats, staleness check, and available tools | | `gitnexus://repo/{name}/clusters` | All functional clusters with cohesion scores | @@ -271,9 +288,9 @@ It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained G **2 MCP prompts** for guided workflows: -| Prompt | What It Does | -| ----------------- | ------------------------------------------------------------------------- | -| `detect_impact` | Pre-commit change analysis — scope, affected processes, risk level | +| Prompt | What It Does | +| --------------- | ------------------------------------------------------------------------- | +| `detect_impact` | Pre-commit change analysis — scope, affected processes, risk level | | `generate_map` | Architecture documentation from the knowledge graph with mermaid diagrams | **4 agent skills** installed to `.claude/skills/` automatically: @@ -360,10 +377,10 @@ npx gitnexus@latest serve The official Docker setup ships **two signed images** orchestrated by `docker-compose.yaml`. Each image is published to both **GitHub Container Registry** (GHCR) and **Docker Hub** — same build, same digest, same Cosign signature — so pick whichever registry you prefer: -| Purpose | GHCR (default in `docker-compose.yaml`) | Docker Hub mirror | -| ---------------------------------------------------------------------- | --------------------------------------------- | ------------------------------------------- | -| CLI / `gitnexus serve` backend (HTTP API on port `4747`, MCP, indexer) | `ghcr.io/abhigyanpatwari/gitnexus:latest` | `akonlabs/gitnexus:latest` | -| Static web UI (port `4173`) | `ghcr.io/abhigyanpatwari/gitnexus-web:latest` | `akonlabs/gitnexus-web:latest` | +| Purpose | GHCR (default in `docker-compose.yaml`) | Docker Hub mirror | +| ---------------------------------------------------------------------- | --------------------------------------------- | ------------------------------ | +| CLI / `gitnexus serve` backend (HTTP API on port `4747`, MCP, indexer) | `ghcr.io/abhigyanpatwari/gitnexus:latest` | `akonlabs/gitnexus:latest` | +| Static web UI (port `4173`) | `ghcr.io/abhigyanpatwari/gitnexus-web:latest` | `akonlabs/gitnexus-web:latest` | > **Heads-up — image rename.** Earlier releases published the web UI under > `ghcr.io/abhigyanpatwari/gitnexus`. Starting with the introduction of the @@ -579,22 +596,22 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas ### Supported Languages -| Language | Imports | Named Bindings | Exports | Heritage | Type Annotations | Constructor Inference | Config | Frameworks | Entry Points | -|----------|---------|----------------|---------|----------|-----------------|---------------------|--------|------------|-------------| -| TypeScript | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| JavaScript | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | ✓ | ✓ | -| Python | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Java | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | -| Kotlin | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | -| C# | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Go | ✓ | — | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Rust | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | -| PHP | ✓ | ✓ | ✓ | — | ✓ | ✓ | ✓ | ✓ | ✓ | -| Ruby | ✓ | — | ✓ | ✓ | — | ✓ | — | ✓ | ✓ | -| Swift | — | — | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| C | — | — | ✓ | — | ✓ | ✓ | — | ✓ | ✓ | -| C++ | — | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | -| Dart | ✓ | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | +| Language | Imports | Named Bindings | Exports | Heritage | Type Annotations | Constructor Inference | Config | Frameworks | Entry Points | +| ---------- | ------- | -------------- | ------- | -------- | ---------------- | --------------------- | ------ | ---------- | ------------ | +| TypeScript | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| JavaScript | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | ✓ | ✓ | +| Python | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Java | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | +| Kotlin | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | +| C# | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Go | ✓ | — | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Rust | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | +| PHP | ✓ | ✓ | ✓ | — | ✓ | ✓ | ✓ | ✓ | ✓ | +| Ruby | ✓ | — | ✓ | ✓ | — | ✓ | — | ✓ | ✓ | +| Swift | — | — | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| C | — | — | ✓ | — | ✓ | ✓ | — | ✓ | ✓ | +| C++ | — | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | +| Dart | ✓ | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | **Imports** — cross-file import resolution · **Named Bindings** — `import { X as Y }` / re-export tracking · **Exports** — public/exported symbol detection · **Heritage** — class inheritance, interfaces, mixins · **Type Annotations** — explicit type extraction for receiver resolution · **Constructor Inference** — infer receiver type from constructor calls (`self`/`this` resolution included for all languages) · **Config** — language toolchain config parsing (tsconfig, go.mod, etc.) · **Frameworks** — AST-based framework pattern detection · **Entry Points** — entry point scoring heuristics @@ -739,16 +756,16 @@ The wiki generator reads the indexed graph structure, groups files into modules ## Tech Stack -| Layer | CLI | Web | -| ------------------------- | ------------------------------------- | --------------------------------------- | +| Layer | CLI | Web | +| ------------------- | ------------------------------------- | --------------------------------------- | | **Runtime** | Node.js (native) | Browser (WASM) | | **Parsing** | Tree-sitter native bindings | Tree-sitter WASM | -| **Database** | LadybugDB native | LadybugDB WASM | +| **Database** | LadybugDB native | LadybugDB WASM | | **Embeddings** | HuggingFace transformers.js (GPU/CPU) | transformers.js (WebGPU/WASM) | | **Search** | BM25 + semantic + RRF | BM25 + semantic + RRF | | **Agent Interface** | MCP (stdio) | LangChain ReAct agent | -| **Visualization** | — | Sigma.js + Graphology (WebGL) | -| **Frontend** | — | React 18, TypeScript, Vite, Tailwind v4 | +| **Visualization** | — | Sigma.js + Graphology (WebGL) | +| **Frontend** | — | React 18, TypeScript, Vite, Tailwind v4 | | **Clustering** | Graphology | Graphology | | **Concurrency** | Worker threads + async | Web Workers + Comlink | @@ -764,12 +781,12 @@ The wiki generator reads the indexed graph structure, groups files into modules ### Recently Completed -- [X] Constructor-Inferred Type Resolution, `self`/`this` Receiver Mapping -- [X] Wiki Generation, Multi-File Rename, Git-Diff Impact Analysis -- [X] Process-Grouped Search, 360-Degree Context, Claude Code Hooks -- [X] Multi-Repo MCP, Zero-Config Setup, 14 Language Support -- [X] Community Detection, Process Detection, Confidence Scoring -- [X] Hybrid Search, Vector Index +- [x] Constructor-Inferred Type Resolution, `self`/`this` Receiver Mapping +- [x] Wiki Generation, Multi-File Rename, Git-Diff Impact Analysis +- [x] Process-Grouped Search, 360-Degree Context, Claude Code Hooks +- [x] Multi-Repo MCP, Zero-Config Setup, 14 Language Support +- [x] Community Detection, Process Detection, Confidence Scoring +- [x] Hybrid Search, Vector Index --- diff --git a/gitnexus/README.md b/gitnexus/README.md index 640053013..61ae131ac 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -359,6 +359,16 @@ npx gitnexus analyze For repositories with very large source files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. The default is **8388608 bytes (8 MB)**. +### Worker pool resilience tuning + +Three env vars expose the pool's resilience layers (respawn budget, cumulative-timeout cap, circuit breaker). Defaults are tuned for typical repos; bump them when an analyze legitimately needs more retries, or lower them to fail-fast on a known-bad shape. + +| Variable | Default | Effect | +| ------------------------------------------------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per slot before the slot is dropped from the active rotation. | +| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. | +| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. | + ## Privacy - All processing happens locally on your machine diff --git a/gitnexus/bench/parse-throughput.md b/gitnexus/bench/parse-throughput.md new file mode 100644 index 000000000..24a7c98b1 --- /dev/null +++ b/gitnexus/bench/parse-throughput.md @@ -0,0 +1,175 @@ +# Parse-throughput benchmark (scaffold) + +> **Status: methodology + harness scaffold, no measurement data yet.** +> The Latest measurement table below contains `_TBD_` placeholders. +> This file ships intentionally without numbers — populating it +> requires a dedicated bench-pass against the U6 fixture (and ideally +> a real-world TS-root-scale repo) on consistent hardware, which is +> tracked as future work rather than gated on PR #1693's merge. +> Until the table is populated, the load-bearing perf-regression +> protection lives in `gitnexus/test/integration/parse-impl-large-fixture.test.ts` +> (U6, 30 s wall-clock budget via `Promise.race`). + +Tracks `runChunkedParseAndResolve` wall-clock + peak heap on a synthetic +fixture so PR #1693's "analyze no longer hangs on TS-root-shaped loads" +claim is measurable, not just asserted by smoke tests. The harness +recipe below is deliberately small enough to re-run in a few minutes +when the bench-pass is undertaken. + +--- + +## Methodology + +### Fixture + +Synthetic TypeScript repo, _not_ a clone of microsoft/TypeScript. CI cost +of cloning real-world repos is prohibitive; the synthetic shape exercises +the same pipeline paths (chunking, deferred extraction, cross-chunk +imports + heritage) without the disk-I/O overhead. Larger numbers can be +manually captured against real repos and cross-referenced here, but the +authoritative regression-tracking shape is the synthetic fixture so runs +are reproducible across hardware. + +The fixture matches the structure pinned by +`gitnexus/test/integration/parse-impl-large-fixture.test.ts` (U6): + +- 15 small modules (`mod0.ts` … `mod14.ts`), one exported function each. +- 1 dense `complex.ts` with 30 functions + 1 class + 1 interface. +- 1 `index.ts` re-exporting every symbol from every module. + +`GITNEXUS_CHUNK_BYTE_BUDGET=64` forces multi-chunk parsing on this small +fixture — without that override the whole thing fits in one chunk and +the deferred-extraction path is not exercised end-to-end. + +### What to measure + +| Metric | How | +| --------------------------- | -------------------------------------------------------------------------------- | +| Wall-clock total | `Date.now()` delta around `runChunkedParseAndResolve` | +| Peak heap | Sample `process.memoryUsage().heapUsed` every 50 ms during the run; keep the max | +| Chunks observed | Count distinct `Parsing chunk X/Y` progress messages | +| `getStats()` final snapshot | Quarantined paths, dropped slots, breaker state | + +### Hardware shape (record alongside each measurement) + +- OS + version +- CPU model + logical core count +- RAM +- Node version +- gitnexus commit SHA (so the snapshot is anchored to a tree, not "main") + +--- + +## Harness recipe + +The U6 test (`test/integration/parse-impl-large-fixture.test.ts`) is the +checked-in mini-benchmark — it exercises the same fixture and bounds the +wall-clock at 30 s via `Promise.race`. To produce a richer snapshot for +this doc, run it under instrumentation: + +```bash +# From the gitnexus/ subdir: +cd gitnexus +# Single-threaded baseline (sequential fallback): +npx vitest run test/integration/parse-impl-large-fixture.test.ts --reporter=verbose + +# Worker-pool path (requires built dist/ — pre-built by `npm run build`): +npm run build && \ + GITNEXUS_WORKER_POOL_SIZE=4 \ + GITNEXUS_PARSE_CHUNK_CONCURRENCY=2 \ + GITNEXUS_VERBOSE=1 \ + npx vitest run test/integration/parse-impl-large-fixture.test.ts --reporter=verbose +``` + +For peak-heap sampling, wrap the dispatch call in a Node script that +polls `process.memoryUsage()`. A future helper at +`gitnexus/bench/scripts/parse-throughput.ts` would automate this — the +plan's stretch goal. Until that lands, capture peak heap manually via: + +```bash +node --inspect=0 \ + --require ./scripts/heap-sampler.js \ + ./node_modules/.bin/vitest run test/integration/parse-impl-large-fixture.test.ts +``` + +--- + +## Latest measurement + +> _No measurement data has been collected yet — this file is the +> methodology + harness scaffold. The single recorded data point is the +> U6 wall-clock smoke baseline below; the worker-pool rows are +> placeholders for future bench-pass output._ + +The U6 integration test (`gitnexus/test/integration/parse-impl-large-fixture.test.ts`) +was observed completing the synthetic fixture in **~6 seconds** under +the sequential path (`skipWorkers: true`) on the development machine, +well under the 30 s `Promise.race` wall-clock budget. That number is a +smoke baseline only — recorded here for reference, not as a regression +target. + +| Path | files/s | wall-clock | peak heap | chunks | quarantined | +| ------------------------------------------ | ------- | -------------------- | --------- | ------ | ----------- | +| Sequential (`skipWorkers: true`, U6 smoke) | _TBD_ | ~6 s _(observation)_ | _TBD_ | 17 | 0 | +| Worker pool, `--workers 4`, concurrency 2 | _TBD_ | _TBD_ | _TBD_ | _TBD_ | 0 | +| Worker pool, `--workers 1`, concurrency 1 | _TBD_ | _TBD_ | _TBD_ | _TBD_ | 0 | + +**Hardware:** _TBD — record OS, CPU, RAM, Node version, gitnexus SHA at +the time of the bench-pass that populates the table above._ + +--- + +## Operator-tuning quick reference + +Cross-links to the env vars documented in the [README](../../README.md#environment-variables). +Use this section as a starting point when the benchmark numbers above +suggest a tuning opportunity for your hardware shape. + +- **CPU-bound, big repo, lots of cores:** raise `GITNEXUS_WORKER_POOL_SIZE` + past the default cap of 16. The 16-worker cap exists because past that + point main-thread merge / extraction dominates; if you've measurably + ruled that out, the env var lifts the cap explicitly. (See + `worker-pool.ts` `DEFAULT_POOL_SIZE_CAP`.) +- **Slow files (large minified JS, deep TS types):** raise + `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` past 30 000 ms. The cumulative + budget is 5× this value (U10 pins this) so a 60 s idle timeout permits + 300 s of total retry-and-split wall-clock before quarantining the file. +- **Constrained container (cgroup CPU limit):** the pool now uses + `os.availableParallelism()` (U3 H2), which honors cgroup limits — no + manual `GITNEXUS_WORKER_POOL_SIZE` override needed unless the auto- + resolved value is too aggressive for your I/O budget. +- **Long-running host (eval-server, MCP daemon) running back-to-back + analyzes:** `--workers` is now threaded through `AnalyzeOptions` + (U2 B2), so per-invocation sizing is honored without `process.env` + state leaking across calls. `GITNEXUS_VERBOSE` is similarly snapshot/ + restore-bracketed. + +--- + +## What this benchmark does NOT measure + +- **Real-repo performance.** The synthetic fixture is sized for CI; it + doesn't exercise the cumulative-load shape (50k files, occasional + pathological file) that drove the original PR #1693 hang report. Real- + repo numbers should be captured ad-hoc against the user's target repo + and cross-referenced here only as supplementary evidence. +- **Worker-pool resilience under real crashes.** That's verified by the + `worker-pool.test.ts` integration tests (real `process.exit`, real + `error` events, real protocol violations) and the unit suite. The + benchmark cares about throughput on the happy path. +- **IPC repack throughput.** Phase 3 of the PR #1693 plan introduces a + transferList + binary wire-format IPC repack (U16-U17). Once that + lands, an `IPC repack` row should be added to the "Latest measurement" + table above with before/after numbers on the same hardware. + +--- + +## Related artifacts + +- Plan: `docs/plans/2026-05-20-001-feat-pr1693-resilience-hardening-and-ipc-repack-plan.md` +- Integration test (mini-benchmark with wall-clock guard): `gitnexus/test/integration/parse-impl-large-fixture.test.ts` (U6) +- Operator env-var reference: `README.md` → Environment variables +- Resilience layer tests: `gitnexus/test/unit/worker-pool-resilience.test.ts`, + `worker-pool-cumulative-timeout.test.ts`, + `worker-pool-windows-quarantine.test.ts`, + `worker-pool-slot-generation.test.ts` diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 70183ecf9..6ea04df05 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -148,7 +148,7 @@ function ensureHeap(): boolean { stdio: 'inherit', env: { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim() }, }); - } catch (e: any) { + } catch (e: unknown) { if (childProcessLikelyOom(e)) { cliError( ` Analysis likely ran out of memory.\n` + @@ -159,11 +159,50 @@ function ensureHeap(): boolean { { recoveryHint: 'heap-oom-respawn' }, ); } - process.exitCode = e.status ?? 1; + const status = + typeof e === 'object' && e !== null && 'status' in e && typeof e.status === 'number' + ? e.status + : 1; + process.exitCode = status; } return true; } +/** + * GITNEXUS_* env vars that `analyzeCommand` writes for backward-compatible + * downstream consumption. Snapshotted at function entry and restored in the + * finally block so that programmatic callers (tests, long-running hosts) + * don't see leaked state across invocations. `GITNEXUS_WORKER_POOL_SIZE` is + * NOT in this list: that knob is threaded through `runFullAnalysis` options + * (see `workerPoolSize` plumbing) so the CLI never has to mutate `process.env` + * for it in the first place. + */ +const ANALYZE_CLI_ENV_KEYS = [ + 'GITNEXUS_VERBOSE', + 'GITNEXUS_MAX_FILE_SIZE', + 'GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', + 'GITNEXUS_EMBEDDING_THREADS', + 'GITNEXUS_EMBEDDING_BATCH_SIZE', + 'GITNEXUS_EMBEDDING_SUB_BATCH_SIZE', + 'GITNEXUS_EMBEDDING_DEVICE', +] as const; + +type AnalyzeEnvSnapshot = Record<(typeof ANALYZE_CLI_ENV_KEYS)[number], string | undefined>; + +const snapshotAnalyzeEnv = (): AnalyzeEnvSnapshot => { + const snap = {} as AnalyzeEnvSnapshot; + for (const k of ANALYZE_CLI_ENV_KEYS) snap[k] = process.env[k]; + return snap; +}; + +const restoreAnalyzeEnv = (snap: AnalyzeEnvSnapshot): void => { + for (const k of ANALYZE_CLI_ENV_KEYS) { + const v = snap[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } +}; + export interface AnalyzeOptions { force?: boolean; repairFts?: boolean; @@ -226,6 +265,8 @@ export interface AnalyzeOptions { maxFileSize?: string; /** Override worker sub-batch idle timeout in seconds. */ workerTimeout?: string; + /** Parse worker pool size; 0 disables workers (sequential fallback). */ + workers?: string; embeddingThreads?: string; embeddingBatchSize?: string; embeddingSubBatchSize?: string; @@ -259,6 +300,22 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption // a stack trace and a non-zero exit code instead of a silent exit 0. installFatalHandlers(); + // Snapshot the GITNEXUS_* env vars that the impl writes for downstream + // consumption, so they don't leak across `analyzeCommand` invocations in + // programmatic callers (tests, long-running hosts). `process.exit(0)` on + // the success path bypasses `finally` — intentional: when the process is + // exiting, restoration is moot. For early-return paths (validation + // errors) and the alreadyUpToDate fast path the finally restores the + // pre-call values. + const envSnap = snapshotAnalyzeEnv(); + try { + await analyzeCommandImpl(inputPath, options); + } finally { + restoreAnalyzeEnv(envSnap); + } +}; + +const analyzeCommandImpl = async (inputPath?: string, options?: AnalyzeOptions): Promise => { if (options?.verbose) { process.env.GITNEXUS_VERBOSE = '1'; } @@ -279,6 +336,26 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption ); } + // `--workers` is threaded through `runFullAnalysis` options → PipelineOptions + // → createWorkerPool, intentionally bypassing the GITNEXUS_WORKER_POOL_SIZE + // env channel so this CLI surface never mutates `process.env` for pool size. + // Tests can therefore re-invoke analyzeCommand with different --workers + // values back-to-back and observe the value they passed, not whatever the + // previous call leaked. + let workerPoolSize: number | undefined; + if (options?.workers !== undefined) { + const parsedWorkers = Number(options.workers); + if (!Number.isInteger(parsedWorkers) || parsedWorkers < 0) { + cliError( + ' --workers must be a non-negative integer. ' + + 'Pass 0 to disable the worker pool (sequential fallback).\n', + ); + process.exitCode = 1; + return; + } + workerPoolSize = parsedWorkers; + } + // Parse `--embeddings [limit]`: `true` → default cap, string → numeric cap // (0 disables the cap entirely). Validated up here so failures match the // sibling-validation pattern (exit before bar.start() — otherwise @@ -551,6 +628,10 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption // be able to accept the duplicate name without also paying the // cost of a full pipeline re-index. See #829 review round 2. allowDuplicateName: options?.allowDuplicateName, + // Worker pool size threaded from --workers, replacing the previous + // GITNEXUS_WORKER_POOL_SIZE env mutation. `undefined` defers to the + // env / auto-formula fallback inside the pipeline. + workerPoolSize, }, { onProgress: (_phase, percent, message) => { @@ -688,7 +769,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption } console.log(''); - } catch (err: any) { + } catch (err: unknown) { clearInterval(elapsedTimer); process.removeListener('SIGINT', sigintHandler); console.log = origLog; @@ -698,7 +779,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption console.error = origError; bar.stop(); - const msg = err.message || String(err); + const msg = err instanceof Error ? err.message : String(err); // Registry name-collision from --name (#829) — surface as an // actionable error rather than a generic stack-trace. diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 9450698d3..31446cf6f 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -71,6 +71,10 @@ program '--worker-timeout ', 'Worker sub-batch idle timeout before retry/fallback. Default: 30.', ) + .option( + '--workers ', + 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).', + ) .option('--embedding-threads ', 'Limit local ONNX embedding CPU threads') .option('--embedding-batch-size ', 'Number of nodes per embedding batch') .option('--embedding-sub-batch-size ', 'Number of chunks per embedding model call') @@ -82,6 +86,11 @@ program ' GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n' + ' GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n' + ' GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n' + + ' GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n' + + ' GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n' + + ' GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n' + + ' GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n' + + ' GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n' + ' GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n' + ' GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n' + '\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n' + diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index 8089dd2f2..6211d371c 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -107,6 +107,24 @@ function prompt(question: string, hide = false): Promise { } export const wikiCommand = async (inputPath?: string, options?: WikiCommandOptions) => { + // Snapshot GITNEXUS_VERBOSE at entry — wikiCommand mutates it (the impl + // below) so cursor-client (process.env-driven) sees the right value during + // this run. Restored in finally so back-to-back wiki calls in long-running + // hosts don't leak verbose state from one invocation to the next. Pairs + // with the same snapshot/restore pattern in `analyzeCommand`. + const originalVerbose = process.env.GITNEXUS_VERBOSE; + try { + await wikiCommandImpl(inputPath, options); + } finally { + if (originalVerbose === undefined) { + delete process.env.GITNEXUS_VERBOSE; + } else { + process.env.GITNEXUS_VERBOSE = originalVerbose; + } + } +}; + +const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions): Promise => { // Set verbose mode globally for cursor-client to pick up if (options?.verbose) { process.env.GITNEXUS_VERBOSE = '1'; diff --git a/gitnexus/src/core/ingestion/languages/typescript/captures.ts b/gitnexus/src/core/ingestion/languages/typescript/captures.ts index b82edcff8..22f48e80d 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/captures.ts @@ -64,7 +64,7 @@ const CALL_TAGS = [ '@reference.call.constructor', ] as const; -function pickFirstDefined(grouped: CaptureMatch, tags: readonly string[]): Capture | undefined { +function pickFirstCapture(grouped: CaptureMatch, tags: readonly string[]): Capture | undefined { for (const tag of tags) { const cap = grouped[tag]; if (cap !== undefined) return cap; @@ -72,6 +72,17 @@ function pickFirstDefined(grouped: CaptureMatch, tags: readonly string[]): Captu return undefined; } +function pickFirstNode( + grouped: Record, + tags: readonly string[], +): SyntaxNode | undefined { + for (const tag of tags) { + const node = grouped[tag]; + if (node !== undefined) return node; + } + return undefined; +} + /** * Drop `@reference.read.member` matches whose underlying `member_expression` * is NOT actually a read context: @@ -113,6 +124,34 @@ function shouldEmitReadMember(memberNode: SyntaxNode): boolean { } } +/** Walks the parent chain from `node` (inclusive), returning the first node + * whose type matches, or null. Faster than `findNodeAtRange` when the caller + * already holds the anchor node — avoids re-scanning the tree from the root. */ +function findSelfOrAncestorOfType(node: SyntaxNode | undefined, type: string): SyntaxNode | null { + if (node === undefined) return null; + let current: SyntaxNode | null = node; + while (current !== null) { + if (current.type === type) return current; + current = current.parent; + } + return null; +} + +/** Walks the parent chain from `node` (inclusive), returning the first node + * whose type is in the set, or null. Plural form of {@link findSelfOrAncestorOfType}. */ +function findSelfOrAncestorOfTypes( + node: SyntaxNode | undefined, + types: readonly string[], +): SyntaxNode | null { + if (node === undefined) return null; + let current: SyntaxNode | null = node; + while (current !== null) { + if (types.includes(current.type)) return current; + current = current.parent; + } + return null; +} + export function emitTsScopeCaptures( sourceText: string, filePath: string, @@ -151,9 +190,11 @@ export function emitTsScopeCaptures( // `@`; we put it back so the central extractor's prefix lookups // (`@scope.`, `@declaration.`, …) work. const grouped: Record = {}; + const groupedNodes: Record = {}; for (const c of m.captures) { const tag = '@' + c.name; grouped[tag] = nodeToCapture(tag, c.node); + groupedNodes[tag] = c.node; } if (Object.keys(grouped).length === 0) continue; @@ -165,6 +206,10 @@ export function emitTsScopeCaptures( if (grouped['@import.statement'] !== undefined) { const stmtCapture = grouped['@import.statement']; const stmtNode = + findSelfOrAncestorOfTypes(groupedNodes['@import.statement'], [ + 'import_statement', + 'export_statement', + ]) ?? findNodeAtRange(tree.rootNode, stmtCapture.range, 'import_statement') ?? findNodeAtRange(tree.rootNode, stmtCapture.range, 'export_statement'); if (stmtNode !== null) { @@ -183,7 +228,9 @@ export function emitTsScopeCaptures( // `splitDynamicImport` branch consumes. if (grouped['@import.dynamic'] !== undefined) { const dynCapture = grouped['@import.dynamic']; - const callNode = findNodeAtRange(tree.rootNode, dynCapture.range, 'call_expression'); + const callNode = + findSelfOrAncestorOfType(groupedNodes['@import.dynamic'], 'call_expression') ?? + findNodeAtRange(tree.rootNode, dynCapture.range, 'call_expression'); if (callNode !== null) { const decomposed = splitImportStatement(callNode); for (const d of decomposed) out.push(d); @@ -197,7 +244,9 @@ export function emitTsScopeCaptures( // we rely on this emit-side filter so the query stays simple. if (grouped['@reference.read.member'] !== undefined) { const anchor = grouped['@reference.read.member']; - const memberNode = findNodeAtRange(tree.rootNode, anchor.range, 'member_expression'); + const memberNode = + findSelfOrAncestorOfType(groupedNodes['@reference.read.member'], 'member_expression') ?? + findNodeAtRange(tree.rootNode, anchor.range, 'member_expression'); if (memberNode === null || !shouldEmitReadMember(memberNode)) { continue; } @@ -208,9 +257,10 @@ export function emitTsScopeCaptures( // overloads — TypeScript supports overload signatures via // function_signature, so `parameterTypes` is populated when // available. - const declAnchor = pickFirstDefined(grouped, FUNCTION_DECL_TAGS); + const declAnchor = pickFirstCapture(grouped, FUNCTION_DECL_TAGS); + const declAnchorNode = pickFirstNode(groupedNodes, FUNCTION_DECL_TAGS); if (declAnchor !== undefined) { - const fnNode = findFunctionNode(tree.rootNode, declAnchor.range); + const fnNode = findFunctionNode(tree.rootNode, declAnchor.range, declAnchorNode); if (fnNode !== null) { const arity = computeTsArityMetadata(fnNode); if (arity.parameterCount !== undefined) { @@ -255,9 +305,11 @@ export function emitTsScopeCaptures( // calls to disambiguate by props-arity, a JSX-aware arity // synthesizer would need to count `jsx_attribute` children of the // opening tag instead of `arguments`. - const callAnchor = pickFirstDefined(grouped, CALL_TAGS); + const callAnchor = pickFirstCapture(grouped, CALL_TAGS); + const callAnchorNode = pickFirstNode(groupedNodes, CALL_TAGS); if (callAnchor !== undefined && grouped['@reference.arity'] === undefined) { const callNode = + findSelfOrAncestorOfTypes(callAnchorNode, ['call_expression', 'new_expression']) ?? findNodeAtRange(tree.rootNode, callAnchor.range, 'call_expression') ?? findNodeAtRange(tree.rootNode, callAnchor.range, 'new_expression'); if (callNode !== null) { @@ -293,7 +345,11 @@ export function emitTsScopeCaptures( // lookup instead of synthesis — covered by `tsReceiverBinding`. const scopeFnAnchor = grouped['@scope.function']; if (scopeFnAnchor !== undefined) { - const fnNode = findFunctionNode(tree.rootNode, scopeFnAnchor.range); + const fnNode = findFunctionNode( + tree.rootNode, + scopeFnAnchor.range, + groupedNodes['@scope.function'], + ); if (fnNode !== null) { const synth = synthesizeTsReceiverBinding(fnNode); if (synth !== null) out.push(synth); @@ -518,7 +574,13 @@ function inferArgType(argNode: SyntaxNode): string { * The `@scope.function` anchor range covers the whole node, but the * tag alone doesn't identify which node type among the many TS * function-likes. */ -function findFunctionNode(rootNode: SyntaxNode, range: Capture['range']): SyntaxNode | null { +function findFunctionNode( + rootNode: SyntaxNode, + range: Capture['range'], + anchorNode?: SyntaxNode, +): SyntaxNode | null { + const fromAnchor = findSelfOrAncestorOfTypes(anchorNode, FUNCTION_NODE_TYPES); + if (fromAnchor !== null) return fromAnchor; for (const nodeType of FUNCTION_NODE_TYPES) { const n = findNodeAtRange(rootNode, range, nodeType); if (n !== null) return n; diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 5cf398bee..28a080eb2 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -832,6 +832,14 @@ const processParsingSequential = async ( // Public API // ============================================================================ +/** + * Per-`WorkerPool` log-dedup state for quarantine reporting. Keyed on the + * pool instance so multiple concurrent pools (test fixtures, future + * multi-pool callers) each get their own seen-set. WeakMap entries vanish + * when the pool is garbage-collected. + */ +const loggedQuarantineByPool = new WeakMap>(); + export const processParsing = async ( graph: KnowledgeGraph, files: { path: string; content: string }[], @@ -874,25 +882,75 @@ export const processParsing = async ( `[scope-resolution prof] worker pool engaged for ${files.length} files — cross-phase tree cache will be empty; scope-resolution re-parses.`, ); } - try { - return await processParsingWithWorkers( - graph, - files, - symbolTable, - astCache, - workerPool, - reportProgress, - outRawResults, - ); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - logger.warn({ message }, 'Worker pool parsing stopped; continuing with sequential parser:'); - reportProgress?.( - lastProgress, - files.length, - `Sequential fallback after worker issue: ${message}`, - ); + // U20 design pivot: the worker pool's resilience layers + // (respawn budget, circuit breaker, quarantine, slot-attribution, + // cumulative timeout) are the SOLE contract for handling worker + // failures. There is no sequential-parser fallback for either + // partial quarantine or full pool failure — the operator must see + // a clear hard signal when workers can't recover, instead of a + // silently-degraded graph from a possibly-crashing main-thread + // sequential parser. A failing tree-sitter native binding that + // quarantined a worker would, under the previous design, re-trigger + // the same SIGSEGV on the main thread; we avoid that risk entirely. + // + // - Partial quarantine: the file is missing from this run's graph; + // the per-chunk warn log below surfaces it; U2's chunk-cache + // write-guard in parse-impl.ts keeps the chunk uncached so the + // next analyze gets a cache miss and a fresh pool retries. + // - Full pool failure: `WorkerPoolDispatchError` propagates from + // `processParsingWithWorkers` up through this function. The + // analyze run errors out instead of falling back to sequential. + const data = await processParsingWithWorkers( + graph, + files, + symbolTable, + astCache, + workerPool, + reportProgress, + outRawResults, + ); + // Session-scoped quarantine (worker-pool resilience Layer 3): surface + // any files this pool has decided are unsafe for workers so the + // operator can see what was skipped. The pool already filtered them + // out of dispatch; we only need to log + progress-report. Quarantine + // is session-scoped per pool instance — a fresh `createWorkerPool` + // call clears it. + // + // Dedup: log full path list only for entries newly quarantined since + // the previous dispatch on the same pool. The per-chunk progress + // message still surfaces the count for UX continuity, but the + // structured `quarantinedFiles` payload is only emitted when there + // is new signal — prevents O(quarantine × chunks) log spam. + const quarantineSnapshot = workerPool.getQuarantinedPaths?.() ?? []; + const quarantineSet = new Set(quarantineSnapshot); + if (quarantineSet.size > 0) { + const quarantinedInChunk = files.filter((file) => quarantineSet.has(file.path)); + if (quarantinedInChunk.length > 0) { + const seenForPool = loggedQuarantineByPool.get(workerPool) ?? new Set(); + const newlyQuarantined = quarantinedInChunk + .map((file) => file.path) + .filter((p) => !seenForPool.has(p)); + for (const p of newlyQuarantined) seenForPool.add(p); + loggedQuarantineByPool.set(workerPool, seenForPool); + if (newlyQuarantined.length > 0) { + logger.warn( + { + newlyQuarantined, + cumulativeQuarantine: quarantineSet.size, + chunkSkipped: quarantinedInChunk.length, + }, + `Worker quarantine: ${newlyQuarantined.length} new file(s) skipped this chunk ` + + `(${quarantinedInChunk.length} skipped total, ${quarantineSet.size} cumulative).`, + ); + } + reportProgress?.( + lastProgress, + files.length, + `${quarantinedInChunk.length} worker-quarantined file(s) skipped`, + ); + } } + return data; } // Fallback: sequential parsing (no pre-extracted data) diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 17cfaab3f..46945d74c 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -55,6 +55,7 @@ import type { ExtractedCall, ExtractedDecoratorRoute, ExtractedFetchCall, + ExtractedImport, ExtractedORMQuery, ExtractedRoute, ExtractedToolDef, @@ -69,6 +70,7 @@ import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { isDev } from '../utils/env.js'; +import { isVerboseIngestionEnabled } from '../utils/verbose.js'; import { synthesizeWildcardImportBindings, needsSynthesis } from './wildcard-synthesis.js'; import { extractORMQueriesInline } from './orm-extraction.js'; @@ -85,11 +87,24 @@ import { logger } from '../../logger.js'; * gives a useful invalidation floor (~1/N chunks on a multi-MB repo) * while keeping worker dispatch overhead under 5% on cold runs. */ -const CHUNK_BYTE_BUDGET = (() => { +/** + * Built-in chunk byte budget when neither `PipelineOptions.chunkByteBudget` + * nor `GITNEXUS_CHUNK_BYTE_BUDGET` is set. Tuned to give a useful + * cache-invalidation floor (~1/N chunks on a multi-MB repo) while keeping + * worker dispatch overhead under 5% on cold runs. Resolution happens at + * call time inside `runChunkedParseAndResolve` (U14 from PR #1693 review) + * — previously this was a module-load IIFE, which froze the env value at + * import time and meant per-call option threading silently no-op'd. + */ +const DEFAULT_CHUNK_BYTE_BUDGET = 2 * 1024 * 1024; + +function resolveChunkByteBudget(options?: PipelineOptions): number { + const opt = options?.chunkByteBudget; + if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0) return opt; const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET); if (Number.isFinite(env) && env > 0) return env; - return 2 * 1024 * 1024; -})(); + return DEFAULT_CHUNK_BYTE_BUDGET; +} // ── Main parse + resolve function ────────────────────────────────────────── @@ -177,18 +192,28 @@ export async function runChunkedParseAndResolve( if (totalParseable === 0) { onProgress({ phase: 'parsing', - percent: 82, + // Skip directly to the end of the parse-phase progress band (M2 from PR + // #1693 review). Parse 20-70%, deferred 70-95%; nothing in either runs + // when there's no parseable file, so jump to 95. + percent: 95, message: 'No parseable files found — skipping parsing phase', stats: { filesProcessed: 0, totalFiles: 0, nodesCreated: graph.nodeCount }, }); } - // Build byte-budget chunks + // Build byte-budget chunks. The budget is resolved per-call (U14): options + // first, then env, then the built-in default. Pre-U14 this was a + // module-load IIFE constant, which froze the env value at import time + // and made `PipelineOptions.chunkByteBudget` silently no-op on warm test + // runs. Resolving in the function body restores per-call configurability + // and matches the pattern used by resolveAutoPoolSize and the U1 + // parseChunkConcurrency resolver. + const chunkByteBudget = resolveChunkByteBudget(options); const chunks: string[][] = []; let currentChunk: string[] = []; let currentBytes = 0; for (const file of parseableScanned) { - if (currentChunk.length > 0 && currentBytes + file.size > CHUNK_BYTE_BUDGET) { + if (currentChunk.length > 0 && currentBytes + file.size > chunkByteBudget) { chunks.push(currentChunk); currentChunk = []; currentBytes = 0; @@ -203,16 +228,22 @@ export async function runChunkedParseAndResolve( if (isDev) { const totalMB = parseableScanned.reduce((s, f) => s + f.size, 0) / (1024 * 1024); logger.info( - `📂 Scan: ${totalFiles} paths, ${totalParseable} parseable (${totalMB.toFixed(0)}MB), ${numChunks} chunks @ ${CHUNK_BYTE_BUDGET / (1024 * 1024)}MB budget`, + `📂 Scan: ${totalFiles} paths, ${totalParseable} parseable (${totalMB.toFixed(0)}MB), ${numChunks} chunks @ ${chunkByteBudget / (1024 * 1024)}MB budget`, ); } - onProgress({ - phase: 'parsing', - percent: 20, - message: `Parsing ${totalParseable} files in ${numChunks} chunk${numChunks !== 1 ? 's' : ''}...`, - stats: { filesProcessed: 0, totalFiles: totalParseable, nodesCreated: graph.nodeCount }, - }); + // Skip the "Parsing N files..." announcement when there's nothing to parse + // — the early-return branch above already emitted percent 95 ("skipping + // parsing phase"), and emitting percent 20 here would regress the + // progress stream non-monotonically (M2 from PR #1693 review). + if (totalParseable > 0) { + onProgress({ + phase: 'parsing', + percent: 20, + message: `Parsing ${totalParseable} files in ${numChunks} chunk${numChunks !== 1 ? 's' : ''}...`, + stats: { filesProcessed: 0, totalFiles: totalParseable, nodesCreated: graph.nodeCount }, + }); + } // Don't spawn workers for tiny repos — overhead exceeds benefit. // Test suites may lower the thresholds via `options.workerThresholdsForTest` @@ -221,18 +252,33 @@ export async function runChunkedParseAndResolve( const MIN_BYTES_FOR_WORKERS = options?.workerThresholdsForTest?.minBytes ?? 512 * 1024; const totalBytes = parseableScanned.reduce((s, f) => s + f.size, 0); - // Create worker pool once, reuse across chunks + // Create worker pool once, reuse across chunks. + // + // `workerPoolSize === 0` is a programmatic equivalent of `skipWorkers: + // true` per the `PipelineOptions.workerPoolSize` contract. Short- + // circuiting here avoids constructing a useless pool that rejects + // every dispatch (with a `Worker pool parsing stopped` warn log per + // chunk) just to fall back to the sequential path via the error + // catch — the gate honors the docstring directly. let workerPool: WorkerPool | undefined; if ( !options?.skipWorkers && + options?.workerPoolSize !== 0 && (totalParseable >= MIN_FILES_FOR_WORKERS || totalBytes >= MIN_BYTES_FOR_WORKERS) ) { try { - let workerUrl = new URL('../workers/parse-worker.js', import.meta.url); + // U20.U3 test-only injection: integration tests pass a custom + // worker script URL via `workerUrlForTest` (mirrors the + // `workerThresholdsForTest` precedent) so they can drive the + // chunk-loop with deterministically-misbehaving workers without + // mocking the module import graph. When unset, the normal src/ + // → dist/ resolution runs. + let workerUrl = + options?.workerUrlForTest ?? new URL('../workers/parse-worker.js', import.meta.url); // When running under vitest, import.meta.url points to src/ where no .js exists. // Fall back to the compiled dist/ worker so the pool can spawn real worker threads. const thisDir = fileURLToPath(new URL('.', import.meta.url)); - if (!fs.existsSync(fileURLToPath(workerUrl))) { + if (!options?.workerUrlForTest && !fs.existsSync(fileURLToPath(workerUrl))) { const distWorker = path.resolve( thisDir, '..', @@ -249,7 +295,7 @@ export async function runChunkedParseAndResolve( workerUrl = pathToFileURL(distWorker); } } - workerPool = createWorkerPool(workerUrl); + workerPool = createWorkerPool(workerUrl, options?.workerPoolSize); } catch (err) { logger.warn( { err: (err as Error).message }, @@ -301,6 +347,16 @@ export async function runChunkedParseAndResolve( const deferredWorkerHeritage: ExtractedHeritage[] = []; const deferredConstructorBindings: FileConstructorBindings[] = []; const deferredAssignments: ExtractedAssignment[] = []; + // Imports accumulated across chunks. Previously processed per-chunk + // via `processImportsFromExtracted` inside the chunk loop, which + // forced workers to sit idle on the main thread's extraction pass + // between chunk dispatches (4-5% CPU utilization symptom). Deferring + // to a single end-of-loop pass lets the worker pool start chunk N+1 + // immediately after chunk N's worker dispatch returns. Resolution is + // strictly-more-information at end-of-loop because graph now has + // every chunk's symbols — improves cross-chunk import targets. + const deferredWorkerImports: ExtractedImport[] = []; + let anyChunkNeedsWildcardSynth = false; // Aggregated per-file ParsedFile artifacts produced by workers' calls // to `extractParsedFile`. Threaded through to the scope-resolution // phase so it can SKIP its own re-extraction on cache hits — this is @@ -317,10 +373,54 @@ export async function runChunkedParseAndResolve( let chunkCacheMisses = 0; try { + // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch + // chunk file contents up to `parseChunkConcurrency` chunks ahead of the + // dispatch cursor so file I/O overlaps with worker compute. Worker + // dispatch itself stays serial because `WorkerPool.dispatch` is not + // reentrant (concurrent calls would race on the shared per-slot + // busy/in-flight state). With concurrency=1 behavior is identical to + // the pure-serial loop. F4: deferred-state aggregation still happens + // in chunkIdx order (the for-loop below iterates sequentially), so + // cross-chunk processors see deterministic input regardless of + // file-read completion order. Honors options.parseChunkConcurrency + // (threaded from the CLI), then GITNEXUS_PARSE_CHUNK_CONCURRENCY env + // (default 2 — matches the help text the CLI advertises). + const parseChunkConcurrency = ((): number => { + const opt = options?.parseChunkConcurrency; + if (typeof opt === 'number' && Number.isInteger(opt) && opt >= 1) return opt; + const env = Number(process.env.GITNEXUS_PARSE_CHUNK_CONCURRENCY); + if (Number.isInteger(env) && env >= 1) return env; + return 2; + })(); + const chunkContentPromises = new Array> | undefined>(numChunks); + const startChunkPrefetch = (i: number): void => { + if (i >= numChunks || chunkContentPromises[i] !== undefined) return; + chunkContentPromises[i] = readFileContents(repoPath, chunks[i]); + }; + for (let i = 0; i < Math.min(parseChunkConcurrency, numChunks); i++) { + startChunkPrefetch(i); + } + + // Hoisted loop-invariant: GITNEXUS_VERBOSE / NODE_ENV are read once + // (not on every chunk). Previously evaluated at the top of the loop + // body, which re-read process.env on every iteration even though + // the env can't change mid-run. + const verboseThroughputLog = isDev || isVerboseIngestionEnabled(); + for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { const chunkPaths = chunks[chunkIdx]; + // Start wall-clock for the per-chunk throughput log emitted at end + // of this iteration. The gate is computed once above; here we just + // sample the clock if the gate is on. Computed when either + // NODE_ENV=development OR the operator passed `--verbose` + // (GITNEXUS_VERBOSE) — the previous `isDev`-only gate meant + // operators running `gitnexus analyze --verbose` in production + // never saw the log (M3 from PR #1693 review). + const chunkStartMs: number | null = verboseThroughputLog ? Date.now() : null; - const chunkContents = await readFileContents(repoPath, chunkPaths); + const chunkContents = await chunkContentPromises[chunkIdx]!; + chunkContentPromises[chunkIdx] = undefined; // release the in-memory copy + startChunkPrefetch(chunkIdx + parseChunkConcurrency); const chunkFiles = chunkPaths .filter((p) => chunkContents.has(p)) .map((p) => ({ path: p, content: chunkContents.get(p)! })); @@ -357,7 +457,11 @@ export async function runChunkedParseAndResolve( const cachedFiles = chunkFiles.length; onProgress({ phase: 'parsing', - percent: Math.round(20 + ((filesParsedSoFar + cachedFiles) / totalParseable) * 62), + // Parse phase covers 20-70 (50 points). Deferred extraction below + // takes 70-95 so the UI advances through the (potentially long) + // resolution stages instead of holding at 82 (M2 from PR #1693 + // review). + percent: Math.round(20 + ((filesParsedSoFar + cachedFiles) / totalParseable) * 50), message: `Parsing chunk ${chunkIdx + 1}/${numChunks} (cache)...`, stats: { filesProcessed: filesParsedSoFar + cachedFiles, @@ -378,7 +482,8 @@ export async function runChunkedParseAndResolve( scopeTreeCache, (current, _total, filePath) => { const globalCurrent = filesParsedSoFar + current; - const parsingProgress = 20 + (globalCurrent / totalParseable) * 62; + // Parse phase covers 20-70 (M2). Deferred extraction handles 70-95. + const parsingProgress = 20 + (globalCurrent / totalParseable) * 50; onProgress({ phase: 'parsing', percent: Math.round(parsingProgress), @@ -399,56 +504,63 @@ export async function runChunkedParseAndResolve( // Persist the raw results for this chunk hash. Sequential path // doesn't populate rawResults (it writes directly to graph), so // small repos without worker pool simply don't cache. That's fine. + // + // U20.U2: refuse the write when any chunk file is in the + // worker pool's cumulative quarantine snapshot. The chunkHash + // is computed from EVERY file in the chunk, but the pool's + // Layer 3 quarantine filters quarantined files out of dispatch + // — so `rawResults` is narrower than the chunkHash key implies. + // Caching it would silently replay incomplete results on the + // next run with unchanged content (the corruption class Codex's + // adversarial review of PR #1693 flagged). + // + // Skipping the write means the next analyze gets a cache miss + // for this chunk and re-dispatches against a fresh worker pool + // (quarantine is session-scoped — `createQuarantine` is called + // per-pool at worker-pool.ts), giving the quarantined file + // another chance. If quarantine fires again, U20.U1's + // sequential gap-fill still produces a complete graph for this + // run; the cache just stays empty for this chunk until a fully- + // clean dispatch lands. if (parseCache && chunkHash && rawResults.length > 0) { - parseCache.entries.set(chunkHash, rawResults); - if (isDev) { - logger.info( - `📦 parse-cache MISS+store: chunk ${chunkIdx + 1}/${numChunks} (${chunkFiles.length} files, ${chunkHash.slice(0, 8)})`, - ); + const quarantineSnapshot = workerPool?.getQuarantinedPaths?.() ?? []; + const quarantineSet = new Set(quarantineSnapshot); + const chunkHadQuarantine = chunkFiles.some((f) => quarantineSet.has(f.path)); + if (chunkHadQuarantine) { + if (isDev) { + const quarantinedInChunk = chunkFiles.filter((f) => quarantineSet.has(f.path)).length; + logger.info( + `📦 parse-cache SKIP: chunk ${chunkIdx + 1}/${numChunks} ` + + `had ${quarantinedInChunk} worker-quarantined file(s); ` + + `next run will rediscover (${chunkHash.slice(0, 8)})`, + ); + } + } else { + parseCache.entries.set(chunkHash, rawResults); + if (isDev) { + logger.info( + `📦 parse-cache MISS+store: chunk ${chunkIdx + 1}/${numChunks} (${chunkFiles.length} files, ${chunkHash.slice(0, 8)})`, + ); + } } } } - const chunkBasePercent = 20 + (filesParsedSoFar / totalParseable) * 62; - + // Per-chunk extraction passes (processImportsFromExtracted, + // processHeritageFromExtracted, processRoutesFromExtracted, + // synthesizeWildcardImportBindings, seedCrossFileReceiverTypes) + // moved out of the chunk loop into a single end-of-loop pass below. + // Reason: per-chunk extraction blocked the chunk loop on + // main-thread work between worker dispatches — workers sat idle + // and total CPU utilization plateaued at 4-5% on multi-core boxes. + // Deferring keeps workers busy chunk-after-chunk; resolution sees + // strictly-more-information (full repo graph) so cross-chunk import + // and heritage targets resolve at least as well as before. if (chunkWorkerData) { - await processImportsFromExtracted( - graph, - allPathObjects, - chunkWorkerData.imports, - ctx, - (current, total) => { - onProgress({ - phase: 'parsing', - percent: Math.round(chunkBasePercent), - message: `Resolving imports (chunk ${chunkIdx + 1}/${numChunks})...`, - detail: `${current}/${total} files`, - stats: { - filesProcessed: filesParsedSoFar, - totalFiles: totalParseable, - nodesCreated: graph.nodeCount, - }, - }); - }, - repoPath, - importCtx, - ); if (chunkNeedsSynthesis[chunkIdx]) { - synthesizeWildcardImportBindings(graph, ctx); - hasSynthesized = true; - } - if (exportedTypeMap.size > 0 && ctx.namedImportMap.size > 0) { - const { enrichedCount } = seedCrossFileReceiverTypes( - chunkWorkerData.calls, - ctx.namedImportMap, - exportedTypeMap, - ); - if (isDev && enrichedCount > 0) { - logger.info( - `🔗 E1: Seeded ${enrichedCount} cross-file receiver types (chunk ${chunkIdx + 1})`, - ); - } + anyChunkNeedsWildcardSynth = true; } + for (const item of chunkWorkerData.imports) deferredWorkerImports.push(item); for (const item of chunkWorkerData.calls) deferredWorkerCalls.push(item); for (const item of chunkWorkerData.heritage) deferredWorkerHeritage.push(item); for (const item of chunkWorkerData.constructorBindings) @@ -463,35 +575,6 @@ export async function runChunkedParseAndResolve( for (const item of chunkWorkerData.assignments) deferredAssignments.push(item); } - await Promise.all([ - processHeritageFromExtracted(graph, chunkWorkerData.heritage, ctx, (current, total) => { - onProgress({ - phase: 'parsing', - percent: Math.round(chunkBasePercent), - message: `Resolving heritage (chunk ${chunkIdx + 1}/${numChunks})...`, - detail: `${current}/${total} records`, - stats: { - filesProcessed: filesParsedSoFar, - totalFiles: totalParseable, - nodesCreated: graph.nodeCount, - }, - }); - }), - processRoutesFromExtracted(graph, chunkWorkerData.routes ?? [], ctx, (current, total) => { - onProgress({ - phase: 'parsing', - percent: Math.round(chunkBasePercent), - message: `Resolving routes (chunk ${chunkIdx + 1}/${numChunks})...`, - detail: `${current}/${total} routes`, - stats: { - filesProcessed: filesParsedSoFar, - totalFiles: totalParseable, - nodesCreated: graph.nodeCount, - }, - }); - }), - ]); - if (chunkWorkerData.fileScopeBindings?.length) { for (const { filePath, bindings } of chunkWorkerData.fileScopeBindings) { if (typeof filePath !== 'string' || filePath.length === 0) continue; @@ -530,6 +613,24 @@ export async function runChunkedParseAndResolve( filesParsedSoFar += chunkFiles.length; astCache.clear(); + + // Throughput observability (U3): emit a per-chunk metrics line + // under verbose ingestion mode so operators can verify CPU + // utilization moved + tune `--workers` / batch sizes without + // guessing. Cheap snapshot — just reads pool closure state. + if (verboseThroughputLog && chunkStartMs !== null) { + const elapsedMs = Date.now() - chunkStartMs; + const filesPerSec = elapsedMs > 0 ? (chunkFiles.length * 1000) / elapsedMs : 0; + const stats = workerPool?.getStats?.(); + const poolFrag = stats + ? ` pool: ${stats.activeSlots}/${stats.size} active, ` + + `${stats.quarantined} quarantined${stats.poolBroken ? ', BROKEN' : ''}` + : ' (sequential)'; + logger.info( + `📊 chunk ${chunkIdx + 1}/${numChunks}: ${chunkFiles.length} files in ${elapsedMs}ms ` + + `(${filesPerSec.toFixed(1)} files/s)${poolFrag}`, + ); + } } if (isDev && parseCache && (chunkCacheHits > 0 || chunkCacheMisses > 0)) { @@ -538,10 +639,129 @@ export async function runChunkedParseAndResolve( ); } + // Deferred end-of-loop extraction (moved out of the per-chunk block): + // 1. processImportsFromExtracted on all chunks' imports + // 2. synthesizeWildcardImportBindings (if any chunk had wildcards) + // 3. seedCrossFileReceiverTypes on deferred calls (depends on + // namedImportMap populated by step 1) + // 4. processHeritageFromExtracted on all chunks' heritage + // 5. processRoutesFromExtracted on all chunks' routes + // Same logic as the prior per-chunk passes, just batched — resolution + // sees the full repo graph instead of just current-and-earlier chunks. + // Deferred extraction band (M2 from PR #1693 review): the 4 stages below + // each get their own 5-10 point slice of the 70-95 range so percent + // advances monotonically through the (potentially long) resolution work + // instead of holding flat at 82. Stages that are skipped (zero-length + // input) leave their band as a no-op jump — the next stage still starts + // at its own band, preserving monotonicity. + // imports: 70 -> 75 (5) + // heritage: 75 -> 80 (5) + // routes: 80 -> 85 (5) + // calls: 85 -> 95 (10) + if (deferredWorkerImports.length > 0) { + await processImportsFromExtracted( + graph, + allPathObjects, + deferredWorkerImports, + ctx, + (current, total) => { + const ratio = total > 0 ? current / total : 1; + onProgress({ + phase: 'parsing', + percent: 70 + Math.round(ratio * 5), + message: 'Resolving imports (all chunks)...', + detail: `${current}/${total} files`, + stats: { + filesProcessed: filesParsedSoFar, + totalFiles: totalParseable, + nodesCreated: graph.nodeCount, + }, + }); + }, + repoPath, + importCtx, + ); + // U15 (lightweight M1): processImportsFromExtracted is the sole + // consumer of `deferredWorkerImports`. Free the array now so the + // GC can reclaim the per-file ExtractedImport records before the + // heavier downstream stages run (heritage, routes, calls). Peak + // accumulator memory drops from O(repo) to O(repo - imports) for + // the remainder of the deferred phase. The future per-chunk + // streaming upgrade can rewrite this with the same correctness + // contract once profile data shows it's warranted. + deferredWorkerImports.length = 0; + } + if (anyChunkNeedsWildcardSynth) { + synthesizeWildcardImportBindings(graph, ctx); + hasSynthesized = true; + } + // L5 from PR #1693 review: populate `exportedTypeMap` from the in-progress + // graph BEFORE `seedCrossFileReceiverTypes` runs. Previously the seeding + // branch below was reached with `exportedTypeMap.size === 0` in the + // worker path (the map was only built at the post-parse block far below, + // AFTER the seeding branch), so the seed dead-coded itself silently and + // call resolution never got the cross-file receiver-type enrichment. + // The post-parse builder still runs as a defensive fallback on the + // sequential path; its `size === 0` guard means we don't pay the cost + // twice on the worker path. + if (exportedTypeMap.size === 0 && graph.nodeCount > 0) { + const graphExports = buildExportedTypeMapFromGraph(graph, ctx.model.symbols); + for (const [fp, exports] of graphExports) exportedTypeMap.set(fp, exports); + } + if (exportedTypeMap.size > 0 && ctx.namedImportMap.size > 0 && deferredWorkerCalls.length > 0) { + const { enrichedCount } = seedCrossFileReceiverTypes( + deferredWorkerCalls, + ctx.namedImportMap, + exportedTypeMap, + ); + if (isDev && enrichedCount > 0) { + logger.info(`🔗 E1: Seeded ${enrichedCount} cross-file receiver types (all chunks)`); + } + } + if (deferredWorkerHeritage.length > 0) { + await processHeritageFromExtracted(graph, deferredWorkerHeritage, ctx, (current, total) => { + const ratio = total > 0 ? current / total : 1; + onProgress({ + phase: 'parsing', + percent: 75 + Math.round(ratio * 5), + message: 'Resolving heritage (all chunks)...', + detail: `${current}/${total} records`, + stats: { + filesProcessed: filesParsedSoFar, + totalFiles: totalParseable, + nodesCreated: graph.nodeCount, + }, + }); + }); + } + if (allExtractedRoutes.length > 0) { + await processRoutesFromExtracted(graph, allExtractedRoutes, ctx, (current, total) => { + const ratio = total > 0 ? current / total : 1; + onProgress({ + phase: 'parsing', + percent: 80 + Math.round(ratio * 5), + message: 'Resolving routes (all chunks)...', + detail: `${current}/${total} routes`, + stats: { + filesProcessed: filesParsedSoFar, + totalFiles: totalParseable, + nodesCreated: graph.nodeCount, + }, + }); + }); + } + const fullWorkerHeritageMap = deferredWorkerHeritage.length > 0 ? buildHeritageMap(deferredWorkerHeritage, ctx, getHeritageStrategyForLanguage) : undefined; + // U15 (lightweight M1): buildHeritageMap is the LAST consumer of the + // raw `deferredWorkerHeritage` records — processCallsFromExtracted + // below reads from the derived `fullWorkerHeritageMap` instead. Free + // the raw heritage array now so the GC can reclaim it before the + // (potentially long) call-resolution stage. processHeritageFromExtracted + // earlier was a read-only consumer (pushed to graph, didn't drain). + deferredWorkerHeritage.length = 0; if (deferredWorkerCalls.length > 0) { await processCallsFromExtracted( @@ -549,9 +769,13 @@ export async function runChunkedParseAndResolve( deferredWorkerCalls, ctx, (current, total) => { + const ratio = total > 0 ? current / total : 1; onProgress({ phase: 'parsing', - percent: 82, + // Calls is the longest deferred stage on real repos — give it the + // 10-point tail 85-95 so the progress bar visibly advances during + // call resolution instead of holding at 82 (M2). + percent: 85 + Math.round(ratio * 10), message: 'Resolving calls (all chunks)...', detail: `${current}/${total} files`, stats: { @@ -576,6 +800,20 @@ export async function runChunkedParseAndResolve( bindingAccumulator, ); } + // U15 (lightweight M1): all three arrays have had their last consumer + // by the time we reach this point — processCallsFromExtracted drained + // `deferredWorkerCalls` and read `deferredConstructorBindings`; + // processAssignmentsFromExtracted drained `deferredAssignments` and + // also read `deferredConstructorBindings`. Free them now so the + // function-scope references die before downstream graph-build / + // scope-resolution starts using its own working memory. Note: arrays + // returned in the function result object (allFetchCalls, + // allExtractedRoutes, allDecoratorRoutes, allToolDefs, allORMQueries, + // allParsedFiles) intentionally stay live — downstream consumers + // need them. + deferredWorkerCalls.length = 0; + deferredConstructorBindings.length = 0; + deferredAssignments.length = 0; } finally { await workerPool?.terminate(); } diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 1ee8e102f..ec5828b3b 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -55,6 +55,16 @@ export interface PipelineOptions { minFiles?: number; minBytes?: number; }; + /** + * @internal Test-only override for the worker script URL the pool + * spawns. When unset, parse-impl resolves `parse-worker.js` from the + * adjacent `workers/` directory (or the compiled `dist/` fallback + * under vitest). Integration tests use this to inject a custom + * worker script that deterministically triggers worker-pool + * resilience paths (e.g., crash-on-poison-file) — same precedent as + * `workerThresholdsForTest`. Do not use from production call sites. + */ + workerUrlForTest?: URL; /** * Incremental-indexing parse cache. When provided: * - The parse phase looks up each chunk's content hash in @@ -68,6 +78,46 @@ export interface PipelineOptions { * See `gitnexus/src/storage/parse-cache.ts`. */ parseCache?: import('../../storage/parse-cache.js').ParseCache; + /** + * Worker pool size override, threaded from the CLI `--workers` flag + * via `AnalyzeOptions`. When set, parse-impl passes this directly to + * `createWorkerPool` so the pool sizing bypasses the env-var fallback + * in `resolveAutoPoolSize`. The env-var channel + * (`GITNEXUS_WORKER_POOL_SIZE`) remains as a back-compat fallback when + * this field is undefined. Setting `workerPoolSize: 0` disables the + * pool entirely (sequential fallback) — equivalent to `skipWorkers` + * but expressed in the same units as `--workers ` so long-running + * hosts (eval-server, MCP daemon) can size per-call without leaking + * `process.env` state across analyze invocations. + */ + workerPoolSize?: number; + /** + * Number of chunks whose file contents may be read into memory in + * parallel while the worker pool is busy dispatching the current + * chunk. Pre-fetching overlaps disk I/O for chunk N+1..N+K with the + * worker compute on chunk N — modest but real wall-clock win on + * repos large enough to chunk. Worker dispatch itself remains serial + * because `WorkerPool.dispatch` is not reentrant (concurrent calls + * would race on the shared per-slot busy/in-flight state). + * + * `1` matches today's pure-serial behavior; `2` is the documented + * default (`GITNEXUS_PARSE_CHUNK_CONCURRENCY`). Falls back to the + * env var when undefined; defaults to 2 when neither is set. + */ + parseChunkConcurrency?: number; + /** + * Byte budget per parse chunk (in bytes). When set, parse-impl uses + * this instead of the `GITNEXUS_CHUNK_BYTE_BUDGET` env var or the + * built-in 2 MB default. Smaller values produce more chunks (finer + * cache-hit granularity, more worker dispatches); larger values + * batch more files per dispatch. + * + * Threading the value through options instead of the env var lets + * tests vary the chunk layout per-call without `vi.resetModules` and + * lets long-running hosts (eval-server, MCP daemon) size per-call + * without leaking `process.env` state across invocations. + */ + chunkByteBudget?: number; } // ── Phase registry ───────────────────────────────────────────────────────── diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index da681b070..eee9593a1 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -301,10 +301,7 @@ export interface ParseWorkerInput { content: string; } -type WorkerIncomingMessage = - | { type: 'sub-batch'; files: ParseWorkerInput[] } - | { type: 'flush' } - | ParseWorkerInput[]; +type WorkerIncomingMessage = { type: 'sub-batch'; files: ParseWorkerInput[] } | { type: 'flush' }; // ============================================================================ // Worker-local parser + language map @@ -1401,6 +1398,15 @@ const processFileGroup = ( // Skip files larger than the max tree-sitter buffer (32 MB) if (getTreeSitterContentByteLength(file.content) > TREE_SITTER_MAX_BUFFER) continue; + // Authoritative in-flight signal for the pool: lets `WorkerPool` exclude + // exactly this file if the worker dies during parse/extract, instead of + // guessing from `items[lastProgress]` (which the language-grouped order + // here would defeat). The pool gracefully ignores this when running an + // older worker build that doesn't emit it. + if (parentPort) { + parentPort.postMessage({ type: 'starting-file', path: file.path }); + } + // Vue SFC preprocessing: extract