diff --git a/.claude/skills/gitnexus-guide/SKILL.md b/.claude/skills/gitnexus-guide/SKILL.md index c96616130..e52560422 100644 --- a/.claude/skills/gitnexus-guide/SKILL.md +++ b/.claude/skills/gitnexus-guide/SKILL.md @@ -81,6 +81,18 @@ list_repos { offset: 400 } → repos 401–437, hasMore false Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. +### Inline staleness signal (`query` / `context` / `impact` / `cypher`) + +These four hot read tools attach a non-blocking `staleness` field to their response when the index is behind the checkout's current HEAD — the same `{ commitsBehind, hint }` shape `list_repos` already reports — so a direct tool call surfaces a behind-HEAD index without a separate `list_repos` call: + +```jsonc +{ /* …the tool's normal result… */ + "staleness": { "commitsBehind": 3, "hint": "⚠️ Index is 3 commits behind HEAD. Run analyze tool to update." } +} +``` + +The field is **absent when the index is current** (or when the freshness check can't run), so its presence is the signal. It is only ever added to object results — raw-array `cypher` output and error envelopes are returned unchanged. `@group`-targeted calls do not carry it (multi-repo staleness is ill-defined). When you see it, the graph may be behind the working tree — re-run `analyze` before trusting blast-radius or dependence answers. + ### Taint findings (`explain`) `explain` returns taint findings recorded by `gitnexus analyze --pdg` — intra-procedural `TAINTED` edges plus cross-function `TAINT_PATH` hops where the interprocedural taint phase found a function-level source→sink chain. Each finding includes a sink category (command-injection, code-injection, path-traversal, sql-injection, xss), source/sink lines, and the ordered hop path with the variable carried on each hop. diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c57c0d6aa..96003dcef 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -39,6 +39,7 @@ ENV BUN_VERSION=${BUN_VERSION} \ TZ=${TZ} \ DEVCONTAINER=true \ NODE_OPTIONS=--max-old-space-size=4096 \ + GITNEXUS_AUTO_HEAP=0 \ POWERLEVEL9K_DISABLE_GITSTATUS=true # Native build toolchain that gitnexus/postinstall needs. It compiles diff --git a/.github/workflows/build-tree-sitter-prebuilds.yml b/.github/workflows/build-tree-sitter-prebuilds.yml index 92f3ef0fc..adf950d0a 100644 --- a/.github/workflows/build-tree-sitter-prebuilds.yml +++ b/.github/workflows/build-tree-sitter-prebuilds.yml @@ -352,7 +352,7 @@ jobs: with: persist-credentials: false # this job uploads artifacts (artipacked) - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 diff --git a/.github/workflows/ci-devcontainer.yml b/.github/workflows/ci-devcontainer.yml index e0d1284a5..4a7eeb39a 100644 --- a/.github/workflows/ci-devcontainer.yml +++ b/.github/workflows/ci-devcontainer.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 - name: Unit-test the host->container config transforms @@ -60,7 +60,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 # Builds the image the same way a developer's "Reopen in Container" does. diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 0106049ae..182c620d2 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: npm @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: npm diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index f58e647b5..12cc53de7 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -428,7 +428,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' cache: npm @@ -446,7 +446,7 @@ jobs: # Switch to the engines-floor Node AFTER building — native deps built on # 22.x load across the whole 22.x ABI line, and nothing installs after this # (so no package-manager cache is needed). - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22.18.0' package-manager-cache: false @@ -549,6 +549,7 @@ jobs: npx vitest run --no-file-parallelism test/integration/cobol-pipeline-benchmark.test.ts test/integration/csharp-pipeline-benchmark.test.ts + test/integration/instance-ownership-pipeline-benchmark.test.ts test/integration/rust-pipeline-benchmark.test.ts test/integration/php-pipeline-benchmark.test.ts test/integration/ruby-pipeline-benchmark.test.ts @@ -587,7 +588,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22.18.0' cache: npm diff --git a/.github/workflows/gitnexus-review-agent.yml b/.github/workflows/gitnexus-review-agent.yml index 88526871e..022954786 100644 --- a/.github/workflows/gitnexus-review-agent.yml +++ b/.github/workflows/gitnexus-review-agent.yml @@ -323,7 +323,7 @@ jobs: - name: Set up pinned Node.js id: setup-node if: steps.context.outputs.ready == 'true' - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22.18.0' diff --git a/.github/workflows/gitnexus-skill-evolution.yml b/.github/workflows/gitnexus-skill-evolution.yml index 4cf601549..bf15b00bc 100644 --- a/.github/workflows/gitnexus-skill-evolution.yml +++ b/.github/workflows/gitnexus-skill-evolution.yml @@ -130,7 +130,7 @@ jobs: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22.18.0' cache: npm diff --git a/.github/workflows/grammar-update-monitor.yml b/.github/workflows/grammar-update-monitor.yml index 64ff20a2d..7380af269 100644 --- a/.github/workflows/grammar-update-monitor.yml +++ b/.github/workflows/grammar-update-monitor.yml @@ -48,7 +48,7 @@ jobs: with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml index 27383cb08..a74152049 100644 --- a/.github/workflows/pr-autofix.yml +++ b/.github/workflows/pr-autofix.yml @@ -59,7 +59,7 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name }} persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: npm diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9be8c4a91..e7b03c198 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -395,7 +395,7 @@ jobs: exit 1 fi - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Node 24 ships with npm >= 11.5.x, which is the minimum that # supports npm Trusted Publishing OIDC. Node 22 ships with npm @@ -881,7 +881,7 @@ jobs: fi - name: Create GitHub Release - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v2 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2 with: tag_name: ${{ steps.vtag-gate.outputs.vtag }} name: >- diff --git a/.github/workflows/skill-sync.yml b/.github/workflows/skill-sync.yml index 48611d544..e9c1a0881 100644 --- a/.github/workflows/skill-sync.yml +++ b/.github/workflows/skill-sync.yml @@ -50,7 +50,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' cache: npm diff --git a/GUARDRAILS.md b/GUARDRAILS.md index aca9f5e37..9005219fd 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -20,6 +20,7 @@ Maintainer may widen scope per task. 3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. 4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. 5. **Preserve embeddings** — plain `npx gitnexus analyze` now preserves any embeddings recorded in the index metadata (`.gitnexus/gitnexus.json`, mirrored to the legacy `meta.json`) — the previous behavior wiped them. Use `--embeddings` to also generate vectors for new/changed nodes; use `--drop-embeddings` only when an explicit wipe is intended (e.g., model swap). +6. **Never `terminate()` a worker that may be inside a native call** — killing a worker thread mid-N-API aborts the entire process (`Napi::Error` → `std::terminate` → SIGABRT, #2432), so a timeout meant to trigger a graceful fallback takes the whole run down instead. Any worker running native code (tree-sitter grammars, LadybugDB, Icebug) must either reach a JS-visible safe point first — the parse pool's `shutdownDrainMs` handshake in `src/core/ingestion/workers/worker-pool.ts` — or be abandoned with `unref()` and left to exit on its own. A one-shot worker that ends after a single `postMessage` needs no `terminate()` at all: it exits by itself. This bites hardest on the path you cannot test locally, because the abort only reproduces once the native module actually loads. --- diff --git a/MIGRATION.md b/MIGRATION.md index f6af6c6a7..ccc94fc35 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -17,7 +17,7 @@ and the caller supplied none of `target_uid` / `file_path` / `kind`, "message": "Found N symbols matching ''. Use target_uid, file_path, or kind to disambiguate.", "target": { "name": "" }, "direction": "upstream", - "impactedCount": 0, + "impactedCount": null, "risk": "UNKNOWN", "candidates": [ { "uid": "...", "name": "...", "kind": "Function", "filePath": "...", "line": 42, "score": 0.76 } @@ -25,6 +25,13 @@ and the caller supplied none of `target_uid` / `file_path` / `kind`, } ``` +> `impactedCount` is `null`, not `0`, on an ambiguous result (#2687): no single +> symbol was resolved, so the blast radius is *undetermined*. A numeric `0` was +> indistinguishable from a genuine "nothing depends on this", so a caller +> testing `impactedCount === 0` read a false all-clear. Read `maxImpactedCount` +> (callgraph ambiguity) or the per-candidate counts in `candidates[]` for the +> real figure. Callers written as `impactedCount || 0` are unaffected. + ### Do I need to migrate? **Probably not, but check for assumptions.** Callers that unconditionally diff --git a/README.md b/README.md index dcca07d56..aa52ab56d 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ flowchart TB | `detect_impact` | Pre-commit change analysis — scope, affected processes, risk level | | `generate_map` | Architecture documentation from the knowledge graph with mermaid diagrams | -### Agent skills installed to `.claude/skills/` automatically +### Agent skills installed to `.claude/skills/` and `.agents/skills/` (if `.agents/` exists) automatically - **Exploring** — navigate unfamiliar code using the knowledge graph - **Debugging** — trace bugs through call chains @@ -198,6 +198,8 @@ flowchart TB **Repo-specific skills** — run `gitnexus analyze --skills` and GitNexus detects the functional areas of your codebase (via Leiden community detection) and generates each one as a direct project skill under `.claude/skills/gitnexus-area-/`. Each skill describes a module's key files, entry points, execution flows, and cross-area connections, and is regenerated on each `--skills` run to stay current. +When a repo contains an `.agents/` directory, the standard and generated skills are also mirrored to `.agents/skills/` (e.g. `.agents/skills/gitnexus-cli/`, `.agents/skills/gitnexus-area-/`) so agents that read repo-local `.agents/skills/` (like Codex) stay in sync. + ## Editor Setup `gitnexus setup` auto-detects your editors and writes the correct global MCP config. Run it once. To configure only selected integrations, pass `--coding-agent`/`-c` with a comma-separated list, e.g. `gitnexus setup -c cursor,codex`. @@ -395,7 +397,7 @@ gitnexus analyze --skills # Generate repo-specific skill files from detec gitnexus analyze --skip-embeddings # Skip embedding generation (faster) gitnexus analyze --embeddings [limit] # Enable embedding generation (slower, better search) gitnexus analyze --skip-agents-md # Preserve custom AGENTS.md/CLAUDE.md gitnexus section edits -gitnexus analyze --skip-skills # Skip installing standard .claude/skills/gitnexus-* skill files +gitnexus analyze --skip-skills # Skip installing standard skill files under .claude/skills/ and .agents/skills/ gitnexus analyze --skip-git # Index folders that are not Git repositories gitnexus analyze --default-branch develop # Branch used in the generated regression-compare example (base_ref) gitnexus analyze --verbose # Log skipped files when parsers are unavailable @@ -451,7 +453,7 @@ Commit a `.gitnexusrc` JSON file at the repo root to preconfigure recurring `ana // over its fix on every analyze. (Alias: "branch".) "defaultBranch": "develop", "skipContextFiles": true, // alias of skipAgentsMd: keep your own AGENTS.md/CLAUDE.md - "skipSkills": true, // don't install standard .claude/skills/gitnexus-* skills + "skipSkills": true, // don't install standard skill files under .claude/skills/ and .agents/skills/ "embeddings": true, // generate embeddings by default "workerTimeout": 60, } @@ -514,7 +516,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget in milliseconds for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. | Slow or heavily loaded hosts where a full pool cold-starting concurrently needs more than 5s, and analyze aborts with "did not report ready within 5000ms". | | `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | | `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). Invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. | | `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | | `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | diff --git a/eval/workflow_bench/learnings.jsonl b/eval/workflow_bench/learnings.jsonl new file mode 100644 index 000000000..7d25e3359 --- /dev/null +++ b/eval/workflow_bench/learnings.jsonl @@ -0,0 +1,2 @@ +{"skill": "gitnexus-work", "date": "2026-07-25", "task": "#2687 const-arrow Const/Function twin fix in parse-worker + MCP impact envelope", "friction": "Phase 2's Build-current/index-current procedure indexes the repo-under-test, which makes CLI-spawning suites (skip-git-cli, cli/tool-no-index-stderr) time out because repo resolution then opens the 237k-node index from that cwd; they pass at the same commit in an unindexed worktree, so the procedure manufactures false regressions in its own final verification.", "suggestion": "Phase 4 should note that CLI-spawn suites can fail solely because the worktree became an indexed repo, and prescribe the A/B check (same commit, unindexed worktree) instead of leaving the executor to conclude a regression."} +{"skill": "gitnexus-work", "date": "2026-07-25", "task": "#2687 same run", "friction": "Phase 2 requires top-level `status: up-to-date` before graph queries, but any uncommitted staged edit makes status report `stale` by design, so the gate is unsatisfiable in the stage -> detect_changes -> commit sequence Phase 3 mandates.", "suggestion": "Scope the up-to-date requirement to index.commit == HEAD + empty incompleteReasons + runnerIdentityStatus current, and state that a `stale` top-level status caused solely by uncommitted working-tree edits is expected at the detect_changes gate."} diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md index c96616130..e52560422 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md @@ -81,6 +81,18 @@ list_repos { offset: 400 } → repos 401–437, hasMore false Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. +### Inline staleness signal (`query` / `context` / `impact` / `cypher`) + +These four hot read tools attach a non-blocking `staleness` field to their response when the index is behind the checkout's current HEAD — the same `{ commitsBehind, hint }` shape `list_repos` already reports — so a direct tool call surfaces a behind-HEAD index without a separate `list_repos` call: + +```jsonc +{ /* …the tool's normal result… */ + "staleness": { "commitsBehind": 3, "hint": "⚠️ Index is 3 commits behind HEAD. Run analyze tool to update." } +} +``` + +The field is **absent when the index is current** (or when the freshness check can't run), so its presence is the signal. It is only ever added to object results — raw-array `cypher` output and error envelopes are returned unchanged. `@group`-targeted calls do not carry it (multi-repo staleness is ill-defined). When you see it, the graph may be behind the working tree — re-run `analyze` before trusting blast-radius or dependence answers. + ### Taint findings (`explain`) `explain` returns taint findings recorded by `gitnexus analyze --pdg` — intra-procedural `TAINTED` edges plus cross-function `TAINT_PATH` hops where the interprocedural taint phase found a function-level source→sink chain. Each finding includes a sink category (command-injection, code-injection, path-traversal, sql-injection, xss), source/sink lines, and the ordered hop path with the variable carried on each hop. diff --git a/gitnexus-shared/src/graph/types.ts b/gitnexus-shared/src/graph/types.ts index 4caf34930..1f500efdc 100644 --- a/gitnexus-shared/src/graph/types.ts +++ b/gitnexus-shared/src/graph/types.ts @@ -180,14 +180,14 @@ export type RelationshipType = | 'ENTRY_POINT_OF' | 'WRAPS' | 'QUERIES' - /** Dependency-injection edge: a consumer class receives every implementer - * of interface `T` via a container-injected collection-typed field - * (`List`, `Set`, `Collection`, or `Map`). Precondition: the - * field carries an injection annotation recognized by a per-language - * matcher registered in `di-extractors/` (Java/Spring today: `@Autowired` - * or `@Inject`; `@Resource` is excluded — by-name-first semantics). - * Source = the consumer Class node (the one owning the field). - * Target = an implementing Class node. + /** Dependency-injection edge: a consumer class receives a likely provider + * through constructor, field, method, or collection injection. A + * per-language resolver identifies the site and provider metadata; the + * shared DI phase uses type heritage, qualifier names, and preferred + * provider markers to resolve it. Ambiguous single injection is represented + * by multiple lower-confidence edges instead of a fabricated exact target. + * Source = the consumer Class node (the one owning the injection site). + * Target = a concrete provider Class node. * Framework specifics live in the `reason` payload (e.g. * `Spring DI: @Autowired List`), not in this type contract. * Lets Cypher queries trace which beans the container injects into a given diff --git a/gitnexus-shared/src/scope-resolution/types.ts b/gitnexus-shared/src/scope-resolution/types.ts index 0956c3db4..ff9e07a05 100644 --- a/gitnexus-shared/src/scope-resolution/types.ts +++ b/gitnexus-shared/src/scope-resolution/types.ts @@ -351,6 +351,11 @@ export interface BindingRef { readonly origin: 'local' | 'import' | 'namespace' | 'wildcard' | 'reexport'; /** Non-null for non-local origins; carries the `ImportEdge` that brought the name into this scope. */ readonly via?: ImportEdge; + /** + * Optional semantic visibility evidence supplied by a language hook. + * Shared resolution consumes this without inspecting language syntax. + */ + readonly visibility?: 'static-member-import'; } // ─── §2.5 TypeRef ─────────────────────────────────────────────────────────── diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index e5fda5365..0ba269217 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -11,7 +11,7 @@ "@langchain/anthropic": "^1.5.1", "@langchain/core": "^1.2.2", "@langchain/google-genai": "^2.2.0", - "@langchain/langgraph": "^1.4.7", + "@langchain/langgraph": "^1.4.8", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.3", "@sigma/edge-curve": "^3.1.0", @@ -29,14 +29,14 @@ "i18next": "^26.3.0", "i18next-browser-languagedetector": "^8.2.1", "langchain": "^1.4.6", - "lru-cache": "^11.5.1", + "lru-cache": "^11.5.2", "lucide-react": "^1.23.0", "mermaid": "^11.15.0", "mnemonist": "^0.40.4", "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.7", - "react-i18next": "^17.0.8", + "react-i18next": "^17.0.10", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -47,7 +47,7 @@ "zod": "^4.4.3" }, "devDependencies": { - "@babel/types": "^7.29.0", + "@babel/types": "^8.0.0", "@playwright/test": "^1.61.1", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", @@ -63,7 +63,7 @@ "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", - "vite": "^8.1.4", + "vite": "^8.1.5", "vitest": "^4.1.10", "wait-on": "^9.0.10" }, @@ -186,13 +186,13 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", - "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", + "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", "dev": true, "license": "MIT", "engines": { - "node": ">=6.9.0" + "node": "^22.18.0 || >=24.11.0" } }, "node_modules/@babel/helper-validator-identifier": { @@ -221,16 +221,17 @@ "node": ">=6.0.0" } }, - "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "node_modules/@babel/parser/node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/types": { + "node_modules/@babel/parser/node_modules/@babel/types": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", @@ -244,6 +245,39 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.0.tgz", + "integrity": "sha512-K8ponJDxBwDHigkeFqaqT5wLGl4bTlwMafR8k7b5CPxr6Ww+UG9ls8Yx6Tcpboxu97eeGVEEyKcHmEyOwN1vSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^8.0.0", + "@babel/helper-validator-identifier": "^8.0.0" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/types/node_modules/@babel/helper-validator-identifier": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.4.tgz", + "integrity": "sha512-4wFaiLd0bVo4cIoTXI3zKI038NIWE/cr3jvBjejOVYVxV/m8Ltav1USiGzG1fmS5J2RhgEOgXNNK46cRPnRsrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, "node_modules/@bcoe/v8-coverage": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", @@ -1138,13 +1172,13 @@ } }, "node_modules/@langchain/langgraph": { - "version": "1.4.7", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.7.tgz", - "integrity": "sha512-2tcyf3QGC7v89kqSxMCtRvzg/3L/4yHtOaWC49A8KieCciWJs7LGaxHoPB6QRxXyUgyR+Zg9Q1ss/XJIE+JuSQ==", + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.8.tgz", + "integrity": "sha512-DN1Np1XefdBEbp1qBKlt39cwoL743AAGpR5Ipja0gY2YbWvsoQnOTIrjnj/orSAhaUYsdTKS8VSWdFzsHZo6Ig==", "license": "MIT", "dependencies": { "@langchain/langgraph-checkpoint": "^1.1.3", - "@langchain/langgraph-sdk": "~1.9.25", + "@langchain/langgraph-sdk": "~1.9.26", "@langchain/protocol": "^0.0.18", "@standard-schema/spec": "1.1.0" }, @@ -1169,9 +1203,9 @@ } }, "node_modules/@langchain/langgraph-sdk": { - "version": "1.9.25", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.25.tgz", - "integrity": "sha512-mRKW8zyQUaHox+HirRFMRrPqOvNbQI3xeXDt6kkk4PbBg77V92bsO1WzUVNrmJ81zCkvxyOrWSK8D6ioCj0a8A==", + "version": "1.9.28", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.28.tgz", + "integrity": "sha512-4j3XuM0PvtmAbL8mPfBS99ez3+ytRfgbOpAR/nOeaejTRF3Q9dNw2QnaGLGng8wLPtGLoSj+SYgUOVxy9Bv9vg==", "license": "MIT", "dependencies": { "@langchain/protocol": "^0.0.18", @@ -1208,9 +1242,9 @@ "license": "MIT" }, "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.0.tgz", - "integrity": "sha512-7NED7xhQ74Ngp4JP/2e0VZHp7vSWfJfqeiR92jPgxsz6m0Se4P03YoTKa9dDXyZ3r6P616gUXttrB6nnHYKang==", + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", "license": "MIT", "dependencies": { "eventemitter3": "^5.0.4", @@ -5672,9 +5706,9 @@ } }, "node_modules/lru-cache": { - "version": "11.5.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", - "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" @@ -5721,6 +5755,30 @@ "source-map-js": "^1.2.1" } }, + "node_modules/magicast/node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/magicast/node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -6826,9 +6884,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "funding": [ { "type": "github", @@ -7216,9 +7274,9 @@ } }, "node_modules/postcss": { - "version": "8.5.16", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", - "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "version": "8.5.22", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.22.tgz", + "integrity": "sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==", "funding": [ { "type": "opencollective", @@ -7235,7 +7293,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -7356,9 +7414,9 @@ } }, "node_modules/react-i18next": { - "version": "17.0.8", - "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.8.tgz", - "integrity": "sha512-0ooKbGLU8JXhe1zwpQUWIeXSgLPOfwJmgheWRIUpcoA0CpyabpGhayjdG+/eA5esC1AQ8h2jWpXjJfzQzeDOCw==", + "version": "17.0.10", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.10.tgz", + "integrity": "sha512-XneHftyYA774MJkkccSkZ5oKrUpCnXIPmxio3wemqrVzCRLWiGXOMbIzObrer03fNDEnm8g8R5yYls4HcE+esg==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.29.2", @@ -7368,7 +7426,7 @@ "peerDependencies": { "i18next": ">= 26.2.0", "react": ">= 16.8.0", - "typescript": "^5 || ^6" + "typescript": "^5 || ^6 || ^7" }, "peerDependenciesMeta": { "react-dom": { @@ -8296,15 +8354,15 @@ } }, "node_modules/vite": { - "version": "8.1.4", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz", - "integrity": "sha512-bTT9PsdWO+MQMNG9ZXIP/qM9wGh37DFxTV/sPq9cFpHr3w4jkgef032PkAL9jAqhk3Nz8NQw3O8n6/xFkqO4QQ==", + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", + "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", "license": "MIT", "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", - "postcss": "^8.5.16", - "rolldown": "~1.1.4", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "bin": { diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index a9aa163da..4416c78fb 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -21,7 +21,7 @@ "@langchain/anthropic": "^1.5.1", "@langchain/core": "^1.2.2", "@langchain/google-genai": "^2.2.0", - "@langchain/langgraph": "^1.4.7", + "@langchain/langgraph": "^1.4.8", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.3", "@sigma/edge-curve": "^3.1.0", @@ -39,14 +39,14 @@ "i18next": "^26.3.0", "i18next-browser-languagedetector": "^8.2.1", "langchain": "^1.4.6", - "lru-cache": "^11.5.1", + "lru-cache": "^11.5.2", "lucide-react": "^1.23.0", "mermaid": "^11.15.0", "mnemonist": "^0.40.4", "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.7", - "react-i18next": "^17.0.8", + "react-i18next": "^17.0.10", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -57,7 +57,7 @@ "zod": "^4.4.3" }, "devDependencies": { - "@babel/types": "^7.29.0", + "@babel/types": "^8.0.0", "@playwright/test": "^1.61.1", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", @@ -73,7 +73,7 @@ "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", - "vite": "^8.1.4", + "vite": "^8.1.5", "vitest": "^4.1.10", "wait-on": "^9.0.10" }, diff --git a/gitnexus/README.md b/gitnexus/README.md index bc7aa5fb5..3812b3efb 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -165,13 +165,20 @@ The result is a **LadybugDB graph database** stored locally in `.gitnexus/` with ### Experimental community detection engine -Community detection uses the bundled Graphology Leiden implementation by default. To test the #2337 Icebug migration path without changing default analyze behavior, set: +> **Experimental — not supported for production indexes.** The Icebug engine is a research path for #2337. It carries no stability guarantee, may change or be removed without a major version, and partitions differently from the default, so switching engines changes community IDs and any generated context keyed on them. Reindex with `graphology` before relying on the output. + +Community detection uses the bundled Graphology Leiden implementation by default. To try the #2337 Icebug path without changing default analyze behavior, install the optional native package alongside GitNexus and set the engine: ```bash +npm i @ladybugmem/icebug GITNEXUS_COMMUNITY_ENGINE=icebug npx gitnexus analyze ``` -Supported values are `graphology`, `icebug`, and `auto`. The Icebug path is an experimental probe: GitNexus does not bundle an Icebug native package yet, and if a separately resolvable module is unavailable or its API does not match the expected `Graph.fromCSR` / `ParallelLeidenView` shape, analyze falls back to Graphology and reports the fallback in progress output. Today `auto` is behaviorally identical to `icebug`: both try Icebug and fall back to Graphology, while `graphology` skips the Icebug probe entirely. +Supported values are `graphology`, `icebug`, and `auto`. Today `auto` is behaviorally identical to `icebug`: both try Icebug and fall back to Graphology, while `graphology` skips Icebug entirely. + +Icebug is **not** a declared dependency — its prebuilds link against system Arrow 24 (`libarrow.so.2400`), OpenMP, and glibc ≥ 2.38, none of which GitNexus can assume. Analyze falls back to Graphology and reports the reason in progress output when the module is missing, fails to load, or predates the `setNumberOfThreads` / `setSeed` controls that reproducible community IDs require (present at [icebug-nodejs](https://github.com/Ladybug-Memory/icebug-nodejs) HEAD, absent from the published 12.8.0 tarball — so the fallback is what you will see today). The engine is pinned to `threads: 1`, `randomize: false` for determinism. + +Note that the bundled Graphology path is no longer the slow option it once was: #2337 removed an accidental O(communities × N) copy in the vendored Leiden. On a synthetic 200k-node / 800k-edge benchmark graph it went from exceeding the 60s timeout to finishing in ~15s. Real projections vary with their degree distribution, so treat that as a direction, not a guarantee. ## MCP Tools @@ -354,6 +361,13 @@ Installed automatically by both `gitnexus analyze` (per-repo) and `gitnexus setu - Node.js >= 22 - Git repository (uses git for commit tracking) +- **Linux: glibc 2.34 or newer** (Ubuntu 22.04+, RHEL/Rocky/Alma 9+, Debian 12+, Fedora 35+). The + LadybugDB native binary ships as a prebuild against that floor, so on an older host it cannot + load and reinstalling does not help — see + [Linux: `GLIBC_2.34' not found`](#linux-glibc_234-not-found). +- **Windows, for full-text search:** the Microsoft Visual C++ 2015-2022 Redistributable (x64) *and* + OpenSSL 3 (`libssl-3-x64.dll`, `libcrypto-3-x64.dll`) resolvable on `PATH` — see + [Windows: full-text search unavailable](#windows-full-text-search-unavailable). ## Aptos / Move builds (`aptos` dist-tag) @@ -481,6 +495,50 @@ pnpm add -g --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=t gitnexus serve ``` +### Linux: `GLIBC_2.34' not found` + +``` +LadybugDB native binary (lbugjs.node) exists but failed to load: + /lib64/libc.so.6: version `GLIBC_2.34' not found (required by .../lbugjs.node) +``` + +The LadybugDB addon ships as a prebuilt binary compiled against **glibc 2.34**. If your +distribution is older (CentOS/RHEL 8 has 2.28, Ubuntu 20.04 has 2.31, Debian 11 has 2.31), the +dynamic loader cannot resolve its symbols. + +**Reinstalling does not help** — every download delivers the same prebuilt binary. The fix is a +newer C library: + +- Run GitNexus on a distribution with glibc 2.34 or newer — Ubuntu 22.04+, RHEL/Rocky/Alma 9+, + Debian 12+, Fedora 35+. +- Or run it in the container image, which bundles a current glibc (see [Docker](#docker)). + +`gitnexus doctor` reports the required and detected glibc versions when this happens +([#2672](https://github.com/abhigyanpatwari/GitNexus/issues/2672)). + +### Windows: full-text search unavailable + +`analyze` completes, but keyword search is degraded and `doctor` shows the FTS extension failing +with Windows error 126 (`The specified module could not be found`). The extension needs two +runtime dependencies Windows does not ship by default: + +1. **Microsoft Visual C++ 2015-2022 Redistributable (x64)** — + +2. **OpenSSL 3** — `libssl-3-x64.dll` and `libcrypto-3-x64.dll`, resolvable on `PATH` + +The redistributable alone is **not** sufficient. If Git for Windows is installed you already have +the OpenSSL DLLs — run `gitnexus` from **Git Bash**, or prepend the directory to `PATH` in the +shell you use: + +```powershell +$env:PATH = "C:\Program Files\Git\mingw64\bin;$env:PATH" +gitnexus analyze --repair-fts +``` + +Without them the index is still built, but without search tables, so `query` returns empty keyword +results until you re-run `gitnexus analyze --repair-fts` from a shell where the DLLs resolve +([#2669](https://github.com/abhigyanpatwari/GitNexus/issues/2669)). + ### Installation fails with native module errors Some optional language grammars (Dart, Proto, Swift, Kotlin) require native compilation. If they fail, GitNexus still works — those languages will be skipped. To skip them intentionally (no C++ toolchain needed), set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before installing. Strict `=1` also disables automatic `move-flow` downloads during analysis; a verified cache entry, explicit `MOVE_FLOW`, or compatible binary on `PATH` remains usable. @@ -541,16 +599,17 @@ GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnex Configure the behavior with these environment variables: -| Variable | Values | Default | Effect | -| -------------------------------------------- | ------------------------------ | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | -| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | -| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | -| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | -| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` uses the bundled default path. `icebug` and `auto` currently behave identically: both try the experimental Icebug CSR path and fall back to Graphology if the optional native module is unavailable or incompatible. | -| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores that native unbounded default; invalid values warn and fall back to the default. | -| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | +| Variable | Values | Default | Effect | +| -------------------------------------------- | ------------------------------ | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | +| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | +| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | +| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | +| `GITNEXUS_STREAM_GRAPH_EMIT` | `0`, `1` | `1` (on) | **On by default** on a full rebuild (`--force`); incremental runs ignore it. Holds structural relationships (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) as CSV-on-disk plus compact in-memory columns instead of as objects in three overlapping indexes, cutting peak in-memory graph heap by ~1.4x at no measurable CPU cost (measured A/B on a synthetic 400k-node / 1.08M-edge graph: 819 MB -> 584 MB, iteration at parity, scaling verified linear from 100k to 800k nodes, with every edge still visible through the graph interface; no end-to-end measurement on a real repository yet). Nothing is traded away — community detection, process extraction, PDG taint summaries and the local-symbol pruner all read a complete relationship set and behave identically. Set to `0` only to bisect a suspected streaming-related fault. | +| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` is the supported default. `icebug` and `auto` are **experimental** and currently behave identically: both try the optional `@ladybugmem/icebug` native Leiden over a CSR export and fall back to Graphology if it is not installed, cannot load, or lacks the deterministic thread/seed controls. Experimental engines partition differently, so community IDs are not comparable across engines. | +| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores that native unbounded default; invalid values warn and fall back to the default. During `analyze` the pool is right-sized to the graph and, on non-4 KiB-page hosts (Apple Silicon 16 KiB, Ascend/aarch64 64 KiB), scaled by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | +| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | ```bash # Offline/airgapped: never reach the network for extensions @@ -570,6 +629,27 @@ GITNEXUS_FTS_CJK_SEGMENTATION=bigram npx gitnexus analyze --force ### Analysis runs out of memory +Memory management is automatic: `analyze` sizes its heap to the machine +(always below physical RAM), caps each parse worker, and — rather than +grinding into a GC death spiral or crash — stops early with a message telling +you the one thing to do. Repeated +`Replacement worker did not report ready within 5000ms` warnings on a large +repository are part of the same picture: memory pressure starving healthy +workers, not a worker bug (#2649). + +If analyze says the repository doesn't fit, do what the message says: + +- **The machine has more memory to give** (a `NODE_OPTIONS` + `--max-old-space-size` pin from your environment is holding analyze back): + re-run without the pin — no flags needed. +- **The machine is the ceiling**: shrink the scope (exclude generated or + vendored directories, below) or use a machine with more RAM. + +Escape hatches (`GITNEXUS_MEMORY=off` to decline the autopilot, +`GITNEXUS_WORKER_HEAP_MB` to size workers yourself) are listed in the +environment-variable table below — +most users never need them. + For very large repositories: ```bash @@ -625,13 +705,16 @@ For repositories with very large source files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BY Four env vars expose the pool's resilience layers (respawn budget, cumulative-timeout cap, circuit breaker, startup handshake). Defaults are tuned for typical repos; bump them when an analyze legitimately needs more retries, or lower them to fail-fast on a known-bad shape. -| Variable | Default | Effect | -| ----------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per slot before the slot is dropped from the active rotation. | -| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. | -| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. | -| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). | +| Variable | Default | Effect | +| ----------------------------------------------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per slot before the slot is dropped from the active rotation. | +| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. | +| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. | +| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). | | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. Raise it on a slow or heavily loaded host where a full pool cold-starting concurrently needs more than 5s. | +| `GITNEXUS_MEMORY` | `off` | unset (autopilot on) | `off` declines GitNexus's memory autopilot: analyze will neither re-run itself with a RAM-aware heap cap nor abort the parse before V8 enters its ineffective-mark-compact death spiral. Use it when you want to drive memory manually; to simply pin a heap size, pass Node's own `--max-old-space-size`, which is already honoured as your decision. | +| `GITNEXUS_WORKER_HEAP_MB` | `clamp(512, RAM/2/poolSize, 4096)` | Per-worker V8 old-generation heap cap (#2649). Bounds pool RSS on large repos; a worker exceeding it dies with a real heap error handled by quarantine/respawn. | +| `GITNEXUS_SERVER_ANALYZE_HEAP_MB` | `min(8192, auto cap)` | Heap for the web/MCP server's forked analyze worker (#2649). Defaults to the historical 8192 MB bounded by the machine/container's RAM-aware auto cap; set an absolute MB value to override. | | `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning (#2432). `0` expires immediately. | ### Graph cleanup tuning diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 81e8b92ec..38cec9153 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -39,20 +39,22 @@ }, "csharp": { "_rebaselined": "#1956 synth-widening: + csharp-qualified-base fixture; the synth now walks record_declaration + struct_declaration base_lists and handles alias_qualified_name (matching the #1940 legacy leg), so record/struct heritage now emits. csharp-record-base gains a record inherits capture. (record->record SAME-namespace EXTENDS is a separate registry resolution gap, tracked as follow-up.) Linear (~1.00). (Earlier #1956: heritage-bearing scale source.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged. | #1924 F16: record primary-constructor base bindings now exclude constructor arguments; capture fingerprint changes, scaling remains linear. | #2036 review follow-up: csharp-record-base now exercises primary-constructor base dispatch end to end; +2 capture groups, scaling remains linear.", - "fingerprint": "75cf380209fa7d1a8a3ec873be1a9424b4e5173be0b08234c2291e8521a9b3c1", + "fingerprint": "e05dc27456bde8175948586c9e7689033a378fa40e9ca4ce78cce41fbea0f2f8", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior f31544530924748f9aa37d11cec570bc10c3ddf9d9b237e6df7a17623fd2bb3a -> 75cf380209fa7d1a8a3ec873be1a9424b4e5173be0b08234c2291e8521a9b3c1; scaling 1.061 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: C# method-group/delegate callable flow facts with invocation-result suppression. Prior 2bb5bc8c19cb8eb08c9590545ad8a1968a7152951f7e12746e2d7901d542fed9 -> f31544530924748f9aa37d11cec570bc10c3ddf9d9b237e6df7a17623fd2bb3a; scaling 1.115 < 1.5.", - "_note": "#2046: F35 qualified-constructor captures now emit @reference.qualified-name + a simple-name @reference.name on `new Ns.Foo()`/`new A.B.Foo()`; namespace_declaration/file_scoped_namespace_declaration now emit @declaration.namespace name captures (feeding the non-destructive namespacePrefix sidecar for `new B.Foo()` same-tail disambiguation). + csharp-interface-only-base and csharp-namespace-qualified-ctor fixtures. Pure capture-additive + fixture-corpus drift; scaling stays linear (~1.11)." + "_note": "#2046: F35 qualified-constructor captures now emit @reference.qualified-name + a simple-name @reference.name on `new Ns.Foo()`/`new A.B.Foo()`; namespace_declaration/file_scoped_namespace_declaration now emit @declaration.namespace name captures (feeding the non-destructive namespacePrefix sidecar for `new B.Foo()` same-tail disambiguation). + csharp-interface-only-base and csharp-namespace-qualified-ctor fixtures. Pure capture-additive + fixture-corpus drift; scaling stays linear (~1.11).", + "_rebaselined_2563_instance_ownership": "#2563: csharp-using-static adds same-file ownership, local-function, overload, partial-class, and cross-namespace same-name coverage. Prior 75cf380209fa7d1a8a3ec873be1a9424b4e5173be0b08234c2291e8521a9b3c1 -> e05dc27456bde8175948586c9e7689033a378fa40e9ca4ce78cce41fbea0f2f8; scaling 1.058 < 1.5." }, "rust": { - "fingerprint": "f7742f65f14d7d6590df7f16303fc3cc9dc0c233cd80bf90c98b084933cd3846", + "fingerprint": "655aed01cf1b6b84fa0c64d48dfb2526ecb67f47d90f0a91edabacd269a212db", "scaling_budget": 1.5, "_rebaselined_dyn_trait_object_2604": "#2604: RUST_SCOPE_QUERY now captures function_signature_item (abstract trait methods, no body) as a scope + declaration, so a &dyn Trait receiver can dispatch a CALLS edge to the trait's own method. Additive capture shift across every bench fixture with a required trait method. Prior df369c5a5f8de7753fc8bab8b4108ef5081750974ea5085ba9a867675ac9eb29 -> f7742f65f14d7d6590df7f16303fc3cc9dc0c233cd80bf90c98b084933cd3846; scaling 1.033 < 1.5.", "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 65e5bca66bb1ca117949409e8fb5c80ee69d6f1b5318908eaaecf08da0482e5c -> df369c5a5f8de7753fc8bab8b4108ef5081750974ea5085ba9a867675ac9eb29; scaling 1.065 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Rust fn-value callable flow facts with invocation/constructor-result suppression. Prior ac610bbe97666bf285923479dd7b43a2fe4c5354aae8df1bcbafdc04fb220f82 -> 65e5bca66bb1ca117949409e8fb5c80ee69d6f1b5318908eaaecf08da0482e5c; scaling 1.024 < 1.5.", "_rebaselined": "#1956 tri-review U1: rust-qualified-trait fixture (scoped + generic-of-scoped impl trait paths); bareTypeIdentifier now resolves scoped_type_identifier bases by their name: tail (additive, no existing-fixture drift); linear (~1.04). #1975: + rust-scoped-impl fixture (impl a::Inner / b::Inner inherent scoped impls) \u2014 legacy @definition.impl scoped arm + findEnclosingClassInfo inherent-impl scoped target; rust scope-extractor captures byte-identical. | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged.", - "_note": "PR #1934: F66/F68 let-binding pattern narrowing; F71 union (Struct-labeled, now materialized via legacy @definition.struct + resolvable); F72 macro FULLY WIRED \u2014 @declaration.macro/@reference.macro + MacroRegistry \u2192 USES edges to Macro nodes (never a same-named fn). + rust-macro / rust-union fixtures and merged with origin/main #1975 rust-scoped-impl; fingerprint re-baselined (scaling ~0.99, fixture_count 126). #1992: + rust-nested-tail-collision-generic and rust-generic-impl-same-method-name (F3) fixtures \u2014 pure fixture-corpus drift, no scope-extractor change; fixture_count 127->129, fingerprint 56ffc1c0->b00aea0f." + "_note": "PR #1934: F66/F68 let-binding pattern narrowing; F71 union (Struct-labeled, now materialized via legacy @definition.struct + resolvable); F72 macro FULLY WIRED \u2014 @declaration.macro/@reference.macro + MacroRegistry \u2192 USES edges to Macro nodes (never a same-named fn). + rust-macro / rust-union fixtures and merged with origin/main #1975 rust-scoped-impl; fingerprint re-baselined (scaling ~0.99, fixture_count 126). #1992: + rust-nested-tail-collision-generic and rust-generic-impl-same-method-name (F3) fixtures \u2014 pure fixture-corpus drift, no scope-extractor change; fixture_count 127->129, fingerprint 56ffc1c0->b00aea0f.", + "_rebaselined_import_disambiguation_2514": "#2514: added rust-import-* and rust-dup-* fixtures under lang-resolution for the range-binding ambiguity latch + import-disambiguated resolution (for-loops / struct destructuring across explicit/aliased/glob use imports). emitRustScopeCaptures is unchanged; the corpus fingerprint shifts purely because the fixture set grew (130 -> 174). Prior f7742f65f14d7d6590df7f16303fc3cc9dc0c233cd80bf90c98b084933cd3846 -> 655aed01cf1b6b84fa0c64d48dfb2526ecb67f47d90f0a91edabacd269a212db; scaling 1.06 < 1.5." }, "php": { "fingerprint": "4a688fa5a7016546f7f3c6d44de023608ae80c5b0e3670c16f6e61b3632608fd", @@ -90,7 +92,7 @@ "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0." }, "java": { - "fingerprint": "d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686", + "fingerprint": "6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata; same-name lexical regions use an O(ancestor-depth) ID-set lookup. Prior d5c59d7dc9e206637515d5aea1163f7c1cdd76410c38c5fe6143d13d19677d6a -> 004a3592998dca1193bd1429a8284513725de7764f2a3eceedaaa984cfd763b4; scaling 0.992 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Java method-reference/SAM callable flow facts with invocation-result suppression. Prior 062d754764aaa8a6772fb90875c710502a63e3e7a300e633942381ed914faada -> d5c59d7dc9e206637515d5aea1163f7c1cdd76410c38c5fe6143d13d19677d6a; scaling 1.074 < 1.5.", @@ -100,7 +102,13 @@ "_rebaselined_2550_instance_model": "PR #2549 (#2550): anonymous class bodies emit synthesized @declaration.class/@declaration.name (Worker$N), an @reference.inherits to the constructed type, and receiver @type-binding.* captures; six new java-* fixtures joined the corpus. Prior f3b4f4b6610e07c3ac90deb1c53d3572b6ad55a36e5d7134984876d30031ff67 -> d79c3b92acfc866094981499b977388ca14f90839bca0c040342ab1cec00aa90; scaling 1.058 < 1.5.", "_rebaselined_2555_enum_constant_bodies": "PR for #2555: enum constant bodies emit synthesized E$N classes + @reference.inherits to the host enum; anonymous naming follows JLS 13.1 immediately-enclosing-type chains INCLUDING anonymous enclosing types (NestHost$1$1, N$1$1); six new java-* fixtures joined the corpus. Prior d79c3b92acfc866094981499b977388ca14f90839bca0c040342ab1cec00aa90 -> 975b68aaac6d06094260fb0c67f9b1bc03692ba7220669d192aca9dccd5fc0ca; scaling 1.05 < 1.5.", "_rebaselined_2564_record_capture": "PR for #2564: JAVA_QUERIES gained a (record_declaration name: (identifier) @name) @definition.record capture, previously entirely missing (record_declaration had no structure-phase capture at all, unlike class/interface/enum) - a record's methods existed as ownerless Method nodes with no HAS_METHOD edge. Two new java-* fixtures (java-record-methods, java-new-expr-chain-call) joined the corpus. Prior 975b68aaac6d06094260fb0c67f9b1bc03692ba7220669d192aca9dccd5fc0ca -> 85fc7af9c3c1bceac76cb4f27214410b04967682a2eaa7e468e26efd1f4e2537; scaling 1.059 < 1.5.", - "_rebaselined_2561_enum_constant_receiver": "PR for #2561: synthesizeJavaAnonymousClassDeclarations now emits a class-scope @type-binding.annotation/name/type per enum constant (constant simple name -> its E$N synthesized class when bodied, else the host enum) so E.CONST.method() resolves through the existing compound-receiver chain walk. Two drivers of the drift, both in the java-enum-constant-body fixture (this bench's corpus IS test/fixtures/lang-resolution): (1) one extra type-binding match per enum_constant from the capture change; (2) review follow-up added a body-less Plain.java enum + EnumConst.dispatchToConstant/dispatchInherited methods (bodied-override, inherited-via-MRO, and body-less dispatch call sites). The review's fail-safe hardening (bodied constant binds ONLY to E$N, never the host enum, when name synthesis fails on a malformed tree) is output-neutral on this well-formed corpus (verified: fingerprint identical with and without it). Prior 85fc7af9c3c1bceac76cb4f27214410b04967682a2eaa7e468e26efd1f4e2537 -> d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686; scaling < 1.5." + "_rebaselined_2561_enum_constant_receiver": "PR for #2561: synthesizeJavaAnonymousClassDeclarations now emits a class-scope @type-binding.annotation/name/type per enum constant (constant simple name -> its E$N synthesized class when bodied, else the host enum) so E.CONST.method() resolves through the existing compound-receiver chain walk. Two drivers of the drift, both in the java-enum-constant-body fixture (this bench's corpus IS test/fixtures/lang-resolution): (1) one extra type-binding match per enum_constant from the capture change; (2) review follow-up added a body-less Plain.java enum + EnumConst.dispatchToConstant/dispatchInherited methods (bodied-override, inherited-via-MRO, and body-less dispatch call sites). The review's fail-safe hardening (bodied constant binds ONLY to E$N, never the host enum, when name synthesis fails on a malformed tree) is output-neutral on this well-formed corpus (verified: fingerprint identical with and without it). Prior 85fc7af9c3c1bceac76cb4f27214410b04967682a2eaa7e468e26efd1f4e2537 -> d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686; scaling < 1.5.", + "_rebaselined_2562_local_classes": "#2562: Java block-local classes, enums, records, and interfaces use source-type-relative JLS 13.1 Host$NLocal identities with javac-compatible per-(host, simple-name) numbering; anonymous numbering remains separate. Lexical aliases begin at each declaration and end with its immediate block. Expanded java-local-class-naming fixtures cover declaration order, disjoint blocks, initializers, lambdas, local type kinds, and recursive local/member/anonymous host chains. Prior d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686 -> 6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197; scaling 1.204 < 1.5." + }, + "java-local-types": { + "fingerprint": "a9ad88de21ca6747a923260dbdf677fb74a004abbf9d57781f745e3a9027530b", + "scaling_budget": 1.5, + "_added": "#2562 performance follow-up: co-scales same-host, same-name local classes and anonymous classes to gate JLS binary-name ordinal allocation. Precomputed per-sequence ordinals reduce the focused 100->800 workload from 176->6655ms to 141->752ms; normalized 250->800 scaling is 1.054." }, "typescript": { "fingerprint": "3280b13d3f9378ab23eee31c2edc779b5a9ae1e7bb510c23a24855b44406d2f4", @@ -125,7 +133,7 @@ "_rebaselined_2550_instance_model": "PR #2549 (#2545/#2551): object literals emit @scope.object. Prior 479927409bbdd9852a36172c8260aa56df260e99129a7a9c20a0d1903dd5538b -> f1ccf42a36895c8e34dcb724286f247d469835f2dcbb23ad3347190adc7fde1c; scaling 1.096 < 1.5." }, "kotlin": { - "fingerprint": "a6fce0dff00e88d41d85023eaf3f35016b5217c7e5225f24a598e4c70bb63091", + "fingerprint": "9f159f8810d342ef1c821f466efd6920dad9a190f06000056e6cd2815861b195", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior bddba25d5a88152bbbee8d70e82c944b5302accb4b625df782adb1d4f7a7ac12 -> e856951c2a779163d555dadc8e1bf59304a86caed78ac1f450d9caa2b50f63d1; scaling 1.090 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Kotlin callable-reference flow facts with invocation-result suppression. Prior 4900431791f2b9280009deb2b82659c26ead8aa6fb8731190a7c505dec5a9041 -> bddba25d5a88152bbbee8d70e82c944b5302accb4b625df782adb1d4f7a7ac12; scaling 0.880 < 1.5.", @@ -133,6 +141,7 @@ "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0.", "_rebaselined_2271": "PR #2271: re-vendored tree-sitter-kotlin 0.3.8 -> unreleased fwcd main c8ac3d26 for `fun interface` support + new kotlin-fun-interface fixture in the corpus. Drift is both corpus-additive (the fixture) and grammar-driven (the new grammar parses `fun interface` as a class_declaration, not an ERROR node). Baselined to the NEW grammar's fingerprint, so this --check passes only once the regenerated prebuilds land \u2014 until then CI loads the committed 0.3.8 binary and the bench is red, same as the kotlin fun-interface integration tests. scaling ~0.83 (linear).", "_rebaselined_2522_review_fixes": "PR #2522 review fixes: fieldless assignment nodes decomposed positionally. Prior e856951c2a779163d555dadc8e1bf59304a86caed78ac1f450d9caa2b50f63d1 -> 4b31f46cfb004ba769a96feeb06ae4ef109c77410f54e7aaab4a688df599b112; scaling ratio re-verified within budget.", - "_rebaselined_2550_instance_model": "PR #2549 (#2545): anonymous object expressions (object_literal) emit @scope.class, and the kotlin-object-literal-scope fixture joined the corpus. Prior 4b31f46cfb004ba769a96feeb06ae4ef109c77410f54e7aaab4a688df599b112 -> a6fce0dff00e88d41d85023eaf3f35016b5217c7e5225f24a598e4c70bb63091; scaling 0.951 < 1.5." + "_rebaselined_2550_instance_model": "PR #2549 (#2545): anonymous object expressions (object_literal) emit @scope.class, and the kotlin-object-literal-scope fixture joined the corpus. Prior 4b31f46cfb004ba769a96feeb06ae4ef109c77410f54e7aaab4a688df599b112 -> a6fce0dff00e88d41d85023eaf3f35016b5217c7e5225f24a598e4c70bb63091; scaling 0.951 < 1.5.", + "_rebaselined_2563_instance_ownership": "#2563: kotlin-instance-ownership adds unrelated, inherited, outer-instance, and anonymous-object coverage. Prior a6fce0dff00e88d41d85023eaf3f35016b5217c7e5225f24a598e4c70bb63091 -> 9f159f8810d342ef1c821f466efd6920dad9a190f06000056e6cd2815861b195; scaling 1.257 < 1.5." } } diff --git a/gitnexus/bench/scope-capture/measure.mjs b/gitnexus/bench/scope-capture/measure.mjs index 953a3d95f..56aa2592d 100644 --- a/gitnexus/bench/scope-capture/measure.mjs +++ b/gitnexus/bench/scope-capture/measure.mjs @@ -264,6 +264,23 @@ const LANGS = [ ` public long getId() { return this.id; }\n` + ` public void setName(String v) { this.name = v; }\n}\n\n`, }, + { + name: 'java-local-types', + emit: emitJavaScopeCaptures, + fixturePrefix: 'java-local', + exts: ['.java'], + file: 'bench-local.java', + header: + 'package generated;\n\nclass Base {}\n\ninterface Marker {}\n\nclass Bench {\n void run() {\n', + // Co-scale both independent ordinal sequences under one host; construction + // and dispatch keep lexical-alias captures hot. The old per-identity + // host-candidate filter made this combined workload quadratic. + unit: (n) => + ` { class Local extends Base implements Marker { long value() { return ${n}L; } } ` + + `new Local().value(); }\n` + + ` Marker marker${n} = new Marker() {};\n`, + footer: ' }\n}\n', + }, { name: 'typescript', emit: emitTsScopeCaptures, @@ -309,7 +326,7 @@ const LANGS = [ function generate(lang, entityCount) { let src = lang.header; for (let i = 0; i < entityCount; i++) src += lang.unit(i); - return src; + return src + (lang.footer ?? ''); } // ---- timing ---- diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index ea146a6c5..86d39f47e 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -3006,11 +3006,12 @@ } }, "node_modules/express-rate-limit": { - "version": "8.5.2", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", - "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.0.tgz", + "integrity": "sha512-XKJXDsASUOo0LLtFwW5hCcQGH0N4WQc/Rn8/Pvoia+TJFOkkFPvrtW9lZOeeNcxQJspvOIERMwiRLsVFlhHEkA==", "license": "MIT", "dependencies": { + "debug": "^4.4.3", "ip-address": "^10.2.0" }, "engines": { diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 54816f34c..b59017dda 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -36,6 +36,19 @@ const PLATFORM_LOGIC = [ // must exercise the Windows backslash branch, so run it on the OS matrix (#2394). 'test/unit/cli-entry.test.ts', 'test/unit/platform-capabilities.test.ts', + // Windows drive-letter case variance in the analyzer runner-identity path + // fields (#2668): normalizeAnalyzerRootPath is a POSIX no-op, so the + // "identity path fields are normalizer-stable" fixpoint guard only bites on + // the windows-latest matrix — it must run there, not just in the Ubuntu + // full-suite where it's trivially green. Deliberately the split-out + // normalization file, NOT analyzer-identity.test.ts: the latter's fixture + // tests compare identity fields against raw temp-dir paths and fail on macOS, + // where /var/... realpaths to /private/var/.... + 'test/unit/analyzer-identity-path-normalization.test.ts', + // `isInside` containment guard vs Windows cross-drive paths: path.relative + // returns the absolute target across drives, so the guard needs isAbsolute. + // Fixture-free and pathApi-injectable, so it is portable to every runner. + 'test/unit/analyzer-identity-is-inside.test.ts', // getconf page-size probe: explicit process.platform gate (win32 short-circuit) // plus a live-probe test whose only real non-4K coverage is macos-arm64's // 16 KiB pages — the exact hardware class #1231 targets (#2424 review). @@ -82,6 +95,13 @@ const PLATFORM_LOGIC = [ // POSIX and Windows — the fail-closed path-claim semantics must hold on the // real windows-latest path implementation (#2419/#2420). 'test/unit/server-api-repo-resolution.test.ts', + // The index write-lock (#2658) selects its backend by process.platform — the + // OS socket lock (Windows named pipe / Linux abstract socket) vs the file + // fallback — and its socket-backend describe block is gated to linux/win32. + // The Ubuntu suite only proves the Linux abstract-socket path, so run it here + // to exercise the Windows named-pipe backend and the macOS file fallback on + // their real platforms (#2658 review H3). + 'test/unit/index-lock.test.ts', ]; // Native LadybugDB integration tests — exercise the @ladybugdb/core @@ -153,6 +173,14 @@ const SPAWN_CLI = [ 'test/integration/antigravity-hook-e2e.test.ts', 'test/unit/local-cli-subprocess.test.ts', 'test/unit/runner-exec-tail.test.ts', + // Real cross-process single-writer lock coordination (#2658): child processes + // contend for the lock and race to reclaim a dead holder. Process spawning, + // kernel socket auto-release (Win named pipe / Linux abstract socket), and the + // FILE-backend rename-steal reclaim (macOS/BSD default) all vary across OSes — + // the exact behaviors the Windows/macOS matrix must prove. macOS timing first + // exposed a file-backend double-admit race here (#2658 review); the reclaim is + // now judgment-verified so a live holder is never displaced. + 'test/integration/analyze-index-lock-concurrency.test.ts', ]; // Worker threads tests — exercise real worker_threads which have diff --git a/gitnexus/src/cli/ai-context.ts b/gitnexus/src/cli/ai-context.ts index 780408451..ec5ae6170 100644 --- a/gitnexus/src/cli/ai-context.ts +++ b/gitnexus/src/cli/ai-context.ts @@ -362,13 +362,32 @@ async function upsertGitNexusSection( } /** - * Install GitNexus skills as direct children of .claude/skills/ - * Works natively with Claude Code, Cursor, and GitHub Copilot + * Some agents read skills from a repo-local `.agents/skills/` directory and + * prefer it over the global `~/.agents/skills/` install. When the repo contains + * an `.agents/` directory, skills written to `.claude/skills/` are mirrored + * there too so those agents serve the up-to-date copies. */ -async function installSkills(repoPath: string): Promise { +export async function shouldMirrorSkillsToAgents(repoPath: string): Promise { + try { + const stat = await fs.stat(path.join(repoPath, '.agents')); + return stat.isDirectory(); + } catch { + return false; + } +} + +/** + * Install GitNexus skills as direct children of .claude/skills/ + * Works natively with Claude Code, Cursor, and GitHub Copilot. + * Mirrored to .agents/skills/ when .agents/ exists. + */ +async function installSkills( + repoPath: string, +): Promise<{ skills: string[]; agentsMirror: boolean }> { const skillsDir = path.join(repoPath, '.claude', 'skills'); const legacySkillsDir = path.join(skillsDir, 'gitnexus'); const installedSkills: string[] = []; + const agentsMirror = await shouldMirrorSkillsToAgents(repoPath); for (const skill of STANDARD_SKILL_CATALOG.filter( (entry) => entry.distributions.project && entry.distributions.npm, @@ -402,6 +421,18 @@ Use GitNexus tools to accomplish this task. } await fs.writeFile(skillPath, skillContent, 'utf-8'); + + // Mirror to .agents/skills/ for agents that read repo-local skills + if (agentsMirror) { + try { + const agentsSkillDir = path.join(repoPath, '.agents', 'skills', skill.name); + await fs.mkdir(agentsSkillDir, { recursive: true }); + await fs.writeFile(path.join(agentsSkillDir, 'SKILL.md'), skillContent, 'utf-8'); + } catch (err) { + logger.warn({ err }, `Warning: Could not mirror skill ${skill.name} to .agents/skills:`); + } + } + installedSkills.push(skill.name); // Previous releases installed these known standard skills one level too @@ -418,7 +449,7 @@ Use GitNexus tools to accomplish this task. } } - return installedSkills; + return { skills: installedSkills, agentsMirror }; } /** @@ -496,9 +527,14 @@ export async function generateAIContextFiles( // Install standard skills directly under .claude/skills/ (unless --skip-skills) if (!options?.skipSkills) { - const installedSkills = await installSkills(repoPath); + const { skills: installedSkills, agentsMirror } = await installSkills(repoPath); if (installedSkills.length > 0) { createdFiles.push(`.claude/skills/gitnexus-*/ (${installedSkills.length} skills)`); + if (agentsMirror) { + createdFiles.push( + `.agents/skills/gitnexus-*/ (${installedSkills.length} skills mirrored for .agents)`, + ); + } } } else { createdFiles.push('.claude/skills/gitnexus-*/ (skipped via --skip-skills)'); diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index c58c4fcab..c0518933a 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -33,7 +33,14 @@ import { assertAnalysisFinalized, type AnalyzerRunnerIdentity, } from '../storage/repo-manager.js'; -import { getGitRoot, hasGitDir, getDefaultBranch } from '../storage/git.js'; +import { + getGitRoot, + hasGitDir, + getDefaultBranch, + selfCommitContextFiles, + snapshotSelfCommitSafety, +} from '../storage/git.js'; +import { IndexLockTimeoutError } from '../storage/index-lock.js'; import { loadAnalyzeConfig, mergeAnalyzeOptions, @@ -47,7 +54,8 @@ import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-si import { warnMissingOptionalGrammars, getOptionalGrammarExtensions } from './optional-grammars.js'; import { glob } from 'glob'; import fs from 'fs/promises'; -import { cliError } from './cli-message.js'; +import { cliError, cliWarn } from './cli-message.js'; +import { heapCapMbFor, memoryAutopilotDisabled } from '../core/ingestion/utils/effective-ram.js'; import { EMBEDDING_DIMS_ERROR, normalizeEmbeddingDims } from './embedding-dims.js'; import { formatElapsed } from './format-elapsed.js'; import { isHfDownloadFailure } from '../core/embeddings/hf-env.js'; @@ -138,25 +146,21 @@ const installFatalHandlers = (): void => { }); }; -/** Historical floor for the re-exec heap cap — the auto-sizer never goes below - * this, so small boxes / CI never regress. */ -const DEFAULT_HEAP_MB = 16384; - /** - * RAM-aware re-exec heap cap (MB): `0.75 × effective RAM`, clamped to - * `>= DEFAULT_HEAP_MB`. Kept BELOW physical RAM on purpose — a cap `>=` RAM makes - * V8 collect lazily and inflate the heap into swap-thrash (observed analyzing the - * Linux kernel at a 30GB cap on a 31GB box). `constrainedBytes` is the cgroup - * limit or `null`; it is honored only as a real, smaller-than-physical cap, because + * RAM-aware re-exec heap cap (MB) — the formula itself is single-sourced in + * `core/ingestion/utils/effective-ram.ts` (`heapCapMbFor`), shared with the + * server's analyze fork. `constrainedBytes` is the cgroup limit or `null`; + * it is honored only as a real, smaller-than-physical cap, because * `process.constrainedMemory()` returns a huge sentinel when UNCONSTRAINED. + * (Observed rationale: a cap ≥ RAM made V8 collect lazily and swap-thrash — + * the #2649 worker-timeout cascade on 16 GB boxes.) */ export function computeHeapCapMb(totalBytes: number, constrainedBytes: number | null): number { const effectiveBytes = constrainedBytes !== null && constrainedBytes > 0 && constrainedBytes < totalBytes ? constrainedBytes : totalBytes; - const effectiveMb = Math.floor(effectiveBytes / (1024 * 1024)); - return Math.max(DEFAULT_HEAP_MB, Math.floor(0.75 * effectiveMb)); + return heapCapMbFor(effectiveBytes); } function readConstrainedBytes(): number | null { @@ -526,21 +530,69 @@ const forceHeapOOMForTestIfEnabled = (): void => { // `gitnexus/src/core/lbug/lbug-config.ts` in sync with this value. const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024; -/** Re-exec the process with the RAM-aware auto heap cap + larger semi-space/stack - * if we're currently below that. A user-supplied NODE_OPTIONS heap wins (no re-exec). */ -async function ensureHeap(): Promise { - const nodeOpts = process.env.NODE_OPTIONS || ''; - if (nodeOpts.includes('--max-old-space-size')) return false; +/** + * Last `--max-old-space-size` value (MB) in a NODE_OPTIONS string, or `null` + * when absent/unparseable. Last occurrence wins, matching V8's own + * later-flag-wins semantics when NODE_OPTIONS repeats a flag. + */ +export function parseMaxOldSpaceMb(nodeOptions: string): number | null { + // V8 accepts `-` and `_` interchangeably in flag names, and Node accepts a + // space-separated value in NODE_OPTIONS — honor every spelling of the pin + // instead of silently overriding it (#2649 review). + const matches = [...nodeOptions.matchAll(/--max[-_]old[-_]space[-_]size(?:=|\s+)(\d+)/g)]; + if (matches.length === 0) return null; + const mb = Number(matches[matches.length - 1][1]); + return Number.isFinite(mb) && mb > 0 ? mb : null; +} - const v8Heap = v8.getHeapStatistics().heap_size_limit; - if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false; +/** Re-exec the process with the RAM-aware auto heap cap + larger semi-space/stack + * if we're currently below that. + * + * Heap-source precedence (#2649): + * - an explicit per-invocation `--max-old-space-size` (execArgv) always wins; + * - `GITNEXUS_MEMORY=off` declines the memory autopilot entirely; + * - an ambient NODE_OPTIONS heap >= the auto cap is honored as-is; + * - an ambient NODE_OPTIONS heap BELOW the auto cap is treated as an + * inherited environment default (devcontainers/CI export one for other + * tooling), not a deliberate per-run choice: warn and respawn with the + * auto cap. Pre-#2649 this returned early and large repos then OOM'd on + * whatever heap the environment happened to specify. */ +async function ensureHeap(): Promise { + // Explicit opt-out disables auto-sizing ENTIRELY — both the ambient-pin + // override and the default v8-limit respawn — and is honored SILENTLY: + // the operator already made the call, and stderr-sensitive consumers + // (test harnesses, scripts, supervisors that track a single PID) rely on + // a quiet, single-process run. + if (memoryAutopilotDisabled()) return false; + const nodeOpts = process.env.NODE_OPTIONS || ''; + if (process.execArgv.some((a) => a.startsWith('--max-old-space-size'))) return false; + + const ambientHeapMb = parseMaxOldSpaceMb(nodeOpts); + if (ambientHeapMb !== null) { + if (ambientHeapMb >= RESPAWN_HEAP_MB) return false; + cliWarn( + ` NODE_OPTIONS pins the heap to ${ambientHeapMb}MB — below the ${RESPAWN_HEAP_MB}MB this machine's RAM supports.\n` + + ` Re-running analyze with the larger auto-sized cap (set GITNEXUS_MEMORY=off to keep the NODE_OPTIONS value).\n`, + ); + } else { + const v8Heap = v8.getHeapStatistics().heap_size_limit; + if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false; + } // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, so pass it // only as a direct CLI argument. --max-semi-space-size IS allowed in NODE_OPTIONS. const cliFlags = [HEAP_FLAG, SEMI_FLAG]; if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG); - const childArgs = [...cliFlags, ...process.argv.slice(1)]; + // Preserve the parent's node flags (execArgv) — dropping them breaks any + // loader-launched CLI: `node --import tsx src/cli/index.ts` respawned + // without `--import tsx` cannot execute TypeScript and dies with a + // swallowed exit 1 (#2649 review). Our heap/semi/stack flags come AFTER + // execArgv so V8's later-flag-wins semantics resolve duplicates our way. + // Inspector flags are the one exception: replaying `--inspect[-brk]` makes + // the child fight the parent for the debug port and die with EADDRINUSE. + const preservedExecArgv = process.execArgv.filter((a) => !a.startsWith('--inspect')); + const childArgs = [...preservedExecArgv, ...cliFlags, ...process.argv.slice(1)]; const childEnv = { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG} ${SEMI_FLAG}`.trim(), @@ -658,6 +710,13 @@ export interface AnalyzeOptions { * default-on case. */ stats?: boolean; + /** + * Opt-in auto-commit of any AGENTS.md/CLAUDE.md changes this `analyze` run + * makes. Scoped to only those two files (never `git add -A`); no-ops + * silently if neither exists, neither changed, or the commit step itself + * fails (e.g. no git identity configured). See #2639. + */ + selfCommit?: boolean; /** Skip installing standard GitNexus skill files directly under .claude/skills/. */ skipSkills?: boolean; /** @@ -1404,6 +1463,15 @@ const analyzeCommandImpl = async ( const bootstrapArgs: [] | [AnalyzerRunnerIdentity] = runnerIdentityAtBootstrap ? [runnerIdentityAtBootstrap] : []; + // #2639 review round 2: snapshot which of AGENTS.md/CLAUDE.md are safe to + // auto-commit BEFORE runFullAnalysis (and the --skills regeneration + // further down) writes to them, so selfCommitContextFiles can tell a + // pre-existing unstaged user edit apart from this run's stats refresh + // and refuse to sweep the former into the latter's commit. + const selfCommitSafety = + options.selfCommit === true + ? snapshotSelfCommitSafety(repoPath, ['AGENTS.md', 'CLAUDE.md']) + : undefined; const result = await runFullAnalysis(repoPath, runOptions, runCallbacks, ...bootstrapArgs); if (result.alreadyUpToDate) { @@ -1448,6 +1516,11 @@ const analyzeCommandImpl = async ( ` Updated base_ref to "${resolvedDefaultBranch}" in ${baseRefRefreshed.join(', ')}\n`, ); } + // #2639: opt-in self-commit of any AGENTS.md/CLAUDE.md churn from this + // fast path (e.g. a base_ref refresh above). Best-effort — never throws. + if (options.selfCommit === true && selfCommitSafety) { + selfCommitContextFiles(repoPath, ['AGENTS.md', 'CLAUDE.md'], selfCommitSafety); + } // Safe to return without process.exit(0) — the early-return path in // runFullAnalysis never opens LadybugDB, so no native handles prevent exit. return; @@ -1537,6 +1610,14 @@ const analyzeCommandImpl = async ( } } + // #2639: opt-in self-commit of any AGENTS.md/CLAUDE.md churn written by + // this run (the primary generateAIContextFiles call inside + // runFullAnalysis, and/or the --skills regeneration above). Best-effort + // — never throws, so a missing git identity etc. can't fail `analyze`. + if (options.selfCommit === true && selfCommitSafety) { + selfCommitContextFiles(repoPath, ['AGENTS.md', 'CLAUDE.md'], selfCommitSafety); + } + const totalTime = ((Date.now() - t0) / 1000).toFixed(1); clearInterval(elapsedTimer); @@ -1563,11 +1644,21 @@ const analyzeCommandImpl = async ( // progress-bar log() that fired mid-run has already scrolled away, so the // degraded-search state must also appear in the final summary (#1161). if (result.ftsSkipped) { - console.log( - `\n Warning: full-text/BM25 search is disabled — the LadybugDB FTS extension was unavailable.\n` + - ` Install it once with network access (GITNEXUS_LBUG_EXTENSION_INSTALL=auto) then rerun, or\n` + - ` run \`gitnexus analyze --repair-fts\` when connected. Run \`gitnexus doctor\` for details.`, - ); + // #2658 review L2: a build/verify failure is NOT an extension-unavailable + // problem — sending the user to install the extension is the wrong remedy. + if (result.ftsSkipReason === 'build-failed') { + console.log( + `\n Warning: full-text/BM25 search is disabled — the search index build failed this run.\n` + + ` The FTS extension is available; rerun \`gitnexus analyze --repair-fts\`. If it persists,\n` + + ` check the disk for space or corruption. Run \`gitnexus doctor\` for details.`, + ); + } else { + console.log( + `\n Warning: full-text/BM25 search is disabled — the LadybugDB FTS extension was unavailable.\n` + + ` Install it once with network access (GITNEXUS_LBUG_EXTENSION_INSTALL=auto) then rerun, or\n` + + ` run \`gitnexus analyze --repair-fts\` when connected. Run \`gitnexus doctor\` for details.`, + ); + } } // Standalone-ingest warnings (skipped/degraded Move packages) share the @@ -1613,6 +1704,22 @@ const analyzeCommandImpl = async ( return; } + // Another analyze held the index lock past the configured wait ceiling + // (#2658, GITNEXUS_INDEX_LOCK_TIMEOUT_MS). The on-disk index is being + // refreshed by the holder — this is a clean, expected condition, not a + // crash, so render the message without a stack trace. + if (err instanceof IndexLockTimeoutError) { + cliError( + ` Another gitnexus analyze (pid ${err.holder.pid} on ${err.holder.hostname}) is ` + + `already refreshing this index and did not finish within the wait window.\n` + + ` The on-disk index is being updated by that run. Retry later, or raise\n` + + ` GITNEXUS_INDEX_LOCK_TIMEOUT_MS to wait longer.\n`, + { recoveryHint: 'index-lock-timeout', holderPid: err.holder.pid }, + ); + process.exitCode = 1; + return; + } + // Finalize invariant failure (#1169) — keep the rich actionable // message intact and write through realStderrWrite so it can't be // erased by a leftover bar refresh on slow terminals. diff --git a/gitnexus/src/cli/cli-message.ts b/gitnexus/src/cli/cli-message.ts index 9d27b6686..61b65dd00 100644 --- a/gitnexus/src/cli/cli-message.ts +++ b/gitnexus/src/cli/cli-message.ts @@ -59,7 +59,8 @@ export type RecoveryHint = | 'npm-resolution' | 'module-not-found' | 'gitnexusrc-invalid' - | 'default-branch-invalid'; + | 'default-branch-invalid' + | 'index-lock-timeout'; /** * Common shape for the optional structured-field bag passed to diff --git a/gitnexus/src/cli/doctor.ts b/gitnexus/src/cli/doctor.ts index 7ec8f30f5..2573dc11d 100644 --- a/gitnexus/src/cli/doctor.ts +++ b/gitnexus/src/cli/doctor.ts @@ -14,10 +14,15 @@ import { import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js'; import { checkLbugNative, + type NativeCheckResult, probeFtsExtensionLoad, probeVectorExtensionLoad, } from '../core/lbug/native-check.js'; -import { getOsPageSize, isPageSizeAwareLadybug } from '../core/lbug/lbug-config.js'; +import { + getEffectiveBufferPoolSize, + getOsPageSize, + isPageSizeAwareLadybug, +} from '../core/lbug/lbug-config.js'; import { diagnoseExtensionLoad } from '../core/lbug/extension-load-error.js'; import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js'; import { t } from './i18n/index.js'; @@ -150,6 +155,49 @@ export function pageSizeDoctorLines( return lines; } +/** + * The hintless buffer-pool doctor line (#2631) — the pool the next Database + * open in THIS process would get. Same plain-params testable-helper shape as + * pageSizeDoctorLines above. `pool` is getEffectiveBufferPoolSize(): `0` is + * the pass-through sentinel for LadybugDB's native 80%-of-RAM default, never + * printed as "0 MiB". `envRaw` (the raw GITNEXUS_LBUG_BUFFER_POOL_SIZE value) + * marks operator-supplied absolute values as "(env override)" — no scaling + * suffix: the hintless default is deliberately unscaled (#2557), and an env + * value is absolute, so a "×N" note would misdescribe both. + */ +export function poolSizeDoctorLine(pool: number, envRaw: string | undefined): string { + const value = pool === 0 ? 'native 80% of RAM' : `${Math.round(pool / (1024 * 1024))} MiB`; + const envNote = envRaw !== undefined && envRaw.trim().length > 0 ? ' (env override)' : ''; + return ` ${padDisplayEnd('pool size', 10)}${value}${envNote}`; +} + +/** + * The `native` status line. Literal label like the page-size and pool-size lines + * above (no i18n key). + * + * A failed check is not automatically a MISSING binary, and saying so is the + * same misdiagnosis #2672 fixed one layer down: on a host whose glibc is too + * old, `lbugjs.node` is present and merely unloadable, so "missing" sent users + * to reinstall a file that was already there — while the detail written to + * stderr right below said the opposite. Render what the check actually found. + */ +export function nativeStatusLine(check: NativeCheckResult): string { + return ` ${padDisplayEnd('native', 10)}${nativeStatusText(check)}`; +} + +function nativeStatusText(check: NativeCheckResult): string { + if (check.ok) return '✓ lbugjs.node loaded'; + switch (check.kind) { + case 'package_missing': + return '✗ @ladybugdb/core not installed'; + case 'load_failed': + return '✗ lbugjs.node present but failed to load'; + default: + // 'binary_missing', and any future kind: the conservative claim. + return '✗ lbugjs.node missing'; + } +} + export const doctorCommand = async () => { const fingerprint = getRuntimeFingerprint(); const capabilities = getRuntimeCapabilities(); @@ -168,11 +216,14 @@ export const doctorCommand = async () => { for (const line of pageSizeDoctorLines(getOsPageSize(), fingerprint.ladybugdb)) { console.log(line); } + // Hintless buffer pool for the next DB open (#2631). Literal label like + // the page size line above (no i18n key). + console.log( + poolSizeDoctorLine(getEffectiveBufferPoolSize(), process.env.GITNEXUS_LBUG_BUFFER_POOL_SIZE), + ); const nativeCheck = checkLbugNative(); - if (nativeCheck.ok) { - console.log(` ${padDisplayEnd('native', 10)}✓ lbugjs.node loaded`); - } else { - console.log(` ${padDisplayEnd('native', 10)}✗ lbugjs.node missing`); + console.log(nativeStatusLine(nativeCheck)); + if (!nativeCheck.ok) { process.stderr.write(`\n${nativeCheck.message?.replace(/^/gm, ' ')}\n\n`); } console.log(` ${label('doctor.labels.onnx', 10)}${fingerprint.onnxruntime ?? 'unknown'}`); diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index ff110615b..58f28d11a 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -57,6 +57,7 @@ const OPTION_DESCRIPTION_KEYS = { 'analyze|--skills': 'help.option.analyze.skills', 'analyze|--skip-agents-md': 'help.option.analyze.skipAgentsMd', 'analyze|--no-stats': 'help.option.analyze.noStats', + 'analyze|--self-commit': 'help.option.analyze.selfCommit', 'analyze|--skip-skills': 'help.option.analyze.skipSkills', 'analyze|--index-only': 'help.option.analyze.indexOnly', 'analyze|--skip-git': 'help.option.skipGit', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 3262a1084..37811cc9e 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -184,8 +184,10 @@ export const en = { 'help.option.analyze.skipAgentsMd': 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md', 'help.option.analyze.noStats': 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md', + 'help.option.analyze.selfCommit': + 'Auto-commit AGENTS.md/CLAUDE.md changes after analyze (opt-in, off by default). Scoped to only those two files (never `git add -A`); no-ops if neither exists, neither changed, or the repo has no git identity configured.', 'help.option.analyze.skipSkills': - 'Skip installing standard GitNexus skill files directly under .claude/skills/. Does not suppress community skills from --skills (those use .claude/skills/gitnexus-area-*). Use --index-only to skip all AI-context file injection.', + 'Skip installing standard GitNexus skill files directly under .claude/skills/ and .agents/skills/. Does not suppress community skills from --skills (those use .claude/skills/gitnexus-area-*). Use --index-only to skip all AI-context file injection.', 'help.option.analyze.indexOnly': 'Pure index mode: skip all file injection (AGENTS.md, CLAUDE.md, skills)', 'help.option.skipGit': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index b0010748e..d41176407 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -175,8 +175,10 @@ export const zhCN = { '根据检测到的社区生成仓库专属 skill 文件（同时设置 --index-only 时无效）。', 'help.option.analyze.skipAgentsMd': '跳过更新 AGENTS.md 和 CLAUDE.md 中的 gitnexus 区块', 'help.option.analyze.noStats': '从 AGENTS.md 和 CLAUDE.md 中省略易变的文件/符号计数', + 'help.option.analyze.selfCommit': + '在 analyze 后自动提交 AGENTS.md/CLAUDE.md 的变更（默认关闭，需显式开启）。仅限这两个文件（绝不使用 `git add -A`）；若两者均不存在、均未变更，或仓库未配置 git 身份，则不执行任何操作。', 'help.option.analyze.skipSkills': - '跳过直接安装在 .claude/skills/ 下的标准 GitNexus skill 文件。不抑制 --skills 生成的社区 skill（位于 .claude/skills/gitnexus-area-*）。使用 --index-only 可跳过所有 AI 上下文文件注入。', + '跳过直接安装在 .claude/skills/ 和 .agents/skills/ 下的标准 GitNexus skill 文件。不抑制 --skills 生成的社区 skill（位于 .claude/skills/gitnexus-area-*）。使用 --index-only 可跳过所有 AI 上下文文件注入。', 'help.option.analyze.indexOnly': '纯索引模式：跳过所有文件注入（AGENTS.md、CLAUDE.md、skills）', 'help.option.skipGit': '将提供的路径/cwd 视为索引根目录，并跳过向上查找 git 根目录', 'help.option.analyze.name': diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 21de56203..0ba1c5548 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -92,9 +92,15 @@ program 'checked-out working tree. Distinct from --default-branch (cosmetic base_ref).', ) .option('--no-stats', 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md') + .option( + '--self-commit', + 'Auto-commit AGENTS.md/CLAUDE.md changes after analyze (opt-in, off by default). ' + + 'Scoped to only those two files (never `git add -A`); no-ops if neither exists, ' + + 'neither changed, or the repo has no git identity configured.', + ) .option( '--skip-skills', - 'Skip installing standard GitNexus skill files directly under .claude/skills/. ' + + 'Skip installing standard GitNexus skill files directly under .claude/skills/ and .agents/skills/. ' + 'Does not suppress community skills from --skills (those use .claude/skills/gitnexus-area-*). ' + 'Use --index-only to skip all AI-context file injection.', ) diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index dc0a0867e..e95ca8048 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -1031,6 +1031,23 @@ export const RENAMED_SKILL_DIRS: Readonly> = { */ export const LEGACY_SKILL_DIR_NAMES: readonly string[] = Object.values(RENAMED_SKILL_DIRS).flat(); +/** Legacy skill dirs found during this run, keyed `oldName|newName` — flushed + * as one grouped notice per rename by {@link flushSkillRenameNotices}. */ +const pendingSkillRenameNotices = new Map(); + +/** Print the collected rename leftovers (one line per rename, all target + * paths grouped) and reset the collector. */ +export function flushSkillRenameNotices(): void { + for (const [key, paths] of pendingSkillRenameNotices) { + const [oldName, skillName] = key.split('|'); + console.log( + ` Note: skill "${oldName}" was renamed to "${skillName}". Left in place ` + + `(delete manually if you have not customized them): ${paths.join(', ')}`, + ); + } + pendingSkillRenameNotices.clear(); +} + /** * Install GitNexus skills to a target directory. * Each skill is installed as {targetDir}/gitnexus-{skillName}/SKILL.md @@ -1102,14 +1119,16 @@ async function installSkillsTo(targetDir: string): Promise { // A directory superseded by a shipped rename is warned about, never // deleted: the installer cannot prove it owns the contents (users // customize installed skills or hand-write their own under these - // names), so an upgrade must not destroy data. + // names), so an upgrade must not destroy data. Collected instead of + // printed here so a multi-tool setup emits one grouped notice per + // rename, not one line per target directory. for (const oldName of RENAMED_SKILL_DIRS[skillName] ?? []) { const legacyDir = path.join(targetDir, oldName); if (await dirExists(legacyDir)) { - console.log( - `[gitnexus] skill "${oldName}" was renamed to "${skillName}"; ` + - `left ${legacyDir} in place — delete it manually if you have not customized it.`, - ); + const key = `${oldName}|${skillName}`; + const paths = pendingSkillRenameNotices.get(key) ?? []; + paths.push(legacyDir); + pendingSkillRenameNotices.set(key, paths); } } installed.push(skillName); @@ -1267,6 +1286,8 @@ export const setupCommand = async (options?: { codingAgent?: string[] | string } } } + flushSkillRenameNotices(); + console.log(''); console.log(' Summary:'); console.log( diff --git a/gitnexus/src/cli/skill-gen.ts b/gitnexus/src/cli/skill-gen.ts index f4c947d63..9e46b1e5c 100644 --- a/gitnexus/src/cli/skill-gen.ts +++ b/gitnexus/src/cli/skill-gen.ts @@ -13,6 +13,7 @@ import { PipelineResult } from '../types/pipeline.js'; import { CommunityNode, CommunityMembership } from '../core/ingestion/community-processor.js'; import { ProcessNode } from '../core/ingestion/process-processor.js'; import { KnowledgeGraph } from '../core/graph/types.js'; +import { shouldMirrorSkillsToAgents } from './ai-context.js'; const GENERATED_SKILL_PREFIX = 'gitnexus-area-'; const MAX_SKILL_NAME_LENGTH = 64; @@ -74,6 +75,12 @@ export const generateSkillFiles = async ( const { communityResult, processResult, graph } = pipelineResult; const outputDir = path.join(repoPath, '.claude', 'skills'); const legacyOutputDir = path.join(outputDir, 'generated'); + // Some agents prioritize repo-local .agents/skills over the global + // ~/.agents/skills install (see shouldMirrorSkillsToAgents). When .agents/ + // exists, mirror the generated community skills there too so those agents + // serve the up-to-date copies. + const agentsOutputDir = path.join(repoPath, '.agents', 'skills'); + let mirrorToAgents = await shouldMirrorSkillsToAgents(repoPath); // Community skills used to live under an undiscoverable `generated/` // grouping directory. Clear that GitNexus-owned legacy output and @@ -95,6 +102,24 @@ export const generateSkillFiles = async ( /* legacy output may not exist */ } + // Mirror cleanup: clear only stale GitNexus-generated community skills under + // .agents/skills/ (reserved gitnexus-area-* namespace), preserving mirrored + // standard skills and any user-authored skills. Never clear the whole root. + if (mirrorToAgents) { + try { + const entries = await fs.readdir(agentsOutputDir, { withFileTypes: true }); + await Promise.all( + entries + .filter((entry) => entry.isDirectory() && entry.name.startsWith(GENERATED_SKILL_PREFIX)) + .map((entry) => + fs.rm(path.join(agentsOutputDir, entry.name), { recursive: true, force: true }), + ), + ); + } catch { + /* mirror root may not exist yet */ + } + } + if (!communityResult || !communityResult.memberships.length) { console.log('\n Skills: no communities detected, skipping skill generation'); return { skills: [], outputPath: outputDir }; @@ -135,6 +160,20 @@ export const generateSkillFiles = async ( // Step 4: Ensure the shared project-skill root exists. Never clear it: it // also contains user-authored and standard GitNexus skills. await fs.mkdir(outputDir, { recursive: true }); + // The .agents/ mirror is a side flow: keep it a weak dependency. If the + // mirror root cannot be created (e.g. `.agents/skills` exists as a file), + // warn and disable mirroring for this run instead of aborting canonical + // community-skill generation. Canonical writes below stay unaffected. + if (mirrorToAgents) { + try { + await fs.mkdir(agentsOutputDir, { recursive: true }); + } catch (err) { + console.log( + `Warning: Could not create mirror root ${agentsOutputDir} — .agents/skills mirroring disabled for this run: ${err}`, + ); + mirrorToAgents = false; + } + } // Step 5: Generate skill files const skills: GeneratedSkillInfo[] = []; @@ -185,6 +224,19 @@ export const generateSkillFiles = async ( await fs.mkdir(skillDir, { recursive: true }); await fs.writeFile(path.join(skillDir, 'SKILL.md'), content, 'utf-8'); + // Mirror to .agents/skills/ for agents that read repo-local skills + // (see mirrorToAgents above). Best-effort: a per-skill mirror failure + // must not abort canonical community-skill generation. + if (mirrorToAgents) { + try { + const agentsSkillDir = path.join(agentsOutputDir, skillName); + await fs.mkdir(agentsSkillDir, { recursive: true }); + await fs.writeFile(path.join(agentsSkillDir, 'SKILL.md'), content, 'utf-8'); + } catch (err) { + console.log(`Warning: Could not mirror skill ${skillName} to .agents/skills: ${err}`); + } + } + const info: GeneratedSkillInfo = { name: skillName, label: community.label, @@ -201,6 +253,11 @@ export const generateSkillFiles = async ( console.log( `\n ${skills.length} skills generated \u2192 .claude/skills/${GENERATED_SKILL_PREFIX}*/`, ); + if (mirrorToAgents) { + console.log( + ` ${skills.length} skills mirrored \u2192 .agents/skills/${GENERATED_SKILL_PREFIX}*/ (.agents)`, + ); + } return { skills, outputPath: outputDir }; }; diff --git a/gitnexus/src/core/analyzer-identity.ts b/gitnexus/src/core/analyzer-identity.ts index a97291871..ec859ed37 100644 --- a/gitnexus/src/core/analyzer-identity.ts +++ b/gitnexus/src/core/analyzer-identity.ts @@ -381,7 +381,14 @@ const LIBC_VARIANT = detectLibcVariant(); function resolveRuntimeVariant(): RuntimeVariant { return { - executablePath: resolveExistingPath(process.execPath), + // Normalized like build.rootPath (#2668): executablePath is a compared + // identity field (only invokedArtifact is stripped in the comparison), and + // process.execPath carries the same Windows drive-letter case ambiguity — + // so leaving it un-normalized would reintroduce the false-stale via runtime. + executablePath: normalizeAnalyzerRootPath( + resolveExistingPath(process.execPath), + process.platform, + ), nodeVersion: process.version, platform: process.platform, architecture: process.arch, @@ -524,6 +531,36 @@ function resolveExistingPath(candidate: string): string { return realpathSync.native(path.resolve(candidate)); } +/** + * Case-stabilize a path's Windows drive letter so two processes that observed + * the same directory under different drive-letter casing (`c:\…` vs `C:\…`) + * produce byte-identical analyzer-identity path fields (#2668). + * + * `realpathSync.native` canonicalizes 8.3 short names and symlinks but does not + * guarantee the drive-letter case it returns — it can preserve whatever casing + * the caller's path carried, and `import.meta.url` casing depends on how each + * entry process (CLI shim vs `npx`/npm wrapper vs server worker) was launched. + * When `analyze` stamps `build.rootPath` under one casing and `status` + * recomputes it under another, `analyzerRunnerIdentitiesEqual` deep-compares + * unequal and `status` reports a freshly-analyzed, untouched repo as stale. + * Uppercasing the drive letter (drive letters are case-insensitive; uppercase + * is the conventional form) collapses that variance. POSIX paths are returned + * unchanged. `platform` is explicit so the transform is unit-testable off + * Windows. + * + * The optional `\\?\` extended-length prefix (which `realpathSync.native` can + * emit for paths over MAX_PATH) is preserved and the drive letter after it is + * still normalized; UNC paths (`\\server\share`, `\\?\UNC\...`) have no drive + * letter and are left untouched. + */ +export function normalizeAnalyzerRootPath(p: string, platform: NodeJS.Platform): string { + if (platform !== 'win32') return p; + return p.replace( + /^(\\\\\?\\)?([a-z]):/, + (_match, prefix: string | undefined, drive: string) => `${prefix ?? ''}${drive.toUpperCase()}:`, + ); +} + function isFile(candidate: string): boolean { try { return statSync(candidate).isFile(); @@ -553,11 +590,30 @@ function manifestLabel(manifest: PackageManifest): string { return `${name}@${version}`; } -function isInside(parent: string, candidate: string): boolean { - const relative = path.relative(parent, candidate); - return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..'); +/** + * Whether `candidate` is `parent` itself or lives beneath it. + * + * The absolute-result rejection is load-bearing on Windows: `path.relative` + * cannot express a relative path between two different drives, so it returns the + * absolute target instead — `path.win32.relative('C:\\parent', 'D:\\other')` is + * `'D:\\other'`. That string does not start with `..`, so the `..` checks alone + * would report an unrelated drive as *inside* the parent. This mirrors the + * containment guards elsewhere in the repo (`server/api.ts`, + * `server/git-clone.ts`, `group/extractors/fs-utils.ts`), which all pair the + * `..` check with `path.isAbsolute`. + * + * `pathApi` is injectable so the win32 semantics are unit-testable from a POSIX + * runner; production callers always use the platform-bound `path`. + */ +function isInside(parent: string, candidate: string, pathApi: typeof path = path): boolean { + const relative = pathApi.relative(parent, candidate); + if (pathApi.isAbsolute(relative)) return false; + return relative === '' || (!relative.startsWith(`..${pathApi.sep}`) && relative !== '..'); } +/** Test seam for {@link isInside} (see `_hashAnalyzerIdentityFramesForTests`). */ +export const _isInsideForTests = isInside; + function resolveBuildRoot(analyzerModulePath: string): { packageRoot: string; buildRoot: string; @@ -570,9 +626,18 @@ function resolveBuildRoot(analyzerModulePath: string): { const packageRoot = path.dirname(cursor); const packageJson = path.join(packageRoot, 'package.json'); if (lstatSync(packageJson).isFile()) { + // Normalize the drive-letter case at this single upstream source so + // every derived identity path field — build.rootPath, identityCacheKey, + // and (via collectDependencyInputs) dependencyRuntime.manifestPath / + // lockfilePath — inherits a case-stable root and analyze-stamp equals + // status-recompute regardless of launch-path casing (#2668). + // Migration: a Windows index stamped before this fix carries the old, + // un-normalized casing, so the first post-upgrade `status` sees one + // spurious "stale" flip — self-healing on the next `analyze`, which + // re-stamps the normalized (idempotent) form. return { - packageRoot, - buildRoot: cursor, + packageRoot: normalizeAnalyzerRootPath(packageRoot, process.platform), + buildRoot: normalizeAnalyzerRootPath(cursor, process.platform), kind: base === 'src' ? 'source' : 'distribution', }; } diff --git a/gitnexus/src/core/graph/graph.ts b/gitnexus/src/core/graph/graph.ts index c906e1b10..1c708a4af 100644 --- a/gitnexus/src/core/graph/graph.ts +++ b/gitnexus/src/core/graph/graph.ts @@ -162,6 +162,11 @@ export const createKnowledgeGraph = (): KnowledgeGraph => { forEachRelationship(fn: (rel: GraphRelationship) => void) { relationshipMap.forEach(fn); }, + forEachRelationshipFields( + fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + ) { + relationshipMap.forEach((rel) => fn(rel.sourceId, rel.targetId, rel.type, rel.confidence)); + }, getNode: (id: string) => nodeMap.get(id), // O(1) count getters - avoid creating arrays just for length diff --git a/gitnexus/src/core/graph/types.ts b/gitnexus/src/core/graph/types.ts index 539f77987..9d9caf12b 100644 --- a/gitnexus/src/core/graph/types.ts +++ b/gitnexus/src/core/graph/types.ts @@ -27,6 +27,19 @@ export interface KnowledgeGraph { iterRelationshipsByType: (type: RelationshipType) => IterableIterator; forEachNode: (fn: (node: GraphNode) => void) => void; forEachRelationship: (fn: (rel: GraphRelationship) => void) => void; + /** + * Zero-allocation relationship scan: fields, not objects (#2680). + * + * The whole-graph scans (the local-symbol pruner, community detection, + * process extraction) read only these four fields, and materializing a + * `GraphRelationship` per edge just to read them dominates iteration cost once + * relationships are held columnar — measured at ~90 ms per analyze on a + * million-edge graph. Prefer this over `forEachRelationship` in any pass that + * walks every edge and needs no other field. + */ + forEachRelationshipFields: ( + fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + ) => void; getNode: (id: string) => GraphNode | undefined; nodeCount: number; relationshipCount: number; @@ -34,5 +47,12 @@ export interface KnowledgeGraph { addRelationship: (relationship: GraphRelationship) => void; removeNode: (nodeId: string) => boolean; removeNodesByFile: (filePath: string) => number; + /** + * Removes the relationship with this id, returning whether it existed. + * + * Implementations that offload relationships out of memory cannot always tell + * "absent" from "already written out" — `GraphEmitSink` deliberately throws + * rather than answering `false` for an edge it can no longer recall (#2680). + */ removeRelationship: (relationshipId: string) => boolean; } diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts index 8fa056a4b..cc6e41c0a 100644 --- a/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts +++ b/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts @@ -2,7 +2,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import type { ClassExtractionConfig } from '../../class-types.js'; -import { synthesizeJavaAnonymousClassName } from '../../utils/ast-helpers.js'; +import { synthesizeJavaTypeIdentity } from '../../utils/ast-helpers.js'; // --------------------------------------------------------------------------- // Java @@ -33,10 +33,10 @@ export const javaClassConfig: ClassExtractionConfig = { 'record_declaration', ], extractName(node) { - if (node.type === 'object_creation_expression' || node.type === 'enum_constant') { - return synthesizeJavaAnonymousClassName(node); - } - return undefined; + return synthesizeJavaTypeIdentity(node)?.name; + }, + extractType(node) { + return synthesizeJavaTypeIdentity(node)?.label; }, // An anonymous body whose name CANNOT be synthesized must not become a // Class node at all. Without this skip, `extract()`'s @@ -50,7 +50,7 @@ export const javaClassConfig: ClassExtractionConfig = { definitionNode !== undefined && (definitionNode.type === 'object_creation_expression' || definitionNode.type === 'enum_constant') && - synthesizeJavaAnonymousClassName(definitionNode) === undefined + synthesizeJavaTypeIdentity(definitionNode) === undefined ); }, }; diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index ff892ae73..80c005143 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -47,9 +47,12 @@ export type CommunityDetectionEngine = CommunityEngine | 'auto'; export interface CommunityDetectionOptions { /** - * Graphology remains the default. `icebug`/`auto` are guarded prototype - * paths for #2337 and fall back to Graphology if the optional native module - * is not available or does not expose the expected API. + * Graphology is the supported default. `icebug`/`auto` are **experimental**: + * they route through the optional `@ladybugmem/icebug` native Leiden (#2337) + * and fall back to Graphology if it is not installed, cannot load, or + * predates the thread/seed controls determinism requires. The two engines + * partition differently, so switching changes community IDs — and with them + * any generated context keyed on those IDs. No stability guarantee. */ engine?: CommunityDetectionEngine; icebug?: { @@ -88,7 +91,8 @@ interface CommunityEngineResult extends LeidenDetailedResult { interface IcebugWorkerSuccess { ok: true; - partition: number[]; + /** `Leiden.getPartition().membership` — a Float64Array over the worker boundary. */ + partition: ArrayLike; modularity: number; } @@ -116,6 +120,12 @@ function createSeededRng(seed: number): () => number { } const COMMUNITY_ENGINE_ENV = 'GITNEXUS_COMMUNITY_ENGINE'; +/** + * Not a declared dependency: the prebuilds need system Arrow 24, libomp and + * glibc >= 2.38, so it stays an opt-in `npm i @ladybugmem/icebug` alongside + * GitNexus rather than 30MB every install pays for. + */ +const ICEBUG_MODULE = '@ladybugmem/icebug'; const DEFAULT_COMMUNITY_ENGINE: CommunityEngine = 'graphology'; const LEIDEN_TIMEOUT_MS = 60_000; const ICEBUG_TIMEOUT_MS = 60_000; @@ -290,14 +300,16 @@ export const buildCommunityProjection = (knowledgeGraph: KnowledgeGraph): Commun const connectedNodes = new Set(); const nodeDegree = new Map(); - knowledgeGraph.forEachRelationship((rel) => { - if (!isClusteringRelationship(rel.type) || rel.sourceId === rel.targetId) return; - if (isLarge && rel.confidence < MIN_CONFIDENCE_LARGE) return; + // Field-wise scan (#2680): this walks every edge and reads only these four, + // so taking objects would allocate one per edge for nothing. + knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + if (!isClusteringRelationship(type) || sourceId === targetId) return; + if (isLarge && confidence < MIN_CONFIDENCE_LARGE) return; - connectedNodes.add(rel.sourceId); - connectedNodes.add(rel.targetId); - nodeDegree.set(rel.sourceId, (nodeDegree.get(rel.sourceId) || 0) + 1); - nodeDegree.set(rel.targetId, (nodeDegree.get(rel.targetId) || 0) + 1); + connectedNodes.add(sourceId); + connectedNodes.add(targetId); + nodeDegree.set(sourceId, (nodeDegree.get(sourceId) || 0) + 1); + nodeDegree.set(targetId, (nodeDegree.get(targetId) || 0) + 1); }); const nodes: CommunityProjectionNode[] = []; @@ -328,12 +340,12 @@ export const buildCommunityProjection = (knowledgeGraph: KnowledgeGraph): Commun const seenEdges = new Set(); const edges: Array = []; - knowledgeGraph.forEachRelationship((rel) => { - if (!isClusteringRelationship(rel.type) || rel.sourceId === rel.targetId) return; - if (isLarge && rel.confidence < MIN_CONFIDENCE_LARGE) return; + knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + if (!isClusteringRelationship(type) || sourceId === targetId) return; + if (isLarge && confidence < MIN_CONFIDENCE_LARGE) return; - const sourceIndex = nodeIndexById.get(rel.sourceId); - const targetIndex = nodeIndexById.get(rel.targetId); + const sourceIndex = nodeIndexById.get(sourceId); + const targetIndex = nodeIndexById.get(targetId); if (sourceIndex === undefined || targetIndex === undefined || sourceIndex === targetIndex) return; @@ -417,6 +429,15 @@ const runCommunityEngine = async ( return runGraphologyLeiden(graph, projection.isLarge, engineRequested); } + // Announced on request, not just on fallback: a run that succeeds is the case + // where the user most needs to know the partition came from the experimental + // engine, since community IDs feed generated context. + onProgress?.( + `Experimental ${engineRequested} community engine requested — unsupported, and its ` + + 'communities will not match the Graphology default.', + 32, + ); + try { return await runIcebugLeiden(projection, engineRequested, options); } catch (error) { @@ -481,10 +502,7 @@ const runIcebugLeiden = async ( if (!Number.isFinite(nativeResult.modularity)) { throw new Error('optional icebug modularity was not finite'); } - if ( - partition.length !== projection.nodes.length || - partition.some((community) => !Number.isSafeInteger(community)) - ) { + if (partition.length !== projection.nodes.length || !isIntegerPartition(partition)) { throw new Error( `optional icebug partition was malformed for ${projection.nodes.length} projected nodes`, ); @@ -500,6 +518,13 @@ const runIcebugLeiden = async ( }; }; +const isIntegerPartition = (partition: ArrayLike): boolean => { + for (let index = 0; index < partition.length; index++) { + if (!Number.isSafeInteger(partition[index])) return false; + } + return true; +}; + const runIcebugWorker = ( nodeCount: number, csr: CommunityCsr, @@ -531,14 +556,21 @@ const runIcebugWorker = ( let settled = false; const timeout = setTimeout(() => { settled = true; - void worker.terminate(); + // Deliberately NOT terminate(): every millisecond of this worker's life is + // spent inside an N-API call (dlopen, GraphR, Leiden, run), and killing a + // thread mid-N-API aborts the whole process — Napi::Error → std::terminate + // → SIGABRT (#2432, see worker-pool.ts `shutdownDrainMs`). A timeout must + // degrade to the Graphology fallback, not take analyze down with it. + // unref() so a wedged native run cannot hold the process open either. + worker.unref(); reject(new Error(`optional icebug community engine timed out after ${ICEBUG_TIMEOUT_MS}ms`)); }, ICEBUG_TIMEOUT_MS); + // No terminate() on the settled paths either: the worker script ends after + // its single postMessage, so the thread exits on its own. worker.once('message', (message: IcebugWorkerSuccess | IcebugWorkerFailure) => { settled = true; clearTimeout(timeout); - void worker.terminate(); if (message.ok === true) { resolve(message); } else { @@ -549,7 +581,6 @@ const runIcebugWorker = ( worker.once('error', (error) => { settled = true; clearTimeout(timeout); - void worker.terminate(); reject(error); }); @@ -565,86 +596,61 @@ const runIcebugWorker = ( }); }; -const ICEBUG_WORKER_SOURCE = ` +/** + * Runs Leiden in a worker so a native crash cannot take the analyze process + * with it. Written against @ladybugmem/icebug's published surface (lib/index.js + * + index.d.ts): `GraphR(n, directed, outIndices, outIndptr)` pins the CSR + * buffers zero-copy, and `Leiden(graph, iterations, randomize, gamma)` — note + * `randomize` precedes `gamma` — returns `{membership, count}` from + * `getPartition()`. + * + * The thread/seed controls are required, not optional: community IDs feed + * generated context, so a build without them would give non-reproducible + * output. They exist at icebug-nodejs HEAD but are missing from the published + * 12.8.0 tarball, so today this guard is what trips and sends us back to + * Graphology. + */ +export const buildIcebugWorkerSource = (moduleSpecifier: string): string => ` const { parentPort, workerData } = require('node:worker_threads'); -const isNumericArrayLike = (value) => - typeof value === 'object' && - value !== null && - 'length' in value && - typeof value.length === 'number'; - -const readPartition = (runner) => { - const candidates = [ - typeof runner.getPartition === 'function' ? runner.getPartition() : runner.partition, - typeof runner.getCommunities === 'function' ? runner.getCommunities() : undefined, - typeof runner.getMembership === 'function' ? runner.getMembership() : undefined, - typeof runner.getMemberships === 'function' ? runner.getMemberships() : undefined, - ]; - - for (const candidate of candidates) { - if (isNumericArrayLike(candidate)) { - return Array.from(candidate, Number); - } - } - - throw new Error('optional icebug ParallelLeidenView did not expose a partition array'); -}; - -const readModularity = (runner) => { - if (typeof runner.getModularity === 'function') return runner.getModularity(); - if (typeof runner.modularity === 'function') return runner.modularity(); - if (typeof runner.modularity === 'number') return runner.modularity; - return 0; -}; - -(async () => { - const imported = await import('icebug'); - const icebug = imported.default ?? imported; - const fromCSR = icebug.Graph?.fromCSR; - const ParallelLeidenView = icebug.community?.ParallelLeidenView; - if (!fromCSR || !ParallelLeidenView) { - throw new Error('optional icebug module does not expose Graph.fromCSR/ParallelLeidenView'); - } +try { + const icebug = require(${JSON.stringify(moduleSpecifier)}); if (typeof icebug.setNumberOfThreads !== 'function' || typeof icebug.setSeed !== 'function') { - throw new Error('optional icebug module does not expose deterministic thread/seed controls'); - } - icebug.setNumberOfThreads(workerData.threads); - icebug.setSeed(workerData.seed, false); - - const nativeGraph = fromCSR(workerData.nodeCount, false, workerData.indices, workerData.indptr); - let runner; - try { - runner = new ParallelLeidenView(nativeGraph, { - iterations: workerData.iterations, - gamma: workerData.gamma, - randomize: workerData.randomize, - }); - } catch { - runner = new ParallelLeidenView( - nativeGraph, - workerData.iterations, - workerData.gamma, - workerData.randomize, + throw new Error( + 'optional icebug build predates the deterministic thread/seed controls (icebug-nodejs#6)', ); } - if (typeof runner.run !== 'function') { - throw new Error('optional icebug ParallelLeidenView does not expose run()'); - } + icebug.setNumberOfThreads(workerData.threads); + icebug.setSeed(workerData.seed, false); + + const graph = new icebug.GraphR( + workerData.nodeCount, + false, + workerData.indices, + workerData.indptr, + ); + const leiden = new icebug.Leiden( + graph, + workerData.iterations, + workerData.randomize, + workerData.gamma, + ); + leiden.run(); - runner.run(); parentPort.postMessage({ ok: true, - partition: readPartition(runner), - modularity: readModularity(runner), + partition: leiden.getPartition().membership, + modularity: leiden.modularity(), }); -})().catch((error) => { +} catch (error) { parentPort.postMessage({ ok: false, error: error instanceof Error ? error.message : String(error) }); -}); +} `; +const ICEBUG_WORKER_SOURCE = buildIcebugWorkerSource(ICEBUG_MODULE); + const normalizePartition = ( projection: CommunityProjection, partition: ArrayLike, diff --git a/gitnexus/src/core/ingestion/di-extractors/index.ts b/gitnexus/src/core/ingestion/di-extractors/index.ts index 0c0869c52..28582e635 100644 --- a/gitnexus/src/core/ingestion/di-extractors/index.ts +++ b/gitnexus/src/core/ingestion/di-extractors/index.ts @@ -1,61 +1,77 @@ /** - * Per-language DI field-matcher registry — the lookup the generic `di` - * pipeline phase uses to decide whether a `Property` node is a - * dependency-injection fan-out candidate. + * Per-language DI resolver registry — the lookup the generic `di` pipeline + * phase uses to discover injection sites and provider metadata on graph nodes. * * Mirrors `scope-resolution/pipeline/registry.ts` (`SCOPE_RESOLVERS`): a - * single-valued `ReadonlyMap` consumed by + * single-valued `ReadonlyMap` consumed by * a framework-neutral phase, so no language or framework names leak into - * shared pipeline code. Adding a framework is two lines: implement a - * `DiFieldMatcher` in `di-extractors/.ts` and register it here. + * shared pipeline code. Adding a framework means implementing a `DiResolver` + * in `di-extractors/.ts` and registering it here. * - * Scope honesty: matchers are per-language *field-injection* matchers. - * Constructor injection (the dominant modern Spring idiom) lives on - * Method/parameter nodes and would require widening the phase's routing — - * deliberately out of scope (see the plan's Deferred work). The registry is - * single-valued per language, matching the `SCOPE_RESOLVERS` shape; widen the - * value type to arrays only when a second same-language framework actually - * lands (a one-line type change then). + * The registry is single-valued per language, matching the `SCOPE_RESOLVERS` + * shape; widen the value type to arrays only when a second same-language + * framework actually lands. Java and Kotlin share Spring's attached metadata + * contract while retaining language-specific syntax capture. */ import { SupportedLanguages } from 'gitnexus-shared'; import type { GraphNode } from 'gitnexus-shared'; -import { springDiFieldMatcher } from './spring.js'; +import { springDiResolver } from './spring.js'; -/** A successful DI field match, produced by a per-language matcher. */ -export interface DiFieldMatch { - /** The element type name `T` — the injected bean interface. */ - elementTypeName: string; +/** A successful injection-site match, produced by a per-language resolver. */ +export interface DiInjectionMatch { + /** The requested dependency type name. */ + targetTypeName: string; + /** A collection receives every matching provider; a single site may need + * framework-specific named/preferred-provider disambiguation. */ + cardinality: 'single' | 'collection'; + /** Statically known provider name requested at the injection site. The + * resolver owns the human-readable explanation of that selection. */ + namedSelection?: { + name: string; + reason: string; + }; /** Human-readable edge reason. Framework specifics (names, idioms, * collection wrapper, gating annotation) live in this payload so the * shared `di` phase stays framework-neutral. */ reason: string; } -/** - * A per-language field-injection matcher: given a `Property` node, return the - * parsed DI match or `null` when the field is not container-injected. The - * matcher receives the whole node (not pre-plucked fields) so the shared - * phase stays ignorant of which properties matter. - */ -export type DiFieldMatcher = (node: GraphNode) => DiFieldMatch | null; +/** Provider metadata used by the shared resolver without naming a framework. */ +export interface DiProviderMatch { + /** Provider names and aliases that can satisfy a named injection. */ + names: readonly string[]; + /** Present when the framework marks this as its preferred candidate. The + * value is appended to the emitted edge reason when it disambiguates. */ + preferenceReason?: string; +} + +/** Per-language DI behavior. Matchers receive whole nodes so the shared phase + * remains ignorant of language/framework-specific property shapes. */ +export interface DiResolver { + matchInjectionSites(node: GraphNode): readonly DiInjectionMatch[]; + matchProvider(node: GraphNode): DiProviderMatch | null; +} /** All `SupportedLanguages` string values, for narrowing raw graph strings. */ const SUPPORTED_LANGUAGE_VALUES: ReadonlySet = new Set(Object.values(SupportedLanguages)); /** * Type guard narrowing an arbitrary graph `language` string to - * `SupportedLanguages`, so `DI_MATCHERS.get()` needs no cast. + * `SupportedLanguages`, so `DI_RESOLVERS.get()` needs no cast. */ export function isSupportedLanguage(value: string): value is SupportedLanguages { return SUPPORTED_LANGUAGE_VALUES.has(value); } -/** Map of `SupportedLanguages` → `DiFieldMatcher`. The `di` phase routes each - * `Property` node here by `node.properties.language`; no entry ⇒ the node is +/** Map of `SupportedLanguages` → `DiResolver`. The `di` phase routes each + * graph node here by `node.properties.language`; no entry ⇒ the node is * skipped. This is the single source of truth for which languages (and, * transitively, frameworks) produce INJECTS edges. */ -export const DI_MATCHERS: ReadonlyMap = new Map< +export const DI_RESOLVERS: ReadonlyMap = new Map< SupportedLanguages, - DiFieldMatcher ->([[SupportedLanguages.Java, springDiFieldMatcher]]); + DiResolver +>([ + [SupportedLanguages.Java, springDiResolver], + [SupportedLanguages.Kotlin, springDiResolver], +]); diff --git a/gitnexus/src/core/ingestion/di-extractors/spring.ts b/gitnexus/src/core/ingestion/di-extractors/spring.ts index 0a6da59ea..44c5b1aca 100644 --- a/gitnexus/src/core/ingestion/di-extractors/spring.ts +++ b/gitnexus/src/core/ingestion/di-extractors/spring.ts @@ -51,13 +51,15 @@ * between `<` and the element) are NOT stripped and fail closed — * acceptable. * - * Registered under `SupportedLanguages.Java` in `./index.ts` (`DI_MATCHERS`); - * language routing is the registry's job, so the matcher itself never reads - * `node.properties.language`. + * Registered for Java and Kotlin in `./index.ts` (`DI_RESOLVERS`); language + * routing is the registry's job, so the matcher itself never reads + * `node.properties.language`. Kotlin's AST-backed class metadata is the + * primary path because Kotlin Property extraction intentionally exposes less + * annotation/type syntax than Java's legacy field contract. */ import type { GraphNode } from 'gitnexus-shared'; -import type { DiFieldMatch, DiFieldMatcher } from './index.js'; +import type { DiInjectionMatch, DiProviderMatch, DiResolver } from './index.js'; import { isDev } from '../utils/env.js'; import { logger } from '../../logger.js'; @@ -84,6 +86,17 @@ const WILDCARD_SUPER_PREFIX = '? super '; * punctuation) fails closed. */ const JAVA_TYPE_NAME_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)*$/; +/** Ephemeral Class-node property populated by Java's post-resolution Spring + * metadata hook. It is consumed in the same pipeline run before persistence. */ +export const SPRING_DI_INJECTION_SITES_PROPERTY = 'springDiInjectionSites'; + +/** Ephemeral Class-node property carrying Spring bean names / @Primary. */ +export const SPRING_DI_PROVIDER_PROPERTY = 'springDiProvider'; + +/** Marker placed on Property nodes whose richer AST-backed field fact was + * attached to the owning Class, suppressing the legacy collection fallback. */ +export const SPRING_DI_CAPTURED_FIELD_PROPERTY = 'springDiCapturedField'; + /** * Split a generic-argument list on TOP-LEVEL commas only, tracking `<`/`>` * bracket depth so nested generics (e.g. the `Pair` key in @@ -181,13 +194,33 @@ export function parseSpringCollectionType( return { collectionType: wrapper, elementTypeName }; } +/** Parse either a supported collect-all type or a standard single bean type. */ +export function parseSpringInjectionType( + rawDeclaredType: string, +): { targetTypeName: string; cardinality: 'single' | 'collection'; displayType: string } | null { + const collection = parseSpringCollectionType(rawDeclaredType); + if (collection !== null) { + return { + targetTypeName: collection.elementTypeName, + cardinality: 'collection', + displayType: `${collection.collectionType}<${collection.elementTypeName}>`, + }; + } + + const normalized = rawDeclaredType.replace(/\s+/g, '').trim(); + if (!JAVA_TYPE_NAME_PATTERN.test(normalized)) return null; + return { targetTypeName: normalized, cardinality: 'single', displayType: normalized }; +} + /** * Match a `Property` node against Spring's collection-injection shape. * * Returns the parsed match (with a Spring-specific human-readable `reason` * payload) or `null` when the field is not container-injected. */ -export const springDiFieldMatcher: DiFieldMatcher = (node: GraphNode): DiFieldMatch | null => { +export const springDiFieldMatcher = ( + node: GraphNode, +): { elementTypeName: string; reason: string } | null => { // Injection-annotation gate: only fields the container actually // injects (@Autowired / @Inject) are candidates. Plain collection // fields are never injected; @Resource is deliberately excluded @@ -220,3 +253,62 @@ export const springDiFieldMatcher: DiFieldMatcher = (node: GraphNode): DiFieldMa reason: `Spring DI: ${matchedAnnotation} ${parsed.collectionType}<${parsed.elementTypeName}>`, }; }; + +function isInjectionMatch(value: unknown): value is DiInjectionMatch { + if (value === null || typeof value !== 'object') return false; + const match = value as Partial; + const namedSelection = match.namedSelection; + return ( + typeof match.targetTypeName === 'string' && + (match.cardinality === 'single' || match.cardinality === 'collection') && + typeof match.reason === 'string' && + (namedSelection === undefined || + (typeof namedSelection === 'object' && + namedSelection !== null && + typeof namedSelection.name === 'string' && + typeof namedSelection.reason === 'string')) + ); +} + +function isProviderMatch(value: unknown): value is DiProviderMatch { + if (value === null || typeof value !== 'object') return false; + const provider = value as Partial; + return ( + Array.isArray(provider.names) && + provider.names.every((name) => typeof name === 'string') && + (provider.preferenceReason === undefined || typeof provider.preferenceReason === 'string') + ); +} + +/** JVM/Spring resolver registered behind the framework-neutral DI seam. */ +export const springDiResolver: DiResolver = { + matchInjectionSites(node): readonly DiInjectionMatch[] { + const matches: DiInjectionMatch[] = []; + + // Preserve the existing Property-node collection contract for hand-built + // graphs and for compatibility with pre-#2414 extraction fixtures. + if (node.label === 'Property' && node.properties[SPRING_DI_CAPTURED_FIELD_PROPERTY] !== true) { + const field = springDiFieldMatcher(node); + if (field !== null) { + matches.push({ + targetTypeName: field.elementTypeName, + cardinality: 'collection', + reason: field.reason, + }); + } + } + + const attached = node.properties[SPRING_DI_INJECTION_SITES_PROPERTY]; + if (Array.isArray(attached)) { + for (const candidate of attached) { + if (isInjectionMatch(candidate)) matches.push(candidate); + } + } + return matches; + }, + + matchProvider(node): DiProviderMatch | null { + const attached = node.properties[SPRING_DI_PROVIDER_PROPERTY]; + return isProviderMatch(attached) ? attached : null; + }, +}; diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 823b3670f..6405b8edf 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -5,7 +5,7 @@ import path from 'path'; import { glob } from 'glob'; import { createIgnoreFilter } from '../../config/ignore-service.js'; -import { logger } from '../logger.js'; +import { warnRespectingProgressBar } from '../logger.js'; /** Lightweight entry — path + size from stat, no content in memory */ export interface ScannedFile { @@ -19,21 +19,8 @@ export interface FilePath { } const READ_CONCURRENCY = 32; -const ANALYZE_PROGRESS_ACTIVE_ENV = 'GITNEXUS_ANALYZE_PROGRESS_ACTIVE'; -const warnLargeFileSkip = (message: string): void => { - if (process.env[ANALYZE_PROGRESS_ACTIVE_ENV] === '1') { - // analyze.ts routes console.warn through the progress bar logger while - // the bar is active. Emitting the operator-facing large-file notice there - // avoids raw pino NDJSON corrupting the one-line progress display in the - // heap-respawn child, whose stderr is intentionally piped for crash - // classification. - // eslint-disable-next-line no-console -- intentionally routed by analyze progress UI - console.warn(message); - return; - } - logger.warn(message); -}; +const warnLargeFileSkip = (message: string): void => warnRespectingProgressBar(message); /** * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded. diff --git a/gitnexus/src/core/ingestion/frameworks/spring/di-metadata.ts b/gitnexus/src/core/ingestion/frameworks/spring/di-metadata.ts new file mode 100644 index 000000000..5083fb4ed --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/di-metadata.ts @@ -0,0 +1,317 @@ +import type { ParsedFile, ScopeId } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { DiInjectionMatch, DiProviderMatch } from '../../di-extractors/index.js'; +import { + parseSpringInjectionType, + SPRING_DI_CAPTURED_FIELD_PROPERTY, + SPRING_DI_INJECTION_SITES_PROPERTY, + SPRING_DI_PROVIDER_PROPERTY, +} from '../../di-extractors/spring.js'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { resolveDefGraphId } from '../../scope-resolution/graph-bridge/ids.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import { createSpringAnnotationNameResolver } from './bean-candidates.js'; +import { SPRING_BEAN_STEREOTYPES } from './bean-catalog.js'; + +export interface SpringDiAnnotationFact { + readonly name: string; + readonly text: string; +} + +export interface SpringDiDependencyFact { + readonly name: string; + readonly rawType: string; + readonly annotations: readonly Annotation[]; +} + +export interface SpringDiInjectionSiteFact< + Annotation extends SpringDiAnnotationFact, + SiteKind extends string, +> { + readonly kind: SiteKind; + readonly memberName: string; + readonly implicitConstructor: boolean; + readonly annotations: readonly Annotation[]; + readonly dependencies: readonly SpringDiDependencyFact[]; +} + +export interface SpringDiClassFact< + Annotation extends SpringDiAnnotationFact, + SiteKind extends string, +> { + readonly classScopeId: ScopeId; + readonly classAnnotations: readonly Annotation[]; + readonly injectionSites: readonly SpringDiInjectionSiteFact[]; +} + +const INJECTION_ANNOTATIONS = new Set([ + 'org.springframework.beans.factory.annotation.Autowired', + 'jakarta.inject.Inject', + 'javax.inject.Inject', +]); + +const QUALIFIER_ANNOTATIONS = new Set([ + 'org.springframework.beans.factory.annotation.Qualifier', + 'jakarta.inject.Named', + 'javax.inject.Named', +]); + +const PRIMARY_ANNOTATIONS = new Set(['org.springframework.context.annotation.Primary']); + +const RESOLVABLE_DI_ANNOTATIONS = new Set([ + ...SPRING_BEAN_STEREOTYPES.keys(), + ...INJECTION_ANNOTATIONS, + ...QUALIFIER_ANNOTATIONS, + ...PRIMARY_ANNOTATIONS, +]); + +const CAPTURE_RELEVANT_ANNOTATIONS = new Set([ + 'Autowired', + 'Inject', + 'Qualifier', + 'Named', + 'Primary', + 'Component', + 'Service', + 'Repository', + 'Controller', + 'RestController', + 'Configuration', +]); + +const STEREOTYPE_SIMPLE_NAMES = new Set( + [...SPRING_BEAN_STEREOTYPES.keys()].map((name) => springAnnotationSimpleName(name)), +); + +export function springAnnotationSimpleName(name: string): string { + const separator = name.lastIndexOf('.'); + return separator === -1 ? name : name.slice(separator + 1); +} + +export function hasSpringDiRelevantAnnotation( + annotations: readonly SpringDiAnnotationFact[], +): boolean { + return annotations.some((annotation) => + CAPTURE_RELEVANT_ANNOTATIONS.has(springAnnotationSimpleName(annotation.name)), + ); +} + +export function hasSpringStereotypeSyntax(annotations: readonly SpringDiAnnotationFact[]): boolean { + return annotations.some((annotation) => + STEREOTYPE_SIMPLE_NAMES.has(springAnnotationSimpleName(annotation.name)), + ); +} + +function staticStringArgument(annotationText: string): string | undefined { + const args = annotationText.match(/\((.*)\)$/s)?.[1]?.trim(); + if (args === undefined) return undefined; + const value = args.replace(/^value\s*=\s*/, '').trim(); + const literal = value.match(/^"((?:\\.|[^"\\])*)"$/s); + if (literal === null) return undefined; + try { + return JSON.parse(`"${literal[1]}"`) as string; + } catch { + return undefined; + } +} + +function defaultBeanName(className: string): string { + if (className.length === 0) return className; + if ( + className.length > 1 && + className[0] !== className[0].toLowerCase() && + className[1] !== className[1].toLowerCase() + ) { + return className; + } + return className[0].toLowerCase() + className.slice(1); +} + +type ParsedSpringInjectionType = NonNullable>; + +export interface SpringDiMetadataAdapter< + Annotation extends SpringDiAnnotationFact, + SiteKind extends string, +> { + getFacts(filePath: string): readonly SpringDiClassFact[]; + isPackageVisibilityIncomplete(filePath: string): boolean; + parseInjectionType(rawType: string): ParsedSpringInjectionType | null; + capturedMemberKind: SiteKind; + isInjectionAnnotationApplicable?( + annotation: Annotation, + site: SpringDiInjectionSiteFact, + ): boolean; + isQualifierAnnotationApplicable?( + annotation: Annotation, + site: SpringDiInjectionSiteFact, + ): boolean; +} + +/** + * Build the post-resolution Spring DI metadata hook shared by language adapters. + * Language adapters retain syntax capture, type normalization, use-site rules, + * and side-channel ownership; this function owns framework semantics only. + */ +export function createSpringDiMetadataAttacher< + Annotation extends SpringDiAnnotationFact, + SiteKind extends string, +>(adapter: SpringDiMetadataAdapter) { + return ( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, + indexes: ScopeResolutionIndexes, + ): void => { + const resolveAnnotation = createSpringAnnotationNameResolver(indexes); + + for (const parsed of parsedFiles) { + const incomplete = adapter.isPackageVisibilityIncomplete(parsed.filePath); + for (const fact of adapter.getFacts(parsed.filePath)) { + const classScope = indexes.scopeTree.getScope(fact.classScopeId); + if (classScope === undefined || classScope.kind !== 'Class') continue; + const classDef = classScope.ownedDefs.find((definition) => definition.type === 'Class'); + if (classDef === undefined) continue; + const graphId = resolveDefGraphId(parsed.filePath, classDef, nodeLookup); + if (graphId === undefined) continue; + const classNode = graph.getNode(graphId); + if (classNode === undefined || classNode.label !== 'Class') continue; + + const resolvedAnnotations = new Map(); + const resolveFact = ( + annotation: Annotation, + enclosingScope: ScopeId | null = classScope.parent, + ): string | undefined => { + const cacheKey = `${enclosingScope ?? ''}\0${annotation.name}`; + if (resolvedAnnotations.has(cacheKey)) return resolvedAnnotations.get(cacheKey); + const resolved = resolveAnnotation( + annotation.name, + parsed, + enclosingScope, + RESOLVABLE_DI_ANNOTATIONS, + incomplete, + ); + resolvedAnnotations.set(cacheKey, resolved); + return resolved; + }; + + const frameworkAnnotations = Array.isArray(classNode.properties.frameworkAnnotations) + ? classNode.properties.frameworkAnnotations.filter( + (annotation): annotation is string => typeof annotation === 'string', + ) + : []; + if (frameworkAnnotations.length > 0) { + const names = new Set(); + let explicitBeanName: string | undefined; + let hasDynamicBeanName = false; + let primary = false; + for (const annotation of fact.classAnnotations) { + const resolved = resolveFact(annotation); + if (resolved === undefined) continue; + if (SPRING_BEAN_STEREOTYPES.has(resolved)) { + const argumentText = annotation.text.match(/\((.*)\)$/s)?.[1]?.trim(); + if (argumentText !== undefined && argumentText.length > 0) { + const staticName = staticStringArgument(annotation.text); + if (staticName === undefined) hasDynamicBeanName = true; + else if (staticName.length > 0) explicitBeanName = staticName; + } + } + if (QUALIFIER_ANNOTATIONS.has(resolved)) { + const qualifier = staticStringArgument(annotation.text); + if (qualifier !== undefined) names.add(qualifier); + } + if (PRIMARY_ANNOTATIONS.has(resolved)) primary = true; + } + if (explicitBeanName !== undefined) names.add(explicitBeanName); + else if (!hasDynamicBeanName) names.add(defaultBeanName(classNode.properties.name)); + const provider: DiProviderMatch = { + names: [...names], + ...(primary ? { preferenceReason: 'selected @Primary' } : {}), + }; + classNode.properties[SPRING_DI_PROVIDER_PROPERTY] = provider; + } + + const matches: DiInjectionMatch[] = []; + const semanticallyOwnedMemberNames = new Set(); + for (const site of fact.injectionSites) { + let injectionAnnotation: Annotation | undefined; + for (const annotation of site.annotations) { + if (adapter.isInjectionAnnotationApplicable?.(annotation, site) === false) continue; + const resolved = resolveFact(annotation, classScope.id); + if (resolved !== undefined && INJECTION_ANNOTATIONS.has(resolved)) { + injectionAnnotation = annotation; + break; + } + } + if (injectionAnnotation === undefined) { + if (!site.implicitConstructor || frameworkAnnotations.length === 0) continue; + } else if (site.kind === adapter.capturedMemberKind) { + // Claim the member only after its injection annotation resolves to + // a recognized FQN. Ambiguous wildcard imports stay unclaimed so + // the legacy collection matcher can fall back. A dynamic qualifier + // later fails closed, but this path still owns the member and must + // suppress that legacy fallback. + semanticallyOwnedMemberNames.add(site.memberName); + } + + for (const dependency of site.dependencies) { + const parsedType = adapter.parseInjectionType(dependency.rawType); + if (parsedType === null) continue; + let qualifierAnnotation: Annotation | undefined; + for (const annotation of dependency.annotations) { + if (adapter.isQualifierAnnotationApplicable?.(annotation, site) === false) continue; + const resolved = resolveFact(annotation, classScope.id); + if (resolved !== undefined && QUALIFIER_ANNOTATIONS.has(resolved)) { + qualifierAnnotation = annotation; + break; + } + } + const qualifier = + qualifierAnnotation === undefined + ? undefined + : staticStringArgument(qualifierAnnotation.text); + // A present-but-dynamic qualifier is not the same as no qualifier. + // Without its value we cannot choose a provider honestly, so fail + // closed instead of emitting the unqualified candidate set. + if (qualifierAnnotation !== undefined && qualifier === undefined) continue; + const trigger = + injectionAnnotation === undefined + ? 'constructor' + : `@${springAnnotationSimpleName(injectionAnnotation.name)} ${site.kind}`; + const location = + site.kind === adapter.capturedMemberKind + ? site.memberName + : `${site.memberName} parameter ${dependency.name}`; + matches.push({ + targetTypeName: parsedType.targetTypeName, + cardinality: parsedType.cardinality, + ...(qualifier === undefined + ? {} + : { + namedSelection: { + name: qualifier, + reason: `qualifier "${qualifier}"`, + }, + }), + reason: `Spring DI: ${trigger} ${location}: ${parsedType.displayType}`, + }); + } + } + if (matches.length > 0) { + classNode.properties[SPRING_DI_INJECTION_SITES_PROPERTY] = matches; + } + + for (const memberName of semanticallyOwnedMemberNames) { + for (const { def } of classScope.bindings.get(memberName) ?? []) { + if (def.ownerId !== classDef.nodeId) continue; + const propertyId = resolveDefGraphId(parsed.filePath, def, nodeLookup); + if (propertyId === undefined) continue; + const property = graph.getNode(propertyId); + if (property?.label === 'Property') { + property.properties[SPRING_DI_CAPTURED_FIELD_PROPERTY] = true; + } + } + } + } + } + }; +} diff --git a/gitnexus/src/core/ingestion/languages/cpp/query.ts b/gitnexus/src/core/ingestion/languages/cpp/query.ts index b50463a7f..52fcc0785 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/query.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/query.ts @@ -98,6 +98,16 @@ const CPP_SCOPE_QUERY = ` declarator: (function_declarator declarator: (identifier) @declaration.name)) @declaration.function +;; Lambda bindings (\`auto f = [](int x){ … };\`). The \`@declaration.function\` +;; anchor sits on the INNER lambda_expression so its range aligns with +;; \`(lambda_expression) @scope.function\` above; otherwise the def is owned by +;; the enclosing scope and calls inside the lambda lose caller attribution. +;; Mirrors the TypeScript arrow patterns (#2687). +(declaration + declarator: (init_declarator + declarator: (identifier) @declaration.name + value: (lambda_expression) @declaration.function)) + ;; ─── Declarations — function definition with pointer return ───────── (function_definition declarator: (pointer_declarator diff --git a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts index 5dab83afe..10aefb45e 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts @@ -615,7 +615,11 @@ export function populateCsharpNamespaceSiblings( } if (seen.has(memberDef.nodeId)) continue; seen.add(memberDef.nodeId); - bucketArr.push({ def: memberDef, origin: 'import' }); + bucketArr.push({ + def: memberDef, + origin: 'import', + visibility: 'static-member-import', + }); } } } diff --git a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts index 19e4ae8e2..29b1a98ed 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts @@ -93,6 +93,7 @@ const csharpScopeResolver: ScopeResolver = { // `(caller, target)` — multiple `g.Greet(...)` sites from Main // yield ONE edge, not one per site. collapseMemberCallsByCallerTarget: true, + freeCallsRequireInstanceOwnership: true, // C# hoists method return-type bindings to the enclosing Module // scope so `propagateImportedReturnTypes` can mirror them across diff --git a/gitnexus/src/core/ingestion/languages/go/query.ts b/gitnexus/src/core/ingestion/languages/go/query.ts index 4246ee3b4..885c19df7 100644 --- a/gitnexus/src/core/ingestion/languages/go/query.ts +++ b/gitnexus/src/core/ingestion/languages/go/query.ts @@ -35,6 +35,25 @@ const GO_SCOPE_QUERY = ` (function_declaration name: (identifier) @declaration.name) @declaration.function +;; Declarations — closure bindings (\`var f = func(){}\`, \`f := func(){}\`). +;; The \`@declaration.function\` anchor sits on the INNER func_literal so its +;; range aligns with the \`(func_literal) @scope.function\` scope above — +;; without that alignment pass2AttachDeclarations owns the def by the module +;; scope and calls inside the closure lose caller attribution. Mirrors the +;; TypeScript \`const f = () => {}\` patterns (#2687). +(var_declaration + (var_spec + name: (identifier) @declaration.name + value: (expression_list (func_literal) @declaration.function))) +(var_declaration + (var_spec_list + (var_spec + name: (identifier) @declaration.name + value: (expression_list (func_literal) @declaration.function)))) +(short_var_declaration + left: (expression_list (identifier) @declaration.name) + right: (expression_list (func_literal) @declaration.function)) + ;; Declarations — method (method_declaration name: (field_identifier) @declaration.name) @declaration.method diff --git a/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts index 348c91653..51a8d6b2e 100644 --- a/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts +++ b/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts @@ -10,6 +10,7 @@ import { } from '../jvm/package-facts.js'; import { getJavaPackageFact, setJavaPackageFact } from './package-facts.js'; import type { JavaSpringConfigConsumerFact } from './spring-config-bindings.js'; +import type { JavaSpringDiClassFact } from './spring-di.js'; export type JavaClassAnnotationFact = ClassAnnotationFact; @@ -18,15 +19,18 @@ export interface JavaCaptureSideChannel { readonly packageFact: JvmPackageFact; readonly classAnnotations: readonly JavaClassAnnotationFact[]; readonly springConfigConsumers?: readonly JavaSpringConfigConsumerFact[]; + readonly springDiFacts?: readonly JavaSpringDiClassFact[]; } const classAnnotations = createClassAnnotationFactStore(); const springConfigConsumers = new Map(); +const springDiFacts = new Map(); /** Clear facts retained by a prior workspace pass in a long-lived process. */ export function clearJavaClassAnnotationFacts(): void { classAnnotations.clear(); springConfigConsumers.clear(); + springDiFacts.clear(); } /** Store the annotation syntax collected by Java's existing scope-query traversal. */ @@ -51,14 +55,32 @@ export function getJavaSpringConfigConsumerFacts( return springConfigConsumers.get(filePath) ?? []; } +export function setJavaSpringDiFacts( + filePath: string, + facts: readonly JavaSpringDiClassFact[], +): void { + if (facts.length === 0) springDiFacts.delete(filePath); + else springDiFacts.set(filePath, facts); +} + +export function getJavaSpringDiFacts(filePath: string): readonly JavaSpringDiClassFact[] { + return springDiFacts.get(filePath) ?? []; +} + /** Snapshot worker-local Java annotation facts for ParsedFile serialization. */ export function collectJavaCaptureSideChannel( filePath: string, ): JavaCaptureSideChannel | undefined { const facts = classAnnotations.get(filePath); const configConsumers = springConfigConsumers.get(filePath) ?? []; + const diFacts = springDiFacts.get(filePath) ?? []; const packageFact = getJavaPackageFact(filePath); - if (facts.length === 0 && configConsumers.length === 0 && packageFact === undefined) { + if ( + facts.length === 0 && + configConsumers.length === 0 && + diFacts.length === 0 && + packageFact === undefined + ) { return undefined; } return { @@ -66,6 +88,7 @@ export function collectJavaCaptureSideChannel( packageFact: packageFact ?? UNKNOWN_JVM_PACKAGE_FACT, classAnnotations: facts, ...(configConsumers.length > 0 ? { springConfigConsumers: configConsumers } : {}), + ...(diFacts.length > 0 ? { springDiFacts: diFacts } : {}), }; } @@ -85,6 +108,7 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void { ) { setJavaClassAnnotationFacts(parsed.filePath, []); setJavaSpringConfigConsumerFacts(parsed.filePath, []); + setJavaSpringDiFacts(parsed.filePath, []); setJavaPackageFact(parsed.filePath, UNKNOWN_JVM_PACKAGE_FACT); return; } @@ -93,6 +117,10 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void { parsed.filePath, Array.isArray(data.springConfigConsumers) ? data.springConfigConsumers : [], ); + setJavaSpringDiFacts( + parsed.filePath, + Array.isArray(data.springDiFacts) ? data.springDiFacts : [], + ); setJavaPackageFact( parsed.filePath, isJvmPackageFact(data.packageFact) ? data.packageFact : UNKNOWN_JVM_PACKAGE_FACT, diff --git a/gitnexus/src/core/ingestion/languages/java/captures.ts b/gitnexus/src/core/ingestion/languages/java/captures.ts index 62d9fe788..dce0b1dca 100644 --- a/gitnexus/src/core/ingestion/languages/java/captures.ts +++ b/gitnexus/src/core/ingestion/languages/java/captures.ts @@ -20,9 +20,10 @@ import { recordClassAnnotationCapture, } from '../../frameworks/spring/bean-candidates.js'; import { + javaLocalTypeDeclarationContainer, nodeIfType, nodeToCapture, - synthesizeJavaAnonymousClassName, + synthesizeJavaTypeIdentity, syntheticCapture, } from '../../utils/ast-helpers.js'; import { splitImportDeclaration } from './import-decomposer.js'; @@ -35,16 +36,22 @@ import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; import { setJavaClassAnnotationFacts, setJavaSpringConfigConsumerFacts, + setJavaSpringDiFacts, } from './capture-side-channel.js'; import { captureJavaPackageFact } from './package-facts.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; import { captureJavaSpringConfigConsumerFacts } from './spring-config-bindings.js'; +import { captureJavaSpringDiClassFact, type JavaSpringDiClassFact } from './spring-di.js'; /** Declaration anchors that carry function-like arity metadata. */ const FUNCTION_DECL_TAGS = ['@declaration.method', '@declaration.constructor'] as const; /** tree-sitter-java node types that the method extractor accepts. */ -const FUNCTION_NODE_TYPES = ['method_declaration', 'constructor_declaration'] as const; +const FUNCTION_NODE_TYPES = [ + 'method_declaration', + 'constructor_declaration', + 'compact_constructor_declaration', +] as const; const JAVA_CALLABLE_CAPTURE_OPTIONS = { functionNodeTypes: new Set([...FUNCTION_NODE_TYPES, 'lambda_expression']), @@ -65,6 +72,26 @@ const JAVA_CALLABLE_CAPTURE_OPTIONS = { normalizeQualifiedName: (raw: string) => raw.replaceAll('::', '.'), } as const; +/** Visibility of a local type begins at its declaration and ends with its + * immediately enclosing block (JLS 6.3). A Java-only synthetic Block scope + * models that range without changing shared resolver selection semantics. */ +function javaLocalTypeVisibilityScope(node: SyntaxNode): CaptureMatch | undefined { + const container = javaLocalTypeDeclarationContainer(node); + if (container === null) return undefined; + return { + '@scope.block': { + name: '@scope.block', + range: { + startLine: node.startPosition.row + 1, + startCol: node.startPosition.column, + endLine: container.endPosition.row + 1, + endCol: container.endPosition.column, + }, + text: node.text, + }, + }; +} + /** Suppress read.member emissions when the field_access is already * covered by a method_invocation (object of a call) or an * assignment_expression (write target). */ @@ -99,6 +126,8 @@ export function emitJavaScopeCaptures( const rawMatches = getJavaScopeQuery().matches(tree.rootNode); const out: CaptureMatch[] = []; const classAnnotations = new Map>(); + const springDiFacts: JavaSpringDiClassFact[] = []; + const springDiClassNodeIds = new Set(); for (const m of rawMatches) { const grouped: Record = {}; @@ -118,6 +147,13 @@ export function emitJavaScopeCaptures( } if (Object.keys(grouped).length === 0) continue; + const springDiClassNode = nodeIfType(nodeMap['@scope.class'], 'class_declaration'); + if (springDiClassNode !== null && !springDiClassNodeIds.has(springDiClassNode.id)) { + springDiClassNodeIds.add(springDiClassNode.id); + const fact = captureJavaSpringDiClassFact(springDiClassNode, filePath); + if (fact !== null) springDiFacts.push(fact); + } + const annotatedClass = grouped['@class-annotation.class']; const annotationName = grouped['@class-annotation.name']; if (annotatedClass !== undefined && annotationName !== undefined) { @@ -125,6 +161,29 @@ export function emitJavaScopeCaptures( continue; } + const typeDeclaration = [ + nodeMap['@declaration.class'], + nodeMap['@declaration.enum'], + nodeMap['@declaration.record'], + nodeMap['@declaration.interface'], + ].find((node): node is SyntaxNode => node !== undefined); + const localTypeIdentity = + typeDeclaration === undefined ? undefined : synthesizeJavaTypeIdentity(typeDeclaration); + if ( + localTypeIdentity?.bindingName !== undefined && + grouped['@declaration.name'] !== undefined && + typeDeclaration !== undefined + ) { + grouped['@declaration.binding-name'] = grouped['@declaration.name']; + grouped['@declaration.name'] = syntheticCapture( + '@declaration.name', + typeDeclaration, + localTypeIdentity.name, + ); + const visibilityScope = javaLocalTypeVisibilityScope(typeDeclaration); + if (visibilityScope !== undefined) out.push(visibilityScope); + } + // Decompose each `import_declaration`. `@import.statement` is captured // directly on the `import_declaration` node. if (grouped['@import.statement'] !== undefined) { @@ -288,6 +347,7 @@ export function emitJavaScopeCaptures( filePath, captureJavaSpringConfigConsumerFacts(tree.rootNode, filePath), ); + setJavaSpringDiFacts(filePath, springDiFacts); return [ ...resolveVarTypeBindings(out), @@ -300,8 +360,8 @@ export function emitJavaScopeCaptures( /** * Synthesize `@declaration.class` matches for anonymous class bodies - * (`new Runnable() { ... }`), named by the same javac-style authority - * (`synthesizeJavaAnonymousClassName` → `Worker$N`) the structure phase + * (`new Runnable() { ... }`), named by the same javac-compatible authority + * (`synthesizeJavaTypeIdentity` → `Worker$N`) the structure phase * uses — the two layers agree by construction (#2550). * * The anchor is the `class_body` node: it shares its range with the @@ -314,13 +374,13 @@ export function emitJavaScopeCaptures( function synthesizeJavaAnonymousClassDeclarations(rootNode: SyntaxNode): CaptureMatch[] { const out: CaptureMatch[] = []; for (const oce of rootNode.descendantsOfType('object_creation_expression')) { - const name = synthesizeJavaAnonymousClassName(oce); - if (name === undefined) continue; + const identity = synthesizeJavaTypeIdentity(oce); + if (identity === undefined) continue; const body = oce.namedChildren.find((c) => c.type === 'class_body'); if (body === undefined) continue; out.push({ '@declaration.class': nodeToCapture('@declaration.class', body), - '@declaration.name': syntheticCapture('@declaration.name', body, name), + '@declaration.name': syntheticCapture('@declaration.name', body, identity.name), }); // Inheritance: the anonymous class extends/implements its constructed @@ -361,7 +421,7 @@ function synthesizeJavaAnonymousClassDeclarations(rootNode: SyntaxNode): Capture out.push({ '@type-binding.annotation': nodeToCapture('@type-binding.annotation', declNode), '@type-binding.name': nodeToCapture('@type-binding.name', varName), - '@type-binding.type': syntheticCapture('@type-binding.type', oce, name), + '@type-binding.type': syntheticCapture('@type-binding.type', oce, identity.name), }); } } @@ -377,11 +437,11 @@ function synthesizeJavaAnonymousClassDeclarations(rootNode: SyntaxNode): Capture const hostEnum = javaEnclosingEnumNameOf(constant); const bodyNode = constant.childForFieldName?.('body'); const isBodied = bodyNode !== null && bodyNode !== undefined && bodyNode.type === 'class_body'; - const bodiedName = synthesizeJavaAnonymousClassName(constant); - if (bodiedName !== undefined && isBodied) { + const bodiedIdentity = synthesizeJavaTypeIdentity(constant); + if (bodiedIdentity !== undefined && isBodied) { out.push({ '@declaration.class': nodeToCapture('@declaration.class', bodyNode), - '@declaration.name': syntheticCapture('@declaration.name', bodyNode, bodiedName), + '@declaration.name': syntheticCapture('@declaration.name', bodyNode, bodiedIdentity.name), }); if (hostEnum !== undefined) { out.push({ @@ -409,7 +469,7 @@ function synthesizeJavaAnonymousClassDeclarations(rootNode: SyntaxNode): Capture // the `object_creation_expression` branch, which skips on synthesis // failure. `hostEnum` is used only for genuinely body-less constants. const constantNameNode = constant.childForFieldName?.('name'); - const constantType = isBodied ? bodiedName : hostEnum; + const constantType = isBodied ? bodiedIdentity?.name : hostEnum; if (constantNameNode !== null && constantNameNode !== undefined && constantType !== undefined) { out.push({ '@type-binding.annotation': nodeToCapture('@type-binding.annotation', constant), diff --git a/gitnexus/src/core/ingestion/languages/java/query.ts b/gitnexus/src/core/ingestion/languages/java/query.ts index 31272c0b7..b51daa4e9 100644 --- a/gitnexus/src/core/ingestion/languages/java/query.ts +++ b/gitnexus/src/core/ingestion/languages/java/query.ts @@ -55,6 +55,7 @@ const JAVA_SCOPE_QUERY = ` (method_declaration) @scope.function (constructor_declaration) @scope.function +(compact_constructor_declaration) @scope.function ;; Declarations — types (class_declaration diff --git a/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts index 52266456b..e79624fea 100644 --- a/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts @@ -31,6 +31,7 @@ import { import { populateJavaPackageSiblings } from './package-siblings.js'; import { attachSpringBeanCandidateMetadata } from './spring-bean-metadata.js'; import { attachJavaSpringConfigBindings } from './spring-config-bindings.js'; +import { attachJavaSpringDiMetadata } from './spring-di.js'; import { applyJavaCaptureSideChannel, clearJavaClassAnnotationFacts, @@ -86,6 +87,7 @@ const javaScopeResolver: ScopeResolver = { populateRangeBindings: populateJavaCrossFileReturnTypes, emitPostResolutionEdges: (graph, parsedFiles, nodeLookup, indexes, ctx) => { attachSpringBeanCandidateMetadata(graph, parsedFiles, nodeLookup, indexes); + attachJavaSpringDiMetadata(graph, parsedFiles, nodeLookup, indexes); attachJavaSpringConfigBindings(graph, parsedFiles, nodeLookup, indexes, ctx); }, }; diff --git a/gitnexus/src/core/ingestion/languages/java/spring-di.ts b/gitnexus/src/core/ingestion/languages/java/spring-di.ts new file mode 100644 index 000000000..2b106060c --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/java/spring-di.ts @@ -0,0 +1,153 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { + createSpringDiMetadataAttacher, + hasSpringDiRelevantAnnotation, + hasSpringStereotypeSyntax, + type SpringDiAnnotationFact, + type SpringDiClassFact, + type SpringDiDependencyFact, + type SpringDiInjectionSiteFact, +} from '../../frameworks/spring/di-metadata.js'; +import { parseSpringInjectionType } from '../../di-extractors/spring.js'; +import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { isJavaPackageSiblingVisibilityIncomplete } from './package-siblings.js'; +import { getJavaSpringDiFacts } from './capture-side-channel.js'; + +export type JavaAnnotationSyntaxFact = SpringDiAnnotationFact; + +export type JavaSpringDependencyFact = SpringDiDependencyFact; + +type JavaSpringInjectionSiteKind = 'field' | 'constructor' | 'method'; + +export type JavaSpringInjectionSiteFact = SpringDiInjectionSiteFact< + JavaAnnotationSyntaxFact, + JavaSpringInjectionSiteKind +>; + +export type JavaSpringDiClassFact = SpringDiClassFact< + JavaAnnotationSyntaxFact, + JavaSpringInjectionSiteKind +>; + +function annotationFacts(node: SyntaxNode): JavaAnnotationSyntaxFact[] { + const facts: JavaAnnotationSyntaxFact[] = []; + for (const child of node.namedChildren) { + if (child.type !== 'modifiers') continue; + for (const modifier of child.namedChildren) { + if (modifier.type !== 'marker_annotation' && modifier.type !== 'annotation') continue; + const nameNode = modifier.childForFieldName('name') ?? modifier.firstNamedChild; + if (nameNode === null) continue; + facts.push({ name: nameNode.text.trim(), text: modifier.text.trim() }); + } + } + return facts; +} + +function dependenciesOf(callable: SyntaxNode): JavaSpringDependencyFact[] { + const parameters = callable.childForFieldName('parameters'); + if (parameters === null) return []; + const dependencies: JavaSpringDependencyFact[] = []; + for (const parameter of parameters.namedChildren) { + if (parameter.type !== 'formal_parameter' && parameter.type !== 'spread_parameter') continue; + const nameNode = parameter.childForFieldName('name'); + const typeNode = parameter.childForFieldName('type'); + if (nameNode === null || typeNode === null) continue; + dependencies.push({ + name: nameNode.text.trim(), + rawType: typeNode.text.trim(), + annotations: annotationFacts(parameter), + }); + } + return dependencies; +} + +/** + * Capture one class already surfaced by Java's scope query. + * + * `captures.ts` calls this from its existing query-match traversal, so Spring + * DI does not perform a second recursive walk from the AST root. + */ +export function captureJavaSpringDiClassFact( + classNode: SyntaxNode, + filePath: string, +): JavaSpringDiClassFact | null { + const body = classNode.childForFieldName('body'); + if (body === null) return null; + const classAnnotations = annotationFacts(classNode); + const injectionSites: JavaSpringInjectionSiteFact[] = []; + + const constructors = body.namedChildren.filter( + (child) => child.type === 'constructor_declaration', + ); + for (const constructor of constructors) { + const annotations = annotationFacts(constructor); + const implicitConstructor = + constructors.length === 1 && + hasSpringStereotypeSyntax(classAnnotations) && + !hasSpringDiRelevantAnnotation(annotations); + if (!implicitConstructor && !hasSpringDiRelevantAnnotation(annotations)) continue; + injectionSites.push({ + kind: 'constructor', + memberName: constructor.childForFieldName('name')?.text.trim() ?? '', + implicitConstructor, + annotations, + dependencies: dependenciesOf(constructor), + }); + } + + for (const member of body.namedChildren) { + if (member.type === 'field_declaration') { + const annotations = annotationFacts(member); + if (!hasSpringDiRelevantAnnotation(annotations)) continue; + const typeNode = member.childForFieldName('type'); + if (typeNode === null) continue; + for (const declarator of member.namedChildren) { + if (declarator.type !== 'variable_declarator') continue; + const nameNode = declarator.childForFieldName('name'); + if (nameNode === null) continue; + injectionSites.push({ + kind: 'field', + memberName: nameNode.text.trim(), + implicitConstructor: false, + annotations, + dependencies: [ + { + name: nameNode.text.trim(), + rawType: typeNode.text.trim(), + annotations, + }, + ], + }); + } + } else if (member.type === 'method_declaration') { + const annotations = annotationFacts(member); + if (!hasSpringDiRelevantAnnotation(annotations)) continue; + injectionSites.push({ + kind: 'method', + memberName: member.childForFieldName('name')?.text.trim() ?? '', + implicitConstructor: false, + annotations, + dependencies: dependenciesOf(member), + }); + } + } + + if (injectionSites.length === 0 && !hasSpringDiRelevantAnnotation(classAnnotations)) return null; + const classCapture = nodeToCapture('@spring-di.class', classNode); + return { + classScopeId: makeScopeId({ filePath, range: classCapture.range, kind: 'Class' }), + classAnnotations, + injectionSites, + }; +} + +/** Attach resolved, framework-private DI metadata to Class nodes. */ +export const attachJavaSpringDiMetadata = createSpringDiMetadataAttacher< + JavaAnnotationSyntaxFact, + JavaSpringInjectionSiteKind +>({ + getFacts: getJavaSpringDiFacts, + isPackageVisibilityIncomplete: isJavaPackageSiblingVisibilityIncomplete, + parseInjectionType: parseSpringInjectionType, + capturedMemberKind: 'field', +}); diff --git a/gitnexus/src/core/ingestion/languages/kotlin.ts b/gitnexus/src/core/ingestion/languages/kotlin.ts index be64475a0..18d4fd9c1 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin.ts @@ -185,10 +185,10 @@ export const kotlinProvider = defineLanguage({ emitScopeCaptures: emitKotlinScopeCaptures, // ── #2195 PDG layer: Kotlin CFG visitor (vendored grammar) ── cfgVisitor: createKotlinCfgVisitor(), - // Worker-side: snapshot companion-scope marks, package visibility, and - // class-annotation facts `emitKotlinScopeCaptures` just populated into plain - // data on `ParsedFile.captureSideChannel`, so the main thread can restore all - // three via `applyCaptureSideChannel` WITHOUT a re-parse (#1983). See + // Worker-side: snapshot companion-scope marks, package visibility, class + // annotations, and Spring DI facts `emitKotlinScopeCaptures` just populated + // into plain data on `ParsedFile.captureSideChannel`, so the main thread can + // restore them via `applyCaptureSideChannel` WITHOUT a re-parse (#1983). See // `kotlin/capture-side-channel.ts`. // `assertCloneable` is a runtime identity; it makes a future non-serializable // value in the side-channel payload a compile error here, at the source, rather diff --git a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts index 52bcec32b..14ea8f122 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts @@ -9,6 +9,8 @@ * from the `@scope.companion` marker capture. * - Spring Bean class-annotation facts collected during the same scope-query * traversal, consumed only after imports and package visibility finalize. + * - Spring DI class facts (constructor/property/method injection syntax), + * resolved and attached only after imports finalize. * - A JVM package fact read from the already-parsed root, so package-sibling * visibility never re-parses Kotlin source on the main thread. * @@ -29,7 +31,8 @@ * The single generic `ParsedFile.captureSideChannel` field is shared with C++, * which is safe because each file is one language (a `.kt` file uses the kotlin * provider, a `.cpp` file the cpp provider). The payload is self-describing - * (`{ kind: 'kotlin', companionScopes, packageFact, classAnnotations }`) so + * (`{ kind: 'kotlin', companionScopes, packageFact, classAnnotations, + * springDiFacts }`) so * `applyKotlinCaptureSideChannel` only restores kotlin state and ignores a * foreign-shaped snapshot. */ @@ -46,8 +49,10 @@ import { } from '../jvm/package-facts.js'; import { getCompanionScopesForFile, markCompanionScope } from './companion-scopes.js'; import { getKotlinPackageFact, setKotlinPackageFact } from './package-facts.js'; +import type { KotlinSpringDiClassFact } from './spring-di.js'; const classAnnotations = createClassAnnotationFactStore(); +const springDiFacts = new Map(); /** * Plain JSON-serializable snapshot of the per-file Kotlin capture-time @@ -63,10 +68,13 @@ export interface KotlinCaptureSideChannel { readonly packageFact: JvmPackageFact; /** Class annotation syntax collected by the existing scope traversal. */ readonly classAnnotations: readonly ClassAnnotationFact[]; + /** Constructor, property, and method injection syntax captured per class. */ + readonly springDiFacts?: readonly KotlinSpringDiClassFact[]; } export function clearKotlinClassAnnotationFacts(): void { classAnnotations.clear(); + springDiFacts.clear(); } export function setKotlinClassAnnotationFacts( @@ -80,6 +88,18 @@ export function getKotlinClassAnnotationFacts(filePath: string): readonly ClassA return classAnnotations.get(filePath); } +export function setKotlinSpringDiFacts( + filePath: string, + facts: readonly KotlinSpringDiClassFact[], +): void { + if (facts.length === 0) springDiFacts.delete(filePath); + else springDiFacts.set(filePath, facts); +} + +export function getKotlinSpringDiFacts(filePath: string): readonly KotlinSpringDiClassFact[] { + return springDiFacts.get(filePath) ?? []; +} + /** * `LanguageProvider.collectCaptureSideChannel` implementation for Kotlin. * Returns `undefined` when this file recorded no side-channel state at all, so @@ -90,8 +110,14 @@ export function collectKotlinCaptureSideChannel( ): KotlinCaptureSideChannel | undefined { const companionScopes = getCompanionScopesForFile(filePath); const annotationFacts = classAnnotations.get(filePath); + const diFacts = springDiFacts.get(filePath) ?? []; const packageFact = getKotlinPackageFact(filePath); - if (companionScopes.length === 0 && annotationFacts.length === 0 && packageFact === undefined) { + if ( + companionScopes.length === 0 && + annotationFacts.length === 0 && + diFacts.length === 0 && + packageFact === undefined + ) { return undefined; } return { @@ -99,6 +125,7 @@ export function collectKotlinCaptureSideChannel( companionScopes, packageFact: packageFact ?? UNKNOWN_JVM_PACKAGE_FACT, classAnnotations: annotationFacts, + ...(diFacts.length > 0 ? { springDiFacts: diFacts } : {}), }; } @@ -121,6 +148,7 @@ export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { !Array.isArray(data.classAnnotations) ) { classAnnotations.set(parsed.filePath, []); + setKotlinSpringDiFacts(parsed.filePath, []); setKotlinPackageFact(parsed.filePath, UNKNOWN_JVM_PACKAGE_FACT); return; } @@ -128,6 +156,10 @@ export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { markCompanionScope(parsed.filePath, scopeId); } classAnnotations.set(parsed.filePath, data.classAnnotations); + setKotlinSpringDiFacts( + parsed.filePath, + Array.isArray(data.springDiFacts) ? data.springDiFacts : [], + ); setKotlinPackageFact( parsed.filePath, isJvmPackageFact(data.packageFact) ? data.packageFact : UNKNOWN_JVM_PACKAGE_FACT, diff --git a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts index 408083780..afcbd703e 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts @@ -18,9 +18,10 @@ import { normalizeKotlinType } from './interpret.js'; import { synthesizeKotlinReceiverBinding } from './receiver-binding.js'; import { getKotlinParser, getKotlinScopeQuery } from './query.js'; import { markCompanionScope } from './companion-scopes.js'; -import { setKotlinClassAnnotationFacts } from './capture-side-channel.js'; +import { setKotlinClassAnnotationFacts, setKotlinSpringDiFacts } from './capture-side-channel.js'; import { captureKotlinPackageFact } from './package-facts.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; +import { captureKotlinSpringDiClassFact, type KotlinSpringDiClassFact } from './spring-di.js'; const FUNCTION_DECL_TAGS = ['@declaration.function'] as const; @@ -83,6 +84,8 @@ export function emitKotlinScopeCaptures( const out: CaptureMatch[] = []; const classAnnotations = new Map>(); + const springDiFacts: KotlinSpringDiClassFact[] = []; + const springDiClassNodeIds = new Set(); const returnTypes = collectKotlinReturnTypeTexts(tree.rootNode); out.push(...synthesizeKotlinLocalAssignmentBindings(tree.rootNode, returnTypes)); out.push(...synthesizeKotlinLoopBindings(tree.rootNode, returnTypes)); @@ -106,6 +109,13 @@ export function emitKotlinScopeCaptures( } if (Object.keys(grouped).length === 0) continue; + const springDiClassNode = nodeIfType(groupedNodes['@scope.class'], 'class_declaration'); + if (springDiClassNode !== null && !springDiClassNodeIds.has(springDiClassNode.id)) { + springDiClassNodeIds.add(springDiClassNode.id); + const fact = captureKotlinSpringDiClassFact(springDiClassNode, filePath); + if (fact !== null) springDiFacts.push(fact); + } + const annotatedClass = grouped['@class-annotation.class']; const annotationName = grouped['@class-annotation.name']; if (annotatedClass !== undefined && annotationName !== undefined) { @@ -288,6 +298,7 @@ export function emitKotlinScopeCaptures( } setKotlinClassAnnotationFacts(filePath, materializeClassAnnotationFacts(classAnnotations)); + setKotlinSpringDiFacts(filePath, springDiFacts); out.push(...synthesizeCallableFlowCaptures(tree.rootNode, KOTLIN_CALLABLE_CAPTURE_OPTIONS)); return out; } diff --git a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts index 0b0381bc2..7d3a78611 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts @@ -22,6 +22,7 @@ import { isKotlinStaticOnly } from './owners.js'; import { populateKotlinPackageSiblings } from './package-siblings.js'; import { attachKotlinSpringBeanCandidateMetadata } from './spring-bean-metadata.js'; import { clearKotlinPackageFacts } from './package-facts.js'; +import { attachKotlinSpringDiMetadata } from './spring-di.js'; /** * Kotlin scope resolver for RFC #909 Ring 3. @@ -120,9 +121,13 @@ export const kotlinScopeResolver: ScopeResolver = { propagatesReturnTypesAcrossImports: true, collapseMemberCallsByCallerTarget: false, hoistTypeBindingsToModule: true, + freeCallsRequireInstanceOwnership: true, postExtractSourceTextPolicy: 'uncached-files', populateNamespaceSiblings: populateKotlinPackageSiblings, - emitPostResolutionEdges: attachKotlinSpringBeanCandidateMetadata, + emitPostResolutionEdges: (graph, parsedFiles, nodeLookup, indexes) => { + attachKotlinSpringBeanCandidateMetadata(graph, parsedFiles, nodeLookup, indexes); + attachKotlinSpringDiMetadata(graph, parsedFiles, nodeLookup, indexes); + }, }; /** diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts new file mode 100644 index 000000000..3efc60522 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts @@ -0,0 +1,299 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { parseSpringInjectionType } from '../../di-extractors/spring.js'; +import { + createSpringDiMetadataAttacher, + hasSpringDiRelevantAnnotation, + hasSpringStereotypeSyntax, + type SpringDiAnnotationFact, + type SpringDiClassFact, + type SpringDiDependencyFact, + type SpringDiInjectionSiteFact, +} from '../../frameworks/spring/di-metadata.js'; +import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { getKotlinSpringDiFacts } from './capture-side-channel.js'; +import { isKotlinPackageSiblingVisibilityIncomplete } from './package-siblings.js'; + +export interface KotlinAnnotationSyntaxFact extends SpringDiAnnotationFact { + readonly useSiteTarget?: string; +} + +export type KotlinSpringDependencyFact = SpringDiDependencyFact; + +type KotlinSpringInjectionSiteKind = 'property' | 'constructor' | 'method'; + +export type KotlinSpringInjectionSiteFact = SpringDiInjectionSiteFact< + KotlinAnnotationSyntaxFact, + KotlinSpringInjectionSiteKind +>; + +export type KotlinSpringDiClassFact = SpringDiClassFact< + KotlinAnnotationSyntaxFact, + KotlinSpringInjectionSiteKind +>; + +const KOTLIN_TYPE_NODES = new Set(['user_type', 'nullable_type', 'function_type']); + +function firstDescendantOfType(node: SyntaxNode, type: string): SyntaxNode | undefined { + const stack = [...node.namedChildren].reverse(); + while (stack.length > 0) { + const current = stack.pop(); + if (current === undefined) continue; + if (current.type === type) return current; + for (let index = current.namedChildren.length - 1; index >= 0; index--) { + const child = current.namedChildren[index]; + if (child !== undefined) stack.push(child); + } + } + return undefined; +} + +function annotationFact(annotation: SyntaxNode): KotlinAnnotationSyntaxFact | null { + const nameNode = firstDescendantOfType(annotation, 'user_type'); + if (nameNode === undefined) return null; + const useSiteTarget = annotation.namedChildren + .find((child) => child.type === 'use_site_target') + ?.text.replace(/:\s*$/, '') + .trim(); + return { + name: nameNode.text.trim(), + text: annotation.text.trim(), + ...(useSiteTarget === undefined || useSiteTarget.length === 0 ? {} : { useSiteTarget }), + }; +} + +function annotationsFromModifierContainer(node: SyntaxNode): KotlinAnnotationSyntaxFact[] { + const facts: KotlinAnnotationSyntaxFact[] = []; + for (const child of node.namedChildren) { + if (child.type !== 'annotation') continue; + const fact = annotationFact(child); + if (fact !== null) facts.push(fact); + } + return facts; +} + +function annotationFacts(node: SyntaxNode): KotlinAnnotationSyntaxFact[] { + const facts: KotlinAnnotationSyntaxFact[] = []; + for (const child of node.namedChildren) { + if (child.type !== 'modifiers' && child.type !== 'parameter_modifiers') continue; + facts.push(...annotationsFromModifierContainer(child)); + } + return facts; +} + +function directTypeNode(node: SyntaxNode): SyntaxNode | undefined { + return node.namedChildren.find((child) => KOTLIN_TYPE_NODES.has(child.type)); +} + +function parameterDependency( + parameter: SyntaxNode, + precedingAnnotations: readonly KotlinAnnotationSyntaxFact[] = [], +): KotlinSpringDependencyFact | null { + const nameNode = parameter.namedChildren.find((child) => child.type === 'simple_identifier'); + const typeNode = directTypeNode(parameter); + if (nameNode === undefined || typeNode === undefined) return null; + return { + name: nameNode.text.trim(), + rawType: typeNode.text.trim(), + annotations: [...precedingAnnotations, ...annotationFacts(parameter)], + }; +} + +function functionDependencies(callable: SyntaxNode): KotlinSpringDependencyFact[] { + const parameters = callable.namedChildren.find( + (child) => child.type === 'function_value_parameters', + ); + if (parameters === undefined) return []; + const dependencies: KotlinSpringDependencyFact[] = []; + let pendingAnnotations: KotlinAnnotationSyntaxFact[] = []; + for (const child of parameters.namedChildren) { + if (child.type === 'parameter_modifiers') { + pendingAnnotations = annotationsFromModifierContainer(child); + continue; + } + if (child.type !== 'parameter') continue; + const dependency = parameterDependency(child, pendingAnnotations); + pendingAnnotations = []; + if (dependency !== null) dependencies.push(dependency); + } + return dependencies; +} + +function primaryConstructorDependencies(constructor: SyntaxNode): KotlinSpringDependencyFact[] { + const dependencies: KotlinSpringDependencyFact[] = []; + for (const parameter of constructor.namedChildren) { + if (parameter.type !== 'class_parameter') continue; + const dependency = parameterDependency(parameter); + if (dependency !== null) dependencies.push(dependency); + } + return dependencies; +} + +function propertyDependency(property: SyntaxNode): KotlinSpringDependencyFact | null { + const variable = property.namedChildren.find((child) => child.type === 'variable_declaration'); + if (variable === undefined) return null; + const nameNode = variable.namedChildren.find((child) => child.type === 'simple_identifier'); + const typeNode = directTypeNode(variable); + if (nameNode === undefined || typeNode === undefined) return null; + const annotations = annotationFacts(property); + return { + name: nameNode.text.trim(), + rawType: typeNode.text.trim(), + annotations, + }; +} + +function isKotlinBeanCandidateClass(classNode: SyntaxNode): boolean { + if (classNode.children.some((child) => child.type === 'interface' || child.type === 'enum')) { + return false; + } + const modifiers = classNode.namedChildren.find((child) => child.type === 'modifiers'); + return !modifiers?.namedChildren.some( + (child) => child.type === 'class_modifier' && child.text.trim() === 'annotation', + ); +} + +/** + * Capture one class already surfaced by Kotlin's scope query. Kotlin-specific + * syntax is normalized here while import/FQN semantics remain deferred until + * post-resolution. + */ +export function captureKotlinSpringDiClassFact( + classNode: SyntaxNode, + filePath: string, +): KotlinSpringDiClassFact | null { + if (!isKotlinBeanCandidateClass(classNode)) return null; + const classAnnotations = annotationFacts(classNode); + const injectionSites: KotlinSpringInjectionSiteFact[] = []; + const body = classNode.namedChildren.find((child) => child.type === 'class_body'); + const primaryConstructor = classNode.namedChildren.find( + (child) => child.type === 'primary_constructor', + ); + const secondaryConstructors = + body?.namedChildren.filter((child) => child.type === 'secondary_constructor') ?? []; + const constructorCount = + (primaryConstructor === undefined ? 0 : 1) + secondaryConstructors.length; + + if (primaryConstructor !== undefined) { + const annotations = annotationFacts(primaryConstructor); + const implicitConstructor = + constructorCount === 1 && + hasSpringStereotypeSyntax(classAnnotations) && + !hasSpringDiRelevantAnnotation(annotations); + if (implicitConstructor || hasSpringDiRelevantAnnotation(annotations)) { + injectionSites.push({ + kind: 'constructor', + memberName: '', + implicitConstructor, + annotations, + dependencies: primaryConstructorDependencies(primaryConstructor), + }); + } + } + + for (const constructor of secondaryConstructors) { + const annotations = annotationFacts(constructor); + const implicitConstructor = + constructorCount === 1 && + hasSpringStereotypeSyntax(classAnnotations) && + !hasSpringDiRelevantAnnotation(annotations); + if (!implicitConstructor && !hasSpringDiRelevantAnnotation(annotations)) continue; + injectionSites.push({ + kind: 'constructor', + memberName: '', + implicitConstructor, + annotations, + dependencies: functionDependencies(constructor), + }); + } + + if (body !== undefined) { + for (const member of body.namedChildren) { + if (member.type === 'property_declaration') { + const annotations = annotationFacts(member); + if (!hasSpringDiRelevantAnnotation(annotations)) continue; + const dependency = propertyDependency(member); + if (dependency === null) continue; + injectionSites.push({ + kind: 'property', + memberName: dependency.name, + implicitConstructor: false, + annotations, + dependencies: [dependency], + }); + } else if (member.type === 'function_declaration') { + const annotations = annotationFacts(member); + if (!hasSpringDiRelevantAnnotation(annotations)) continue; + const name = + member.namedChildren.find((child) => child.type === 'simple_identifier')?.text.trim() ?? + ''; + injectionSites.push({ + kind: 'method', + memberName: name, + implicitConstructor: false, + annotations, + dependencies: functionDependencies(member), + }); + } + } + } + + if (injectionSites.length === 0 && !hasSpringDiRelevantAnnotation(classAnnotations)) return null; + const classCapture = nodeToCapture('@spring-di.class', classNode); + return { + classScopeId: makeScopeId({ filePath, range: classCapture.range, kind: 'Class' }), + classAnnotations, + injectionSites, + }; +} + +function isApplicableInjectionAnnotation( + annotation: KotlinAnnotationSyntaxFact, + site: KotlinSpringInjectionSiteFact, +): boolean { + if (annotation.useSiteTarget === undefined) return true; + if (site.kind === 'constructor') return annotation.useSiteTarget === 'constructor'; + if (site.kind === 'property') { + return annotation.useSiteTarget === 'field' || annotation.useSiteTarget === 'set'; + } + return false; +} + +function isApplicableQualifierAnnotation( + annotation: KotlinAnnotationSyntaxFact, + site: KotlinSpringInjectionSiteFact, +): boolean { + if (annotation.useSiteTarget === undefined) return true; + if (site.kind === 'property') { + return ( + annotation.useSiteTarget === 'field' || + annotation.useSiteTarget === 'param' || + annotation.useSiteTarget === 'setparam' + ); + } + return annotation.useSiteTarget === 'param'; +} + +function parseKotlinSpringInjectionType(rawType: string) { + // Kotlin nullable suffixes, type projections, and mutable collection aliases + // do not change the JVM bean type selected by Spring. Normalize only those + // surface forms; stars, function types, arrays, and nested generic elements + // still fail closed in the shared parser. + const normalized = rawType + .replace(/\bMutable(List|Set|Collection|Map)(?=\s*<)/g, '$1') + .replace(/([<,])\s*(?:out|in)\s+/g, '$1') + .replace(/\?(?=\s*(?:[>,]|$))/g, ''); + return parseSpringInjectionType(normalized); +} + +/** Attach resolved, framework-private DI metadata to Kotlin Class nodes. */ +export const attachKotlinSpringDiMetadata = createSpringDiMetadataAttacher< + KotlinAnnotationSyntaxFact, + KotlinSpringInjectionSiteKind +>({ + getFacts: getKotlinSpringDiFacts, + isPackageVisibilityIncomplete: isKotlinPackageSiblingVisibilityIncomplete, + parseInjectionType: parseKotlinSpringInjectionType, + capturedMemberKind: 'property', + isInjectionAnnotationApplicable: isApplicableInjectionAnnotation, + isQualifierAnnotationApplicable: isApplicableQualifierAnnotation, +}); diff --git a/gitnexus/src/core/ingestion/languages/python/query.ts b/gitnexus/src/core/ingestion/languages/python/query.ts index 530e78af3..b1892ec20 100644 --- a/gitnexus/src/core/ingestion/languages/python/query.ts +++ b/gitnexus/src/core/ingestion/languages/python/query.ts @@ -22,6 +22,15 @@ const PYTHON_SCOPE_QUERY = ` (function_definition name: (identifier) @declaration.name) @declaration.function +;; Lambda bindings (\`f = lambda x: x\`). The \`@declaration.function\` anchor +;; sits on the INNER lambda so its range aligns with \`(lambda) @scope.function\` +;; above; otherwise the def is owned by the module scope and calls inside the +;; lambda lose caller attribution. Mirrors the TypeScript arrow patterns (#2687). +(expression_statement + (assignment + left: (identifier) @declaration.name + right: (lambda) @declaration.function)) + (assignment left: (identifier) @declaration.name) @declaration.variable diff --git a/gitnexus/src/core/ingestion/languages/rust/range-binding.ts b/gitnexus/src/core/ingestion/languages/rust/range-binding.ts index 4c7224333..593716cf9 100644 --- a/gitnexus/src/core/ingestion/languages/rust/range-binding.ts +++ b/gitnexus/src/core/ingestion/languages/rust/range-binding.ts @@ -5,6 +5,7 @@ import { getTreeSitterBufferSize } from '../../constants.js'; import { parseSourceSafe, ParseTimeoutError } from '../../../tree-sitter/safe-parse.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { logger } from '../../../logger.js'; +import { lookupBindingsAt } from '../../scope-resolution/scope/walkers.js'; /** * Populate type bindings for patterns and iterators that the tree-sitter @@ -16,9 +17,54 @@ import { logger } from '../../../logger.js'; * Runs in Phase 2 (after propagateImportedReturnTypes) so all cross-file * type bindings are available for lookup. */ +type RustTree = ReturnType['parse']>; + +/** + * Hold parsed trees for reuse across both prepass loops only when the whole + * Rust source fits this budget. Trees are much larger than their source, so a + * modest source cap keeps peak held-tree memory bounded; larger repos fall + * back to re-parsing per loop (unchanged RSS). + */ +const TREE_REUSE_SOURCE_BUDGET_BYTES = 16 * 1024 * 1024; + +/** + * Parse `filePath`'s source once, honoring the caller's `treeCache` and, when + * provided, an in-function `store` so the two prepass loops share a single + * parse instead of re-parsing every file. Returns null when the source is + * missing or parsing times out. + */ +function getOrParseTree( + parser: ReturnType, + filePath: string, + ctx: { + readonly fileContents: ReadonlyMap; + readonly treeCache?: { get(filePath: string): unknown }; + }, + store: Map | undefined, +): RustTree | null { + const cached = (ctx.treeCache?.get(filePath) ?? store?.get(filePath)) as RustTree | undefined; + if (cached !== undefined) return cached; + const sourceText = ctx.fileContents.get(filePath); + if (sourceText === undefined) return null; + let tree: RustTree; + try { + tree = parseSourceSafe(parser, sourceText, undefined, { + bufferSize: getTreeSitterBufferSize(sourceText), + }); + } catch (err) { + if (err instanceof ParseTimeoutError) { + logger.warn({ file: filePath }, 'rust range-binding: parse timed out, skipping file'); + return null; + } + throw err; + } + store?.set(filePath, tree); + return tree; +} + export function populateRustRangeBindings( parsedFiles: readonly ParsedFile[], - _indexes: ScopeResolutionIndexes, + indexes: ScopeResolutionIndexes, ctx: { readonly fileContents: ReadonlyMap; readonly treeCache?: { get(filePath: string): unknown }; @@ -26,45 +72,45 @@ export function populateRustRangeBindings( ): void { const parser = getRustParser(); const allReturnTypes = new Map(); + const ambiguousReturnTypes = new Set(); const allFieldTypes = new Map>(); + const ambiguousFieldTypes = new Set(); + // Per-defining-file, un-collapsed, FULL-generic return/field types. When a + // bare name is ambiguous (#2514) but the call site's `use` import pins a + // single definition, we resolve that definition's file here and read its + // untruncated type so a generic `Vec` element type survives (#2514 + // follow-up: import-disambiguated duplicates resolve like the compiler). + const returnTypeByFile = new Map>(); + const fieldTypeByFile = new Map>>(); + // Parse each file once and reuse across both loops when the workspace fits + // the byte budget; otherwise re-parse per loop to bound RSS (see helper). + let totalSourceBytes = 0; + for (const parsed of parsedFiles) { + totalSourceBytes += ctx.fileContents.get(parsed.filePath)?.length ?? 0; + } + const treeStore: Map | undefined = + totalSourceBytes <= TREE_REUSE_SOURCE_BUDGET_BYTES ? new Map() : undefined; for (const parsed of parsedFiles) { - const sourceText = ctx.fileContents.get(parsed.filePath); - if (sourceText === undefined) continue; - - const cachedTree = ctx.treeCache?.get(parsed.filePath) as - | ReturnType - | undefined; - let tree: ReturnType; - if (cachedTree !== undefined) { - tree = cachedTree; - } else { - try { - tree = parseSourceSafe(parser, sourceText, undefined, { - bufferSize: getTreeSitterBufferSize(sourceText), - }); - } catch (err) { - if (err instanceof ParseTimeoutError) { - logger.warn( - { file: parsed.filePath }, - 'rust range-binding: parse timed out, skipping file', - ); - continue; - } - throw err; - } - } + const tree = getOrParseTree(parser, parsed.filePath, ctx, treeStore); + if (tree === null) continue; for (const fn of tree.rootNode.descendantsOfType('function_item')) { const nameNode = fn.childForFieldName('name'); const retType = fn.childForFieldName('return_type'); if (nameNode !== null && retType !== null) { const name = nameNode.text; + // Ambiguity is a latch, not a toggle: once a name has two or more + // workspace definitions it stays ambiguous for the rest of the + // prepass, regardless of duplicate count or file order (#2514). if (allReturnTypes.has(name)) { allReturnTypes.delete(name); - } else { + ambiguousReturnTypes.add(name); + } else if (!ambiguousReturnTypes.has(name)) { allReturnTypes.set(name, retType.text); } + // Full-generic record per defining file for import-disambiguated lookup. + recordByFile(returnTypeByFile, parsed.filePath, name, retType.text); } } @@ -82,11 +128,16 @@ export function populateRustRangeBindings( } if (fields.size > 0) { const name = nameNode.text; + // Same ambiguity latch as return types (#2514): a third same-named + // struct must not restore a resolvable global field map. if (allFieldTypes.has(name)) { allFieldTypes.delete(name); - } else { + ambiguousFieldTypes.add(name); + } else if (!ambiguousFieldTypes.has(name)) { allFieldTypes.set(name, fields); } + // Full-generic record per defining file for import-disambiguated lookup. + recordByFile(fieldTypeByFile, parsed.filePath, name, fields); } } @@ -99,39 +150,32 @@ export function populateRustRangeBindings( } for (const parsed of parsedFiles) { - const sourceText = ctx.fileContents.get(parsed.filePath); - if (sourceText === undefined) continue; - - const cachedTree = ctx.treeCache?.get(parsed.filePath) as - | ReturnType - | undefined; - let tree: ReturnType; - if (cachedTree !== undefined) { - tree = cachedTree; - } else { - try { - tree = parseSourceSafe(parser, sourceText, undefined, { - bufferSize: getTreeSitterBufferSize(sourceText), - }); - } catch (err) { - if (err instanceof ParseTimeoutError) { - logger.warn( - { file: parsed.filePath }, - 'rust range-binding: parse timed out, skipping file', - ); - continue; - } - throw err; - } - } + const tree = getOrParseTree(parser, parsed.filePath, ctx, treeStore); + if (tree === null) continue; const scopeMap = new Map(parsed.scopes.map((s) => [s.id, s])); const moduleScope = parsed.scopes.find((s) => s.kind === 'Module'); if (moduleScope === undefined) continue; - processForLoops(tree.rootNode, parsed, scopeMap, moduleScope, allReturnTypes); + processForLoops( + tree.rootNode, + parsed, + scopeMap, + moduleScope, + allReturnTypes, + indexes, + returnTypeByFile, + ); processPatternBindings(tree.rootNode, parsed, scopeMap, moduleScope); - processStructDestructuring(tree.rootNode, parsed, scopeMap, moduleScope, allFieldTypes); + processStructDestructuring( + tree.rootNode, + parsed, + scopeMap, + moduleScope, + allFieldTypes, + indexes, + fieldTypeByFile, + ); processPendingAssignments( tree.rootNode, parsed, @@ -196,12 +240,88 @@ function normalizeFieldType(text: string): string { return t.trim(); } +/** Get-or-create the inner map for `file` and record `name -> value`. */ +function recordByFile( + byFile: Map>, + file: string, + name: string, + value: V, +): void { + let inner = byFile.get(file); + if (inner === undefined) { + inner = new Map(); + byFile.set(file, inner); + } + inner.set(name, value); +} + +/** Final segment of a dot-joined qualified name (`a.make` -> `make`), or the + * bare name when the def carries no qualifier. */ +function simpleName(qualifiedName: string | undefined, bareName: string): string { + if (qualifiedName === undefined) return bareName; + const dot = qualifiedName.lastIndexOf('.'); + return dot === -1 ? qualifiedName : qualifiedName.slice(dot + 1); +} + +/** Distinct `(file, name)` definitions, in first-seen order. */ +function uniqueDefs( + defs: readonly { file: string; name: string }[], +): { file: string; name: string }[] { + const seen = new Set(); + const out: { file: string; name: string }[] = []; + for (const d of defs) { + const key = `${d.file} ${d.name}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(d); + } + return out; +} + +/** + * Resolve `name` at `moduleScope` to the value recorded in `byFile` for the one + * definition visible here, or null when zero or several are visible (which + * keeps the #2514 ambiguity latch). Mirrors Rust name resolution: explicit + * `use`/re-export imports and local defs shadow `use x::*` globs, so a glob is + * consulted only when no explicit binding names `name`, and even then only when + * exactly one glob-target file actually defines it. + */ +function resolveImportedDef( + name: string, + moduleScope: Scope, + indexes: ScopeResolutionIndexes, + byFile: ReadonlyMap>, +): V | null { + const explicit = uniqueDefs( + lookupBindingsAt(moduleScope.id, name, indexes) + .filter((r) => r.origin === 'import' || r.origin === 'reexport' || r.origin === 'local') + .map((r) => ({ file: r.def.filePath, name: simpleName(r.def.qualifiedName, name) })), + ); + const defs = + explicit.length > 0 + ? explicit + : uniqueDefs( + (indexes.imports.get(moduleScope.id) ?? []) + .filter( + (e) => + e.kind === 'wildcard-expanded' && + e.targetFile !== null && + byFile.get(e.targetFile)?.has(name) === true, + ) + .map((e) => ({ file: e.targetFile as string, name })), + ); + if (defs.length !== 1) return null; + return byFile.get(defs[0].file)?.get(defs[0].name) ?? null; +} + function processForLoops( root: SyntaxNode, parsed: ParsedFile, scopeMap: ReadonlyMap, moduleScope: Scope, allReturnTypes: ReadonlyMap, + indexes: ScopeResolutionIndexes, + returnTypeByFile: ReadonlyMap>, ): void { for (const forNode of root.descendantsOfType('for_expression')) { const patternNode = forNode.childForFieldName('pattern'); @@ -217,6 +337,8 @@ function processForLoops( scopeMap, moduleScope, allReturnTypes, + indexes, + returnTypeByFile, ); if (elementType === null) continue; @@ -331,7 +453,9 @@ function processStructDestructuring( parsed: ParsedFile, scopeMap: ReadonlyMap, moduleScope: Scope, - allFieldTypes?: ReadonlyMap>, + allFieldTypes: ReadonlyMap>, + indexes: ScopeResolutionIndexes, + fieldTypeByFile: ReadonlyMap>>, ): void { for (const letNode of root.descendantsOfType('let_declaration')) { const patternNode = letNode.childForFieldName('pattern'); @@ -356,7 +480,13 @@ function processStructDestructuring( let fieldType = lookupFieldType(typeName, fieldName, parsed, scopeMap, moduleScope); if (fieldType === null) { - fieldType = allFieldTypes?.get(typeName)?.get(fieldName) ?? null; + fieldType = allFieldTypes.get(typeName)?.get(fieldName) ?? null; + } + if (fieldType === null) { + // Import-disambiguated duplicate struct (#2514 follow-up): the global + // field map is ambiguous, but a `use` import pins one definition. + const fields = resolveImportedDef(typeName, moduleScope, indexes, fieldTypeByFile); + fieldType = fields?.get(fieldName) ?? null; } if (fieldType !== null) { injectTypeBinding(targetScope, fieldName, fieldType); @@ -481,7 +611,9 @@ function resolveIterableElementType( parsed: ParsedFile, scopeMap: ReadonlyMap, moduleScope: Scope, - allReturnTypes?: ReadonlyMap, + allReturnTypes: ReadonlyMap, + indexes: ScopeResolutionIndexes, + returnTypeByFile: ReadonlyMap>, ): string | null { let iterableNode = valueNode; if (iterableNode.type === 'reference_expression') { @@ -506,10 +638,16 @@ function resolveIterableElementType( } if (func.type === 'identifier') { - const crossFileReturn = allReturnTypes?.get(func.text); + const crossFileReturn = allReturnTypes.get(func.text); if (crossFileReturn !== undefined) return unwrapGeneric(crossFileReturn); const rawReturn = lookupRawFunctionReturnType(func.text, valueNode); if (rawReturn !== null) return unwrapGeneric(rawReturn); + // Import-disambiguated duplicate: the bare-name map is ambiguous (#2514) + // but a `use` import pins one definition. Read its FULL return type + // here, BEFORE the scope-binding lookup below, because that binding is + // generic-truncated (`Vec` becomes `Vec`), losing the element. + const importedReturn = resolveImportedDef(func.text, moduleScope, indexes, returnTypeByFile); + if (importedReturn !== null) return unwrapGeneric(importedReturn); const returnType = lookupReturnTypeInScopes(func.text, parsed, scopeMap, moduleScope); if (returnType !== null) return unwrapGeneric(returnType); } diff --git a/gitnexus/src/core/ingestion/local-symbol-pruner.ts b/gitnexus/src/core/ingestion/local-symbol-pruner.ts index 3ff876b44..24e9733ff 100644 --- a/gitnexus/src/core/ingestion/local-symbol-pruner.ts +++ b/gitnexus/src/core/ingestion/local-symbol-pruner.ts @@ -1,4 +1,4 @@ -import type { GraphNode, GraphRelationship, NodeLabel } from 'gitnexus-shared'; +import type { GraphNode, NodeLabel, RelationshipType } from 'gitnexus-shared'; import type { KnowledgeGraph } from '../graph/types.js'; import { parseTruthyEnv } from './utils/env.js'; @@ -30,9 +30,13 @@ const isLocalValueCandidate = (node: GraphNode): boolean => { // True when `rel` is the structural `File -> DEFINES -> candidate` edge. Callers // guard on the candidate already being the edge target, so only the source label // needs checking here. -const isFileDefinesEdge = (graph: KnowledgeGraph, rel: GraphRelationship): boolean => { - if (rel.type !== 'DEFINES') return false; - return graph.getNode(rel.sourceId)?.label === 'File'; +const isFileDefinesEdge = ( + graph: KnowledgeGraph, + type: RelationshipType, + sourceId: string, +): boolean => { + if (type !== 'DEFINES') return false; + return graph.getNode(sourceId)?.label === 'File'; }; export const pruneLocalValueSymbols = ( @@ -51,21 +55,21 @@ export const pruneLocalValueSymbols = ( if (candidateIds.size === 0) return emptyStats(false); const candidatesWithSemanticEdges = new Set(); - for (const rel of graph.iterRelationships()) { + // Field-wise scan (#2680): a whole-graph walk that reads only these three, so + // materializing a relationship object per edge would be pure overhead. + graph.forEachRelationshipFields((sourceId, targetId, type) => { // Any outgoing edge from a candidate is a semantic edge: the only structural // edge a block-local value symbol carries is the incoming File -> DEFINES, on // which the candidate is the target, never the source. - if (candidateIds.has(rel.sourceId)) { - candidatesWithSemanticEdges.add(rel.sourceId); + if (candidateIds.has(sourceId)) { + candidatesWithSemanticEdges.add(sourceId); } // An incoming edge is semantic unless it is the structural File -> DEFINES. - if (candidateIds.has(rel.targetId)) { - if (!isFileDefinesEdge(graph, rel)) { - candidatesWithSemanticEdges.add(rel.targetId); - } + if (candidateIds.has(targetId) && !isFileDefinesEdge(graph, type, sourceId)) { + candidatesWithSemanticEdges.add(targetId); } - } + }); let prunedNodes = 0; for (const candidateId of candidateIds) { diff --git a/gitnexus/src/core/ingestion/pipeline-phases/di.ts b/gitnexus/src/core/ingestion/pipeline-phases/di.ts index 784e65bc4..1a8f789fc 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/di.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/di.ts @@ -1,91 +1,91 @@ /** * Phase: di * - * Framework-neutral dependency-injection resolution. Routes `Property` nodes - * by `properties.language` to the per-language field matchers registered in - * `di-extractors/` (`DI_MATCHERS` — same registry seam shape as - * `SCOPE_RESOLVERS`), then fans each match out to `INJECTS` edges from the - * consumer Class node to every Class implementing the matched element - * interface. - * - * This file names NO language or framework: which fields count as - * container-injected — and why — is entirely the registered matcher's - * business (see `di-extractors/` for the matchers and their semantics, - * including deliberate annotation exclusions). The matcher also supplies the - * human-readable edge `reason`, so framework specifics stay in the payload, - * never in this phase. - * - * The resolution uses ONLY graph data — Property nodes, `HAS_PROPERTY` edges, - * `IMPLEMENTS` edges, and Interface nodes. No filesystem access is performed: - * the structural information was already extracted by earlier parse / - * structure phases. - * - * Interface resolution is scoped to the CANDIDATE'S OWN language and prefers - * qualified names: a dotted element type resolves via the language's - * `qualifiedName` index; a bare simple name resolves only while unique within - * that language. Ambiguous names — simple OR qualified (a qualifiedName has - * no file-path component, so the same package+name duplicated across monorepo - * modules collides too) — fail CLOSED — no edge, never - * last-writer-wins — but observably: skips are counted in the phase output's - * `ambiguousSkipped` and named in an isDev debug log, so "no DI fields" is - * distinguishable from "all candidates ambiguous". Same-package/import-aware - * disambiguation is a documented follow-up (see the plan's Deferred work). + * Framework-neutral dependency-injection resolution. Per-language resolvers + * identify injection sites and provider metadata; this phase performs only + * graph-level type/heritage resolution and emits Class -> Class INJECTS edges. * * @deps mro - * @reads graph (Property nodes, HAS_PROPERTY edges, IMPLEMENTS edges, Interface nodes) + * @reads graph (Class/Interface/member nodes and heritage/ownership edges) * @writes graph (INJECTS edges) */ -import type { SupportedLanguages } from 'gitnexus-shared'; +import type { GraphNode, SupportedLanguages } from 'gitnexus-shared'; import type { PipelinePhase, PipelineContext } from './types.js'; -import { DI_MATCHERS, isSupportedLanguage } from '../di-extractors/index.js'; +import { + DI_RESOLVERS, + isSupportedLanguage, + type DiInjectionMatch, + type DiProviderMatch, +} from '../di-extractors/index.js'; import { isDev } from '../utils/env.js'; import { logger } from '../../logger.js'; export interface DIOutput { injectsEdges: number; + /** Kept for output compatibility; now counts every matched injection site. */ fieldsScanned: number; - /** Candidates skipped because their element type name — bare simple name - * or dotted qualified name — matched more than one Interface within the - * candidate's language (fail-closed). */ + /** Sites skipped because the requested type name itself was ambiguous. */ ambiguousSkipped: number; + /** Single-valued sites represented by multiple low-confidence candidates. */ + ambiguousInjections: number; } -/** Sentinel marking an interface name (simple or qualified) claimed by more - * than one Interface node within a language — resolution must fail closed. */ const AMBIGUOUS: unique symbol = Symbol('ambiguous'); -/** Per-language interface lookup: qualified names resolve exactly; bare - * simple names resolve only while unique within the language. Both indexes - * fail closed on their own duplicates. */ -interface InterfaceIndex { - /** `properties.qualifiedName` → Interface node id (when extracted — e.g. - * package-qualified for languages with a file-scope package declaration), - * or {@link AMBIGUOUS} once a second Interface claims the same qualified - * name in the same language — realistic in monorepos, where the same - * package+name is duplicated across modules or main/test source roots - * (a qualifiedName carries no file-path component). */ +interface NameIndex { byQualifiedName: Map; - /** `properties.name` → Interface node id, or {@link AMBIGUOUS} once a - * second same-name Interface appears in the same language. */ bySimpleName: Map; } -/** A Property node a registered matcher accepted as a DI fan-out candidate. */ -interface CandidateField { - propertyId: string; - /** The candidate's language — interface resolution (Pass 3) looks up ONLY - * this language's interface index. */ +interface CandidateSite extends DiInjectionMatch { + siteNodeId: string; language: SupportedLanguages; - elementTypeName: string; - /** Matcher-supplied edge reason (carries the framework specifics). */ +} + +interface PendingEdge { + sourceId: string; + targetId: string; + confidence: number; reason: string; } +function emptyNameIndex(): NameIndex { + return { byQualifiedName: new Map(), bySimpleName: new Map() }; +} + +function addIndexedName(index: NameIndex, node: GraphNode): void { + const qualifiedName = node.properties.qualifiedName; + if (typeof qualifiedName === 'string') { + index.byQualifiedName.set( + qualifiedName, + index.byQualifiedName.has(qualifiedName) ? AMBIGUOUS : node.id, + ); + } + const simpleName = node.properties.name; + index.bySimpleName.set(simpleName, index.bySimpleName.has(simpleName) ? AMBIGUOUS : node.id); +} + +function resolveIndexedName(index: NameIndex | undefined, name: string) { + if (index === undefined) return undefined; + return name.includes('.') ? index.byQualifiedName.get(name) : index.bySimpleName.get(name); +} + +function providerCandidates( + ids: ReadonlySet, + providers: ReadonlyMap, +): string[] { + const all = [...ids]; + const recognized = all.filter((id) => providers.has(id)); + // Recall-first fallback: provider metadata can be incomplete (custom + // registration mechanisms and legacy indexes can omit it). Prefer + // framework-recognized providers when present, but keep structurally valid + // candidates when none are known instead of dropping the injection entirely. + return recognized.length > 0 ? recognized : all; +} + export const diPhase: PipelinePhase = { name: 'di', - // Depends on `mro` for ordering: heritage edges (IMPLEMENTS/EXTENDS) must be - // fully populated before we resolve interface→implementer fan-out. deps: ['mro'], async execute(ctx: PipelineContext): Promise { @@ -96,174 +96,193 @@ export const diPhase: PipelinePhase = { stats: { filesProcessed: 0, totalFiles: 0, nodesCreated: ctx.graph.nodeCount }, }); - // ── Pass 1: route Property nodes to registered per-language matchers ─── - // Early-exit optimization: if no registered matcher accepts any Property - // node, skip all index construction. This makes the phase a no-op on - // repos with no DI-matched fields (no IMPLEMENTS / HAS_PROPERTY scans). - const candidates: CandidateField[] = []; - + const candidates: CandidateSite[] = []; + const providers = new Map(); ctx.graph.forEachNode((node) => { - if (node.label !== 'Property') return; const language = node.properties.language; if (language === undefined || !isSupportedLanguage(language)) return; - const matcher = DI_MATCHERS.get(language); - if (matcher === undefined) return; - const match = matcher(node); - if (match === null) return; - candidates.push({ - propertyId: node.id, - language, - elementTypeName: match.elementTypeName, - reason: match.reason, - }); + const resolver = DI_RESOLVERS.get(language); + if (resolver === undefined) return; + + const provider = resolver.matchProvider(node); + if (provider !== null) providers.set(node.id, provider); + for (const match of resolver.matchInjectionSites(node)) { + candidates.push({ ...match, siteNodeId: node.id, language }); + } }); if (candidates.length === 0) { - return { injectsEdges: 0, fieldsScanned: 0, ambiguousSkipped: 0 }; + return { + injectsEdges: 0, + fieldsScanned: 0, + ambiguousSkipped: 0, + ambiguousInjections: 0, + }; } - // ── Pass 2: build single-pass reverse indexes ───────────────────────── - - // interfaceNodeId → Set (reverse of IMPLEMENTS edge) - // IMPLEMENTS edges go Class→Interface, so target is the interface. - // Keyed by node id — globally unique — so this index needs no language - // scoping; only NAME-based lookups (below) do. const interfaceToImplementers = new Map>(); for (const rel of ctx.graph.iterRelationshipsByType('IMPLEMENTS')) { - const implementerId = rel.sourceId; // Class - const interfaceId = rel.targetId; // Interface - let set = interfaceToImplementers.get(interfaceId); - if (set === undefined) { - set = new Set(); - interfaceToImplementers.set(interfaceId, set); + const set = interfaceToImplementers.get(rel.targetId) ?? new Set(); + set.add(rel.sourceId); + interfaceToImplementers.set(rel.targetId, set); + } + + const memberToClass = new Map(); + for (const relationType of ['HAS_PROPERTY', 'HAS_METHOD'] as const) { + for (const rel of ctx.graph.iterRelationshipsByType(relationType)) { + memberToClass.set(rel.targetId, rel.sourceId); } - set.add(implementerId); } - // propertyNodeId → consumerClassId (reverse of HAS_PROPERTY edge) - // HAS_PROPERTY edges go Class→Property, so target is the property. - const propertyToClass = new Map(); - for (const rel of ctx.graph.iterRelationshipsByType('HAS_PROPERTY')) { - propertyToClass.set(rel.targetId, rel.sourceId); - } - - // language → InterfaceIndex (from Interface-labeled nodes). Scoped per - // language so an Interface in one language can never satisfy a candidate - // from another. Within a language, a name resolves only while unique — - // a second Interface claiming the same simple OR qualified name flips - // that entry to AMBIGUOUS and resolution fails closed (never - // last-writer-wins). - // Index only languages that can resolve: an Interface in a language with - // no candidate can never be looked up in Pass 3. - const candidateLanguages = new Set(candidates.map((c) => c.language)); - const interfacesByLanguage = new Map(); + const candidateLanguages = new Set(candidates.map((candidate) => candidate.language)); + const interfacesByLanguage = new Map(); + const classesByLanguage = new Map(); + const classNodes = new Map(); ctx.graph.forEachNode((node) => { - if (node.label !== 'Interface') return; + if (node.label !== 'Class' && node.label !== 'Interface') return; const language = node.properties.language; - if (typeof language !== 'string') return; // no language ⇒ unindexable - if (!candidateLanguages.has(language)) return; - let index = interfacesByLanguage.get(language); - if (index === undefined) { - index = { byQualifiedName: new Map(), bySimpleName: new Map() }; - interfacesByLanguage.set(language, index); - } - // `qualifiedName` reaches NodeProperties through the extensible index - // signature, so narrow it explicitly (no `any`). - const qualifiedName = node.properties.qualifiedName; - if (typeof qualifiedName === 'string') { - index.byQualifiedName.set( - qualifiedName, - index.byQualifiedName.has(qualifiedName) ? AMBIGUOUS : node.id, - ); - } - const simpleName = node.properties.name; - index.bySimpleName.set(simpleName, index.bySimpleName.has(simpleName) ? AMBIGUOUS : node.id); + if (typeof language !== 'string' || !candidateLanguages.has(language)) return; + const indexes = node.label === 'Class' ? classesByLanguage : interfacesByLanguage; + const index = indexes.get(language) ?? emptyNameIndex(); + addIndexedName(index, node); + indexes.set(language, index); + if (node.label === 'Class') classNodes.set(node.id, node); }); - // ── Pass 3: emit INJECTS edges ──────────────────────────────────────── - let injectsEdges = 0; let ambiguousSkipped = 0; - const ambiguousElementTypes = new Set(); - const seenEdges = new Set(); + let ambiguousInjections = 0; + const ambiguousTypeNames = new Set(); + const pending = new Map(); + + const queueEdge = (edge: PendingEdge): void => { + if (edge.sourceId === edge.targetId) return; + const id = `INJECTS:${edge.sourceId}->${edge.targetId}`; + const existing = pending.get(id); + if (existing === undefined || edge.confidence > existing.confidence) pending.set(id, edge); + }; for (const candidate of candidates) { - // Resolve the consumer Class that owns this Property. - const consumerClassId = propertyToClass.get(candidate.propertyId); - if (!consumerClassId) continue; + const siteNode = ctx.graph.getNode(candidate.siteNodeId); + const consumerClassId = + siteNode?.label === 'Class' ? siteNode.id : memberToClass.get(candidate.siteNodeId); + if (consumerClassId === undefined) continue; - // Resolve the element type name via the CANDIDATE'S OWN language index - // only — a same-named Interface in another language never participates. - const index = interfacesByLanguage.get(candidate.language); - if (index === undefined) continue; - - // A dotted element type is a qualified name (e.g. `com.a.Shape`) — - // exact qualifiedName lookup, unaffected by simple-name ambiguity. - // A bare name uses the simple-name index. BOTH lookups fail CLOSED - // on their own ambiguity (a qualified name too can be claimed twice — - // same package+name across monorepo modules): no edge (never - // last-writer-wins), but counted and logged so the skip is - // observable. Same-package/import-aware disambiguation is a - // deliberate follow-up (plan: Deferred work). - let interfaceId: string | undefined; - if (candidate.elementTypeName.includes('.')) { - const entry = index.byQualifiedName.get(candidate.elementTypeName); - if (entry === AMBIGUOUS) { - ambiguousSkipped++; - ambiguousElementTypes.add(candidate.elementTypeName); - continue; - } - interfaceId = entry; - } else { - const entry = index.bySimpleName.get(candidate.elementTypeName); - if (entry === AMBIGUOUS) { - ambiguousSkipped++; - ambiguousElementTypes.add(candidate.elementTypeName); - continue; - } - interfaceId = entry; + const classEntry = resolveIndexedName( + classesByLanguage.get(candidate.language), + candidate.targetTypeName, + ); + const interfaceEntry = resolveIndexedName( + interfacesByLanguage.get(candidate.language), + candidate.targetTypeName, + ); + if ( + classEntry === AMBIGUOUS || + interfaceEntry === AMBIGUOUS || + (classEntry !== undefined && interfaceEntry !== undefined) + ) { + // A simple/qualified name claimed by both a Class and an Interface is + // type-ambiguous too. Fail closed rather than guessing which Java type + // the injection site meant; import-aware disambiguation is not + // available in this graph-only phase. This intentionally applies to + // legacy collection sites too: a Class/Interface collision no longer + // fans out through the interface on a simple-name guess. + ambiguousSkipped++; + ambiguousTypeNames.add(candidate.targetTypeName); + continue; } - if (interfaceId === undefined) continue; - // Fan out to every class implementing that interface. - const implementers = interfaceToImplementers.get(interfaceId); - if (!implementers) continue; + const structural = new Set(); + if (typeof classEntry === 'string') structural.add(classEntry); + if (typeof interfaceEntry === 'string') { + for (const id of interfaceToImplementers.get(interfaceEntry) ?? []) structural.add(id); + } + structural.delete(consumerClassId); + if (structural.size === 0) continue; - for (const implId of implementers) { - // Skip self-edges: a class never injects its own bean into itself. - if (implId === consumerClassId) continue; + let viable = providerCandidates(structural, providers); + const namedSelection = candidate.namedSelection; + if (namedSelection !== undefined) { + viable = viable.filter( + (id) => providers.get(id)?.names.includes(namedSelection.name) === true, + ); + if (viable.length === 0) continue; + } - // Dedup-safe edge ID: deterministic from (consumer, implementer). - const edgeId = `INJECTS:${consumerClassId}->${implId}`; - if (seenEdges.has(edgeId)) continue; - seenEdges.add(edgeId); + if (candidate.cardinality === 'collection') { + const confidence = namedSelection === undefined ? 0.8 : 0.9; + const suffix = namedSelection === undefined ? '' : `; ${namedSelection.reason}`; + for (const targetId of viable) { + queueEdge({ + sourceId: consumerClassId, + targetId, + confidence, + reason: candidate.reason + suffix, + }); + } + continue; + } - ctx.graph.addRelationship({ - id: edgeId, + if (viable.length === 1) { + const suffix = namedSelection === undefined ? '' : `; ${namedSelection.reason}`; + queueEdge({ sourceId: consumerClassId, - targetId: implId, - type: 'INJECTS', - confidence: 0.8, - // Matcher-supplied reason — names the framework and the annotation - // actually found on the field (see di-extractors/). - reason: candidate.reason, + targetId: viable[0], + confidence: namedSelection === undefined ? 0.9 : 0.95, + reason: candidate.reason + suffix, }); - injectsEdges++; + continue; } + + const preferred = viable.flatMap((id) => { + const reason = providers.get(id)?.preferenceReason; + return reason === undefined ? [] : [{ id, reason }]; + }); + if (namedSelection === undefined && preferred.length === 1) { + const selected = preferred[0]; + queueEdge({ + sourceId: consumerClassId, + targetId: selected.id, + confidence: 0.95, + reason: `${candidate.reason}; ${selected.reason}`, + }); + continue; + } + + ambiguousInjections++; + const candidateNames = viable + .map((id) => classNodes.get(id)?.properties.name ?? id) + .sort() + .join(', '); + for (const targetId of viable) { + queueEdge({ + sourceId: consumerClassId, + targetId, + confidence: 0.5, + reason: `${candidate.reason}; ambiguous candidates: ${candidateNames}`, + }); + } + } + + for (const [id, edge] of pending) { + ctx.graph.addRelationship({ id, type: 'INJECTS', ...edge }); } if (isDev && ambiguousSkipped > 0) { - // One aggregated debug line (not per-candidate spam): duplicate simple - // names are NORMAL in large repos, but the skip must stay observable. logger.debug( - `🧩 DI: ${ambiguousSkipped} candidate(s) skipped — ambiguous element interface name(s): ${[...ambiguousElementTypes].sort().join(', ')}`, + `DI: ${ambiguousSkipped} site(s) skipped because requested type names were ambiguous: ${[...ambiguousTypeNames].sort().join(', ')}`, ); } - if (isDev && (injectsEdges > 0 || ambiguousSkipped > 0)) { + if (isDev && (pending.size > 0 || ambiguousInjections > 0)) { logger.info( - `🧩 DI: ${injectsEdges} INJECTS edges from ${candidates.length} injection-annotated collection fields (${ambiguousSkipped} ambiguous skipped)`, + `DI: ${pending.size} INJECTS edges from ${candidates.length} injection sites (${ambiguousInjections} ambiguous single-site resolutions)`, ); } - return { injectsEdges, fieldsScanned: candidates.length, ambiguousSkipped }; + return { + injectsEdges: pending.size, + fieldsScanned: candidates.length, + ambiguousSkipped, + ambiguousInjections, + }; }, }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index a9d601f11..55611848a 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -95,7 +95,9 @@ import { import type { KnowledgeGraph } from '../../graph/types.js'; import type { PipelineOptions } from '../pipeline.js'; import fs from 'node:fs'; +import { effectiveRamBytes, memoryAutopilotDisabled } from '../utils/effective-ram.js'; import path from 'node:path'; +import v8 from 'node:v8'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { isDev } from '../utils/env.js'; @@ -111,6 +113,81 @@ import { isDebugHeapEnabled, logHeapProbe } from '../utils/heap-probe.js'; import { logger } from '../../logger.js'; // ── Constants ────────────────────────────────────────────────────────────── +/** + * Heap-scale guardrail constants (#2649). Measured on a Linux-kernel analyze: + * ~75 graph nodes per PARSEABLE file (~5M nodes / ~65k parseable files; + * validated against heap probes at chunks 25/50/75 of 113 — the first + * calibration divided by total scanned files and under-projected by ~30%), + * main-thread heap per node. C-heavy corpus; other language mixes vary — these + * feed a WARNING and an emergency abort, never a hard admission gate, so + * estimate error only shifts when the operator hears about the problem, not + * whether analyze runs. + * + * RECALIBRATED for streamed structural emit (#2680), which is on by default for + * full rebuilds and holds relationships out of the JS heap. The original 2250 + * was measured against the object-based graph; an A/B at 400k nodes / 1.08M + * edges put streaming at 1.40x smaller (819 MB -> 584 MB), so the corpus- + * calibrated figure is divided by that ratio: 2250 / 1.40 ~= 1600. Scaling the + * measured constant rather than substituting a synthetic one keeps #2649's + * kernel calibration intact and changes only the one thing that actually moved. + * + * If streaming is disabled (GITNEXUS_STREAM_GRAPH_EMIT=0, or any non-force run) + * this UNDER-projects by ~40%, so the preflight warning may stay quiet on a repo + * that then struggles. That is the safe direction to be wrong in: the abort + * below reads LIVE heap use, not this projection, so it still catches the real + * condition — only the early warning is affected. + */ +const PROJECTED_NODES_PER_FILE = 75; +const PROJECTED_HEAP_BYTES_PER_NODE = 1600; +/** Warn at scan end when the projection crosses this share of the heap limit. */ +const PREFLIGHT_WARN_FRACTION = 0.85; +/** + * Abort the chunk loop when live heap use crosses this share of the limit. + * Above ~0.95 V8 enters the ineffective-mark-compact death spiral (2s+ GC + * pauses that also falsely idle-timeout healthy workers, #2649); 0.92 leaves + * one chunk's worth of headroom to fail with an actionable message instead. + * `GITNEXUS_MEMORY=off` declines the abort (proceed-at-own-risk). + */ +const HEAP_ABORT_FRACTION = 0.92; + +/** Projected main-thread heap need for the parse phase (#2649). */ +export function projectParseHeapNeedBytes(parseableFileCount: number): number { + return parseableFileCount * PROJECTED_NODES_PER_FILE * PROJECTED_HEAP_BYTES_PER_NODE; +} + +/** True when the mid-loop heap guard should abort the parse (#2649). */ +export function shouldAbortForHeapPressure(heapUsedBytes: number, heapLimitBytes: number): boolean { + if (memoryAutopilotDisabled()) return false; + return heapUsedBytes > heapLimitBytes * HEAP_ABORT_FRACTION; +} + +/** + * The ONE action a user should take when this repository doesn't fit the + * current heap (#2649). Users hitting memory limits are already frustrated — + * a menu of env knobs at that moment is noise. Branch on whether the machine + * itself has more memory to give: if this process's limit sits well below + * what the RAM-aware auto-sizer would grant (an inherited NODE_OPTIONS pin or + * explicit flag), the fix is to drop the pin — gitnexus sizes itself. + * Otherwise the machine is the ceiling and only scope or hardware helps. + * Escape hatches (GITNEXUS_MEMORY etc.) stay in the README env table. + */ +export function heapPressureRemedy(heapLimitBytes: number): string { + // Effective RAM honors a real cgroup limit — raw os.totalmem() told users + // inside an 8GB-limited container on a 64GB host that "this machine has + // more memory available", an advice loop with no exit (#2649 review). + const autoCapBytes = effectiveRamBytes() * 0.75; + if (heapLimitBytes < autoCapBytes * 0.9) { + return ( + `This machine has more memory available: re-run without the --max-old-space-size ` + + `pin (NODE_OPTIONS or node flag) — gitnexus sizes its heap to the machine automatically.` + ); + } + return ( + `This machine is at its memory ceiling: exclude generated or vendored directories ` + + `via .gitnexusignore, or analyze on a machine with more memory.` + ); +} + /** Max bytes of source content to load per parse chunk. * * Memory bound for the worker pool dispatch + a granularity knob for @@ -516,6 +593,22 @@ export async function runChunkedParseAndResolve( MIN_SUB_BATCH_BYTES, Math.ceil(chunkByteBudget / (effectivePoolSize * TARGET_JOBS_PER_WORKER)), ); + // Heap-scale guardrails (#2649), measured on a Linux-kernel analyze + // (94,773 files): ~55 graph nodes per parseable file and ~2.2KB of + // main-thread heap per node, linear across 113 chunks (see + // docs/plans/2026-07-23-gitnexus-plan-large-repo-analyze-oom.md §2). + // Estimates, not contracts — used only to warn early (preflight) and to + // convert a certain multi-minute GC death spiral into an immediate + // actionable error (mid-loop guard). + const projectedHeapNeedBytes = projectParseHeapNeedBytes(parseableScanned.length); + const heapLimitBytes = v8.getHeapStatistics().heap_size_limit; + if (projectedHeapNeedBytes > heapLimitBytes * PREFLIGHT_WARN_FRACTION) { + logger.warn( + `Large repository: analyzing ${parseableScanned.length} files needs roughly ${Math.round(projectedHeapNeedBytes / 1024 / 1024 / 1024)}GB of memory, ` + + `but Node is limited to ${Math.round(heapLimitBytes / 1024 / 1024 / 1024)}GB — analyze may stop early. ${heapPressureRemedy(heapLimitBytes)}`, + ); + } + const chunks: string[][] = []; let currentChunk: string[] = []; let currentBytes = 0; @@ -869,6 +962,18 @@ export async function runChunkedParseAndResolve( `nodes=${graph.nodeCount} parsedFiles=${allParsedFiles.length}`, ); } + // #2649 mid-loop heap guard: fail actionably BEFORE V8 enters the + // ineffective-mark-compact death spiral (which also falsely times out + // healthy workers). The pool is torn down by this function's finally. + const heapUsedNow = process.memoryUsage().heapUsed; + const heapLimitNow = v8.getHeapStatistics().heap_size_limit; + if (shouldAbortForHeapPressure(heapUsedNow, heapLimitNow)) { + throw new Error( + `Analyze stopped before running out of memory: ${Math.round(heapUsedNow / 1024 / 1024)}MB of the ` + + `${Math.round(heapLimitNow / 1024 / 1024)}MB Node heap in use at parse chunk ${chunkIdx + 1}/${numChunks} (#2649). ` + + heapPressureRemedy(heapLimitNow), + ); + } const chunkPaths = chunks[chunkIdx]; // Start wall-clock for the per-chunk throughput log emitted at end // of this iteration. The gate is computed once above; here we just diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 983e67a0f..ee62df8d8 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -61,6 +61,8 @@ export interface ParseOutput { model: MutableSemanticModel; /** Pass-through: all file paths for downstream phases. */ readonly allPaths: readonly string[]; + /** Pass-through: shared `allPathSet` from structure (built once, not per-phase). */ + readonly allPathSet: ReadonlySet; /** Pass-through: total file count for progress reporting. */ totalFiles: number; /** @@ -89,6 +91,12 @@ export const parsePhase: PipelinePhase = { ctx: PipelineContext, deps: ReadonlyMap>, ): Promise { + // Begin streamed structural emit (#2680), if enabled. Deliberately here and + // not at graph construction: the pre-parse phases are not all write-only — + // `mapCobolToGraph` scans CALLS edges and removes the unresolved ones — and + // nothing before parse produces bulk edge volume anyway. + ctx.graphEmit?.beginStreaming(); + const structure = getPhaseOutput(deps, 'structure'); const { totalFiles } = structure; @@ -101,6 +109,8 @@ export const parsePhase: PipelinePhase = { const allPaths = ingested.size ? structure.allPaths.filter((p) => !ingested.has(p)) : structure.allPaths; + // Keep the O(1) lookup set consistent with the filtered path list. + const allPathSet = ingested.size ? new Set(allPaths) : structure.allPathSet; const result = await runChunkedParseAndResolve( ctx.graph, @@ -116,6 +126,7 @@ export const parsePhase: PipelinePhase = { return { ...result, allPaths, + allPathSet, totalFiles, }; }, diff --git a/gitnexus/src/core/ingestion/pipeline-phases/types.ts b/gitnexus/src/core/ingestion/pipeline-phases/types.ts index 17786ff4c..7958e404f 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/types.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/types.ts @@ -14,6 +14,7 @@ * - Each phase is independently testable with mocked inputs */ +import type { GraphEmitControl } from '../../lbug/graph-emit-sink.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import type { PipelineProgress } from 'gitnexus-shared'; import type { PipelineOptions } from '../pipeline.js'; @@ -32,6 +33,12 @@ export interface PipelineContext { readonly options?: PipelineOptions; /** Pipeline start timestamp (for elapsed-time logging). */ readonly pipelineStart: number; + /** + * Streamed structural emit (#2680), present only when `streamGraphEmit` is on. + * `parse` calls `beginStreaming()` at its start; `pruneLocalSymbols` consults + * `hasStreamedSemanticEdge()`. Absent ⇒ everything stays in the graph. + */ + readonly graphEmit?: GraphEmitControl; } // ── Phase result wrapper ─────────────────────────────────────────────────── diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index e7fe1177a..8ff2c9ed4 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -16,6 +16,7 @@ */ import { createKnowledgeGraph } from '../graph/graph.js'; +import { GraphEmitSink, type GraphEmitManifest } from '../lbug/graph-emit-sink.js'; import { type PipelineProgress } from 'gitnexus-shared'; import { PipelineResult } from '../../types/pipeline.js'; import { @@ -146,6 +147,22 @@ export interface PipelineOptions< * whole-graph emit. */ streamPdgEmit?: boolean; + /** + * Streamed structural graph emit (#2680). When true, relationships that no + * mid-pipeline phase reads back (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) are + * streamed to CSV-on-disk from the parse boundary onward instead of being + * retained in the in-memory graph — measured ~2.9x reduction of graph heap. + * + * NOT free: the `communities`, `processes`, `taintSummaries` and + * `callSummaries` phases all consume the whole CALLS graph and are disabled + * under this flag. The caller (`run-analyze`) gates it to full rebuilds. + * Requires `graphEmitCsvDir`. + */ + streamGraphEmit?: boolean; + /** Directory for the streamed structural CSVs. Required when + * `streamGraphEmit` is on; supplied by the caller, which owns storage-path + * resolution (and its native-safe relocation). */ + graphEmitCsvDir?: string; /** Streamed PDG-emit write buffer (rows) when `streamPdgEmit` is on (#2202). * `undefined` ⇒ `DEFAULT_PDG_EMIT_CHUNK_ROWS`. Memory-only; does not affect * emitted bytes. */ @@ -309,15 +326,46 @@ export const runPipelineFromRepo = async < const graph = createKnowledgeGraph(); const pipelineStart = Date.now(); + // Streamed structural emit (#2680). The sink is a write-routing façade over + // `graph`; it streams nothing until `beginStreaming()` fires at the parse + // boundary. + // + // A missing `graphEmitCsvDir` is a caller bug, not a reason to quietly skip + // streaming: this is on by default, so a programmatic host that builds its own + // `PipelineOptions` (eval-server, the MCP daemon, a test) would otherwise ask + // for streaming, silently not get it, and still see a successful run. Fail + // loudly instead — the whole point of the surrounding work is that a degraded + // outcome must never look like a clean one. + let graphEmitSink: GraphEmitSink | undefined; + if (options?.streamGraphEmit === true) { + if (options.graphEmitCsvDir === undefined) { + throw new Error( + 'streamGraphEmit was requested but graphEmitCsvDir is missing. The caller owns ' + + 'storage-path resolution (see resolveNativeSafeStorageDir in run-analyze.ts); ' + + 'pass the directory, or leave streamGraphEmit unset to run without streaming.', + ); + } + graphEmitSink = new GraphEmitSink(graph, options.graphEmitCsvDir); + } + const phases = buildPhaseList(options); - const results = await runPipeline(phases, { - repoPath, - graph, - onProgress, - options, - pipelineStart, - }); + let graphEmitManifest: GraphEmitManifest | undefined; + let results; + try { + results = await runPipeline(phases, { + repoPath, + graph: graphEmitSink ?? graph, + onProgress, + options, + pipelineStart, + graphEmit: graphEmitSink, + }); + graphEmitManifest = graphEmitSink?.finalize(); + } finally { + // Release per-pair fds when the pipeline threw before finalize ran. + graphEmitSink?.close(); + } // Extract final results for the PipelineResult contract const { totalFiles, usedWorkerPool } = getPhaseOutput<{ @@ -336,7 +384,12 @@ export const runPipelineFromRepo = async < // Streamed PDG-emit manifest (#2202): present only when streaming was on. const pdgEmitManifest = scopeResolutionOutput.pdgEmitManifest; - if (!options?.skipGraphPhases) { + // Presence check, not `!skipGraphPhases`: phases can now be filtered out by + // any `enabledWhen` predicate (streamGraphEmit disables communities/processes + // too), and `getPhaseOutput` THROWS on a phase that was never resolved. Keying + // off the options flag alone made every filtered-out combination crash here + // rather than return undefined results. + if (results.has('communities') && results.has('processes')) { communityResult = getPhaseOutput(results, 'communities').communityResult; processResult = getPhaseOutput(results, 'processes').processResult; } @@ -356,9 +409,16 @@ export const runPipelineFromRepo = async < }); return { + // The RAW graph, deliberately — NOT `graphEmitSink`. Phases above received + // the sink so their reads are complete, but `loadGraphToLbug` feeds this to + // `streamAllCSVsToDisk`, and the sink's complete iterator would then emit + // every streamed edge a SECOND time on top of the per-pair CSVs the sink + // already wrote and the manifest already COPYs. Returning the sink here + // silently doubles every streamed relationship in the persisted graph. graph, repoPath, totalFileCount: totalFiles, + graphEmitManifest, communityResult, processResult, resolutionOutcomes, diff --git a/gitnexus/src/core/ingestion/process-processor.ts b/gitnexus/src/core/ingestion/process-processor.ts index d8a5e751d..fe75dcac4 100644 --- a/gitnexus/src/core/ingestion/process-processor.ts +++ b/gitnexus/src/core/ingestion/process-processor.ts @@ -248,12 +248,12 @@ const buildCallsAdjacency = ( else bucket.push(value); }; - for (const rel of graph.iterRelationships()) { - if (rel.type === 'CALLS' && rel.confidence >= MIN_TRACE_CONFIDENCE) { - push(forward, rel.sourceId, rel.targetId); - push(reverse, rel.targetId, rel.sourceId); - } - } + // Field-wise scan (#2680) — whole-graph walk, four fields, no object needed. + graph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + if (type !== 'CALLS' || confidence < MIN_TRACE_CONFIDENCE) return; + push(forward, sourceId, targetId); + push(reverse, targetId, sourceId); + }); return { forward, reverse }; }; diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index 9fe8ae8b9..2022ced64 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -705,6 +705,7 @@ function parseJsonStringArrayCapture( function deriveDeclarationName(match: CaptureMatch, def: SymbolDefinition): string | undefined { const nameCap = + match['@declaration.binding-name'] ?? match['@declaration.name'] ?? match[ Object.keys(match).find((k) => k.startsWith('@declaration.') && k.endsWith('.name')) ?? '' diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts index b39ff8689..03b6c0d2d 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts @@ -30,6 +30,10 @@ interface Target { readonly def: SymbolDefinition; } +// Shared miss results for the fixpoint's read paths — callers only iterate. +const EMPTY_TARGETS: ReadonlyMap = new Map(); +const EMPTY_CELLS: ReadonlySet = new Set(); + interface FileFact { readonly filePath: string; readonly site: CallableFlowSite; @@ -40,13 +44,22 @@ interface FileInvoke { readonly site: CallableFlowInvokeSite; } -export interface CallableValueFlowWarning { +interface RawCallableValueFlowWarning { readonly language: string; readonly context: string; readonly candidateCount: number; readonly cap: number; } +export interface CallableValueFlowWarning extends RawCallableValueFlowWarning { + /** Number of internal cells represented by this aggregate warning. */ + readonly occurrences: number; + /** Number of source contexts represented by this aggregate warning. */ + readonly distinctContexts: number; + /** Bounded diagnostic sample; stdout receives one aggregate, not every site. */ + readonly contextSamples: readonly string[]; +} + export interface CallableValueFlowResult { readonly emitted: number; readonly resolvedInvokes: number; @@ -55,6 +68,46 @@ export interface CallableValueFlowResult { readonly iterations: number; } +/** Collapse internal-cell overflows to one bounded warning per language/cap. */ +export function aggregateCallableValueFlowWarnings( + warnings: Iterable, +): CallableValueFlowWarning[] { + const grouped = new Map< + string, + RawCallableValueFlowWarning & { + // Re-declared mutable so occurrences can raise it (the raw field is readonly). + candidateCount: number; + occurrences: number; + allContexts: Set; + contextSamples: string[]; + } + >(); + for (const warning of warnings) { + const key = `${warning.language}\0${warning.cap}`; + const current = grouped.get(key); + if (!current) { + grouped.set(key, { + ...warning, + occurrences: 1, + allContexts: new Set([warning.context]), + contextSamples: [warning.context], + }); + continue; + } + + current.candidateCount = Math.max(current.candidateCount, warning.candidateCount); + current.occurrences++; + current.allContexts.add(warning.context); + if (current.contextSamples.length < 5 && !current.contextSamples.includes(warning.context)) { + current.contextSamples.push(warning.context); + } + } + return [...grouped.values()].map(({ allContexts, ...warning }) => ({ + ...warning, + distinctContexts: allContexts.size, + })); +} + export interface EmitCallableValueFlowInput { readonly graph: KnowledgeGraph; readonly scopes: ScopeResolutionIndexes; @@ -66,6 +119,12 @@ export interface EmitCallableValueFlowInput { readonly isCallableValueTarget?: (def: SymbolDefinition) => boolean; readonly hasFileLocalCallableLinkage?: (def: SymbolDefinition) => boolean; readonly onWarn?: (warning: CallableValueFlowWarning) => void; + /** + * Precomputed {@link collectDeferredIndirectSites} result for the same + * files/scopes. The orchestrator already needs it to build skip sets; + * threading it here avoids a second whole-repo scan. Recomputed when absent. + */ + readonly canonicalInvokeKeys?: ReadonlySet; } /** Position key shared with the existing free/reference skip-set contract. */ @@ -76,6 +135,34 @@ export function callableFlowSiteKey( return `${filePath}:${range.startLine}:${range.startCol}`; } +function warningContext(filePath: string, site: CallableFlowSite): string { + let range: { readonly startLine: number; readonly startCol: number }; + switch (site.kind) { + case 'seed': + case 'copy': + case 'alias': + case 'address': + case 'load': + range = site.destination.atRange; + break; + case 'store': + range = site.pointer.atRange; + break; + case 'formal': + range = site.ownerRange; + break; + case 'argument': + case 'invoke': + range = site.callSite; + break; + default: { + const exhaustive: never = site; + throw new Error(`Unhandled callable-flow site kind: ${String(exhaustive)}`); + } + } + return `${site.kind}:${callableFlowSiteKey(filePath, range)}`; +} + /** * Return only invoke sites that join to a canonical call ReferenceSite. * Malformed/stale facts never suppress ordinary resolution. @@ -140,7 +227,8 @@ function flowCellOperand(site: CallableFlowSite): CallableFlowOperand | undefine export function emitCallableValueFlow(input: EmitCallableValueFlowInput): CallableValueFlowResult { const facts: FileFact[] = []; const invokes: FileInvoke[] = []; - const canonicalInvokeKeys = collectDeferredIndirectSites(input.parsedFiles, input.scopes); + const canonicalInvokeKeys = + input.canonicalInvokeKeys ?? collectDeferredIndirectSites(input.parsedFiles, input.scopes); let unmatchedInvokes = 0; for (const parsed of input.parsedFiles) { for (const site of parsed.callableFlowSites ?? []) { @@ -161,7 +249,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab const addressesByBinding = new Map>(); const overflowedTargets = new Set(); const overflowedAddresses = new Set(); - const overflowWarnings = new Map(); + const overflowWarnings = new Map(); const rawGraphTargets = buildGraphTargetIndex( input.scopes, input.nodeLookup, @@ -311,7 +399,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab ): { readonly targets: ReadonlyMap; readonly overflow: boolean } => { watch('target', key); return { - targets: targetsByBinding.get(key) ?? new Map(), + targets: targetsByBinding.get(key) ?? EMPTY_TARGETS, overflow: overflowedTargets.has(key), }; }; @@ -321,7 +409,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab ): { readonly cells: ReadonlySet; readonly overflow: boolean } => { watch('address', key); return { - cells: addressesByBinding.get(key) ?? new Set(), + cells: addressesByBinding.get(key) ?? EMPTY_CELLS, overflow: overflowedAddresses.has(key), }; }; @@ -476,10 +564,10 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab for (const fact of facts) { const site = fact.site; - const context = `${fact.site.kind}:${fact.filePath}`; switch (site.kind) { case 'copy': case 'alias': { + const context = warningContext(fact.filePath, site); addWorkItem(() => { const source = bindingKey(fact.filePath, site.source); const destination = bindingKey(fact.filePath, site.destination); @@ -493,6 +581,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab break; } case 'load': { + const context = warningContext(fact.filePath, site); addWorkItem(() => { const destination = bindingKey(fact.filePath, site.destination); const reached = reachedCells(fact.filePath, site.pointer); @@ -509,6 +598,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab break; } case 'store': { + const context = warningContext(fact.filePath, site); addWorkItem(() => { const sourceTargets = operandTargets(fact.filePath, site.source); const reached = reachedCells(fact.filePath, site.pointer); @@ -585,7 +675,11 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab targetIds.add(id); } for (const id of dynamicCallees.get(callKey)?.keys() ?? []) targetIds.add(id); - let hasIndexedFormal = [...targetIds].some((id) => indexedFormals(id).length > 0); + const hasAnyIndexedFormal = (): boolean => { + for (const id of targetIds) if (indexedFormals(id).length > 0) return true; + return false; + }; + let hasIndexedFormal = hasAnyIndexedFormal(); if (!hasIndexedFormal && site.directCalleeName !== undefined) { for (const target of resolveSeedCandidates( fact.filePath, @@ -601,7 +695,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab )) { targetIds.add(target.id); } - hasIndexedFormal = [...targetIds].some((id) => indexedFormals(id).length > 0); + hasIndexedFormal = hasAnyIndexedFormal(); } const history = dynamicTargetHistory.get(callKey); const callOverflow = @@ -684,7 +778,12 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab ); } - for (const warning of overflowWarnings.values()) input.onWarn?.(warning); + // A large generated bundle can create thousands of distinct binding cells + // at the same source site. Preserve the causal evidence while emitting one + // structured warning per site instead of one line per internal cell. + for (const warning of aggregateCallableValueFlowWarnings(overflowWarnings.values())) { + input.onWarn?.(warning); + } // No partial graph output when a hostile/corrupt fact graph exhausts the // bounded work budget. The caller receives a warning; NOTE this is not diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 2f0262410..4e32a4c4e 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -18,6 +18,7 @@ */ import type { + DefId, ParameterTypeClass, ParsedFile, Reference, @@ -37,11 +38,13 @@ import type { import { resolveCallerGraphId, resolveDefGraphId } from '../graph-bridge/ids.js'; import type { CalleeIdSink } from '../graph-bridge/callee-id-sink.js'; import { + findAllCallableBindingCandidatesInScope, findAllCallableBindingsInScope, findCallableBindingInScope, findCallableBindingsAndAdlBlocker, findEnclosingClassDef, resolveInheritanceBaseInScope, + type CallableBindingCandidate, } from '../scope/walkers.js'; import { isOverloadAmbiguousAfterNormalization, @@ -131,8 +134,46 @@ export function emitFreeCallFallback( options.isCallableVisibleFromCaller === undefined ? new Map() : undefined; + const enclosingInstanceOwnerByScope = + options.freeCallsRequireInstanceOwnership === true + ? new Map() + : undefined; + const reachableInstanceOwnersByOwner = + options.freeCallsRequireInstanceOwnership === true + ? new Map>() + : undefined; + const instanceOwnerKey = (ownerId: string): string => { + const owner = scopes.defs.get(ownerId as DefId); + const qualifiedName = owner?.qualifiedName; + if (qualifiedName === undefined || qualifiedName === '') return ownerId; + const namespacePrefix = owner.namespacePrefix ?? ''; + return `${namespacePrefix.length}:${namespacePrefix}:${qualifiedName}`; + }; + const isReachableInstanceOwner = (scopeId: ScopeId, ownerId: string): boolean => { + let enclosing = enclosingInstanceOwnerByScope?.get(scopeId); + if (enclosing === undefined) { + enclosing = findEnclosingClassDef(scopeId, scopes) ?? null; + enclosingInstanceOwnerByScope?.set(scopeId, enclosing); + } + if (enclosing === null) return false; + + let owners = reachableInstanceOwnersByOwner?.get(enclosing.nodeId); + if (owners === undefined) { + const mutableOwners = new Set([instanceOwnerKey(enclosing.nodeId)]); + for (const inheritedOwnerId of scopes.methodDispatch.mroFor(enclosing.nodeId)) { + mutableOwners.add(instanceOwnerKey(inheritedOwnerId)); + } + owners = mutableOwners; + reachableInstanceOwnersByOwner?.set(enclosing.nodeId, owners); + } + return owners.has(instanceOwnerKey(ownerId)); + }; for (const parsed of parsedFiles) { + const bindingCandidatesByScope = + options.freeCallsRequireInstanceOwnership === true + ? new Map>() + : undefined; for (const site of parsed.referenceSites) { if (site.kind !== 'call') continue; if (site.explicitReceiver !== undefined) continue; @@ -202,11 +243,61 @@ export function emitFreeCallFallback( // (local shadows import). When a conversion-rank function is // available AND the binding scope contains multiple overloads, // refine with narrowOverloadCandidates (#1578). - fnDef = findCallableBindingInScope(site.inScope, site.name, scopes); + let bindingCandidates: readonly CallableBindingCandidate[] | undefined; + if (bindingCandidatesByScope !== undefined) { + let byName = bindingCandidatesByScope.get(site.inScope); + if (byName === undefined) { + byName = new Map(); + bindingCandidatesByScope.set(site.inScope, byName); + } + bindingCandidates = byName.get(site.name); + if (bindingCandidates === undefined) { + bindingCandidates = findAllCallableBindingCandidatesInScope( + site.inScope, + site.name, + scopes, + ); + byName.set(site.name, bindingCandidates); + } + } + let eligibleBindingCandidates: readonly CallableBindingCandidate[] | undefined; + if (bindingCandidates === undefined) { + fnDef = findCallableBindingInScope(site.inScope, site.name, scopes); + } else { + eligibleBindingCandidates = bindingCandidates.filter((candidate) => { + const def = candidate.def; + if ( + def.type !== 'Method' || + def.ownerId === undefined || + def.filePath !== parsed.filePath + ) { + return true; + } + const ownerReachable = isReachableInstanceOwner(site.inScope, def.ownerId); + const staticallyImported = candidate.bindings.some( + (binding) => binding.visibility === 'static-member-import', + ); + return ownerReachable || staticallyImported; + }); + fnDef = eligibleBindingCandidates[0]?.def; + if (fnDef === undefined && bindingCandidates.length > 0) { + recordSuppressedOutcome(options.recordResolutionOutcome, { + phase: 'free-call-fallback', + filePath: parsed.filePath, + name: site.name, + range: site.atRange, + reason: 'free-call-instance-ownership', + candidates: bindingCandidates.map((candidate) => candidate.def), + }); + } + } if ( fnDef !== undefined && options.isBuiltInName?.(site.name) === true && fnDef.filePath === parsed.filePath && + eligibleBindingCandidates?.some((candidate) => + candidate.bindings.some((binding) => binding.visibility === 'static-member-import'), + ) !== true && !hasGenuineLexicalBinding(site.inScope, site.name, scopes) ) { // A platform/language built-in (e.g. `fetch`, `setTimeout`) @@ -234,48 +325,14 @@ export function emitFreeCallFallback( // stopped resolving (verified via a scratch probe fixture). fnDef = undefined; } - // Instance-ownership gate (#2550). Placement matters: after the - // scope-chain lookup, BEFORE overload narrowing -- a suppressed - // candidate must not participate in overload selection. The - // legitimate same-class bare call already resolved earlier via - // `pickImplicitThisOverload`; an inherited bare call passes the - // MRO arm here; what remains is the finalize-bucket leak (an - // unrelated same-file method matched by bare name). - // - // Same-file only (mirrors the #2545 guard's load-bearing - // condition): the `materializeBindings` bucket is per-file, so - // the leak is ALWAYS same-file. A cross-file Method match here - // came through a genuine import channel -- e.g. the arity- - // narrowing parity fixtures resolve a bare `writeAudit(u)` to - // an imported class's method, which must keep working - // (suppressing it broke `java.test.ts`'s arity-filtering suite, - // verified empirically). if ( fnDef !== undefined && - options.freeCallsRequireInstanceOwnership === true && - fnDef.type === 'Method' && - fnDef.ownerId !== undefined && - fnDef.filePath === parsed.filePath + (options.conversionRankFn !== undefined || bindingCandidates !== undefined) ) { - const enclosing = findEnclosingClassDef(site.inScope, scopes); - const ownerReachable = - enclosing !== undefined && - (enclosing.nodeId === fnDef.ownerId || - scopes.methodDispatch.mroFor(enclosing.nodeId).includes(fnDef.ownerId)); - if (!ownerReachable) { - recordSuppressedOutcome(options.recordResolutionOutcome, { - phase: 'free-call-fallback', - filePath: parsed.filePath, - name: site.name, - range: site.atRange, - reason: 'free-call-instance-ownership', - candidates: [fnDef], - }); - fnDef = undefined; - } - } - if (fnDef !== undefined && options.conversionRankFn !== undefined) { - const allCallables = findAllCallableBindingsInScope(site.inScope, site.name, scopes); + const allCallables = + eligibleBindingCandidates === undefined + ? findAllCallableBindingsInScope(site.inScope, site.name, scopes) + : eligibleBindingCandidates.map((candidate) => candidate.def); if (allCallables.length > 1) { const narrowed = narrowOverloadCandidates( allCallables, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index e47498ebb..e7ff4a021 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -82,6 +82,7 @@ import { callableFlowSiteKey, collectDeferredIndirectSites, emitCallableValueFlow, + type CallableValueFlowWarning, } from '../passes/callable-value-flow.js'; import type { ScopeResolver } from '../contract/scope-resolver.js'; import { findEnclosingClassDef, resolveInheritanceBaseInScope } from '../scope/walkers.js'; @@ -92,7 +93,37 @@ import { parseTruthyEnv } from '../../utils/env.js'; import { TransitionalScopeTree } from '../../../../storage/scope-index-store.js'; import { forceGc } from '../../../../storage/parsedfile-store.js'; -import { logger } from '../../../logger.js'; +import { logger, warnRespectingProgressBar } from '../../../logger.js'; + +/** Exported for the boundary test in run-progress.test.ts. */ +export const MAX_PROGRESS_WARNING_CONTEXT_CHARS = 160; + +/** Escape control bytes and bound one context shown beside the live bar. */ +export function formatScopeResolutionWarningContext(context: string): string { + const escaped = JSON.stringify(context).slice(1, -1); + if (escaped.length <= MAX_PROGRESS_WARNING_CONTEXT_CHARS) return escaped; + return `${escaped.slice(0, MAX_PROGRESS_WARNING_CONTEXT_CHARS - 3)}...`; +} + +/** One-line progress warning for property-dispatch fan-out drops. */ +function formatPropertyDispatchProgress( + language: string, + skippedKeys: number, + fanoutCap: number, + skippedKeyNames: readonly string[], +): string { + return ` Warning: property dispatch (${language}) skipped ${skippedKeys} key(s) above fan-out cap ${fanoutCap}; no CALLS were synthesized. Sample: ${skippedKeyNames + .slice(0, 5) + .join(', ')}`; +} + +/** One-line progress warning for callable-value-flow candidate-set overflows. */ +function formatCallableValueFlowProgress(warning: CallableValueFlowWarning): string { + return ` Warning: callable value flow (${warning.language}) skipped ${warning.occurrences} candidate set(s) across ${warning.distinctContexts} context(s) above cap ${warning.cap}; no partial CALLS were emitted. Sample: ${warning.contextSamples + .slice(0, 2) + .map(formatScopeResolutionWarningContext) + .join(', ')}`; +} /** * Emit one class-owned inheritance edge directly (the inheritance pre-pass is @@ -711,6 +742,7 @@ export function runScopeResolution( propagateImportedReturnTypes(parsedFiles, indexes, workspaceIndex); } + const tRangeBindStart = PROF ? process.hrtime.bigint() : 0n; if (provider.populateRangeBindings !== undefined) { provider.populateRangeBindings(parsedFiles, indexes, { fileContents: getFileContents(), @@ -879,14 +911,23 @@ export function runScopeResolution( // Never drop dispatch coverage silently: a hook table larger than the // fan-out cap means member calls through those keys get no synthesized // CALLS — the #2437 false-safe gap reappears for exactly those keys. - logger.warn( + warnRespectingProgressBar( + formatPropertyDispatchProgress( + provider.language, + propertyDispatch.skippedKeys, + MAX_PROPERTY_DISPATCH_FANOUT, + propertyDispatch.skippedKeyNames, + ), { - lang: provider.language, - skippedKeys: propertyDispatch.skippedKeys, - skippedKeyNames: propertyDispatch.skippedKeyNames, - fanoutCap: MAX_PROPERTY_DISPATCH_FANOUT, + fields: { + lang: provider.language, + skippedKeys: propertyDispatch.skippedKeys, + skippedKeyNames: propertyDispatch.skippedKeyNames, + fanoutCap: MAX_PROPERTY_DISPATCH_FANOUT, + }, + message: + 'property-dispatch: keys over the fan-out cap were dropped (no CALLS synthesized for them)', }, - 'property-dispatch: keys over the fan-out cap were dropped (no CALLS synthesized for them)', ); } const callableValueFlow = @@ -904,15 +945,17 @@ export function runScopeResolution( parsedFiles: emitParsedFiles, nodeLookup: postHeritageNodeLookup, calleeIds: calleeIdAccumulator, + canonicalInvokeKeys: deferredIndirectSites, language: provider.language, collapseByCallerTarget: provider.collapseMemberCallsByCallerTarget === true, isCallableValueTarget: provider.isCallableValueTarget, hasFileLocalCallableLinkage: provider.hasFileLocalCallableLinkage, onWarn: (warning) => - logger.warn( - warning, - 'callable-value-flow: candidate set exceeded the cap; no partial CALLS emitted', - ), + warnRespectingProgressBar(formatCallableValueFlowProgress(warning), { + fields: warning, + message: + 'callable-value-flow: candidate set exceeded the cap; grouped occurrences emitted no partial CALLS', + }), }); const importsEmitted = callableFlowOnly ? 0 @@ -1309,6 +1352,7 @@ export function runScopeResolution( `[scope-resolution prof] extract=${ns(tStart, tExtract).toFixed(0)}ms` + ` finalize=${ns(tExtract, tFinalize).toFixed(0)}ms` + ` propagate=${ns(tFinalize, tPropagate).toFixed(0)}ms` + + ` rangeBind=${ns(tRangeBindStart, tPropagate).toFixed(1)}ms` + ` resolve=${ns(tPropagate, tResolve).toFixed(0)}ms` + ` emit=${ns(tResolve, tEnd).toFixed(0)}ms` + // pdg ⊆ emit: the M2 reaching-defs share of the emit bucket (#2082 U4). diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts index 1d371b925..d022092cc 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts @@ -668,6 +668,69 @@ export function findCallableBindingInScope( return findAllCallableBindingsInScope(startScope, callableName, scopes)[0]; } +export interface CallableBindingCandidate { + readonly def: SymbolDefinition; + /** Every visibility path for this definition, in binding precedence order. */ + readonly bindings: readonly BindingRef[]; +} + +function collectCallableBindingCandidates( + sources: readonly (readonly BindingRef[] | undefined)[], +): readonly CallableBindingCandidate[] { + const byNodeId = new Map(); + for (const source of sources) { + if (source === undefined) continue; + for (const binding of source) { + const def = binding.def; + if (def.type !== 'Function' && def.type !== 'Method' && def.type !== 'Constructor') continue; + const existing = byNodeId.get(def.nodeId); + if (existing === undefined) { + byNodeId.set(def.nodeId, { def, bindings: [binding] }); + } else { + existing.bindings.push(binding); + } + } + } + return [...byNodeId.values()]; +} + +/** + * Binding-aware callable lookup for consumers that need visibility evidence. + * Unlike `lookupBindingsAt`, duplicate definitions retain every binding path, + * so a weaker augmentation can contribute provenance even when a finalized + * binding remains the candidate's canonical definition. + */ +export function findAllCallableBindingCandidatesInScope( + startScope: ScopeId, + callableName: string, + scopes: ScopeResolutionIndexes, +): readonly CallableBindingCandidate[] { + let currentId: ScopeId | null = startScope; + const visited = new Set(); + while (currentId !== null) { + if (visited.has(currentId)) return []; + visited.add(currentId); + const scope = scopes.scopeTree.getScope(currentId); + if (scope === undefined) return []; + + if (scope.kind !== 'Object') { + const lexical = collectCallableBindingCandidates([scope.bindings.get(callableName)]); + if (lexical.length > 0) return lexical; + + const candidates = collectCallableBindingCandidates([ + scopes.bindings.get(currentId)?.get(callableName), + scopes.bindingAugmentations.get(currentId)?.get(callableName), + collectNamespaceFqnBindings(currentId, callableName, scopes), + scopes.workspaceFqnBindings?.get(callableName), + ]); + if (candidates.length > 0) return candidates; + } + + currentId = scope.parent; + } + return []; +} + /** * Look up all callable bindings (Function/Method/Constructor) by name * from the nearest scope in the chain that binds `callableName`. diff --git a/gitnexus/src/core/ingestion/tree-sitter-queries.ts b/gitnexus/src/core/ingestion/tree-sitter-queries.ts index 9de207f1d..7eef9021c 100644 --- a/gitnexus/src/core/ingestion/tree-sitter-queries.ts +++ b/gitnexus/src/core/ingestion/tree-sitter-queries.ts @@ -699,6 +699,17 @@ export const PYTHON_QUERIES = ` (assignment left: (identifier) @name)) @definition.variable +; Lambda bindings: \`f = lambda x: x\` binds a CALLABLE, so it emits Function +; rather than Variable, matching what TS/JS already do for \`const f = () => {}\`. +; This aligns the LABEL only — call resolution runs off the scope-resolution +; query, which still models the binding as a value, so \`f()\` does not resolve +; here yet. Overlap with the assignment pattern above is collapsed by the +; parse-worker dedup (#2687). +(expression_statement + (assignment + left: (identifier) @name + right: (lambda))) @definition.function + ; Write access: obj.field = value (assignment left: (attribute @@ -868,6 +879,25 @@ export const GO_QUERIES = ` ; Short variable declaration: x := 5 (short_var_declaration left: (expression_list (identifier) @name)) @definition.variable +; Closure bindings: \`var f = func(){}\` / \`f := func(){}\` bind a CALLABLE, so +; they emit Function, not Variable — the same convention TS/JS already use for +; \`const f = () => {}\`. This aligns the LABEL only — call resolution runs off +; the scope-resolution query, which still models the binding as a value, so +; \`f()\` does not resolve here yet. Overlap with the value patterns above is +; collapsed by the parse-worker dedup (#2687). +(var_declaration + (var_spec + name: (identifier) @name + value: (expression_list (func_literal)))) @definition.function +(var_declaration + (var_spec_list + (var_spec + name: (identifier) @name + value: (expression_list (func_literal))))) @definition.function +(short_var_declaration + left: (expression_list (identifier) @name) + right: (expression_list (func_literal))) @definition.function + ; Struct literal construction: User{Name: "Alice"} (composite_literal type: (type_identifier) @call.name) @call @@ -1031,6 +1061,16 @@ export const CPP_QUERIES = ` declarator: (init_declarator declarator: (identifier) @name)) @definition.variable +; Lambda bindings: \`auto f = [](int x){ … };\` binds a CALLABLE, so it emits +; Function rather than Variable, matching TS/JS. This aligns the LABEL only — +; call resolution runs off the scope-resolution query, which still models the +; binding as a value, so \`f()\` does not resolve here yet. Overlap with the +; pattern above is collapsed by the parse-worker dedup (#2687). +(declaration + declarator: (init_declarator + declarator: (identifier) @name + value: (lambda_expression))) @definition.function + ; Structured bindings: auto [a, b] = makePair(); (one @name per bound identifier) (declaration declarator: (init_declarator @@ -1379,6 +1419,16 @@ export const KOTLIN_QUERIES = ` (variable_declaration (simple_identifier) @name)) @definition.property +; Lambda bindings: \`val f = { x -> x }\` binds a CALLABLE, so it emits Function +; rather than Property, matching TS/JS. This aligns the LABEL only — call +; resolution runs off the scope-resolution query, which still models the binding +; as a value, so \`f()\` does not resolve here yet. Overlap with the property +; pattern above is collapsed by the parse-worker dedup (#2687). +(property_declaration + (variable_declaration + (simple_identifier) @name) + (lambda_literal)) @definition.function + ; ── Destructuring declarations (F51, issue #1919) ──────────────────────── ; "val (a, b) = pair" binds several names through a multi_variable_declaration ; (NOT a variable_declaration), which the property rule above misses. Emit one @@ -1503,6 +1553,15 @@ export const SWIFT_QUERIES = ` ; Properties (stored and computed) (property_declaration (pattern (simple_identifier) @name)) @definition.property +; Closure bindings: \`let f = { ... }\` binds a CALLABLE, so it emits Function +; rather than Property, matching TS/JS. This aligns the LABEL only — call +; resolution runs off the scope-resolution query, which still models the binding +; as a value, so \`f()\` does not resolve here yet. Overlap with the property +; pattern above is collapsed by the parse-worker dedup (#2687). +(property_declaration + name: (pattern (simple_identifier) @name) + value: (lambda_literal)) @definition.function + ; Protocol property requirements (F75): "var title: String { get }" parses to a ; protocol_property_declaration (NOT property_declaration). Its name is a ; "name:" pattern field wrapping a value_binding_pattern + the bound @@ -1659,6 +1718,16 @@ export const DART_QUERIES = ` (initialized_identifier_list (initialized_identifier (identifier) @name)) @definition.variable) +; Closure bindings: \`var f = (x) => x;\` binds a CALLABLE, so it emits Function +; rather than Variable, matching TS/JS. This aligns the LABEL only — call +; resolution runs off the scope-resolution query, which still models the binding +; as a value, so \`f()\` does not resolve here yet. Overlap with the pattern +; above is collapsed by the parse-worker dedup (#2687). +(program + (initialized_identifier_list + (initialized_identifier + (identifier) @name + (function_expression))) @definition.function) (program (static_final_declaration_list (static_final_declaration diff --git a/gitnexus/src/core/ingestion/type-extractors/jvm.ts b/gitnexus/src/core/ingestion/type-extractors/jvm.ts index 28f1e5797..9788f3d8e 100644 --- a/gitnexus/src/core/ingestion/type-extractors/jvm.ts +++ b/gitnexus/src/core/ingestion/type-extractors/jvm.ts @@ -1,8 +1,4 @@ -import { - findChild, - synthesizeJavaAnonymousClassName, - type SyntaxNode, -} from '../utils/ast-helpers.js'; +import { findChild, synthesizeJavaTypeIdentity, type SyntaxNode } from '../utils/ast-helpers.js'; import type { LanguageTypeConfig, ParameterExtractor, @@ -40,7 +36,7 @@ const JAVA_DECLARATION_NODE_TYPES: ReadonlySet = new Set([ const anonymousInitializerTypeName = (declarator: SyntaxNode): string | undefined => { const valueNode = declarator.childForFieldName('value'); if (!valueNode || valueNode.type !== 'object_creation_expression') return undefined; - return synthesizeJavaAnonymousClassName(valueNode); + return synthesizeJavaTypeIdentity(valueNode)?.name; }; /** Java: Type x = ...; Type x; */ diff --git a/gitnexus/src/core/ingestion/utils/ast-helpers.ts b/gitnexus/src/core/ingestion/utils/ast-helpers.ts index edfe2dda8..0eafe74c5 100644 --- a/gitnexus/src/core/ingestion/utils/ast-helpers.ts +++ b/gitnexus/src/core/ingestion/utils/ast-helpers.ts @@ -8,6 +8,7 @@ import { templateArgumentsIdTag, } from './template-arguments.js'; import { splitQualifiedName } from './qualified-name.js'; +import { isOverloadableCallable } from './callable-labels.js'; /** Tree-sitter AST node. Re-exported for use across ingestion modules. */ export type SyntaxNode = Parser.SyntaxNode; @@ -110,24 +111,6 @@ const isConcreteTypedefCapture = (captureMap: Record): boole ); }; -export const buildConcreteTypedefDefinitionRanges = ( - matches: readonly QueryMatchLike[], -): Set => { - const ranges = new Set(); - for (const match of matches) { - const captureMap: Record = {}; - for (const capture of match.captures) { - captureMap[capture.name] = capture.node; - } - - const definitionNode = getDefinitionNodeFromCaptures(captureMap); - if (definitionNode && isConcreteTypedefCapture(captureMap)) { - ranges.add(nodeRangeKey(definitionNode)); - } - } - return ranges; -}; - export const isSuppressedConcreteTypedefDuplicate = ( captureMap: Record, concreteTypedefRanges: ReadonlySet, @@ -140,6 +123,129 @@ export const isSuppressedConcreteTypedefDuplicate = ( ); }; +/** + * Graph labels produced by a value capture (`@definition.const` / + * `@definition.static` / `@definition.variable`) — a binding that holds a value. + * + * `Property` is deliberately NOT here. It outranks these: Python matches both + * `@definition.property` (annotated) and `@definition.variable` (bare) on one + * assignment, and the property must win so a typed class attribute keeps its + * `Property` node and its owning `HAS_PROPERTY` edge. `Property` is instead + * suppressed only by a *callable* claim — see {@link buildDefinitionNameClaims}. + */ +const VALUE_DEFINITION_LABELS: ReadonlySet = new Set([ + 'Const', + 'Static', + 'Variable', +]); + +/** True when `label` is the kind of node a value capture emits. */ +export const isValueDefinitionLabel = (label: NodeLabel): boolean => + VALUE_DEFINITION_LABELS.has(label); + +/** + * One pass over a file's matches: definition-name claims by rank, plus the + * concrete-typedef ranges the loop's separate typedef guard consumes. + */ +export interface DefinitionPreScan { + /** + * Keys claimed by any non-value capture — consulted by `Const`/`Static`/ + * `Variable`. Includes `Property`, so an annotated Python attribute still + * beats the bare-assignment `Variable` capture on the same statement. + */ + readonly nonValue: ReadonlySet; + /** + * Keys claimed by a *callable* capture (`Function`/`Method`/`Constructor`) — + * consulted by `Property`. Narrower than `nonValue` on purpose: a `Property` + * must be collapsible by a callable (Kotlin `val f = { … }`, Swift + * `let f = { … }`) without being collapsible by its own claim. + */ + readonly callable: ReadonlySet; + /** Ranges of `type_definition` nodes that already emit a concrete struct/enum. */ + readonly concreteTypedefRanges: ReadonlySet; +} + +/** + * Pre-scan `matches` for the `${definitionNode.startIndex}:${name}` keys already + * claimed by a higher-ranked definition capture, so the parse-worker's duplicate + * suppression is order-independent. + * + * Rank, highest first: callable (`Function`/`Method`/`Constructor`) → `Property` + * → value (`Const`/`Static`/`Variable`). A capture is dropped only when a + * STRICTLY higher rank claimed the same declaration node and name, so no capture + * can suppress itself and no rank can suppress a peer. + * + * ## Why this exists (#2687) + * + * `const X = () => {}` matches BOTH `@definition.function` and + * `@definition.const` on the same `lexical_declaration`. Only one graph node + * should survive — the `Function`, because that is what `CALLS` edges target. + * The parse-worker's in-loop dedup intends exactly that, but only the value + * branch consults its `processedDefinitionNodes` set, so suppression worked only + * if the function match happened to be processed first. It is not: tree-sitter + * completes the const pattern at `@name`, while the function pattern must also + * match the trailing `(arrow_function)` / `(function_expression)` value, so the + * const match is yielded FIRST and the edgeless `Const:` twin escaped. + * + * Consulting this set makes the outcome independent of match order. + * + * ## Keying + * + * Keys are `startIndex:name`, never `startIndex` alone — a multi-name + * declaration (`const a = 1, b = () => {}`) shares ONE definition node, and a + * bare-index key would wrongly suppress `a`'s legitimate `Const` node. + * + * Labels come from {@link getLabelFromCaptures}, the same function the main loop + * uses, so the pre-scan and the loop can never disagree about what counts as a + * value capture — including when a provider's `labelOverride` reclassifies one. + * A match that resolves to a value label registers nothing, so a match can never + * suppress itself. + * + * Language-agnostic: keyed off capture names and labels only. + * + * Also collects the concrete-typedef ranges that suppress the analogous + * typedef/struct duplicate, so both suppression sets come from one traversal. + */ +export const buildDefinitionPreScan = ( + matches: readonly QueryMatchLike[], + provider: LanguageProvider, +): DefinitionPreScan => { + const nonValue = new Set(); + const callable = new Set(); + const concreteTypedefRanges = new Set(); + for (const match of matches) { + // ONE capture-map build per match feeds both suppression sets. These used + // to be two independent passes over `matches` (each rebuilding this object) + // on the hot per-file parse path. + const captureMap: Record = {}; + for (const capture of match.captures) { + captureMap[capture.name] = capture.node; + } + + const definitionNode = getDefinitionNodeFromCaptures(captureMap); + if (definitionNode === null) continue; + + if (isConcreteTypedefCapture(captureMap)) { + concreteTypedefRanges.add(nodeRangeKey(definitionNode)); + } + + // No `@name` capture means nothing a lower-ranked capture could collide + // with — a value or property pattern always binds a name. Checked before + // `getLabelFromCaptures` so a nameless match never pays for label + // resolution (which can reach a provider's `labelOverride`). + const nameNode = captureMap['name']; + if (nameNode === undefined) continue; + + const label = getLabelFromCaptures(captureMap, provider); + if (label === null || isValueDefinitionLabel(label)) continue; + + const key = `${definitionNode.startIndex}:${nameNode.text}`; + nonValue.add(key); + if (isOverloadableCallable(label)) callable.add(key); + } + return { nonValue, callable, concreteTypedefRanges }; +}; + /** * Node types that represent function/method definitions across languages. * Used by parent-walk in call-processor, parse-worker, and type-env to detect @@ -405,31 +511,43 @@ export interface EnclosingClassInfo { const MAX_ENCLOSING_WALK_ITERATIONS = 4096; /** - * Synthesize a javac-style name for a Java anonymous class body: - * `new Runnable() { ... }` inside top-level class `Worker` becomes - * `Worker$1` (`$N` = 1-based source order of anonymous bodies within the - * top-level class). Returns undefined when the node is not an - * `object_creation_expression` carrying a `class_body` child — which also - * keeps this a no-op for C#, whose `object_creation_expression` uses - * `initializer_expression`, never `class_body` (#2550). - * - * The SAME name must be produced by every layer that keys the anonymous - * class (structure-phase node id, enclosing-owner walk, scope-side - * declaration synthesis, receiver typeBinding) — they agree by all calling - * this one helper. + * GitNexus's source-type-relative Java identity for local and anonymous + * types. It follows javac's `$N` allocation but intentionally omits the + * package prefix because graph ids already include the source file path. */ -/** Type-declaration node types that can host (and name) a Java anonymous - * class body. Naming follows JLS 13.1: the binary name is the - * IMMEDIATELY enclosing type's binary name + `$N`, so the synthesized - * name is the `$`-joined chain of enclosing host names - * (`EnumWrap$Mode$1`), numbered per immediate host in source order. */ -const JAVA_ANON_HOST_TYPES = new Set([ - 'class_declaration', - 'enum_declaration', - 'interface_declaration', - 'record_declaration', +export interface JavaSynthesizedTypeIdentity { + readonly name: string; + readonly label: 'Class' | 'Enum' | 'Record' | 'Interface'; + readonly bindingName?: string; +} + +/** Named Java declarations that can host, or themselves be, local types. */ +const JAVA_NAMED_TYPE_NODE_LABELS = new Map([ + ['class_declaration', 'Class'], + ['enum_declaration', 'Enum'], + ['interface_declaration', 'Interface'], + ['record_declaration', 'Record'], ]); +const JAVA_ANON_HOST_TYPES = new Set(JAVA_NAMED_TYPE_NODE_LABELS.keys()); +const JAVA_LOCAL_TYPE_CONTAINERS = new Set([ + 'block', + 'constructor_body', + 'switch_block_statement_group', +]); + +/** A legal local type declaration is a class, enum, record, or interface + * directly occupying a block-statement position. Annotation interfaces are + * deliberately excluded: javac rejects local annotation declarations. */ +export const javaLocalTypeDeclarationContainer = (node: SyntaxNode): SyntaxNode | null => { + if (!JAVA_NAMED_TYPE_NODE_LABELS.has(node.type)) return null; + const parent = node.parent; + return parent !== null && JAVA_LOCAL_TYPE_CONTAINERS.has(parent.type) ? parent : null; +}; + +const isJavaLocalTypeNode = (node: SyntaxNode): boolean => + javaLocalTypeDeclarationContainer(node) !== null; + /** The two Java anonymous-class-body shapes (#2550/#2555): an * `object_creation_expression` with a `class_body` child * (`new Runnable() { ... }`), and an `enum_constant` with a `body:` @@ -439,10 +557,7 @@ const isJavaAnonymousBodyNode = (node: SyntaxNode): boolean => node.namedChildren?.some((c: SyntaxNode) => c.type === 'class_body') === true) || (node.type === 'enum_constant' && node.childForFieldName?.('body')?.type === 'class_body'); -/** Nearest ancestor of `node` that is an enclosing TYPE per JLS 13.1 — - * a named host declaration OR another anonymous body (both shapes). - * Anonymous ancestors chain through: an anon inside an anon is - * `Host$1$1`, and an anon inside an enum constant body is `E$1$1`. */ +/** Nearest ancestor of `node` that is an enclosing type per JLS 13.1. */ const nearestJavaEnclosingType = (node: SyntaxNode): SyntaxNode | null => { let cursor: SyntaxNode | null = node.parent; let iterations = 0; @@ -454,83 +569,142 @@ const nearestJavaEnclosingType = (node: SyntaxNode): SyntaxNode | null => { return null; }; -/** Per-parse-tree memo of anonymous-body numbering: tree → (startIndex → - * synthesized name). Keyed by the tree OBJECT via WeakMap so entries die - * with the parse; without it every call re-scans the host subtree - * (`descendantsOfType`), and the helper is called from four independent - * layers per anonymous body — quadratic on anon-heavy files (old-style - * listener-per-widget Java). */ -const javaAnonNameMemo = new WeakMap>(); +interface JavaTypeIdentityState { + readonly byStart: Map; + readonly ordinalByStart: Map; +} -export const synthesizeJavaAnonymousClassName = (node: SyntaxNode): string | undefined => { - if (!isJavaAnonymousBodyNode(node)) return undefined; +/** Parse-tree-bounded memo. Sequence ordinals are built once per tree, avoiding + * a host-candidate scan for every extraction/ownership consumer. */ +const javaTypeIdentityMemo = new WeakMap(); - const tree = (node as { tree?: object }).tree; - if (tree !== undefined) { - const cached = javaAnonNameMemo.get(tree)?.get(node.startIndex); - if (cached !== undefined) return cached; +const javaHostKey = (node: SyntaxNode): string => `${node.type}:${node.startIndex}`; + +const javaIdentityCandidatesBelow = (root: SyntaxNode): SyntaxNode[] => { + const seen = new Set(); + const candidates: SyntaxNode[] = []; + for (const type of [ + 'object_creation_expression', + 'enum_constant', + ...JAVA_NAMED_TYPE_NODE_LABELS.keys(), + ]) { + for (const candidate of root.descendantsOfType?.(type) ?? []) { + if (!isJavaAnonymousBodyNode(candidate) && !isJavaLocalTypeNode(candidate)) continue; + const key = javaHostKey(candidate); + if (seen.has(key)) continue; + seen.add(key); + candidates.push(candidate); + } } + return candidates.sort((left, right) => left.startIndex - right.startIndex); +}; - // JLS 13.1: the binary name is the IMMEDIATELY ENCLOSING TYPE's binary - // name + `$N`. The enclosing type may itself be anonymous — then its - // own synthesized name is the prefix (recursion, memo-bounded): - // `NestHost$1$1` for an anon inside an anon, `E$1$1` for an anon - // inside an enum constant body. For a named enclosing type the prefix - // is the `$`-joined chain of named hosts (`EnumWrap$Mode`). +const buildJavaTypeIdentityState = (root: SyntaxNode): JavaTypeIdentityState => { + const ordinalByStart = new Map(); + const sequenceCounts = new Map(); + for (const candidate of javaIdentityCandidatesBelow(root)) { + const host = nearestJavaEnclosingType(candidate); + if (host === null) continue; + const isAnonymous = isJavaAnonymousBodyNode(candidate); + const bindingName = isAnonymous ? '' : candidate.childForFieldName?.('name')?.text; + // Anonymous types deliberately use the empty sequence key; malformed named + // declarations must not enter that sequence. + if (!isAnonymous && !bindingName) continue; + const sequenceKey = `${javaHostKey(host)}:${bindingName}`; + const ordinal = (sequenceCounts.get(sequenceKey) ?? 0) + 1; + sequenceCounts.set(sequenceKey, ordinal); + ordinalByStart.set(candidate.startIndex, ordinal); + } + return { byStart: new Map(), ordinalByStart }; +}; + +const javaTypeIdentityStateFor = (node: SyntaxNode): JavaTypeIdentityState => { + const tree = (node as { tree?: { rootNode?: SyntaxNode } }).tree; + if (tree === undefined) { + const host = nearestJavaEnclosingType(node); + return buildJavaTypeIdentityState(host ?? node); + } + let state = javaTypeIdentityMemo.get(tree); + if (state === undefined) { + state = buildJavaTypeIdentityState(tree.rootNode ?? node); + javaTypeIdentityMemo.set(tree, state); + } + return state; +}; + +/** Source-type-relative binary name of a Java enclosing type, including + * synthesized local/anonymous hosts and named member-type chains. */ +const javaBinaryNameOfType = (node: SyntaxNode): string | undefined => { + if (isJavaAnonymousBodyNode(node) || isJavaLocalTypeNode(node)) { + return synthesizeJavaTypeIdentity(node)?.name; + } + if (!JAVA_ANON_HOST_TYPES.has(node.type)) return undefined; + const simpleName = node.childForFieldName?.('name')?.text; + if (simpleName === undefined || simpleName.length === 0) return undefined; const enclosing = nearestJavaEnclosingType(node); + if (enclosing === null) return simpleName; + const enclosingName = javaBinaryNameOfType(enclosing); + return enclosingName === undefined ? undefined : `${enclosingName}$${simpleName}`; +}; + +/** + * Authoritative Java local/anonymous type identity. + * + * JLS 13.1 defines the shape and immediate-host prefix. OpenJDK javac's + * Check.localClassName allocates N independently for each + * (enclosing binary name, local simple name) pair; anonymous types use the + * empty simple name and therefore have their own sequence. Package names are + * omitted from this project identity because graph ids already include the + * file path. + */ +export const synthesizeJavaTypeIdentity = ( + node: SyntaxNode, +): JavaSynthesizedTypeIdentity | undefined => { + const localLabel = JAVA_NAMED_TYPE_NODE_LABELS.get(node.type); + const isLocal = localLabel !== undefined && isJavaLocalTypeNode(node); + const isAnonymous = isJavaAnonymousBodyNode(node); + const enclosing = nearestJavaEnclosingType(node); + const memberSimpleName = + !isLocal && !isAnonymous && localLabel !== undefined + ? node.childForFieldName?.('name')?.text + : undefined; + const synthesizedHostIdentity = + memberSimpleName !== undefined && enclosing !== null + ? synthesizeJavaTypeIdentity(enclosing) + : undefined; + if (!isLocal && !isAnonymous && synthesizedHostIdentity === undefined) return undefined; if (enclosing === null) return undefined; - let prefix: string; - if (isJavaAnonymousBodyNode(enclosing)) { - const enclosingName = synthesizeJavaAnonymousClassName(enclosing); - if (enclosingName === undefined) return undefined; - prefix = enclosingName; - } else { - const hostNames: string[] = []; - let cursor: SyntaxNode | null = enclosing; - let iterations = 0; - while (cursor) { - if (++iterations > MAX_ENCLOSING_WALK_ITERATIONS) return undefined; - if (JAVA_ANON_HOST_TYPES.has(cursor.type)) { - const hostName = cursor.childForFieldName?.('name')?.text; - if (hostName === undefined || hostName.length === 0) return undefined; - hostNames.unshift(hostName); - } - cursor = cursor.parent; - } - prefix = hostNames.join('$'); + + const state = javaTypeIdentityStateFor(node); + const cached = state.byStart.get(node.startIndex); + if (cached !== undefined) return cached; + + const prefix = javaBinaryNameOfType(enclosing); + if (prefix === undefined) return undefined; + + if (memberSimpleName !== undefined) { + const identity: JavaSynthesizedTypeIdentity = { + name: `${prefix}$${memberSimpleName}`, + label: localLabel!, + bindingName: memberSimpleName, + }; + state.byStart.set(node.startIndex, identity); + return identity; } - // All anonymous bodies (both shapes) whose immediately enclosing TYPE - // is THIS one, in source order. `descendantsOfType` over the subtree - // also finds bodies belonging to nested enclosing types — filter them - // out by re-deriving each candidate's own enclosing type. - const candidates = [ - ...(enclosing.descendantsOfType?.('object_creation_expression') ?? []), - ...(enclosing.descendantsOfType?.('enum_constant') ?? []), - ] - .filter(isJavaAnonymousBodyNode) - .filter((c: SyntaxNode) => { - const host = nearestJavaEnclosingType(c); - return ( - host !== null && host.startIndex === enclosing.startIndex && host.type === enclosing.type - ); - }) - .sort((a: SyntaxNode, b: SyntaxNode) => a.startIndex - b.startIndex); + const bindingName = isLocal ? node.childForFieldName?.('name')?.text : undefined; + if (isLocal && !bindingName) return undefined; - if (tree !== undefined) { - let byStart = javaAnonNameMemo.get(tree); - if (byStart === undefined) { - byStart = new Map(); - javaAnonNameMemo.set(tree, byStart); - } - for (let i = 0; i < candidates.length; i++) { - byStart.set(candidates[i]!.startIndex, `${prefix}$${i + 1}`); - } - return byStart.get(node.startIndex); - } - const index = candidates.findIndex((c: SyntaxNode) => c.startIndex === node.startIndex); - if (index === -1) return undefined; - return `${prefix}$${index + 1}`; + const ordinal = state.ordinalByStart.get(node.startIndex); + if (ordinal === undefined) return undefined; + + const identity: JavaSynthesizedTypeIdentity = { + name: `${prefix}$${ordinal}${bindingName ?? ''}`, + label: isAnonymous ? 'Class' : localLabel!, + ...(bindingName === undefined ? {} : { bindingName }), + }; + state.byStart.set(node.startIndex, identity); + return identity; }; export const findEnclosingClassInfo = ( @@ -605,12 +779,12 @@ export const findEnclosingClassInfo = ( // enum constant, and every C# `object_creation_expression`), so the // walk continues unchanged for those — including on to // `enum_declaration`, which sits in CLASS_CONTAINER_TYPES below. - if (current.type === 'object_creation_expression' || current.type === 'enum_constant') { - const anonName = synthesizeJavaAnonymousClassName(current); - if (anonName !== undefined) { + if (isJavaAnonymousBodyNode(current) || JAVA_ANON_HOST_TYPES.has(current.type)) { + const identity = synthesizeJavaTypeIdentity(current); + if (identity !== undefined) { return { - classId: generateId('Class', `${filePath}:${anonName}`), - className: anonName, + classId: generateId(identity.label, `${filePath}:${identity.name}`), + className: identity.name, }; } } diff --git a/gitnexus/src/core/ingestion/utils/effective-ram.ts b/gitnexus/src/core/ingestion/utils/effective-ram.ts new file mode 100644 index 000000000..c2503ba97 --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/effective-ram.ts @@ -0,0 +1,67 @@ +import os from 'node:os'; + +/** + * Effective RAM in bytes: physical total, or a REAL smaller cgroup limit + * (#2649). `process.constrainedMemory()` returns a huge sentinel when + * unconstrained, and only the leaf cgroup's limit is visible (parent-slice + * caps are not) — so a smaller-than-physical value is trusted and anything + * else falls back to `os.totalmem()`. Mirrors `computeHeapCapMb`'s + * constrained handling in `cli/analyze.ts`; container-blind sizing told + * users "this machine has more memory" inside an 8GB-limited container on + * a 64GB host, and sized worker heap caps past the whole container. + */ +export function effectiveRamBytes(): number { + const total = os.totalmem(); + const constrained = + typeof process.constrainedMemory === 'function' ? process.constrainedMemory() : undefined; + return typeof constrained === 'number' && constrained > 0 && constrained < total + ? constrained + : total; +} + +/** Historical floor for the auto heap cap — applied only up to 0.80 × RAM + * (a floor at or above physical memory swap-thrashes instead of OOMing, + * #2649). */ +const HEAP_FLOOR_MB = 16384; + +/** + * The RAM-aware heap cap formula (#2649), single-sourced here so the CLI + * respawn (`computeHeapCapMb` in `cli/analyze.ts`), and the server's + * analyze fork size from the same rule: `0.75 × effective RAM`, raised to + * the floor when RAM allows, never above `0.80 × effective RAM`. + */ +export function heapCapMbFor(effectiveBytes: number): number { + const effectiveMb = Math.floor(effectiveBytes / (1024 * 1024)); + return Math.min( + Math.max(HEAP_FLOOR_MB, Math.floor(0.75 * effectiveMb)), + Math.floor(0.8 * effectiveMb), + ); +} + +/** + * True when the operator has turned GitNexus's memory autopilot off + * (`GITNEXUS_MEMORY=off`). + * + * One switch for one concern. Memory management has two automatic behaviours — + * re-running analyze with a RAM-aware heap cap, and aborting the parse before + * V8's ineffective-mark-compact death spiral — and an operator who wants to + * drive manually wants both off, not one. They were previously two separate + * variables (`GITNEXUS_AUTO_HEAP`, `GITNEXUS_HEAP_GUARD`), which is three knobs + * for one intent once the worker-heap override is counted; neither had shipped, + * so this consolidates them rather than deprecating anything. + * + * Note the ordinary way to pin the heap is Node's own `--max-old-space-size`, + * which `ensureHeap` already honours as the operator's decision. This switch is + * for declining the autopilot WITHOUT naming a size. + * + * Lives here beside the cap formula so policy and its escape hatch are + * single-sourced. Read every call (not memoized) so tests can stub the env. + */ +export function memoryAutopilotDisabled(): boolean { + return process.env.GITNEXUS_MEMORY === 'off'; +} + +/** The cap for THIS machine/container: `heapCapMbFor(effectiveRamBytes())`. */ +export function autoHeapCapMb(): number { + return heapCapMbFor(effectiveRamBytes()); +} diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index bf0e19f07..4e35f72d2 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -78,7 +78,7 @@ try { } catch {} import { getLanguageFromFilename } from 'gitnexus-shared'; import { - buildConcreteTypedefDefinitionRanges, + buildDefinitionPreScan, FUNCTION_NODE_TYPES, findAncestorBeforeBoundary, getDefinitionNodeFromCaptures, @@ -89,6 +89,7 @@ import { genericFuncName, inferFunctionLabel, isSuppressedConcreteTypedefDuplicate, + isValueDefinitionLabel, isQualifiableScopeLabel, qualifyRustImplTargetByModScope, CLASS_CONTAINER_TYPES, @@ -1313,10 +1314,15 @@ const processFileGroup = ( ); continue; } - const concreteTypedefRanges = buildConcreteTypedefDefinitionRanges(matches); - const provider = getProvider(language); + // #2687: ONE pass over `matches` yields both suppression sets — the + // definition-name claims by rank (callable > Property > value), so the dedup + // below cannot depend on tree-sitter's match order, and the concrete-typedef + // ranges the typedef guard consumes. + const definitionPreScan = buildDefinitionPreScan(matches, provider); + const concreteTypedefRanges = definitionPreScan.concreteTypedefRanges; + // Produce the `ParsedFile` for the scope-resolution pipeline HERE, reusing // the tree we just parsed (no second tree-sitter parse). Scope-resolution // consumes these via the disk-backed parsedfile-store instead of @@ -1967,19 +1973,41 @@ const processFileGroup = ( // Dedup: variable captures (Const/Static/Variable) may overlap with higher-priority // captures (e.g. `const fn = () => {}` matches both @definition.function and @definition.const). // Multi-name declarations share the same definition node, so include the emitted name. + // + // `processedDefinitionNodes` alone only suppressed the value twin when the + // function-like match happened to be processed FIRST — and it is not. + // tree-sitter completes `@definition.const` at `@name`, while + // `@definition.function` must also match the trailing arrow / function + // expression, so the const match is yielded first and its edgeless twin + // escaped (#2687). `definitionPreScan` is the order-independent view of + // the same claim, pre-scanned over `matches` before this loop and ranked so + // a capture is dropped only by a STRICTLY higher-ranked claimant. + // + // It also replaces the old bare-`startIndex` claim, which was too coarse: + // a callable declared FIRST in a multi-name declaration + // (`const cb = () => 1, SIBLING = 2`) registered the shared definition + // node and silently dropped every later sibling on it. Both keys are now + // name-scoped, so siblings survive in either declarator order. + // + // The long-term collapse seam for this duplicate class is + // `selectNodeBearingDef` (#1876, still unwired); this pre-scan is the local + // form that keeps the hot loop single-pass. Keep them in sync if #1876 lands. if (definitionNode) { - const definitionBaseKey = `${definitionNode.startIndex}`; - if (nodeLabel === 'Const' || nodeLabel === 'Static' || nodeLabel === 'Variable') { - const definitionNameKey = `${definitionBaseKey}:${nodeName}`; + const definitionNameKey = `${definitionNode.startIndex}:${nodeName}`; + if (isValueDefinitionLabel(nodeLabel)) { if ( - processedDefinitionNodes.has(definitionBaseKey) || - processedDefinitionNodes.has(definitionNameKey) + processedDefinitionNodes.has(definitionNameKey) || + definitionPreScan.nonValue.has(definitionNameKey) ) { continue; } processedDefinitionNodes.add(definitionNameKey); - } else { - processedDefinitionNodes.add(definitionBaseKey); + } else if (nodeLabel === 'Property' && definitionPreScan.callable.has(definitionNameKey)) { + // Only a CALLABLE collapses a property. Consulting the wider + // `nonValue` set here would let a property suppress itself, and would + // let an annotated Python attribute lose to its own bare-assignment + // twin — the property must outrank `Variable`, not tie with it. + continue; } } diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index 9cf40a1f9..de102ebef 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -1,5 +1,6 @@ import { Worker } from 'node:worker_threads'; import os from 'node:os'; +import { effectiveRamBytes } from '../utils/effective-ram.js'; import fs from 'node:fs'; import { fileURLToPath } from 'node:url'; @@ -224,6 +225,13 @@ export interface WorkerPoolOptions { * code should leave this unset. */ workerFactory?: (workerUrl: URL) => Worker; + /** + * Test-only injection point for the main-thread stall probe (#2649): + * returns cumulative event-loop stall in ms. When provided, the pool + * skips its heartbeat tracker and reads this instead. Production code + * should leave this unset. + */ + stallMsProbe?: () => number; /** * Storage path for the disk-backed ParsedFile store (#1983 parallel * serialization). When set, it is baked into every spawned worker's @@ -811,7 +819,7 @@ function waitForWorkerReady(worker: Worker, readyTimeoutMs: number): Promise( * single non-cloneable value can't masquerade as a worker death and exhaust a * slot's respawn budget here. */ + +/** + * Main-thread stall tracking (#2649). Near the V8 heap limit, multi-second + * mark-compact pauses freeze the main thread's message processing, so a + * healthy worker's `progress` messages sit unread and the worker LOOKS idle — + * the idle-timeout path then splits/retires it, and the respawn storm ends in + * "Replacement worker did not report ready". A 250ms unref'd heartbeat + * accumulates observed event-loop drift; the idle-timeout handler credits + * that stall once per job instead of retiring a worker the main thread + * starved. The floor filters scheduler jitter from real stalls. + */ +const HEARTBEAT_INTERVAL_MS = 250; +const HEARTBEAT_STALL_FLOOR_MS = 100; +/** Fraction of the idle-timeout budget that must be main-thread stall before + * the timeout is credited and re-armed instead of acted on. */ +const STALL_CREDIT_FRACTION = 0.5; + +export function startHeartbeatStallTracker(): { read: () => number; stop: () => void } { + let totalStallMs = 0; + let last = Date.now(); + const handle = setInterval(() => { + const now = Date.now(); + const drift = now - last - HEARTBEAT_INTERVAL_MS; + if (drift > HEARTBEAT_STALL_FLOOR_MS) totalStallMs += drift; + last = now; + }, HEARTBEAT_INTERVAL_MS); + handle.unref?.(); + return { read: () => totalStallMs, stop: () => clearInterval(handle) }; +} + +/** + * Per-worker V8 old-generation heap cap in MB (#2649). Without one, worker + * isolates inherit an unbounded default and a full pool can inflate process + * RSS past physical RAM on large repos. Half of RAM split across the pool, + * clamped to [512, 4096] MB — generous for the per-sub-batch working set + * (jobs are byte-budgeted), and a worker that does exceed it dies with a + * real heap error surfaced by the stderr-tail machinery + the + * quarantine/respawn path, instead of silently dragging the host into swap. + * `GITNEXUS_WORKER_HEAP_MB` overrides the formula. Exported for unit tests. + */ +export function resolveWorkerHeapCapMb(poolSize: number): number { + return ( + positiveInteger(process.env.GITNEXUS_WORKER_HEAP_MB) ?? + Math.min(4096, Math.max(512, Math.floor(effectiveRamBytes() / (1024 * 1024) / 2 / poolSize))) + ); +} + export const createWorkerPool = ( workerUrl: URL, poolSize?: number, @@ -957,6 +1012,24 @@ export const createWorkerPool = ( parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg ? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines } : undefined; + const workerHeapCapMb = resolveWorkerHeapCapMb(size); + // The 512MB per-worker floor exists so a worker can parse anything real, + // but on a very small container a large pool of floored workers can still + // overcommit total memory (#2649 review). Behavior is unchanged — deaths + // are attributed and quarantine converges — but say so up front, with the + // two levers, instead of letting the operator discover it from worker OOMs. + const poolCommitMb = workerHeapCapMb * size; + const effectiveMb = Math.floor(effectiveRamBytes() / (1024 * 1024)); + if (poolCommitMb > 0.6 * effectiveMb) { + logger.warn( + { poolSize: size, workerHeapCapMb, effectiveMb }, + `Worker pool may overcommit memory: ${size} workers × ${workerHeapCapMb}MB heap cap exceeds 60% of the ${effectiveMb}MB available to this process. Reduce GITNEXUS_WORKER_POOL_SIZE or set GITNEXUS_WORKER_HEAP_MB.`, + ); + } + // #2649 stall probe: test seam wins; production uses the heartbeat tracker. + const stallTracker = options?.stallMsProbe + ? { read: options.stallMsProbe, stop: (): void => undefined } + : startHeartbeatStallTracker(); const spawnWorker = options?.workerFactory ?? ((url: URL) => @@ -976,7 +1049,7 @@ export const createWorkerPool = ( // nesting levels (far beyond any hand-written code); a deeper machine- // generated nest is still caught per-function (buildFunctionCfg's R4 // try/catch) and only that function's PDG is skipped, never a crash. - resourceLimits: { stackSizeMb: 16 }, + resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: workerHeapCapMb }, })); /** Spawn + wire stdio capture/forwarding in one step (used by all spawn sites). */ const spawnAndCapture = (url: URL): Worker => { @@ -1843,10 +1916,28 @@ export const createWorkerPool = ( maybeDone(); }; + let stallCreditUsed = false; + let stallAtArm = 0; const resetIdleTimer = () => { if (idleTimer) clearTimeout(idleTimer); + stallAtArm = stallTracker.read(); idleTimer = setTimeout(() => { if (!settled) { + // #2649: when at least STALL_CREDIT_FRACTION of the timeout + // window was main-thread stall (GC pressure near the heap + // limit), the worker's progress messages were starved, not + // absent — credit the stall once per job and re-arm instead + // of splitting/retiring a healthy worker. + const stallMs = stallTracker.read() - stallAtArm; + if (!stallCreditUsed && stallMs >= job.timeoutMs * STALL_CREDIT_FRACTION) { + stallCreditUsed = true; + logger.warn( + { workerIndex, stallMs: Math.round(stallMs), timeoutMs: job.timeoutMs }, + `Worker ${workerIndex} idle timeout overlapped a main-thread stall (GC pressure); re-arming once instead of retiring.`, + ); + resetIdleTimer(); + return; + } settled = true; cleanup(); inFlightProgress[workerIndex] = 0; @@ -2084,10 +2175,22 @@ export const createWorkerPool = ( // the `{type:'error'}` message, the event delivers a real Error whose // `.stack` is the worker-side frame — carry it so the surfaced reason // points at the actual failure site, not just `err.message` (#2068). - void recoverAndResume( - workerErrorReason(workerIndex, err.message, err.stack), - resolveExcludePaths(), - ); + // A worker dying on ITS OWN heap cap (#2649) must be attributable to + // that cap, not read as generic quarantine noise — name the cap and + // its override so an oversized-but-legitimate file (e.g. under a + // raised GITNEXUS_MAX_FILE_SIZE) is a one-env-var fix. + // The 'error' event does not guarantee a well-formed Error: the + // structured-clone failure path can deliver a value with no + // `message` — guard every property access or the handler itself + // throws and the pool hangs instead of recovering. + const isWorkerHeapOom = + (err as NodeJS.ErrnoException | undefined)?.code === 'ERR_WORKER_OUT_OF_MEMORY' || + (typeof err?.message === 'string' && + err.message.includes('ERR_WORKER_OUT_OF_MEMORY')); + const reason = isWorkerHeapOom + ? `${workerErrorReason(workerIndex, err.message, err.stack)} (worker hit its ${workerHeapCapMb}MB heap cap — raise with GITNEXUS_WORKER_HEAP_MB)` + : workerErrorReason(workerIndex, err.message, err.stack); + void recoverAndResume(reason, resolveExcludePaths()); } }; @@ -2154,6 +2257,7 @@ export const createWorkerPool = ( const terminate = async (): Promise => { terminated = true; + stallTracker.stop(); // Cancel any in-flight startup backoff so its ref'd timer doesn't keep the // event loop alive after terminate; each cancel resolves the awaiting sleep // and the slot loop then sees `terminated` and gives up (#1741). diff --git a/gitnexus/src/core/lbug/extension-load-error.ts b/gitnexus/src/core/lbug/extension-load-error.ts index 712730164..f8213ef4d 100644 --- a/gitnexus/src/core/lbug/extension-load-error.ts +++ b/gitnexus/src/core/lbug/extension-load-error.ts @@ -127,13 +127,25 @@ const VC_REDIST_INSTALL_HINT = 'the Microsoft Visual C++ 2015-2022 Redistributable (x64) from ' + 'https://aka.ms/vs/17/release/vc_redist.x64.exe'; +// Git for Windows already ships the OpenSSL 3 DLLs in its mingw64 bin directory, +// so the identical command that fails in PowerShell succeeds in Git Bash (#2669 +// reporter, who had the VC++ redist installed and still failed until that +// directory was on PATH). Deliberately a fixed system path and never a +// user-profile one: remedy text is NOT path-redacted (fts-indexes.ts redacts +// only the reason), and fts-degraded-warning.test.ts asserts that no +// `C:\Users\…` path ever reaches a user through this surface. +const GIT_BASH_OPENSSL_HINT = + ' If Git for Windows is installed you already have those DLLs: run the same command from Git Bash, ' + + 'or prepend "C:\\Program Files\\Git\\mingw64\\bin" to PATH.'; + // MSVC-first per DuckDB's canonical answer for this exact error; OpenSSL second. const windowsMissingDependencyRemedy = (label: string): string => `The ${label} extension is present but a required runtime library is missing (Windows error 126). ` + 'Reinstalling the extension will NOT help. Install ' + VC_REDIST_INSTALL_HINT + '; if the error persists, the extension also needs OpenSSL 3 ' + - '(libcrypto-3-x64.dll / libssl-3-x64.dll) on the DLL search path.'; + '(libcrypto-3-x64.dll / libssl-3-x64.dll) on the DLL search path.' + + GIT_BASH_OPENSSL_HINT; const posixMissingDependencyRemedy = (label: string): string => `The ${label} extension is present but a shared library it depends on could not be loaded (named in ` + @@ -205,7 +217,8 @@ const structuralMissingDependencyRemedy = (label: string): string => `The ${label} extension file is valid, so the failure is a missing or incompatible runtime dependency, ` + 'not the extension itself — reinstalling will NOT help. On Windows, install ' + VC_REDIST_INSTALL_HINT + - ' and ensure OpenSSL 3 is available; on Linux/macOS install the shared library named in the error above.'; + ' and ensure OpenSSL 3 is available; on Linux/macOS install the shared library named in the error above.' + + GIT_BASH_OPENSSL_HINT; /** * Pull the extension file path out of lbug's load error. lbug's wrapper is diff --git a/gitnexus/src/core/lbug/graph-emit-sink.ts b/gitnexus/src/core/lbug/graph-emit-sink.ts new file mode 100644 index 000000000..b814decf6 --- /dev/null +++ b/gitnexus/src/core/lbug/graph-emit-sink.ts @@ -0,0 +1,633 @@ +/** + * Streaming structural graph-emit sink (issue #2680). + * + * `analyze` holds the whole `KnowledgeGraph` on the main thread for the entire + * pipeline, so peak heap is O(repo) — ~2.1 KB/node at Linux-kernel scale + * (#2649). Measurement on a kernel-shaped synthetic graph (400k nodes, + * 2.7 edges/node) says where that goes: + * + * nodes only ....... 367 B/node + * nodes + edges .... 2075 B/node <- reproduces the #2649 figure + * + * So **relationships are ~83% of graph heap** (~646 B/edge), and that is what + * this sink removes. 646 B for an object holding four short strings is the cost + * of storing every edge four times over — `relationshipMap`, a + * `relationshipsByType` bucket, and both endpoints' `edgeIdsByNode` Sets — plus + * an `id` that concatenates both endpoint ids. (Dropping just the two redundant + * indexes was measured too: 174 of 648 B/edge, ~1.3x. Not enough on its own.) + * + * Nodes are deliberately NOT streamed: they are the other 17%, and two + * scope-resolution index builders (`buildGraphNodeLookup`, + * `buildGraphCallableAnchorIndex`) scan them. + * + * ## How much this actually saves — read this before quoting a number + * + * Measured A/B against the object-based graph, 400k nodes / 1.08M edges, all + * edges streamable (the worst case for this design): **823 MB -> 626 MB, ~1.3x**, + * with all 1.08M edges still visible through `iterRelationships`. + * + * That is well short of the ~2.9x a naive `0.17 + 0.83 * 0.21` retained-share + * calculation suggests, and the gap is deliberate: this sink is *lossless*, so + * it pays for the {@link streamedIds} dedup Set (one unique id string per + * streamed edge) and the columns above. An earlier revision hit a bigger number + * by disabling community detection, process extraction and the taint fixpoint — + * which is why it could not be the default. 1.3x with nothing traded away is the + * honest figure; if a future change needs more, the next lever is dedup keyed on + * the interned column triple rather than on id strings (it must first be shown + * not to alter the emitted row SET). + * + * ## What it costs — measured, not assumed + * + * Measured on the same 400k-node / 1.08M-edge graph, all edges streamable, each + * arm running what its own consumers actually call: + * + * heap 819 MB -> 584 MB (1.40x better) + * scans ~78 ms -> ~88 ms (parity, within run-to-run noise) + * + * Linear in both: verified at 100k/200k/400k/800k nodes, per-edge scan cost flat + * (~13 ns both arms) and the heap ratio drifting only 1.7x -> 1.5x as interner + * indices gain digits. No super-linear term, so a larger repo costs + * proportionally more, not disproportionately. + * + * The dedup key encodes its tail SEGMENT COUNT, which measurably costs ~66 MB + * here versus omitting it. That is not optional: without it a one-segment tail + * `:7` and a two-segment `:7:0` collapse onto one key and an edge is silently + * dropped (regression test in graph-emit-sink.test.ts). + * + * Getting there took three measured steps, because the naive version was 6.8x + * WORSE (651 ms) — reads rebuild objects, and a real analyze performs SIX full + * relationship scans (the pruner, community detection x2, process extraction x2, + * and the taint fixpoint's CALLS pass): + * + * 1. The ~150-character synthesized `id` was built eagerly on every read — 6.5M + * concatenations for a field no in-pipeline consumer reads. Isolating it + * showed 436 ms of the regression. It is now a lazy prototype getter on + * {@link StreamedRelationship}. + * 2. Generator and iterator-protocol overhead: {@link forEachRelationship} loops + * the columns directly, and {@link iterRelationships} reuses one + * iterator-result record. Note a hand-rolled iterator allocating a fresh + * `{value, done}` per edge measured WORSE (252 ms) than the generator it + * replaced, so the obvious rewrite is not the one that shipped. + * 3. The remaining ~90 ms was object allocation itself, irreducible while the + * read API returns objects — so the five whole-graph scans moved to + * `forEachRelationshipFields`, which passes the four fields they actually + * read as primitives and allocates nothing. See + * {@link GraphEmitSink.forEachRelationshipFields}. + * + * The last allocating scan is the taint fixpoint's `iterRelationshipsByType` + * pass; it is one scan of six and accounts for the small residual. Give it a + * by-type field variant only if a measurement says it matters. + * + * It is in any case NOT O(chunk) — node identity and the resolution registries + * stay O(repo). True O(chunk) needs DB-side resolution and Leiden (#2337), at + * which point this sink should be deleted rather than extended. + * + * ## Correctness contract + * + * Structural sibling of {@link PdgEmitSink}, and reuses its row builder + * (`buildRelRow`), header (`REL_CSV_HEADER`), label derivation (`getNodeLabel`) + * and `RelPairRouter` validity check, so the streamed row SET equals the + * whole-graph emit's and the bulk COPY loads the same rows. Set-level, not + * byte-level: rows stream in emit order and are not re-sorted under + * `GITNEXUS_SORT_GRAPH_OUTPUT`. + */ +import fs from 'fs'; +import path from 'path'; +import type { GraphNode, GraphRelationship, RelationshipType } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../graph/types.js'; +import { REL_CSV_HEADER, buildRelRow } from './csv-generator.js'; +import { getNodeLabel } from './rel-pair-routing.js'; +import { NODE_TABLES } from './schema.js'; +import { DEFAULT_EMIT_CHUNK_ROWS, SyncCsvWriter } from './sync-csv-writer.js'; + +/** + * Relationship types that MUST stay in the in-memory graph because a phase + * running while streaming is active reads them back. + * + * Derived from an exhaustive audit of every relationship read site under + * `gitnexus/src/` (`iterRelationshipsByType` / `iterRelationships` / + * `forEachRelationship` / `removeRelationship`), not from intuition — an + * earlier draft of this list carried 14 types, 5 of which no reachable phase + * reads. Every entry below names its reader: + * + * EXTENDS, IMPLEMENTS - mro-processor, scope-resolution/passes/mro, + * receiver-bound-calls, pipeline/run.ts, cpp + * member-lookup, and 9 language scope-resolvers + * HAS_METHOD - mro-processor, di phase + * HAS_PROPERTY - di phase, ruby scope-resolver, spring config-bindings + * METHOD_OVERRIDES, + * METHOD_IMPLEMENTS - mro-processor + * DEFINES - local-symbol-pruner's isFileDefinesEdge test + * INJECTS - di phase fan-out + * ENTRY_POINT_OF - process-processor's collectExplicitEntryPointIds + * (compiler-declared roots written pre-parse by the + * standalone Move ingest, read back in `processes`) + * IMPORTS - move-linker's file-import linking pass + * + * Deliberately NOT retained: STEP_IN_PROCESS / MEMBER_OF (written only by the + * `processes` / `communities` phases, which the streaming flag disables), + * TAINT_PATH / CALL_SUMMARY (their phases are likewise gated off under the + * flag), and HANDLES_ROUTE / HANDLES_TOOL (written by `routes`/`tools`, never + * read back mid-pipeline). + * + * Adding a relationship type that a phase reads back WITHOUT adding it here is + * a silent-wrong-graph bug, not a crash — and NOTHING automated catches it. + * The differential round-trip test cannot: `addRelationship` partitions edges + * between the graph and the CSVs, and the union of a partition is invariant + * under where the partition line falls, so that test stays green no matter how + * this set is drawn. Only the read-site audit protects this invariant; re-run it + * (grep iterRelationshipsByType / iterRelationships / forEachRelationship / + * removeRelationship across src/) when adding a phase or a relationship type. + */ +export const RETAINED_REL_TYPES: ReadonlySet = new Set([ + 'EXTENDS', + 'IMPLEMENTS', + 'HAS_METHOD', + 'HAS_PROPERTY', + 'METHOD_OVERRIDES', + 'METHOD_IMPLEMENTS', + 'DEFINES', + 'INJECTS', + 'ENTRY_POINT_OF', + 'IMPORTS', +]); + +/** + * COPY manifest produced by {@link GraphEmitSink.finalize}. + * + * Only `relsByPair` — this PR does not stream node rows, so a `nodeFiles` + * dimension would be permanently empty. Note that unlike `PdgEmitManifest`, + * these pair keys DO collide with the whole-graph emit's (streamed `CALLS` is + * `Function|Function`, same as retained edges), so `loadGraphToLbug` must + * APPEND these files to the pair rather than reject them as a collision. + */ +export interface GraphEmitManifest { + /** pairKey (`From|To`) -> per-pair edge CSV. */ + readonly relsByPair: Map; + /** Total streamed rows, for the buffer-pool size hint (#2631 path). */ + readonly totalRows: number; +} + +/** + * The slice of the sink that pipeline phases drive. Declared here, next to the + * implementation, and imported as a type by `pipeline-phases/types.ts` so the + * phase layer depends on this narrow capability rather than on two loose + * callbacks bolted onto the context. + */ +export interface GraphEmitControl { + /** Start routing non-retained relationships to disk (see {@link GraphEmitSink.beginStreaming}). */ + beginStreaming(): void; +} + +/** + * A streamed edge, rebuilt for a read. + * + * A class, not an object literal, for two reasons that both showed up in + * measurement. Its shape is fixed, so V8 keeps one hidden class across millions + * of instances; and `id` is a PROTOTYPE getter, so the ~150-character + * concatenation happens only if a caller actually reads it — which none of the + * in-pipeline consumers do. Building it eagerly cost 436 ms of a 555 ms + * iteration regression across the six full scans an analyze performs (measured + * at 400k nodes / 1.08M edges); deferring it gives that back. + */ +class StreamedRelationship implements GraphRelationship { + /** Constant for streamed edges — see the note on the columns about why + * `reason` is not retained. The persisted CSV row keeps the real value. */ + readonly reason = 'streamed'; + + constructor( + readonly sourceId: string, + readonly targetId: string, + readonly type: RelationshipType, + readonly confidence: number, + private readonly ix: number, + ) {} + + /** Deterministic and unique — the column index disambiguates two streamed + * edges that share (type, source, target). Lazily built; nothing in the + * pipeline reads it. */ + get id(): string { + return `${this.type}:${this.sourceId}->${this.targetId}#${this.ix}`; + } +} + +/** Thrown when a consumer removes a relationship that already streamed to + * disk. Silently no-oping would let a mutating consumer (e.g. the COBOL + * cross-program CALL resolver) corrupt the persisted graph undetected. */ +export class StreamedRelationshipRemovalError extends Error { + constructor(relationshipId: string) { + super( + `Cannot remove relationship "${relationshipId}": it has already been streamed to ` + + `CSV and cannot be recalled. A phase that removes relationships must run before ` + + `the GraphEmitSink is installed (see the parse-boundary construction in pipeline.ts).`, + ); + this.name = 'StreamedRelationshipRemovalError'; + } +} + +/** + * Write-routing graph façade. Construct one per analyze run at the PARSE + * boundary — not at `createKnowledgeGraph()` — so the pre-parse phases + * (`structure`, `springConfig`, `markdown`, `cobol`) complete their + * read-modify-delete passes against a fully in-memory graph. Call + * {@link finalize} once after the pipeline, before `loadGraphToLbug`. + */ +export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { + private readonly validTables: Set; + private readonly relWriters = new Map(); + /** + * Ids of relationships already streamed. `KnowledgeGraph.addRelationship` + * drops duplicate ids first-writer-wins, and COPY into a PK-bearing table + * would violate on a repeat, so the sink must dedup itself — unlike + * `PdgEmitSink`, whose emit loop guarantees per-file uniqueness upstream. + * + * ponytail: O(streamed-edges) id strings retained. That is ~a tenth of full + * edge retention (the objects, both endpoint index Sets, and the type bucket + * all go away), but it is not O(chunk). Upgrade path if it ever dominates: + * a per-pair sorted-run dedup on disk, or hashing ids into a Bloom filter + * with an exact fallback. + */ + private readonly streamedIds = new Set(); + /** + * Streamed edges, kept as parallel columns so the sink can still answer a + * COMPLETE relationship read (see {@link iterRelationships}). Only the four + * fields any consumer of these edges actually reads are retained — + * `sourceId`, `targetId`, `type`, `confidence` — audited across + * community-processor, process-processor, taint-summaries and the pruner. + * + * `id`, `reason` and `step` are deliberately NOT kept. Every relationship id + * is a unique long string, and retaining ids is exactly what made an earlier + * fully-columnar attempt LOSE to the object-based graph (measured 838 MB vs + * 822 MB at 400k nodes / 1.08M edges). Keeping ids out of the heap is where + * the saving comes from, so a read synthesizes a deterministic id instead — + * safe because `buildRelRow` never persists `rel.id` and no consumer keys on + * it (audited). + * + * The dropped `reason`/`step` are safe too, but for a different reason worth + * stating: the PERSISTED row keeps their true values, because `buildRelRow` is + * handed the original relationship on the way through. Only in-memory reads + * see the `'streamed'` placeholder, and the in-pipeline consumers of streamed + * edges read neither field. So e.g. the `ACCESSES reason: 'read'|'write'` + * distinction that MCP queries rely on survives in the database. A future + * in-pipeline consumer needing `reason` or `step` on a streamed edge must add + * the column, not trust the placeholder. + * + * Node ids are interned; the strings are shared by reference with the node + * map's, so interning adds bookkeeping, not new text. + */ + private readonly nodeIds = new Map(); + private readonly nodeIdByIx: string[] = []; + private readonly srcIx: number[] = []; + private readonly tgtIx: number[] = []; + private readonly relTypes: RelationshipType[] = []; + private readonly confidences: number[] = []; + private finalized = false; + /** + * Streaming is OFF until {@link beginStreaming} is called by `parse`. + * + * The pre-parse phases are not all write-only: `mapCobolToGraph` scans + * `CALLS` edges and REMOVES the unresolved ones after adding resolved + * replacements (cobol-processor.ts). If the sink streamed from + * construction, that scan would see an empty set, no COBOL cross-program + * call would ever resolve, and the removal would be a silent no-op. Nothing + * before parse produces bulk edge volume, so deferring costs nothing. + */ + private armed = false; + /** + * First writer-construction failure (`fs.openSync` throwing on e.g. EMFILE). + * It happens inside the `SyncCsvWriter` constructor before a writer object + * exists to carry poison, so it is held at sink level and folded into the + * {@link finalize} error check — otherwise an open failure mid-emit would be + * swallowed by a caller's try/catch and silently drop the rest of the rows. + */ + private openFailure: unknown | undefined = undefined; + + constructor( + private readonly real: KnowledgeGraph, + private readonly csvDir: string, + private readonly chunkRows: number = DEFAULT_EMIT_CHUNK_ROWS, + ) { + this.validTables = new Set(NODE_TABLES as readonly string[]); + // Own directory, distinct from the PDG sink's: PdgEmitSink wipes and + // recreates its dir on construction and opens with O_EXCL, so a shared dir + // would destroy the other sink's manifest on a combined --pdg run. + fs.rmSync(csvDir, { recursive: true, force: true }); + fs.mkdirSync(csvDir, { recursive: true }); + } + + // ── routed writes ────────────────────────────────────────────────────────── + + /** Nodes are never streamed (see the file header) — always the real graph. */ + addNode(node: GraphNode): void { + this.real.addNode(node); + } + + /** + * Start streaming. Called once, by the `parse` phase, for the reason on + * {@link armed}. + */ + beginStreaming(): void { + this.armed = true; + } + + /** + * Exact dedup key, built to hold no reference to the relationship id. + * + * An id embeds both node ids in full — ~200 characters on this repo — and the + * only information it adds beyond `(type, source, target)` is a short trailing + * disambiguator, e.g. `emit-references.ts` appends `:line:col` so two calls + * between the same pair at different sites stay distinct. The endpoints are + * already interned for the columns, so the key reuses those indices and parses + * the tail into NUMBERS. + * + * Numbers matter for more than size: a key built by slicing or replacing + * inside a long string is a V8 sliced/cons string that keeps its parent alive, + * so the 200-character id would never be freed and the memory saving would + * silently fail to materialize. Parsing to numbers severs that link. + * + * Falls back to the full id when the tail is not a numeric `:a:b` form (other + * id shapes exist, e.g. `rel:contains:` has no tail). Correctness first: an + * unrecognized shape is stored exactly, just without the saving. + */ + private dedupKey(rel: GraphRelationship, srcIx: number, tgtIx: number): string { + const afterTarget = rel.id.lastIndexOf(rel.targetId); + if (afterTarget >= 0) { + const tail = rel.id.slice(afterTarget + rel.targetId.length); + if (tail.length === 0) return `${srcIx}|${tgtIx}|${rel.type}`; + // `:1483:6` -> two integers. Any non-numeric segment falls through. + if (tail.charCodeAt(0) === 58 /* ':' */) { + let a = 0; + let b = 0; + let seen = 0; + let ok = true; + for (const part of tail.slice(1).split(':')) { + const n = Number(part); + if (part.length === 0 || !Number.isInteger(n)) { + ok = false; + break; + } + if (seen === 0) a = n; + else if (seen === 1) b = n; + else { + ok = false; + break; + } + seen++; + } + // `seen` is part of the key: without it a one-segment tail `:7` (b + // defaults to 0) and a two-segment `:7:0` produce the same key, and the + // second edge is silently discarded as a duplicate. Distinct ids must + // never collapse — that is a lost relationship with no error. + if (ok) return `${srcIx}|${tgtIx}|${rel.type}|${seen}|${a}|${b}`; + } + } + return rel.id; + } + + private internNode(id: string): number { + const existing = this.nodeIds.get(id); + if (existing !== undefined) return existing; + const ix = this.nodeIdByIx.length; + this.nodeIdByIx.push(id); + this.nodeIds.set(id, ix); + return ix; + } + + /** Rebuild a streamed edge; its id is synthesized lazily, not stored. */ + private streamedAt(ix: number): GraphRelationship { + return new StreamedRelationship( + this.nodeIdByIx[this.srcIx[ix]], + this.nodeIdByIx[this.tgtIx[ix]], + this.relTypes[ix], + this.confidences[ix], + ix, + ); + } + + addRelationship(relationship: GraphRelationship): void { + if (!this.armed || RETAINED_REL_TYPES.has(relationship.type)) { + this.real.addRelationship(relationship); + return; + } + // Mirror KnowledgeGraph.addRelationship's first-writer-wins dedup. + + const fromLabel = getNodeLabel(relationship.sourceId); + const toLabel = getNodeLabel(relationship.targetId); + // Skip edges whose endpoint labels are not valid node tables — mirrors + // `RelPairRouter` exactly so the streamed set matches the whole-graph set. + if (!this.validTables.has(fromLabel) || !this.validTables.has(toLabel)) return; + + const pairKey = `${fromLabel}|${toLabel}`; + let writer = this.relWriters.get(pairKey); + if (writer === undefined) { + try { + writer = new SyncCsvWriter( + path.join(this.csvDir, `rel_${fromLabel}_${toLabel}.csv`), + REL_CSV_HEADER, + this.chunkRows, + ); + } catch (e) { + this.openFailure ??= e; + throw e; + } + this.relWriters.set(pairKey, writer); + } + // Intern first so the dedup key can reuse the indices. + const srcIx = this.internNode(relationship.sourceId); + const tgtIx = this.internNode(relationship.targetId); + const key = this.dedupKey(relationship, srcIx, tgtIx); + if (this.streamedIds.has(key)) return; + this.streamedIds.add(key); + + writer.addRow(buildRelRow(relationship)); + this.srcIx.push(srcIx); + this.tgtIx.push(tgtIx); + this.relTypes.push(relationship.type); + this.confidences.push(relationship.confidence); + } + + /** Flush + close every writer and return the COPY manifest. Every fd is + * closed even when a writer is poisoned; any IO fault — an in-flight write, + * a final-flush failure, or a writer-open failure (EMFILE) — is surfaced + * loudly here so a disk-full / out-of-fds run never hands a truncated CSV to + * the bulk COPY. */ + finalize(): GraphEmitManifest { + if (this.finalized) throw new Error('GraphEmitSink.finalize() called twice'); + this.finalized = true; + + const errors: unknown[] = []; + if (this.openFailure !== undefined) errors.push(this.openFailure); + + const relsByPair = new Map(); + let totalRows = 0; + for (const [pairKey, writer] of this.relWriters) { + writer.close(); + if (writer.poison !== undefined) errors.push(writer.poison); + relsByPair.set(pairKey, { csvPath: writer.csvPath, rows: writer.rows }); + totalRows += writer.rows; + } + + if (errors.length > 0) { + const first = errors[0]; + throw new Error( + `GraphEmitSink: ${errors.length} streamed CSV writer(s) hit an IO error ` + + `(disk-full / out-of-fds) during the emit — the persisted graph would be ` + + `truncated, so the run is failed rather than COPYing a partial CSV: ${ + first instanceof Error ? first.message : String(first) + }`, + ); + } + + return { relsByPair, totalRows }; + } + + /** Best-effort fd release for the error path — when the pipeline throws + * before {@link finalize} runs, the caller's `finally` calls this so the + * per-pair fds never leak. Idempotent with finalize via `finalized`. */ + close(): void { + if (this.finalized) return; + this.finalized = true; + for (const writer of this.relWriters.values()) { + try { + writer.close(); + } catch { + /* best-effort */ + } + } + } + + // ── delegated reads / retained mutations ─────────────────────────────────── + + get nodes(): GraphNode[] { + return this.real.nodes; + } + get relationships(): GraphRelationship[] { + return [...this.iterRelationships()]; + } + iterNodes(): IterableIterator { + return this.real.iterNodes(); + } + /** + * Retained edges followed by the streamed ones, so every consumer sees a + * complete graph and no phase needs to know streaming happened. This is what + * lets streaming be the default. + * + * Hand-rolled rather than a generator: a generator pays per-`yield` machinery + * on every one of millions of edges, and the pruner and process extraction + * walk this three times per analyze. + */ + iterRelationships(): IterableIterator { + const retained = this.real.iterRelationships(); + const self = this; + let ix = 0; + // One reused result record. The iterator protocol lets the producer hand + // back the same object each step — `for…of` reads `value`/`done` and drops + // it immediately — and allocating a fresh one per edge cost more than the + // generator it replaced. + const result: { value: GraphRelationship | undefined; done: boolean } = { + value: undefined, + done: true, + }; + const it: IterableIterator = { + next(): IteratorResult { + const fromReal = retained.next(); + if (fromReal.done !== true) { + result.value = fromReal.value; + result.done = false; + return result as IteratorResult; + } + if (ix < self.srcIx.length) { + result.value = self.streamedAt(ix++); + result.done = false; + return result as IteratorResult; + } + result.value = undefined; + result.done = true; + return result as IteratorResult; + }, + [Symbol.iterator]() { + return it; + }, + }; + return it; + } + + *iterRelationshipsByType(type: RelationshipType): IterableIterator { + yield* this.real.iterRelationshipsByType(type); + if (RETAINED_REL_TYPES.has(type)) return; // never streamed — skip the scan + for (let ix = 0; ix < this.srcIx.length; ix++) { + if (this.relTypes[ix] === type) yield this.streamedAt(ix); + } + } + forEachNode(fn: (node: GraphNode) => void): void { + this.real.forEachNode(fn); + } + /** + * The fast path: streamed edges are read straight out of the columns, so a + * whole-graph scan allocates NOTHING. This is what keeps iteration at parity + * with the object-based graph despite holding relationships columnar. + */ + forEachRelationshipFields( + fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + ): void { + this.real.forEachRelationshipFields(fn); + for (let ix = 0; ix < this.srcIx.length; ix++) { + fn( + this.nodeIdByIx[this.srcIx[ix]], + this.nodeIdByIx[this.tgtIx[ix]], + this.relTypes[ix], + this.confidences[ix], + ); + } + } + + /** Direct loop rather than delegating to {@link iterRelationships}: this is + * the form community detection uses (twice), and skipping the generator and + * iterator protocol is measurably cheaper on a million-edge scan. */ + forEachRelationship(fn: (rel: GraphRelationship) => void): void { + this.real.forEachRelationship(fn); + for (let ix = 0; ix < this.srcIx.length; ix++) fn(this.streamedAt(ix)); + } + getNode(id: string): GraphNode | undefined { + return this.real.getNode(id); + } + get nodeCount(): number { + return this.real.nodeCount; + } + /** Retained edges only — streamed edges are gone from the heap by design. + * `run-analyze.ts` sizes the LadybugDB buffer pool from this, so it adds + * the manifest's `totalRows` back in (the hint only ever shrinks the pool, + * so under-reporting would starve the COPY at exactly the scale this + * feature targets). */ + get relationshipCount(): number { + return this.real.relationshipCount + this.srcIx.length; + } + removeNode(nodeId: string): boolean { + return this.real.removeNode(nodeId); + } + removeNodesByFile(filePath: string): number { + return this.real.removeNodesByFile(filePath); + } + /** + * Deliberately conservative. The dedup Set holds compact keys derived from a + * relationship's endpoints ({@link dedupKey}), and a bare id alone cannot be + * turned back into one — so a streamed edge is not directly identifiable here. + * + * Rather than risk the silent case (returning `false` for an edge that IS on + * disk and cannot be recalled), anything the real graph does not hold is + * treated as possibly-streamed once streaming has begun, and fails loudly. A + * genuinely-absent id therefore throws too, where the object-based graph would + * return `false`; that is acceptable because the only production caller is the + * COBOL resolver, which runs BEFORE the sink is armed and so takes the branch + * below. + * + * NOTE this diverges from {@link KnowledgeGraph.removeRelationship}, which + * returns `false` for an id it does not hold. Pinned by a test so the + * divergence stays deliberate. + */ + removeRelationship(relationshipId: string): boolean { + if (this.real.removeRelationship(relationshipId)) return true; + if (this.srcIx.length > 0) throw new StreamedRelationshipRemovalError(relationshipId); + return false; + } +} diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 769549492..8e558c072 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -20,6 +20,7 @@ import { NodeTableName, } from './schema.js'; import { streamAllCSVsToDisk, type StreamedCSVResult } from './csv-generator.js'; +import type { GraphEmitManifest } from './graph-emit-sink.js'; import type { PdgEmitManifest } from './pdg-emit-sink.js'; import { getNodeLabel as deriveNodeLabel, type WriteStreamFactory } from './rel-pair-routing.js'; import { EMBEDDABLE_LABELS, type CachedEmbedding } from '../embeddings/types.js'; @@ -37,6 +38,7 @@ import { isDbBusyError, isOpenRetryExhausted, isWalCorruptionError, + bufferPoolExhaustionRemedy, openLbugConnection, sleep, toNativeSafePath, @@ -46,6 +48,7 @@ import { type LbugConnectionHandle, } from './lbug-config.js'; import { + cleanQuarantinedMissingShadowWals, finalizeLbugSidecarsAfterClose, guardWalQuarantine, isMissingShadowSidecarError, @@ -55,6 +58,7 @@ import { quarantineWalForMissingShadow, renameFailureMessage, shadowSidecarRecoveryMessage, + sidecarPreflightDisabled, } from './sidecar-recovery.js'; import { logger } from '../logger.js'; @@ -822,6 +826,30 @@ const doInitLbug = async (dbPath: string, readOnly: boolean = false) => { // ------------------------------------------------------------------------- const releaseInitLock = await acquireInitLock(dbPath); try { + // Reclaim missing-shadow WAL quarantines from a PRIOR crash (#2637). + // LadybugDB renames an unrecoverable WAL aside as + // `${dbPath}.wal.missing-shadow.-` (quarantineWalForMissingShadow) + // instead of deleting it. Once quarantined it is permanently detached from + // the live store and never reopened, so reclaiming it is safe regardless of + // whether the main DB file exists this run — unlike the orphan-sidecar + // cleanup below, this must NOT be gated on "main DB missing": a quarantine + // event and a healthy main DB are independent facts. Never let a reclaim + // failure (e.g. a transient EBUSY from an antivirus scan) block DB startup. + if (!sidecarPreflightDisabled()) { + try { + const reclaimed = await cleanQuarantinedMissingShadowWals(dbPath); + for (const file of reclaimed) { + logger.warn( + `GitNexus: reclaimed quarantined WAL ${path.basename(file)} from a prior crash`, + ); + } + } catch (err) { + logger.warn( + `GitNexus: failed to reclaim missing-shadow WAL quarantines: ${summarizeError(err)}`, + ); + } + } + // Crash-recovery cleanup: if the main DB file is missing, stale sidecars // from an interrupted run can block fresh opens indefinitely. try { @@ -953,7 +981,14 @@ const copyNodeCSVs = async ( const copyQuery = getCopyQuery(table, normalizeCopyPath(csvPath)); await copyCsvWithRetry(targetConn, copyQuery, (retryErr) => { const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); - throw new Error(`COPY failed for ${table}: ${retryMsg.slice(0, 200)}`); + // Pool exhaustion gets a remedy (#2631): the raw binder text gives the + // operator nothing to act on, and on non-4K-page hosts (Ascend aarch64, + // Apple Silicon) the pool bills up to pageSize/4KiB x faster than the + // sizing was calibrated for — name the knob and the mechanism. + const remedy = bufferPoolExhaustionRemedy(retryMsg); + throw new Error( + `COPY failed for ${table}: ${retryMsg.slice(0, 200)}${remedy ? ` ${remedy}` : ''}`, + ); }); } }; @@ -984,6 +1019,15 @@ export const loadGraphToLbug = async ( * emits none — the manifest is the sole source and there is no double-COPY. */ pdgEmitManifest?: PdgEmitManifest, + /** + * Streamed structural-emit manifest (#2680). Unlike {@link pdgEmitManifest}, + * these pair keys are NOT disjoint from the whole-graph emit's: a streamed + * `CALLS` edge is `Function|Function`, exactly like the retained edges + * `streamAllCSVsToDisk` just wrote. So these files are APPENDED as additional + * COPY jobs for the same pair rather than merged into `relsByPair` (a Map, + * which holds one CSV per pair and would silently drop one of them). + */ + graphEmitManifest?: GraphEmitManifest, ) => { if (!conn) { throw new Error('LadybugDB not initialized. Call initLbug first.'); @@ -1123,16 +1167,32 @@ export const loadGraphToLbug = async ( let tCopyRels = tCopyNodes; let tFallback = tCopyNodes; - const insertedRels = totalValidRels; + // One COPY job per CSV FILE, not per label pair. The whole-graph emit writes + // at most one file per pair, but the streamed structural manifest (#2680) can + // contribute a second file for a pair the whole-graph emit also wrote — both + // must load. `relsByPair` stays a one-file-per-pair Map so the PDG merge above + // and every other consumer are untouched. + const copyJobs: Array<{ pairKey: string; csvPath: string; rows: number }> = []; + for (const [pairKey, meta] of relsByPair) { + copyJobs.push({ pairKey, csvPath: meta.csvPath, rows: meta.rows }); + } + if (graphEmitManifest) { + for (const [pairKey, meta] of graphEmitManifest.relsByPair) { + copyJobs.push({ pairKey, csvPath: meta.csvPath, rows: meta.rows }); + } + } + + const insertedRels = totalValidRels + (graphEmitManifest?.totalRows ?? 0); const warnings: string[] = []; + let poolRemedyIssued = false; if (insertedRels > 0) { - log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`); + log(`Loading edges: ${insertedRels.toLocaleString()} across ${copyJobs.length} CSV files`); let pairIdx = 0; let failedPairEdges = 0; const failedPairCsvPaths = new Set(); - for (const [pairKey, { csvPath: pairCsvPath, rows }] of relsByPair) { + for (const { pairKey, csvPath: pairCsvPath, rows } of copyJobs) { pairIdx++; const [fromLabel, toLabel] = pairKey.split('|'); const normalizedPath = normalizeCopyPath(pairCsvPath); @@ -1140,7 +1200,7 @@ export const loadGraphToLbug = async ( const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`; if (pairIdx % 5 === 0 || rows > 1000) { - log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`); + log(`Loading edges: ${pairIdx}/${copyJobs.length} files (${fromLabel} -> ${toLabel})`); } // Use the captured `writeConn` (not the module-level `conn`) for the rel @@ -1151,6 +1211,17 @@ export const loadGraphToLbug = async ( await copyCsvWithRetry(writeConn, copyQuery, (retryErr) => { const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); warnings.push(`${fromLabel}->${toLabel} (${rows} edges): ${retryMsg.slice(0, 80)}`); + // One remedy per bulk load, not per pair (#2631): pool exhaustion + // repeats for every remaining pair once it starts. logger.warn, not + // just warnings.push — the returned warnings array has no consumer at + // any call site, so a push alone would leave the remedy invisible + // while the row-by-row fallback quietly degrades the load. + const remedy = poolRemedyIssued ? undefined : bufferPoolExhaustionRemedy(retryMsg); + if (remedy) { + poolRemedyIssued = true; + warnings.push(remedy); + logger.warn(remedy); + } failedPairEdges += rows; failedPairCsvPaths.add(pairCsvPath); }); diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index c838c674d..19bfa7826 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -345,18 +345,84 @@ const parseBufferPoolSize = (raw: string | undefined): number | undefined => { return Math.floor(parsed); }; +/** + * The buffer-manager frame size compiled into every shipped `@ladybugdb/core` + * binary (`LBUG_PAGE_SIZE_LOG2 = 12` in the engine's CMake) — frames are 4 KiB + * on every platform, independent of the OS page size. + */ +const LBUG_ASSUMED_FRAME_SIZE = 4096; + +/** + * How much the OS page size amplifies buffer-pool consumption (#2631). + * + * LadybugDB's VM region charges pool budget per DISCARD GRANULE, not per + * frame: `discardGranuleSize = max(frameSize, osPageSize)` (vm_region.cpp), + * `claimFrame` bills the whole granule when its first 4 KiB frame becomes + * resident, and `releaseFrame` refunds only when the granule's LAST frame + * leaves. On a 64 KiB-page kernel (aarch64 openEuler — Ascend hosts) that is + * 16 frames per granule: scattered access is billed up to 16× its real bytes, + * and whole eviction passes can evict frames yet refund nothing — which is + * exactly the engine's "buffer pool is full and no memory could be freed" + * throw. Apple Silicon macOS (16 KiB pages) is the same mechanism at 4×. + * + * So the ANALYZE-path pool sizes (the per-element estimate, the COPY-safety + * floor, and the cap the hint is clamped against) are scaled by this ratio: + * the budget must cover worst-case granule charging or COPY dies on non-4K + * hosts with a pool that would be ample on x86. The hintless default + * (defaultBufferPoolSize — MCP serve, doctor, native-check) is deliberately + * NOT scaled: the pool is a native eager allocation committed at DB open + * (measured — see POOL_BYTES_PER_ELEMENT below), so scaling the global + * default would revert the #2557 OOM cap on every 16 KiB/64 KiB host. If the + * engine ever charges per-frame (or ships page-size-matched frames), this + * collapses back to 1 and the scaling disappears. + * + * Fail-safe: an undetectable page size (win32 — where the granule mechanism + * is absent anyway — or a failed `getconf`) means ratio 1, i.e. today's + * behavior. + */ +export const granuleRatio = (pageSize: number | undefined = getOsPageSize()): number => { + if (pageSize === undefined || !Number.isFinite(pageSize)) return 1; + return Math.max(1, Math.floor(pageSize / LBUG_ASSUMED_FRAME_SIZE)); +}; + +/** + * Hintless pool default — MCP serve, doctor, native-check, any open without a + * per-run hint. Deliberately UNSCALED (#2557): the pool is an eager native + * allocation at DB open, so a page-size-scaled default would hand a + * long-lived `gitnexus mcp` on a 16 KiB/64 KiB host up to 80% of RAM — the + * exact OOM exposure the 2 GiB cap was added to remove. + */ const defaultBufferPoolSize = (): number => Math.min(DEFAULT_BUFFER_POOL_CAP, Math.max(BUFFER_POOL_FLOOR, Math.floor(os.totalmem() * 0.8))); /** - * Clamp an adaptive pool request to [ADAPTIVE_POOL_FLOOR, default]. The lower - * bound keeps LadybugDB's COPY viable; the upper bound (defaultBufferPoolSize) - * means the hint can only shrink the pool from today's default and can never - * exceed the 2 GiB / 80%-RAM cap — and on a machine whose default is below the - * COPY floor, the default wins, so the pool is never over-committed. + * Upper bound for the ANALYZE-path (hinted) pool: the #2557 cap scaled by the + * granule ratio, still bounded by 80% of RAM. Scaling only this bound — and + * not defaultBufferPoolSize — is what lets the #2631 fix take effect during + * the bulk COPY without touching hintless opens: with an unscaled cap the + * min() below would clamp the scaled COPY floor straight back to 2 GiB. */ -const clampBufferPool = (bytes: number): number => - Math.min(defaultBufferPoolSize(), Math.max(ADAPTIVE_POOL_FLOOR, Math.floor(bytes))); +const scaledAnalyzePoolCap = (pageSize: number | undefined): number => + Math.min( + DEFAULT_BUFFER_POOL_CAP * granuleRatio(pageSize), + Math.max(BUFFER_POOL_FLOOR, Math.floor(os.totalmem() * 0.8)), + ); + +/** + * Clamp an adaptive pool request to [ADAPTIVE_POOL_FLOOR × granuleRatio, + * scaledAnalyzePoolCap]. The lower bound keeps LadybugDB's COPY viable + * (scaled because the granule accounting inflates consumption on non-4K + * hosts, see granuleRatio); the upper bound means the hint can never exceed + * the page-size-scaled #2557 cap or 80% of RAM — and on a machine whose cap + * is below the COPY floor, the cap wins, so the pool is never over-committed. + * On 4 KiB hosts (ratio 1) this is byte-identical to clamping against the + * hintless default. + */ +const clampBufferPool = (bytes: number, pageSize: number | undefined = getOsPageSize()): number => + Math.min( + scaledAnalyzePoolCap(pageSize), + Math.max(ADAPTIVE_POOL_FLOOR * granuleRatio(pageSize), Math.floor(bytes)), + ); /** * Buffer-pool bytes to provision per graph element (node + relationship). @@ -377,13 +443,22 @@ const POOL_BYTES_PER_ELEMENT = 4 * 1024; /** * Size the buffer pool to an estimated graph size (node + relationship count), - * clamped to [ADAPTIVE_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can - * only *shrink* the pool from the default — never above the 2 GiB / 80%-RAM cap, - * never below the COPY-safety floor — so no repo is under-sized or gets more - * than the default it would have today. + * clamped to [ADAPTIVE_POOL_FLOOR, scaledAnalyzePoolCap], with every term + * scaled by granuleRatio (#2631): on non-4K hosts the engine bills pool + * budget per OS-page-sized granule, so the same graph consumes up to + * pageSize/4096 × the budget it needs on x86. On 4 KiB hosts the ratio is 1 + * and this is byte-identical to the pre-#2631 behavior. The estimate is never + * above the page-size-scaled #2557 cap bounded by 80% of RAM, never below the + * scaled COPY-safety floor; the hintless default stays unscaled. + * + * `pageSize` is a test seam (the pageSizeDoctorLines convention); production + * callers omit it and get the memoized real OS page size. */ -export const estimateBufferPool = (graphElementCount: number): number => - clampBufferPool(graphElementCount * POOL_BYTES_PER_ELEMENT); +export const estimateBufferPool = ( + graphElementCount: number, + pageSize: number | undefined = getOsPageSize(), +): number => + clampBufferPool(graphElementCount * POOL_BYTES_PER_ELEMENT * granuleRatio(pageSize), pageSize); /** * Optional per-run buffer-pool size hint (bytes). The analyze orchestrator sets @@ -422,12 +497,64 @@ const resolveBufferManagerSize = (): number => { if (raw.trim().length > 0) { logger.warn( { rawValue: raw, fallback: defaultBufferPoolSize() }, - `Ignoring invalid GITNEXUS_LBUG_BUFFER_POOL_SIZE=${raw}; expected integer >= 0 (bytes; 0 restores the native 80%-of-RAM default); falling back to min(2 GiB, 80% of RAM).`, + `Ignoring invalid GITNEXUS_LBUG_BUFFER_POOL_SIZE=${raw}; expected integer >= 0 (bytes; 0 restores the native 80%-of-RAM default); falling back to the platform default pool size.`, ); } return defaultBufferPoolSize(); }; +/** + * Doctor-facing view of the pool size the next Database open would get + * (#2631): env override > clamped hint > unscaled hintless default. Read-only; + * doctor prints it next to the page-size lines so support triage sees the + * sizing inputs at a glance. `0` is the pass-through sentinel for LadybugDB's + * native 80%-of-RAM default — callers must label it, not print "0 MiB". + */ +export const getEffectiveBufferPoolSize = (): number => resolveBufferManagerSize(); + +/** + * Matches the engine's buffer-pool exhaustion throw (buffer_manager.cpp: + * "Unable to allocate memory! The buffer pool is full and no memory could be + * freed!"). Distinct from isLbugPageSizeFrameError above, which matches the + * madvise/frame-release failure class. + */ +const BUFFER_POOL_EXHAUSTION_RE = /buffer pool is full|unable to allocate memory/i; + +const formatMiB = (bytes: number): string => `${Math.round(bytes / (1024 * 1024))} MiB`; + +/** + * Actionable remedy for a buffer-pool exhaustion error (#2631), or undefined + * when `message` is not that class. Cause → consequence → remedy, the + * diagnoseExtensionLoad convention: names the effective pool, the override + * knob, and — on non-4K hosts — the granule amplification that makes the + * budget exhaust early (the reporter's Ascend/aarch64 64 KiB kernel billed a + * pool up to 16× faster than the same analyze on x86). + */ +export const bufferPoolExhaustionRemedy = ( + message: string, + pageSize: number | undefined = getOsPageSize(), +): string | undefined => { + if (!BUFFER_POOL_EXHAUSTION_RE.test(message)) return undefined; + const ratio = granuleRatio(pageSize); + const pool = resolveBufferManagerSize(); + // 0 is the pass-through sentinel (GITNEXUS_LBUG_BUFFER_POOL_SIZE=0 → + // LadybugDB's native 80%-of-RAM default) — "0 MiB" would be nonsense in the + // very triage text this remedy exists to provide. + const poolLabel = pool === 0 ? "LadybugDB's native 80%-of-RAM default" : formatMiB(pool); + const pageNote = + ratio > 1 + ? ` This host's ${(pageSize ?? 0) / 1024} KiB OS page size makes the engine bill pool ` + + `memory in ${(pageSize ?? 0) / 1024} KiB granules — up to ${ratio}× faster budget use ` + + `than a 4 KiB-page host running the same analyze.` + : ''; + return ( + `The LadybugDB buffer pool (${poolLabel}) was exhausted during the bulk COPY.` + + pageNote + + ` Set GITNEXUS_LBUG_BUFFER_POOL_SIZE= to raise it (e.g. ${4 * 1024 * 1024 * 1024}` + + ` for 4 GiB); 0 restores LadybugDB's native 80%-of-RAM default.` + ); +}; + /** Matches WAL corruption errors from the LadybugDB engine. */ const WAL_CORRUPTION_RE = /corrupt(ed)?\s+wal|invalid\s+wal\s+record|wal.*corrupt|checksum.*wal/i; @@ -514,8 +641,12 @@ const LBUG_PAGE_COMBO_RE = /unsupported page size combination/i; * True when `err` looks like the LadybugDB buffer manager failing to release * frame memory — the failure mode of a 4 KiB page-size assumption on a * 16 KiB/64 KiB-page kernel (#1231). Deliberately does NOT match the - * generic "buffer pool is full" exhaustion error, which is a sizing - * problem, not a page-size one. + * generic "buffer pool is full" exhaustion error: that one is handled as a + * SIZING problem — though since #2631 we know page size drives sizing too + * (the engine bills pool budget per OS-page-sized discard granule, so non-4K + * hosts exhaust the same budget up to pageSize/4096× earlier; see + * granuleRatio, which scales the pool accordingly, and + * bufferPoolExhaustionRemedy, which explains it to the operator). */ export const isLbugPageSizeFrameError = (err: unknown): boolean => { if (!err) return false; @@ -542,6 +673,16 @@ export const isPageSizeAwareLadybug = (version: string | undefined): boolean => // because analyze error paths and doctor may both ask, and getconf forks. let cachedOsPageSize: number | null | undefined; +/** + * Test seam (the `_captureLogger` convention): pin the memoized OS page size + * so sizing tests are host-independent — without this they would silently + * drift on 16 KiB-page Apple Silicon runners. `number` pins a value, `null` + * pins "undetectable", `undefined` clears the memo so the next call re-probes. + */ +export const _setOsPageSizeForTests = (pageSize: number | null | undefined): void => { + cachedOsPageSize = pageSize; +}; + /** * OS memory page size in bytes, or `undefined` when it cannot be determined * (Windows, missing getconf, sandboxed exec). Node exposes no page-size API, @@ -580,11 +721,6 @@ export const getOsPageSize = (): number | undefined => { return cachedOsPageSize ?? undefined; }; -/** Exported only for unit tests — clears the getconf probe cache. */ -export const _resetOsPageSizeCacheForTest = (): void => { - cachedOsPageSize = undefined; -}; - type LbugModule = typeof lbug; export interface LbugDatabaseOptions { diff --git a/gitnexus/src/core/lbug/native-check.ts b/gitnexus/src/core/lbug/native-check.ts index a9971a5e9..f518af05a 100644 --- a/gitnexus/src/core/lbug/native-check.ts +++ b/gitnexus/src/core/lbug/native-check.ts @@ -1,11 +1,27 @@ import fs from 'fs'; import path from 'path'; import { createRequire } from 'node:module'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; + +/** Cap the out-of-process native load probe so a hung filesystem cannot wedge a + * CLI startup gate (same bounding rationale as the extension probe below). */ +const NATIVE_LOAD_PROBE_TIMEOUT_MS = 15_000; + +/** + * Why the native check failed. A failed check is NOT necessarily a missing + * binary — the package may be absent, the binary may be absent, or a binary that + * is right there may fail to load (host glibc too old, truncated download). + * Callers that render a status line must tell those apart: reporting all of them + * as "missing" sends users to reinstall a file they already have (#2672). + */ +export type NativeCheckFailureKind = 'package_missing' | 'binary_missing' | 'load_failed'; export interface NativeCheckResult { ok: boolean; binaryPath?: string; message?: string; + /** Set only when `ok` is false. */ + kind?: NativeCheckFailureKind; } export function checkLbugNative(overridePkgDir?: string): NativeCheckResult { @@ -21,6 +37,7 @@ export function checkLbugNative(overridePkgDir?: string): NativeCheckResult { } catch { return { ok: false, + kind: 'package_missing', message: [ 'LadybugDB package (@ladybugdb/core) is not installed.', '', @@ -35,6 +52,7 @@ export function checkLbugNative(overridePkgDir?: string): NativeCheckResult { return { ok: false, binaryPath, + kind: 'binary_missing', message: [ 'LadybugDB native binary (lbugjs.node) is missing.', '', @@ -59,35 +77,168 @@ export function checkLbugNative(overridePkgDir?: string): NativeCheckResult { }; } - try { - const _require = createRequire(import.meta.url); - _require(binaryPath); - } catch (err: unknown) { - const nativeError = err instanceof Error ? err.message : String(err); + // Validate loadability in a THROWAWAY CHILD PROCESS, not in-process. A merely + // truncated or corrupted .node (valid header, missing pages) does not throw a + // catchable error — it SIGBUSes the dynamic loader mid-dlopen, which would take + // the whole CLI down with a raw exit 135 and no guidance (#2441). Loading it in + // a child lets us observe that crash (a non-zero exit or a kill signal) and turn + // it into the same actionable failure as a clean load error. The child requires + // the binary by absolute path, exactly as the former in-process load did. + const probe = spawnSync(process.execPath, ['-e', 'require(process.argv[1])', binaryPath], { + encoding: 'utf8', + timeout: NATIVE_LOAD_PROBE_TIMEOUT_MS, + stdio: ['ignore', 'ignore', 'pipe'], + // Run as Node even if process.execPath is an Electron/embedder binary. + env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, + }); + + // Only a child that actually RAN and failed proves the binary is bad. If the + // probe could not run at all — a spawn error or a timeout, e.g. a sandbox that + // forbids subprocesses or a non-Node execPath — we could not test the binary, + // so we stay out of the way and let the command's own load be the authority + // rather than condemn a healthy binary. (#2441 still holds: a genuinely broken + // binary loaded in-process later still exits non-zero.) + if (probe.error || probe.status === 0) { + return { ok: true, binaryPath }; + } + + // One failure class is NOT repairable by reinstalling: a host whose glibc is + // older than the prebuilt binary requires. Every download ships the same + // binary, so the generic advice below sends the user around a loop that always + // ends here (#2672). Branch before it, and only here — on the arm where the + // probe actually ran and failed, so an unrunnable probe still fails open above. + const glibcExplanation = glibcTooOldMessage(probe.stderr ?? ''); + if (glibcExplanation !== null) { return { ok: false, binaryPath, + kind: 'load_failed', message: [ 'LadybugDB native binary (lbugjs.node) exists but failed to load:', - ` ${nativeError}`, + ` ${describeNativeLoadFailure(probe)}`, '', - 'This can happen with a truncated file, ABI mismatch, or wrong-platform binary.', - '', - 'To repair:', - ` node ${path.join(pkgDir, 'install.js')}`, - '', - 'If install scripts were skipped (pnpm dlx / pnpx / ignore-scripts):', - ' pnpm --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter \\', - ' dlx gitnexus@latest serve', - ' pnpm add -g --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter gitnexus', - '', - 'If using bun, add to package.json and reinstall:', - ' "trustedDependencies": ["@ladybugdb/core"]', + glibcExplanation, ].join('\n'), }; } - return { ok: true, binaryPath }; + return { + ok: false, + binaryPath, + kind: 'load_failed', + message: [ + 'LadybugDB native binary (lbugjs.node) exists but failed to load:', + ` ${describeNativeLoadFailure(probe)}`, + '', + 'This can happen with a truncated file, ABI mismatch, or wrong-platform binary.', + '', + 'To repair:', + ` node ${path.join(pkgDir, 'install.js')}`, + '', + 'If install scripts were skipped (pnpm dlx / pnpx / ignore-scripts):', + ' pnpm --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter \\', + ' dlx gitnexus@latest serve', + ' pnpm add -g --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter gitnexus', + '', + 'If using bun, add to package.json and reinstall:', + ' "trustedDependencies": ["@ladybugdb/core"]', + ].join('\n'), + }; +} + +/** + * Describe a child-observed native load failure. Reached only after a probe that + * actually ran and failed: a fatal signal (SIGBUS/SIGSEGV ⇒ truncated/corrupt + * binary), otherwise the child's own load error lifted from its stderr. + */ +function describeNativeLoadFailure(probe: SpawnSyncReturns): string { + if (probe.signal) { + return `crashed while loading (signal ${probe.signal}) — the binary is likely truncated or corrupted`; + } + const lines = (probe.stderr ?? '') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + const errorLine = lines.find((line) => /^\w*Error: /.test(line)); + return ( + errorLine?.replace(/^\w*Error:\s*/, '') ?? + lines.at(-1) ?? + `exited with code ${probe.status ?? 'unknown'}` + ); +} + +/** + * A `GLIBC_` token. The dynamic loader names the first unresolved + * versioned symbol as ``version `GLIBC_2.34' not found (required by …)``, but we + * key on the token plus a "not found" line rather than on glibc's exact + * backtick/apostrophe quoting: if that wording ever changes, this degrades to + * the generic failure message instead of misfiring. + */ +const GLIBC_VERSION_TOKEN = /GLIBC_(\d+(?:\.\d+)+)/g; + +/** Numeric dotted-segment order — glibc 2.9 is OLDER than 2.34, not newer. */ +function compareDottedVersions(a: string, b: string): number { + const left = a.split('.').map((part) => Number.parseInt(part, 10)); + const right = b.split('.').map((part) => Number.parseInt(part, 10)); + for (let i = 0; i < Math.max(left.length, right.length); i += 1) { + const diff = (left[i] ?? 0) - (right[i] ?? 0); + if (diff !== 0) return diff; + } + return 0; +} + +/** + * This host's runtime glibc, or null when Node cannot report it (musl builds, + * embedders without `process.report`). Read locally rather than through + * analyzer-identity's `detectLibcVariant`: that module is deliberately reached + * via dynamic import from the CLI lazy actions, and this file is the + * dependency-light startup gate that must not pull it in. + */ +function hostGlibcVersion(): string | null { + try { + const report = process.report?.getReport() as + | { header?: { glibcVersionRuntime?: unknown } } + | undefined; + const runtime = report?.header?.glibcVersionRuntime; + return typeof runtime === 'string' && runtime.length > 0 ? runtime : null; + } catch { + // Report generation is optional on some embedded Node builds; an unknown + // host version still leaves the required version worth printing. + return null; + } +} + +/** + * Explain a glibc-too-old native load failure, or null when the probe's stderr + * describes something else. + * + * Reinstalling cannot fix this class — the package ships one prebuilt binary per + * platform — so the caller must NOT fall through to the reinstall instructions + * (#2672). Exported for direct unit testing: a real `GLIBC_2.34' not found` + * cannot be provoked on a host whose glibc is new enough to run the tests. + */ +export function glibcTooOldMessage(stderr: string): string | null { + const required = stderr + .split('\n') + .filter((line) => /not found/i.test(line)) + .flatMap((line) => [...line.matchAll(GLIBC_VERSION_TOKEN)].map((match) => match[1])) + .sort(compareDottedVersions) + .at(-1); + if (required === undefined) return null; + + const host = hostGlibcVersion(); + return [ + "This host's C library (glibc) is older than the prebuilt binary requires.", + ` required: glibc ${required} or newer`, + ` this host: ${host === null ? 'glibc version could not be determined' : `glibc ${host}`}`, + '', + 'Reinstalling will NOT help — every download ships the same prebuilt binary.', + '', + 'Options:', + ` - Run GitNexus on a distribution with glibc ${required} or newer`, + ' (Ubuntu 22.04+, RHEL/Rocky/Alma 9+, Debian 12+, Fedora 35+).', + ' - Or use the GitNexus container image, which bundles a current glibc.', + ].join('\n'); } export interface FtsProbeResult { diff --git a/gitnexus/src/core/lbug/pdg-emit-sink.ts b/gitnexus/src/core/lbug/pdg-emit-sink.ts index 79cc05f58..79f8b899f 100644 --- a/gitnexus/src/core/lbug/pdg-emit-sink.ts +++ b/gitnexus/src/core/lbug/pdg-emit-sink.ts @@ -54,6 +54,7 @@ import { buildRelRow, } from './csv-generator.js'; import { getNodeLabel } from './rel-pair-routing.js'; +import { DEFAULT_EMIT_CHUNK_ROWS, SyncCsvWriter } from './sync-csv-writer.js'; import { NODE_TABLES, type NodeTableName } from './schema.js'; /** @@ -73,103 +74,9 @@ const PDG_EDGE_TYPES: ReadonlySet = new Set( ]); /** Default streamed-write buffer (rows). Matches the whole-graph emit's - * `FLUSH_EVERY` order of magnitude; overridable via `GITNEXUS_PDG_EMIT_CHUNK_SIZE`. */ -export const DEFAULT_PDG_EMIT_CHUNK_ROWS = 500; - -/** - * Synchronous buffered CSV writer. Buffers up to `chunkRows` rows, then issues - * one `fs.writeSync` straight to the OS (no in-process stream buffer). Header - * is written into the buffer at construction and is NOT counted in `rows` - * (matching `BufferedCSVWriter` semantics, so manifest row counts line up). - */ -class SyncCsvWriter { - private fd: number; - private buf: string[] = []; - private readonly chunkRows: number; - rows = 0; - /** - * First IO error this writer hit (a `fs.writeSync` short-write loop throwing - * on e.g. disk-full). Once poisoned the writer refuses further rows and - * skips its final flush; the sink surfaces it from {@link PdgEmitSink.finalize} - * so a truncated CSV is never handed to the bulk COPY (#2202 review #4). A - * streamed-write failure is an IO fault, not the CFG-logic error that the - * emit loop's per-file try/catch is built to swallow — poisoning routes it - * past that catch to a loud failure. - */ - poison: unknown | undefined = undefined; - - constructor( - readonly csvPath: string, - header: string, - chunkRows: number, - ) { - // Guard a 0/negative buffer: the flush modulo would never fire and `buf` - // would grow unbounded, defeating the whole point of streaming. - this.chunkRows = Math.max(1, chunkRows); - // Exclusive create (O_EXCL): the streamed-CSV dir is wiped + recreated fresh - // by the PdgEmitSink constructor before any writer opens a file, so the path - // never pre-exists — 'wx' both matches that invariant and refuses to follow - // a pre-planted symlink at the path (CWE-377 / CodeQL js/insecure-temporary-file). - this.fd = fs.openSync(csvPath, 'wx'); - this.buf.push(header); - } - - addRow(row: string): void { - // A poisoned writer is dead — stop buffering so memory can't grow on a - // writer whose fd is already in a bad state; finalize will report the fault. - if (this.poison !== undefined) return; - this.buf.push(row); - this.rows++; - // Flush on DATA-row count, not buffer length: the header occupies buf[0] - // until the first flush, so a `buf.length >= chunkRows` test would fire one - // row early on the first chunk. Counting rows makes every flush exactly - // `chunkRows` rows. - if (this.rows % this.chunkRows === 0) this.flushOrPoison(); - } - - /** Flush, recording (and re-throwing) any IO error as poison. Re-throwing - * lets the immediate caller log the per-file failure; the persisted `poison` - * is the backstop that makes finalize fail loudly even when that throw is - * swallowed by the emit loop's CFG try/catch. */ - private flushOrPoison(): void { - try { - this.flush(); - } catch (e) { - this.poison ??= e; - throw e; - } - } - - private flush(): void { - if (this.buf.length === 0) return; - const data = Buffer.from(this.buf.join('\n') + '\n', 'utf8'); - // fs.writeSync can return a short byte count; loop until the whole buffer - // lands so a partial write never truncates a CSV row mid-field. - let offset = 0; - while (offset < data.length) { - offset += fs.writeSync(this.fd, data, offset, data.length - offset); - } - this.buf.length = 0; - } - - /** Flush remaining rows (unless already poisoned) and close the fd. Never - * throws: a final-flush IO error is recorded as poison and the fd is still - * closed, so a write error neither leaks an fd nor escapes here — the sink - * reads {@link poison} after closing every writer and fails loudly then. */ - close(): void { - try { - if (this.poison === undefined) this.flush(); - } catch (e) { - this.poison ??= e; - } finally { - try { - fs.closeSync(this.fd); - } catch { - /* fd may already be invalid after an IO fault — nothing to recover */ - } - } - } -} + * `FLUSH_EVERY` order of magnitude; overridable via `GITNEXUS_PDG_EMIT_CHUNK_SIZE`. + * Aliases the shared default in `sync-csv-writer.ts` (#2680 extraction). */ +export const DEFAULT_PDG_EMIT_CHUNK_ROWS = DEFAULT_EMIT_CHUNK_ROWS; /** * COPY manifest produced by {@link PdgEmitSink.finalize}. Shaped to merge @@ -374,6 +281,11 @@ export class PdgEmitSink implements KnowledgeGraph { forEachRelationship(fn: (rel: GraphRelationship) => void): void { this.real.forEachRelationship(fn); } + forEachRelationshipFields( + fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + ): void { + this.real.forEachRelationshipFields(fn); + } getNode(id: string): GraphNode | undefined { return this.real.getNode(id); } diff --git a/gitnexus/src/core/lbug/sidecar-recovery.ts b/gitnexus/src/core/lbug/sidecar-recovery.ts index c00c1aa7d..882041747 100644 --- a/gitnexus/src/core/lbug/sidecar-recovery.ts +++ b/gitnexus/src/core/lbug/sidecar-recovery.ts @@ -60,7 +60,7 @@ export const isMissingFsError = (err: unknown): boolean => const missing = isMissingFsError; -const sidecarPreflightDisabled = (): boolean => +export const sidecarPreflightDisabled = (): boolean => /^(1|true|yes|on)$/i.test(process.env.GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT ?? ''); export const statIfExists = async (filePath: string): Promise<{ size: number } | null> => { diff --git a/gitnexus/src/core/lbug/sync-csv-writer.ts b/gitnexus/src/core/lbug/sync-csv-writer.ts new file mode 100644 index 000000000..fecff951b --- /dev/null +++ b/gitnexus/src/core/lbug/sync-csv-writer.ts @@ -0,0 +1,110 @@ +/** + * Synchronous buffered CSV writer, shared by the streaming emit sinks. + * + * Extracted verbatim from `pdg-emit-sink.ts` (issue #2202) so the structural + * `GraphEmitSink` (#2680) reuses the same buffering and IO-fault discipline + * instead of duplicating ~90 lines of it. No behaviour change: `PdgEmitSink` + * imports this class and is otherwise untouched. + * + * Why synchronous? The emit loops these sinks sit under are synchronous — there + * is no `await` point to drain an async stream, so a `WriteStream` would + * accumulate unwritten chunks in process memory across millions of rows, + * defeating the RSS bound this exists to provide. `fs.writeSync` goes straight + * to the OS; resident memory is bounded to one `chunkRows` buffer. This mirrors + * the sync-shard pattern in `storage/parsedfile-store.ts`. + */ + +import fs from 'fs'; + +/** Default streamed-write buffer (rows), shared by both sinks. */ +export const DEFAULT_EMIT_CHUNK_ROWS = 500; + +export class SyncCsvWriter { + private fd: number; + private buf: string[] = []; + private readonly chunkRows: number; + rows = 0; + /** + * First IO error this writer hit (a `fs.writeSync` short-write loop throwing + * on e.g. disk-full). Once poisoned the writer refuses further rows and + * skips its final flush; the owning sink surfaces it from its `finalize()` + * so a truncated CSV is never handed to the bulk COPY (#2202 review #4). A + * streamed-write failure is an IO fault, not the logic error that the emit + * loops' per-file try/catch is built to swallow — poisoning routes it past + * that catch to a loud failure. + */ + poison: unknown | undefined = undefined; + + constructor( + readonly csvPath: string, + header: string, + chunkRows: number, + ) { + // Guard a 0/negative buffer: the flush modulo would never fire and `buf` + // would grow unbounded, defeating the whole point of streaming. + this.chunkRows = Math.max(1, chunkRows); + // Exclusive create (O_EXCL): the streamed-CSV dir is wiped + recreated fresh + // by the owning sink's constructor before any writer opens a file, so the + // path never pre-exists — 'wx' both matches that invariant and refuses to + // follow a pre-planted symlink at the path (CWE-377 / CodeQL + // js/insecure-temporary-file). + this.fd = fs.openSync(csvPath, 'wx'); + this.buf.push(header); + } + + addRow(row: string): void { + // A poisoned writer is dead — stop buffering so memory can't grow on a + // writer whose fd is already in a bad state; finalize will report the fault. + if (this.poison !== undefined) return; + this.buf.push(row); + this.rows++; + // Flush on DATA-row count, not buffer length: the header occupies buf[0] + // until the first flush, so a `buf.length >= chunkRows` test would fire one + // row early on the first chunk. Counting rows makes every flush exactly + // `chunkRows` rows. + if (this.rows % this.chunkRows === 0) this.flushOrPoison(); + } + + /** Flush, recording (and re-throwing) any IO error as poison. Re-throwing + * lets the immediate caller log the per-file failure; the persisted `poison` + * is the backstop that makes finalize fail loudly even when that throw is + * swallowed by an emit loop's try/catch. */ + private flushOrPoison(): void { + try { + this.flush(); + } catch (e) { + this.poison ??= e; + throw e; + } + } + + private flush(): void { + if (this.buf.length === 0) return; + const data = Buffer.from(this.buf.join('\n') + '\n', 'utf8'); + // fs.writeSync can return a short byte count; loop until the whole buffer + // lands so a partial write never truncates a CSV row mid-field. + let offset = 0; + while (offset < data.length) { + offset += fs.writeSync(this.fd, data, offset, data.length - offset); + } + this.buf.length = 0; + } + + /** Flush remaining rows (unless already poisoned) and close the fd. Never + * throws: a final-flush IO error is recorded as poison and the fd is still + * closed, so a write error neither leaks an fd nor escapes here — the owning + * sink reads {@link poison} after closing every writer and fails loudly then. */ + close(): void { + try { + if (this.poison === undefined) this.flush(); + } catch (e) { + this.poison ??= e; + } finally { + try { + fs.closeSync(this.fd); + } catch { + /* fd may already be invalid after an IO fault — nothing to recover */ + } + } + } +} diff --git a/gitnexus/src/core/logger.ts b/gitnexus/src/core/logger.ts index bf32f1045..95547dfb9 100644 --- a/gitnexus/src/core/logger.ts +++ b/gitnexus/src/core/logger.ts @@ -286,6 +286,34 @@ export const logger = new Proxy({} as Logger, { }, }) as Logger; +/** + * Env flag the analyze CLI sets while its live progress bar owns the + * terminal (it reroutes console.warn through the bar logger; see + * `cli/analyze.ts`). + */ +export const ANALYZE_PROGRESS_ACTIVE_ENV = 'GITNEXUS_ANALYZE_PROGRESS_ACTIVE'; + +/** + * Emit an operator-facing warning without corrupting analyze's live progress + * bar. While the bar is active, the one-line progress message goes through + * console.warn (routed into the bar by the analyze CLI) — raw pino NDJSON + * would corrupt the one-line display, including in the heap-respawn child + * whose stderr is piped for crash classification. Otherwise the structured + * Pino record is emitted, falling back to the progress message when no + * structured form is given. + */ +export function warnRespectingProgressBar( + progressMessage: string, + structured?: { readonly fields: object; readonly message: string }, +): void { + if (process.env[ANALYZE_PROGRESS_ACTIVE_ENV] === '1') { + console.warn(progressMessage); + return; + } + if (structured) logger.warn(structured.fields, structured.message); + else logger.warn(progressMessage); +} + /** * Shape of a parsed pino record. `level`, `time`, and `msg` are always * present; `name` is set when emitted from a named child logger; arbitrary diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 0180e22bf..a83576f3f 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -11,7 +11,9 @@ import path from 'path'; import fs from 'fs/promises'; +import { randomUUID } from 'node:crypto'; import { retryRename } from '../storage/fs-atomic.js'; +import { acquireIndexLock } from '../storage/index-lock.js'; import { runPipelineFromRepo } from './ingestion/pipeline.js'; import { isMoveCompilerInputPath, @@ -48,10 +50,15 @@ import { LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js'; -import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js'; +import { + estimateBufferPool, + setBufferPoolSizeHint, + resolveNativeSafeStorageDir, +} from './lbug/lbug-config.js'; import { escapeCypherString } from './lbug/cypher-escape.js'; import { buildSearchIndexesOrDegrade, + ftsFailureIsFatal, createSearchFTSIndexes, dropSearchFTSIndexes, initialiseSearchFTSStemmer, @@ -289,6 +296,11 @@ export interface AnalyzeOptions { * `DEFAULT_PDG_EMIT_CHUNK_ROWS`. May also be set via * `GITNEXUS_PDG_EMIT_CHUNK_SIZE`. Memory-only (#2202). */ pdgEmitChunkSize?: number; + /** Streamed structural graph emit (#2680). Honored only on a full rebuild + * (`force === true`). May also be enabled via `GITNEXUS_STREAM_GRAPH_EMIT`. + * Trades community detection, process extraction and PDG taint summaries for + * a ~2.9x reduction of in-memory graph heap. */ + streamGraphEmit?: boolean; /** * Default branch threaded into generated AGENTS.md / CLAUDE.md so the * regression-compare example uses the configured branch instead of a @@ -376,6 +388,15 @@ export interface AnalyzeResult { * in the CLI summary — same rationale as the FTS warning (#1161). */ ingestWarnings?: readonly string[]; + /** + * Why FTS was skipped, when `ftsSkipped` is true (#2658 review L2): + * `extension-unavailable` (the LadybugDB FTS extension could not load — the + * offline-first case, remedied by installing it) vs `build-failed` (the + * extension loaded but the index build/verify failed non-fatally — remedied by + * `--repair-fts`, not by installing the extension). Lets the CLI show the + * correct recovery hint instead of always blaming a missing extension. + */ + ftsSkipReason?: 'extension-unavailable' | 'build-failed'; /** * True when the index this run produced/validated is the flat workspace * slot (#2106 R2, inverted by #2354 to follow the checked-out branch). @@ -591,6 +612,38 @@ export const resolveStreamPdgEmit = (options: { options.force === true && (options.streamPdgEmit === true || parseTruthyEnv(process.env.GITNEXUS_STREAM_PDG_EMIT)); +/** + * Resolve whether streamed structural graph emit is on for this run (#2680). + * + * **On by default.** It costs nothing observable: the sink answers a complete + * relationship read, so community detection, process extraction, the taint + * fixpoint and the local-symbol pruner all behave exactly as they do without it + * — the edges simply live in columns and on disk instead of as objects. There is + * no reason to make a user opt in to using less memory. + * + * Two conditions still bound it: + * + * - `force === true`. Sound only on a full rebuild, because the incremental + * writeback (`extractChangedSubgraph`) reads relationships back out of the + * in-memory graph. Same gate, and same reason, as {@link resolveStreamPdgEmit}. + * - `GITNEXUS_STREAM_GRAPH_EMIT=0` (or an explicit `streamGraphEmit: false`) + * turns it off. The escape hatch exists for bisecting a suspected + * streaming-related fault, not as a routine choice. + * + * Memory-only: not part of {@link resolvePdgConfig}, so toggling never trips + * `pdgModeMismatch`. Read every call (not memoized) so `vi.stubEnv` works. + */ +export const resolveStreamGraphEmit = (options: { + force?: boolean; + streamGraphEmit?: boolean; +}): boolean => { + if (options.force !== true) return false; + if (options.streamGraphEmit !== undefined) return options.streamGraphEmit; + // Unset ⇒ on. Set ⇒ honour it, so `=0` / `=false` is the escape hatch. + const raw = process.env.GITNEXUS_STREAM_GRAPH_EMIT; + return raw === undefined || raw === '' ? true : parseTruthyEnv(raw); +}; + /** * Resolve the streamed PDG-emit write-buffer size (#2202). Explicit option wins * over `GITNEXUS_PDG_EMIT_CHUNK_SIZE`; `undefined` ⇒ the sink's @@ -643,34 +696,179 @@ export const pdgModeMismatch = (recorded: RepoMeta['pdg'], options: PdgOptions): return false; }; +/** + * The storage paths + resolved branch placement a run will write to. Computed + * once, up front, so the `runFullAnalysis` wrapper can lock the ACTUAL write + * directory (#2658). `metaDir` — not `getStoragePaths(repoPath, options.branch)` + * — is the lock scope: a `--branch X` that owns the flat slot resolves to the + * flat `.gitnexus`, so scoping off the raw option would lock the wrong dir. + */ +interface WriteTarget { + storagePath: string; + repoHasGit: boolean; + currentCommit: string; + checkedOutBranch: string | null; + branchLabel: string | null; + placement: { branch?: string }; + lbugPath: string; + metaPath: string; + metaDir: string; +} + +/** + * Resolve which storage slot this analyze writes to, including branch + * placement (#2106/#2354). Extracted from the top of the pipeline so the lock + * scope (`metaDir`) is known before the lock is acquired. Throws the same + * `--branch` / checked-out mismatch error the pipeline used to throw inline, so + * that failure still surfaces before any lock is taken. + */ +async function resolveWriteTarget(repoPath: string, options: AnalyzeOptions): Promise { + // `storagePath` is ALWAYS the flat `.gitnexus` — content-addressed caches + // (parse-cache, parsedfile-store) and kuzu-migration cleanup live there and + // are shared across branches (#2106 KTD7). + const { storagePath } = getStoragePaths(repoPath); + const repoHasGit = hasGitDir(repoPath); + const currentCommit = repoHasGit ? getCurrentCommit(repoPath) : ''; + // Normalize the auto-detected branch the same way an explicit `--branch` is + // validated (#2106 R1): a git ref the branch-name rules forbid becomes `null` + // → the flat slot, matching that a later `--branch ` query would + // also be rejected. A normal ref round-trips index-time/query-time labels. + const checkedOutBranch = repoHasGit + ? (sanitizeDetectedBranch(getCurrentBranch(repoPath)) ?? null) + : null; + // Analyze indexes the working tree, not an arbitrary ref. An explicit + // `--branch X` while a DIFFERENT branch Y is checked out would write Y's + // content into X's slot, corrupting X (#2106). Refuse the mismatch. Detached + // HEAD / non-git (checkedOutBranch === null) still allow an explicit label. + if (options.branch && checkedOutBranch && options.branch !== checkedOutBranch) { + throw new Error( + `--branch "${options.branch}" does not match the checked-out branch "${checkedOutBranch}". ` + + `Check out "${options.branch}" before indexing it, or omit --branch to index the current branch.`, + ); + } + const branchLabel = options.branch ?? checkedOutBranch; + const placement = options.branch ? await resolveBranchPlacement(repoPath, branchLabel) : {}; + const { lbugPath, metaPath } = getStoragePaths(repoPath, placement.branch); + return { + storagePath, + repoHasGit, + currentCommit, + checkedOutBranch, + branchLabel, + placement, + lbugPath, + metaPath, + metaDir: path.dirname(metaPath), + }; +} + +/** + * Run the full analysis under an exclusive, index-directory-scoped write lock + * (#2658). A second concurrent `analyze` on the same slot waits here for the + * first to finish, then falls through to the normal freshness check inside — + * so a run whose work the holder already did returns `alreadyUpToDate` in + * seconds instead of rebuilding (single-flight coalescing), while a run for a + * genuinely-changed tree does one follow-up incremental. No new flag: waiting + * is the default, which is what hook-driven re-index wants. + * + * The lock is held by whichever process runs the pipeline (the heap-respawn + * child, or the original) — see index-lock.ts for why ownership lives with the + * writer, not a supervising parent. Released as soon as the write completes or + * throws; the post-analysis steps in the CLI (skills, registry) run lock-free. + */ export async function runFullAnalysis( repoPath: string, options: AnalyzeOptions, callbacks: AnalyzeCallbacks, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, +): Promise { + // Validate operator-provided FTS config before anything else — a typo fails + // here in ms, without taking the lock. (createSearchFTSIndexes reuses the + // cached value via getSearchFTSStemmer.) + initialiseSearchFTSStemmer(); + initialiseSearchFTSCjkSegmentation(); + // Scope the degraded-parse log throttle to this run (module-level counter + // would otherwise stay saturated on a reused process). + resetDegradedParseCounter(); + + const log = (msg: string) => callbacks.onLog?.(msg); + const acquireOpts = { + log, + onWaitStart: () => + callbacks.onProgress('lock', 0, 'Waiting for another analyze to finish on this index…'), + }; + + let writeTarget = await resolveWriteTarget(repoPath, options); + let lock = await acquireIndexLock(writeTarget.metaDir, acquireOpts); + try { + // #2658 review H2: acquireIndexLock can wait up to the timeout ceiling, + // during which git HEAD/branch — and thus the resolved write slot — may + // change (a commit lands, a branch is switched, or another writer adopts the + // flat slot). The pre-wait snapshot must NOT be reused: re-resolve UNDER the + // lock so the freshness check (`existingMeta.lastCommit === currentCommit`) + // and the meta stamps see current git state, honoring the module's "re-check + // freshness after acquiring" contract. If the slot itself moved we hold the + // WRONG lock — release and re-acquire the correct one. Bounded so a + // pathologically churning checkout can't loop forever; after the cap we + // proceed on the current lock. The loop is INSIDE the try so a re-resolve + // that throws (e.g. a `--branch` that stopped matching the now-switched + // checkout) still releases the held lock via `finally` (no leak). + const MAX_RELOCK = 3; + for (let attempt = 0; attempt < MAX_RELOCK; attempt++) { + const fresh = await resolveWriteTarget(repoPath, options); + if (fresh.metaDir === writeTarget.metaDir) { + writeTarget = fresh; // same slot — adopt the freshly-read commit/branch/placement + break; + } + log( + `Index write target moved while waiting for the lock ` + + `(${writeTarget.metaDir} → ${fresh.metaDir}); re-acquiring the correct slot.`, + ); + lock.release(); + writeTarget = fresh; + lock = await acquireIndexLock(fresh.metaDir, acquireOpts); + if (attempt === MAX_RELOCK - 1) { + log('Index write target still moving after repeated re-acquire; proceeding on this lock.'); + } + } + return await runFullAnalysisInner( + repoPath, + options, + callbacks, + writeTarget, + runnerIdentityAtBootstrap, + ); + } finally { + lock.release(); + } +} + +async function runFullAnalysisInner( + repoPath: string, + options: AnalyzeOptions, + callbacks: AnalyzeCallbacks, + writeTarget: WriteTarget, + runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { const log = (msg: string) => callbacks.onLog?.(msg); const progress = (phase: string, percent: number, message: string) => callbacks.onProgress(phase, percent, message); - // Resolve + validate operator-provided FTS config once, before the expensive - // parse/load phases. A typo fails here in ms; createSearchFTSIndexes reuses - // the cached value via getSearchFTSStemmer. - initialiseSearchFTSStemmer(); - initialiseSearchFTSCjkSegmentation(); + // Streamed structural emit (#2680), resolved once so the pipeline flag and the + // CSV-dir resolution below cannot disagree. + const streamGraphEmitActive = resolveStreamGraphEmit(options); - // Scope the degraded-parse log throttle to this run. On a reused process - // (e.g. tests, or any host that calls runFullAnalysis more than once) the - // module-level counter would otherwise stay saturated and suppress every - // degraded-parse log after the first run. The per-parse worker holds its own - // counter in its own module instance and is process-scoped, so no separate - // worker-side reset is needed (see safe-parse.ts ParseTimeoutError contract). - resetDegradedParseCounter(); + // FTS-config validation and the degraded-parse counter reset happen in the + // `runFullAnalysis` wrapper (before the lock is taken). - // `storagePath` is ALWAYS the flat `.gitnexus` — content-addressed caches - // (parse-cache, parsedfile-store) and the kuzu-migration cleanup live there - // and are shared across branches (#2106 KTD7). - const { storagePath } = getStoragePaths(repoPath); + // Write target (storage paths + resolved branch placement) was computed by + // the `runFullAnalysis` wrapper — which needs `metaDir` up front to acquire + // the exclusive index lock BEFORE any of the freshness/write work below + // (#2658). `storagePath` is ALWAYS the flat `.gitnexus`; `placement.branch` + // selects a `branches//` sub-slot only for an explicit `--branch` that + // does not own the flat slot. See resolveWriteTarget for the full contract. + const { storagePath, repoHasGit, currentCommit, branchLabel, placement, lbugPath, metaDir } = + writeTarget; // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open // (e.g. the embeddings-cache open) falls back to the default until the hint is @@ -683,44 +881,6 @@ export async function runFullAnalysis( log('Migrating from KuzuDB to LadybugDB — rebuilding index...'); } - const repoHasGit = hasGitDir(repoPath); - const currentCommit = repoHasGit ? getCurrentCommit(repoPath) : ''; - - // ── #2106/#2354: resolve which branch slot this run writes to ───────── - // `branchLabel` is the branch identity recorded in meta.json (incl. the - // flat workspace slot). `placement.branch` is undefined for the flat slot - // (the lbug/meta paths stay byte-identical to single-branch behavior) and - // set for a `branches//` sub-directory. Only an explicit `--branch` - // can route to a sub-directory; a plain analyze ALWAYS targets the flat - // slot, which follows the checked-out working tree (#2354) — the - // auto-detected branch (null for detached HEAD / non-git) is recorded as - // the slot's informational label only. - // Normalize the auto-detected branch the same way an explicit `--branch` is - // validated (#2106 R1): a git ref the branch-name rules forbid (backtick, - // `~ ^ : ? *`, leading `-`, `..`) becomes `null` → the flat slot, matching - // that a later `--branch ` query would also be rejected. A normal - // ref passes through unchanged so index-time and query-time labels round-trip. - const checkedOutBranch = repoHasGit - ? (sanitizeDetectedBranch(getCurrentBranch(repoPath)) ?? null) - : null; - // Analyze indexes the working tree, not an arbitrary ref. An explicit - // `--branch X` while a DIFFERENT branch Y is checked out would write Y's - // content (and Y's commit) into X's index slot, corrupting X (#2106). Refuse - // the mismatch. Detached HEAD / non-git (checkedOutBranch === null) still - // allow an explicit label so CI checkouts can name their snapshot. - if (options.branch && checkedOutBranch && options.branch !== checkedOutBranch) { - throw new Error( - `--branch "${options.branch}" does not match the checked-out branch "${checkedOutBranch}". ` + - `Check out "${options.branch}" before indexing it, or omit --branch to index the current branch.`, - ); - } - const branchLabel = options.branch ?? checkedOutBranch; - const placement = options.branch ? await resolveBranchPlacement(repoPath, branchLabel) : {}; - const { lbugPath, metaPath } = getStoragePaths(repoPath, placement.branch); - // metaPath now points to the metadata file (gitnexus.json) in a branch-specific directory. - // metaDir is the directory containing the metadata file (and branch-specific DBs). - const metaDir = path.dirname(metaPath); - // Keep gitnexus.json and the legacy meta.json mirror in sync (fresher // indexedAt wins; nothing is deleted). Best-effort: loadMeta has its own // legacy fallback, so a reconciliation failure (read-only mount, full disk) @@ -1363,6 +1523,16 @@ export async function runFullAnalysis( // offloaded BasicBlock layer. Memory-only; byte-identical output. streamPdgEmit: resolveStreamPdgEmit(options), pdgEmitChunkSize: resolvePdgEmitChunkSize(options), + // Streamed structural emit (#2680) — same full-rebuild gate as the PDG + // toggle above, for the same incremental-writeback reason. + streamGraphEmit: streamGraphEmitActive, + // Resolved ONLY when streaming is active: on a Windows non-ASCII storage + // path this helper mkdtempSyncs a real directory, so evaluating it + // unconditionally would leak one temp dir per analyze even with the flag + // off. The PDG sibling resolves inside its guard for the same reason. + graphEmitCsvDir: streamGraphEmitActive + ? resolveNativeSafeStorageDir(storagePath, 'graph-csv') + : undefined, fetchWrappers: options.fetchWrappers, }, ).finally(() => moveFlowClient?.shutdown()); @@ -1490,7 +1660,12 @@ export async function runFullAnalysis( log('atomic-incremental: live index carries orphan sidecars — using in-place writeback'); } const useAtomicSwap = (isFullRebuild || atomicIncremental) && (posixSwap || windowsSwapOk); - const buildPath = useAtomicSwap ? `${lbugPath}.new` : lbugPath; + // #2658: a per-run staging name (was the fixed `lbug.new`). Even under the + // single-writer lock, a unique name means a crashed run's half-built staging + // file can never be mistaken for — or clobber — a live run's; the lock's + // orphan sweep (sweepStagingArtifacts) reclaims stragglers on the next + // acquire. The `.staging.` prefix is what that sweep matches. + const buildPath = useAtomicSwap ? `${lbugPath}.staging.${randomUUID()}` : lbugPath; if (isIncremental && hashDiff) { log( @@ -1566,7 +1741,15 @@ export async function runFullAnalysis( // the pool; env override / no-hint paths are unchanged. See // resolveBufferManagerSize / estimateBufferPool. setBufferPoolSizeHint( - estimateBufferPool(pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount), + estimateBufferPool( + pipelineResult.graph.nodeCount + + pipelineResult.graph.relationshipCount + + // Streamed edges left the heap but still get COPYed, so they are part of + // the real load volume (#2680). The hint only ever SHRINKS the pool, so + // omitting them would starve the COPY at exactly the scale streaming + // exists to serve. + (pipelineResult.graphEmitManifest?.totalRows ?? 0), + ), ); // Full rebuild (POSIX) builds into the temp `buildPath`; incremental and @@ -2009,6 +2192,7 @@ export async function runFullAnalysis( progress('lbug', pct, msg); }, pipelineResult.pdgEmitManifest, + pipelineResult.graphEmitManifest, ); } @@ -2032,6 +2216,11 @@ export async function runFullAnalysis( // build/verify step itself fails, so capabilities.fts.status / ftsSkipped // stay honest even though that failure no longer aborts the whole analyze. let ftsReady = ftsAvailable; + // Why FTS ended up skipped (#2658 review L2): extension-unavailable up front, + // or build-failed in the degrade branch below. + let ftsSkipReason: 'extension-unavailable' | 'build-failed' | undefined = ftsAvailable + ? undefined + : 'extension-unavailable'; if (ftsAvailable) { // Degrade rather than throw: createSearchFTSIndexes re-tokenizes every // stored row on every run, so a native tokenizer error on a single @@ -2047,8 +2236,24 @@ export async function runFullAnalysis( }); if (ftsResult.ok) { progress('fts', 90, 'Search indexes ready'); + } else if (ftsFailureIsFatal(ftsResult.failureClass, useAtomicSwap)) { + // #2658: an IO/rename/checkpoint/corruption failure while building FTS + // is a genuinely broken build on this disk — not a concurrent writer + // (the single-writer lock rules that out). ONLY fatal on the atomic-swap + // path: the graph was built into a throwaway staging DB, so throwing + // before the swap abandons the staging file and leaves the previous live + // index intact. On an in-place build the live DB is already mutated and + // cannot be rolled back by throwing (see ftsFailureIsFatal) — those + // degrade in the branch below instead. + throw new Error( + `Search index build failed with an integrity error and the analysis was aborted ` + + `to avoid publishing a broken index: ${ftsResult.error}. The previous index is ` + + `left intact. Re-run \`gitnexus analyze\`; if it persists, check the disk for space ` + + `or corruption.`, + ); } else { ftsReady = false; + ftsSkipReason = 'build-failed'; log( `FTS index build failed (${ftsResult.error}) — keyword search degraded this run. ` + 'Graph and embeddings analysis completed successfully. Run `gitnexus analyze --repair-fts` to retry.', @@ -2723,6 +2928,7 @@ export async function runFullAnalysis( pipelineResult, ftsSkipped: !ftsReady, ingestWarnings: pipelineResult.standaloneIngest.ingestWarnings, + ftsSkipReason: ftsReady ? undefined : ftsSkipReason, isPrimaryBranch: !placement.branch, }; } catch (err) { diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index 1207861b9..ab098a68c 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -193,9 +193,81 @@ export async function verifySearchFTSIndexes( return missing; } +/** + * Why an FTS build failed, so the caller can react correctly (#2658): + * + * - `capability`: the environment can't support FTS this run, or a single + * pre-existing row can't be tokenized (#2544/#2546 "Invalid UTF-8"). The + * graph/embeddings work is sound — degrade keyword search and keep exit 0. + * - `integrity`: an IO / rename / checkpoint / corruption failure while + * writing the index. With the single-writer lock (#2658) this is no longer + * "some other analyze racing us" — it's a genuinely broken build on this + * disk, so the run must fail loudly rather than publish a clean-looking + * index whose search silently never worked. + */ +export type FtsBuildFailureClass = 'capability' | 'integrity'; + +// Checked before integrity signatures: a row-level tokenizer error that happens +// to mention an integrity word still degrades (it isn't a broken build). +const FTS_CAPABILITY_SIGNATURES = ['invalid utf-8', 'failed calling lower', 'tokeniz'] as const; +// IO / durability / corruption signatures that mean the build itself broke. +// Deliberately SPECIFIC (#2658 review L1): generic OS errors a capability/config +// failure can also carry — bare 'no such file or directory' (ENOENT, e.g. a +// missing FTS extension asset) and 'bad file descriptor'/'ebadf' — are NOT here, +// so an ambiguous failure degrades (the pre-#2658 safe behavior) instead of +// newly aborting the whole analyze. A genuine write/rename/checkpoint integrity +// failure still matches via 'error renaming' / 'io exception' / 'checkpoint' +// (the #2658 repro message "Error renaming … : No such file or directory" hits +// both 'io exception' and 'error renaming'). +const FTS_INTEGRITY_SIGNATURES = [ + 'io exception', + 'i/o error', + 'io error', + 'error renaming', + 'checkpoint', + 'corrupt', + 'no space', + 'enospc', + 'double free', + 'segmentation', +] as const; + +/** + * Classify an FTS build failure message. Defaults to `capability` (degrade) — + * only clearly-integrity failures escalate, so the long-standing resilience to + * row-level tokenizer errors is preserved and we never newly fail a run on an + * unrecognised message. + */ +export const classifyFtsBuildError = (message: string): FtsBuildFailureClass => { + const m = message.toLowerCase(); + if (FTS_CAPABILITY_SIGNATURES.some((s) => m.includes(s))) return 'capability'; + if (FTS_INTEGRITY_SIGNATURES.some((s) => m.includes(s))) return 'integrity'; + return 'capability'; +}; + +/** + * Whether an FTS build failure should ABORT the analyze (throw before publish) + * rather than degrade to a search-less-but-queryable index (#2658). + * + * Only an `integrity` failure on the atomic-swap path is fatal: there the graph + * was built into a throwaway staging DB, so throwing abandons the staging file + * and leaves the previous live index intact. On an in-place build + * (`useAtomicSwap === false`: incremental, Windows default) the graph DML + * already mutated the LIVE database, so there is nothing to roll back by + * throwing — degrading to a queryable index with FTS marked unavailable is + * strictly better than exiting mid-finalization over a dirty, partially-indexed + * live DB. `capability` failures always degrade. + */ +export const ftsFailureIsFatal = ( + failureClass: FtsBuildFailureClass | undefined, + useAtomicSwap: boolean, +): boolean => failureClass === 'integrity' && useAtomicSwap; + export interface BuildSearchIndexesResult { ok: boolean; error?: string; + /** Present only when `ok` is false. See {@link FtsBuildFailureClass}. */ + failureClass?: FtsBuildFailureClass; } /** @@ -216,10 +288,15 @@ export async function buildSearchIndexesOrDegrade( await createSearchFTSIndexes(options); const missing = await verifySearchFTSIndexes(executeQuery); if (missing.length > 0) { - return { ok: false, error: `missing indexes after build: ${missing.join(', ')}` }; + // Structural incompleteness with no thrown error — treat as capability + // (degrade), matching prior behavior; a broken *write* surfaces as a + // thrown IO/checkpoint error below and is classified integrity there. + const error = `missing indexes after build: ${missing.join(', ')}`; + return { ok: false, error, failureClass: classifyFtsBuildError(error) }; } return { ok: true }; } catch (e) { - return { ok: false, error: e instanceof Error ? e.message : String(e) }; + const error = e instanceof Error ? e.message : String(e); + return { ok: false, error, failureClass: classifyFtsBuildError(error) }; } } diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 14121736a..66ca657a2 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -71,7 +71,11 @@ import { isSupportedCjkSegmentationMode, MAX_CJK_SEGMENTATION_QUERY_LENGTH, } from '../../core/search/cjk-segmentation.js'; -import { checkStalenessAsync, checkCwdMatch } from '../../core/git-staleness.js'; +import { + checkStalenessAsync, + checkCwdMatch, + type StalenessInfo, +} from '../../core/git-staleness.js'; import { logger } from '../../core/logger.js'; import { isLocalEmbeddingRuntimeBlockerMessage, @@ -111,6 +115,13 @@ import { type PdgLayerStatus, } from './pdg-impact.js'; +/** + * Candidate `type`s that label enrichment newly populates (#2687). Before that, + * these surfaced as `''`, which several resolution gates read as "kind unknown". + * Anything keyed on the empty string must name these explicitly. + */ +const VALUE_CANDIDATE_TYPES: ReadonlySet = new Set(['Const', 'Variable', 'Static']); + /** Real source-file extensions (`.ts`, `.py`, …) from the resolver's list, * excluding the empty entry and the `/index.*` forms — used to decide whether * an `explain` target is a file path vs a (possibly dotted) symbol name. */ @@ -749,12 +760,60 @@ interface MoveResourceRow { fieldList: unknown; } +/** + * #2655: a tool result can carry a `staleness` field only if it is a plain + * object that isn't an error envelope and doesn't already carry one. Raw-array + * results (non-tabular `cypher` rows) are excluded because the CLI's `--limit` + * and other consumers branch on `Array.isArray`, so wrapping them would break + * that contract. Shared by `attachToolStaleness` and the dispatch site, which + * uses it to skip the freshness `git` spawn for results that can't carry it. + */ +function canCarryStaleness(result: unknown): result is Record { + return ( + result !== null && + typeof result === 'object' && + !Array.isArray(result) && + !('error' in result) && + !('staleness' in result) + ); +} + +/** + * #2655: attach a non-blocking `staleness` signal to a tool result when the + * index is behind HEAD, mirroring the `list_repos` `{commitsBehind, hint}` + * shape. Only ever ADDS a field to a carryable object result (see + * {@link canCarryStaleness}) — it never changes an existing result's shape. + */ +export function attachToolStaleness( + result: unknown, + staleness: StalenessInfo | undefined, +): unknown { + if (!staleness?.isStale || !canCarryStaleness(result)) { + return result; + } + return { + ...result, + staleness: { commitsBehind: staleness.commitsBehind, hint: staleness.hint }, + }; +} + export class LocalBackend { + private static readonly TOOL_STALENESS_TTL_MS = 5000; private repos: Map = new Map(); private contextCache: Map = new Map(); private initializedRepos: Set = new Set(); private reinitPromises: Map> = new Map(); private lastStalenessCheck: Map = new Map(); + // #2655: commit-behind freshness for the hot read tools. Stores the IN-FLIGHT + // promise (not just a timestamp) so N concurrent tool calls arriving before + // the first `git rev-list` resolves share one subprocess instead of each + // spawning their own; the resolved value is reused for TOOL_STALENESS_TTL_MS. + // Keyed by lbugPath (like lastStalenessCheck) — NOT repoPath — because flat + // and branch handles for one repo share a repoPath but carry different + // lastCommit values, so a repoPath key would serve one handle's freshness for + // the other; lbugPath is unique per flat/branch index. + private toolStalenessCache: Map }> = + new Map(); // Last meta.indexedAt observed for an open pool, keyed by lbugPath. Keyed by // pool (not stored on the handle) because branch handles are produced fresh // by applyBranchScope on every resolveRepo call, so mutating the handle would @@ -1101,6 +1160,7 @@ export class LocalBackend { if (liveLbugPaths.has(key)) continue; this.initializedRepos.delete(key); this.lastStalenessCheck.delete(key); + this.toolStalenessCache.delete(key); this.lastObservedIndexedAt.delete(key); this.lastObservedDbIdentity.delete(key); this.reinitPromises.delete(key); @@ -1768,6 +1828,60 @@ export class LocalBackend { // ─── Tool Dispatch ─────────────────────────────────────────────── + /** + * #2655: attach a commits-behind freshness signal to a hot-read-tool result, + * skipping the `git` spawn entirely for results that can't carry it (error + * envelopes, arrays, non-objects — see {@link canCarryStaleness}) so an + * error-returning call pays nothing. + */ + private async withToolStaleness(repo: RepoHandle, result: unknown): Promise { + if (!canCarryStaleness(result)) return result; + // Defensive: `checkStalenessAsync` self-catches today, but a rejection here + // must never fail the tool — degrade to no-staleness. Paired with the + // evict-on-reject in `stalenessForTool`, a transient failure also can't + // poison the TTL cache entry (#2655 review F1). + const staleness = await this.stalenessForTool(repo).catch(() => undefined); + return attachToolStaleness(result, staleness); + } + + /** + * #2655: commits-behind freshness for the hot read tools, deduped per index. + * Returns a shared in-flight promise so concurrent tool calls spawn at most + * one `git rev-list` per index per TTL window; the resolved value is cached + * for TOOL_STALENESS_TTL_MS. Keyed by lbugPath so flat and branch handles + * (same repoPath, different lastCommit) don't share an entry. Non-blocking by + * construction: `checkStalenessAsync` swallows git failures to + * `{ isStale: false }`, so a git error never fails the tool — it just omits + * the `staleness` field. + */ + private stalenessForTool(repo: RepoHandle): Promise { + const now = Date.now(); + const cached = this.toolStalenessCache.get(repo.lbugPath); + if (cached && now - cached.at < LocalBackend.TOOL_STALENESS_TTL_MS) { + return cached.value; + } + // Evict the entry if the check rejects so a transient failure isn't served + // (as a permanently-rejecting promise) for the rest of the TTL window; the + // next call then re-runs. A resolving promise is never evicted, so happy-path + // dedup is untouched (#2655 review F1). `Promise.resolve` wraps the call so a + // non-thenable return can't throw at this boundary — a no-op for the real + // async `checkStalenessAsync`, robust defense-in-depth otherwise. + const entry: { at: number; value: Promise } = { + at: now, + // Only evict if THIS entry is still current — a later call may have + // installed a fresh (resolving) entry for the same key before a slow + // rejection lands, and that newer entry must not be dropped. + value: Promise.resolve(checkStalenessAsync(repo.repoPath, repo.lastCommit)).catch((err) => { + if (this.toolStalenessCache.get(repo.lbugPath) === entry) { + this.toolStalenessCache.delete(repo.lbugPath); + } + throw err; + }), + }; + this.toolStalenessCache.set(repo.lbugPath, entry); + return entry.value; + } + async callTool(method: string, params: any): Promise { if (method === 'list_repos') { // Paginated tool surface (#2119). `listRepos()` is unchanged for internal @@ -1810,19 +1924,19 @@ export class LocalBackend { switch (method) { case 'query': - return this.query(repo, p); + return this.withToolStaleness(repo, await this.query(repo, p)); case 'cypher': { const raw = await this.cypher(repo, p); - return this.formatCypherAsMarkdown(raw); + return this.withToolStaleness(repo, this.formatCypherAsMarkdown(raw)); } case 'context': - return this.context(repo, p); + return this.withToolStaleness(repo, await this.context(repo, p)); case 'explain': return this.explain(repo, p); case 'pdg_query': return this.pdgQuery(repo, p); case 'impact': - return this.impact(repo, p as unknown as ImpactParams); + return this.withToolStaleness(repo, await this.impact(repo, p as unknown as ImpactParams)); case 'detect_changes': return this.detectChanges(repo, p); case 'check': @@ -2806,10 +2920,15 @@ export class LocalBackend { * Patch the `type` field on candidates whose `labels(n)[0]` projection * came back empty — a known LadybugDB behaviour for several node types. * - * Uses one scoped UNION query across the five priority labels rather - * than per-candidate round-trips, so cost is a single DB call regardless - * of how many candidates need enrichment. No-op when every candidate - * already has a non-empty type. + * Uses one scoped UNION query across the priority labels rather than + * per-candidate round-trips, so cost is a single DB call regardless of how + * many candidates need enrichment. No-op when every candidate already has a + * non-empty type. + * + * The value labels (`Const` / `Variable` / `Static`) are included because a + * value candidate otherwise surfaces with `kind: ""` — which reads as + * "unknown kind" and, worse, makes the `kind` disambiguation hint unable to + * filter it out (#2687). * * Failures are swallowed: label enrichment is an optimisation for * downstream scoring and #480 Class/Interface BFS seeding; if it fails @@ -2834,6 +2953,12 @@ export class LocalBackend { MATCH (n:\`Method\`) WHERE n.id IN $ids RETURN n.id AS id, 'Method' AS label UNION ALL MATCH (n:\`Constructor\`) WHERE n.id IN $ids RETURN n.id AS id, 'Constructor' AS label + UNION ALL + MATCH (n:\`Const\`) WHERE n.id IN $ids RETURN n.id AS id, 'Const' AS label + UNION ALL + MATCH (n:\`Variable\`) WHERE n.id IN $ids RETURN n.id AS id, 'Variable' AS label + UNION ALL + MATCH (n:\`Static\`) WHERE n.id IN $ids RETURN n.id AS id, 'Static' AS label `, { ids }, ); @@ -3008,7 +3133,7 @@ export class LocalBackend { // types (notably Class), which left downstream consumers (impact's // Class/Interface BFS seed, the kind-priority scoring bonus) unable to // distinguish a Class target from "unknown kind". One scoped UNION - // across the five priority labels patches the type in-place without + // across the priority labels patches the type in-place without // per-candidate round-trips. await this.enrichCandidateLabels(repo, normalized); @@ -3020,7 +3145,15 @@ export class LocalBackend { // the `type === 'Constructor'` gate still correctly triggers when a // Class and its Constructor share the name. if (!hints.kind && normalized.length > 1) { - const ambiguousType = normalized.some((s) => s.type === '' || s.type === 'Constructor'); + // A value candidate (`Const`/`Variable`/`Static`) used to reach here with + // `type === ''`, which is what kept this gate true for a `class Foo` + + // `const Foo` pair and let the collapse resolve it to the Class. Label + // enrichment now fills those in (#2687), so they must be named explicitly + // or the collapse silently stops firing and confident resolutions become + // `ambiguous` across every resolver-backed tool. + const ambiguousType = normalized.some( + (s) => s.type === '' || s.type === 'Constructor' || VALUE_CANDIDATE_TYPES.has(s.type), + ); if (ambiguousType) { const candidateIds = normalized.map((s) => s.id).filter(Boolean); for (const label of ['Class', 'Interface']) { @@ -5224,10 +5357,12 @@ export class LocalBackend { target: { name: target }, direction, totalCandidates: outcome.candidates.length, - // No single resolved symbol → impactedCount stays 0 / risk UNKNOWN - // (UNKNOWN must never read as "safe to refactor"). No callgraph - // fan-out runs, so there is no per-candidate blast radius here yet. - impactedCount: 0, + // No single resolved symbol → the blast radius is UNDETERMINED, not + // zero. `null` (not 0) because no callgraph fan-out runs on this path, + // so there is not even a `maxImpactedCount` to correct a numeric zero + // against — it would be indistinguishable from a genuine "nothing + // depends on this" (#2687). + impactedCount: null, risk: 'UNKNOWN', ...(truncated && { candidatesTruncated: true }), candidates: shown.map((c) => ({ @@ -5343,12 +5478,15 @@ export class LocalBackend { // so consumers (CLI formatter) need this to report "N of M" honestly (#2129 // review F11; the CLI previously read the truncated array length). totalCandidates: outcome.candidates.length, - // `impactedCount` stays 0 and `risk` stays UNKNOWN — there is no single - // resolved symbol, and UNKNOWN must NOT read as "safe to refactor". The - // real blast radius is surfaced per-candidate plus `maxImpactedCount` / - // `maxRisk` so a real caller can never hide behind the ambiguous zero - // (#2129). - impactedCount: 0, + // `impactedCount` is `null` — UNDETERMINED, not zero — and `risk` stays + // UNKNOWN, because there is no single resolved symbol. #2129 hoisted + // `maxImpactedCount` / `maxRisk` here so a real caller could not hide + // behind the ambiguous zero, but the zero itself remained + // byte-identical to a genuine "nothing depends on this": a consumer + // testing `impactedCount === 0` still read a confident all-clear + // without ever looking at `candidates[]`. `null` cannot be mistaken for + // a measured zero, while `|| 0` consumers are unchanged (#2687). + impactedCount: null, risk: 'UNKNOWN', maxImpactedCount, maxRisk, diff --git a/gitnexus/src/server/analyze-launch.ts b/gitnexus/src/server/analyze-launch.ts index b505cdfcf..720ebe7f8 100644 --- a/gitnexus/src/server/analyze-launch.ts +++ b/gitnexus/src/server/analyze-launch.ts @@ -23,6 +23,7 @@ import { registryPathEquals, } from '../storage/repo-manager.js'; import { logger } from '../core/logger.js'; +import { autoHeapCapMb } from '../core/ingestion/utils/effective-ram.js'; import type { JobManager } from './analyze-job.js'; import type { WorkerMessage } from './analyze-worker.js'; @@ -151,12 +152,20 @@ export function createLaunchAnalysisWorker(deps: LaunchDeps) { ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href] : []; + // Worker heap: 8192MB historical default, but never above what this + // machine/container actually has (#2649 review — a fixed 8192 inside a + // smaller cgroup limit died to the kernel with a misleading remedy). + // GITNEXUS_SERVER_ANALYZE_HEAP_MB overrides as an absolute value. + const envHeapMb = Number(process.env.GITNEXUS_SERVER_ANALYZE_HEAP_MB); + const workerHeapMb = + Number.isInteger(envHeapMb) && envHeapMb > 0 ? envHeapMb : Math.min(8192, autoHeapCapMb()); + const forkWorker = () => { const currentJob = jobManager.getJob(job.id); if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed') return; const child = fork(workerPath, [], { - execArgv: [...tsxHookArgs, '--max-old-space-size=8192'], + execArgv: [...tsxHookArgs, `--max-old-space-size=${workerHeapMb}`], stdio: ['ignore', 'pipe', 'pipe', 'ipc'], }); diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index 5c35d69a3..547b29dbd 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -16,6 +16,9 @@ import type { AnalyzeOptions } from '../core/run-analyze.js'; import type { WorkerMessage } from './analyze-worker.js'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js'; +// Value import (instanceof): index-lock is a lightweight storage primitive +// (node:fs/net/crypto only), so this does NOT pull in run-analyze/repo-manager. +import { IndexLockTimeoutError } from '../storage/index-lock.js'; export interface WorkerAnalysisDeps { runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis; @@ -74,7 +77,13 @@ export async function runWorkerAnalysis( } catch (err: unknown) { // Report the failure to the parent over IPC (the parent surfaces the message). const message = err instanceof Error ? err.message : 'Analysis failed'; - terminal = { type: 'error', message }; + // #2658 review M2: a lock-wait timeout is transient contention (another + // analyze held the single-writer lock), not a broken build — tag it so the + // parent can surface a retry signal instead of an opaque hard failure. + terminal = + err instanceof IndexLockTimeoutError + ? { type: 'error', message, code: 'index-lock-timeout', retryable: true } + : { type: 'error', message }; } // P3 (#2264): only report if a SIGTERM cancellation hasn't already claimed the diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index 37ae1713b..08b32b9d6 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -40,6 +40,16 @@ export interface CompleteMessage { export interface ErrorMessage { type: 'error'; message: string; + /** + * Machine-readable failure code for a parent that wants to branch instead of + * only surfacing the string. `index-lock-timeout` (#2658 review M2) means + * another analyze held the single-writer lock past the wait ceiling — a + * transient, retryable condition, not a broken build. Absent for a generic + * failure. + */ + code?: 'index-lock-timeout'; + /** True when the failure is expected to clear on retry (e.g. lock contention). */ + retryable?: boolean; } /** Child → parent IPC messages. Shared with the parent-side launcher. */ diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 4b4fdd0f1..585322b11 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -1,7 +1,8 @@ import { execFileSync, execSync } from 'child_process'; -import { statSync } from 'fs'; +import { statSync, existsSync } from 'fs'; import path from 'path'; import os from 'os'; +import { logger } from '../core/logger.js'; // Git utilities for repository detection, commit tracking, and diff analysis @@ -10,10 +11,13 @@ const chompGitOutput = (value: Buffer): string => value.toString().replace(/\r?\ /** * True when the working tree has uncommitted changes that analyze would * re-index, even at a matching HEAD. Excludes the paths GitNexus writes during - * analyze (.gitnexus/, .claude/, .cursor/, AGENTS.md, CLAUDE.md) so its own - * output never counts as dirty (regression vs PR #1233 behavior). Conservative - * on any git failure. Shared so `analyze`'s fast-path gate and `status`'s - * freshness report agree on what "dirty" means. + * analyze (.gitnexus/, .claude/, .cursor/, AGENTS.md, CLAUDE.md, and the + * repo-local .agents/ mirror) so its own output never counts as dirty + * (regression vs PR #1233 behavior). The entire .agents/ tree is excluded, + * matching the .claude/ treatment, because the skill mirror writes across + * .agents/skills/ and deeper paths. Conservative on any git failure. Shared + * so `analyze`'s fast-path gate and `status`'s freshness report agree on what + * "dirty" means. */ export const isWorkingTreeDirty = (repoPath: string): boolean => { try { @@ -32,6 +36,8 @@ export const isWorkingTreeDirty = (repoPath: string): boolean => { ':(exclude).cursor/**', ':(exclude)AGENTS.md', ':(exclude)CLAUDE.md', + ':(exclude).agents', + ':(exclude).agents/**', ], { cwd: repoPath, @@ -46,6 +52,123 @@ export const isWorkingTreeDirty = (repoPath: string): boolean => { } }; +/** + * Snapshot, per candidate file, whether it is safe for `selfCommitContextFiles` + * to auto-commit — call this BEFORE `analyze` writes AGENTS.md/CLAUDE.md. + * A file is safe when it does not exist yet (first-time creation, the normal + * case) or is currently clean (`git status --porcelain` reports nothing for + * it). A file that already has an uncommitted user edit is unsafe: without + * this check `selfCommitContextFiles` cannot tell that edit apart from the + * stats refresh `analyze` is about to write, and would silently sweep both + * into one generated-looking commit. Fails closed — a git failure marks the + * file unsafe rather than assuming it's clean. See #2639 review round 2. + */ +export const snapshotSelfCommitSafety = ( + repoPath: string, + candidateFiles: string[], +): Map => { + const safety = new Map(); + for (const name of candidateFiles) { + if (!existsSync(path.join(repoPath, name))) { + safety.set(name, true); + continue; + } + try { + const status = execFileSync('git', ['status', '--porcelain', '--', name], { + cwd: repoPath, + stdio: ['ignore', 'pipe', 'ignore'], + windowsHide: true, + encoding: 'utf8', + }); + safety.set(name, status.trim().length === 0); + } catch { + safety.set(name, false); + } + } + return safety; +}; + +/** + * Best-effort auto-commit for the AGENTS.md/CLAUDE.md files `analyze --self-commit` + * just (re)wrote. Filters `candidateFiles` down to the ones that actually exist + * under `repoPath` AND were marked safe by `snapshotSelfCommitSafety` — a file + * that already had an uncommitted edit before this run is skipped (logged), + * never swept into the generated commit. Never `git add -A`. `git status + * --porcelain` (not `diff --quiet`) is deliberate: a first-time `analyze` run + * creates AGENTS.md/CLAUDE.md fresh, and untracked files never show up in + * `git diff`, only in `git status` — the same reason `isWorkingTreeDirty` + * above uses `--porcelain`. If `git commit` fails after `git add` already + * staged the safe files (e.g. missing git identity), the staged files are + * reset back to unstaged so the user's index isn't silently left mutated. + * No-ops silently (never throws) when: none of the candidate files exist or + * are safe, none changed, or any git step fails. Must never fail the + * surrounding `analyze` run. See #2639. + */ +export const selfCommitContextFiles = ( + repoPath: string, + candidateFiles: string[], + preRunSafety: Map, +): void => { + const existing = candidateFiles.filter((name) => existsSync(path.join(repoPath, name))); + if (existing.length === 0) return; + + const safe = existing.filter((name) => preRunSafety.get(name) === true); + const skippedDirty = existing.filter((name) => preRunSafety.get(name) !== true); + if (skippedDirty.length > 0) { + logger.warn( + { files: skippedDirty }, + 'gitnexus: --self-commit skipping file(s) with uncommitted changes from before this analyze run', + ); + } + if (safe.length === 0) return; + + try { + const status = execFileSync('git', ['status', '--porcelain', '--', ...safe], { + cwd: repoPath, + stdio: ['ignore', 'pipe', 'ignore'], + windowsHide: true, + encoding: 'utf8', + }); + if (status.trim().length === 0) return; // nothing to commit + } catch { + return; // git failed (not a repo, git missing, etc.) — nothing to do + } + + try { + execFileSync('git', ['add', '--', ...safe], { + cwd: repoPath, + stdio: 'ignore', + windowsHide: true, + }); + } catch (err) { + logger.warn({ err, files: safe }, 'gitnexus: --self-commit failed to stage context files'); + return; + } + + try { + execFileSync( + 'git', + ['commit', '-m', 'chore(gitnexus): refresh index stats [skip ci]', '--', ...safe], + { cwd: repoPath, stdio: 'ignore', windowsHide: true }, + ); + } catch (err) { + // Commit failed after `git add` already staged `safe` (e.g. missing git + // identity). Restore the index to its pre-add state for exactly those + // files rather than leaving them silently staged — `analyze` reporting + // "success" must not leave the user's index mutated. + try { + execFileSync('git', ['reset', '--', ...safe], { + cwd: repoPath, + stdio: 'ignore', + windowsHide: true, + }); + } catch { + /* best-effort restore; nothing more we can do */ + } + logger.warn({ err, files: safe }, 'gitnexus: --self-commit failed to commit context files'); + } +}; + export const isGitRepo = (repoPath: string): boolean => { try { execSync('git rev-parse --is-inside-work-tree', { diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts new file mode 100644 index 000000000..c67750266 --- /dev/null +++ b/gitnexus/src/storage/index-lock.ts @@ -0,0 +1,722 @@ +/** + * Cross-process single-writer lock for a GitNexus index directory (#2658). + * + * `analyze` is the only writer of a `.gitnexus/` (or `branches//`) slot, + * but nothing stopped two `analyze` runs — e.g. two editor/agent SessionStart + * hooks firing on the same repo at once — from wiping and rebuilding the same + * store concurrently. They raced on `lbug` and its sidecars, wasted N× CPU + * producing one index, and left orphaned WAL fragments (#2637). This module + * gives the write path an exclusive, index-directory-scoped lock so a second + * writer waits for the first instead of colliding; after acquiring, the caller + * re-runs its normal freshness check, so a run whose work the holder already + * did exits up-to-date rather than rebuilding (single-flight coalescing). + * + * Ownership lives with the process that runs the pipeline (the heap-respawn + * child when a respawn happens, the original otherwise) — NOT a supervising + * parent — so the entity the OS tracks for liveness is always the real writer. + * See run-analyze.ts for the acquire site. + * + * TWO BACKENDS behind the {@link acquireIndexLock} seam: + * + * - **socket** (Windows named pipe / Linux abstract socket, via `net`) — the + * preferred, KERNEL-OWNED lock. Holding it = holding a listening endpoint the + * kernel binds to this process; `EADDRINUSE` therefore means a *live* holder, + * and the kernel drops the binding the instant the holder exits for ANY reason + * (clean exit, crash, OOM, SIGKILL). That makes it provably race-free: no + * stale detection, no pid-reuse guess, no takeover, and — since the endpoint + * lives outside the index dir — no filesystem write, so it works unchanged on + * a read-only index mount. This is the same class of kernel object as the + * Windows named mutex the issue's reporter used as an external workaround, but + * built from Node's stdlib `net`, so it adds NO native dependency and cannot + * break `npx gitnexus` install anywhere. + * + * - **file** (`O_EXCL` pidfile) — the portable fallback for macOS/BSD (no + * abstract sockets; filesystem sockets don't release cleanly on death) and + * for any environment where the socket backend can't bind. It uses pid- + * liveness staleness, an atomic rename-steal reclaim, bounded malformed-file + * handling, read-only tolerance, and a finite wait timeout (a reused pid can + * masquerade as live where process start-time isn't verifiable, so waiting is + * bounded rather than a hang). Its stale-takeover has an irreducible narrow + * race — inherent to file-based advisory locks — which is precisely why the + * socket backend is preferred; only a kernel primitive closes it. + * + * Scope: cross-process, same logical index dir. The file backend never steals a + * foreign-host lock (pid liveness is meaningless across hosts); the socket + * backend is single-host by nature. The motivating case (local hook-driven + * re-index) is single-host. See AcquireOptions.timeoutMs for the wait ceiling. + */ +import { + openSync, + writeSync, + closeSync, + readFileSync, + unlinkSync, + renameSync, + existsSync, + mkdirSync, + readdirSync, + realpathSync, +} from 'node:fs'; +import net from 'node:net'; +import path from 'node:path'; +import os from 'node:os'; +import { randomBytes, randomUUID, createHash } from 'node:crypto'; + +const LOCK_FILENAME = 'analyze.lock'; +const LOCK_RECORD_VERSION = 1 as const; + +/** Base poll interval while waiting for a live holder; jittered per attempt. */ +const DEFAULT_POLL_MS = 250; +/** How often to re-emit the "still waiting for pid N" diagnostic. */ +const DIAGNOSTIC_INTERVAL_MS = 15_000; +/** + * Default wait ceiling (10 min). Generous enough to sit behind a normal + * analyze, finite so a pid-reuse ghost on a platform without start-time + * verification can't wedge acquisition forever (see AcquireOptions.timeoutMs). + * A repo whose analyze legitimately runs longer can raise + * GITNEXUS_INDEX_LOCK_TIMEOUT_MS (or set it ≤ 0 for unbounded). + */ +const DEFAULT_TIMEOUT_MS = 600_000; +/** + * How long a lock file must stay unreadable (empty/partial JSON) before we + * treat it as a crash orphan and reclaim it. Tolerates the microsecond + * create→write→close window of a *live* owner (see acquireIndexLock), so we + * never steal a lock that is a poll-interval away from being written. Scaled + * off the poll interval, floored at 1s. + */ +const malformedGraceMs = (pollMs: number): number => Math.max(1000, pollMs * 2); + +/** + * On-disk lock record. `token` proves ownership on release/steal; `startTime` + * (Linux only) defends against pid reuse; `invocationId` is a human-traceable + * id distinct from the security-irrelevant `token`. + */ +export interface LockRecord { + v: typeof LOCK_RECORD_VERSION; + pid: number; + hostname: string; + /** /proc//stat starttime (clock ticks) on Linux; null where unavailable. */ + startTime: string | null; + token: string; + invocationId: string; + acquiredAt: string; +} + +export interface IndexLockHandle { + /** Our own record — `invocationId` is shown to waiters as the holder id. */ + readonly record: LockRecord; + /** Idempotent; only removes the lock file if it still carries our token. */ + release(): void; +} + +export interface AcquireOptions { + log?: (msg: string) => void; + /** + * Give up waiting after this long (ms), throwing {@link IndexLockTimeoutError}. + * Default: {@link DEFAULT_TIMEOUT_MS} ({@link resolveTimeoutMs}). A finite + * default is deliberate: on platforms without process start-time verification + * (anything but Linux — see {@link readProcStartTime}) a crashed holder whose + * pid was reused by an unrelated long-lived process reads as a live holder and + * would otherwise block acquisition forever. Timing out is safe — it stops + * *waiting*, never *steals* a possibly-live holder — and names the holder so + * the caller can retry. Override (including to unbounded, value ≤ 0) via + * GITNEXUS_INDEX_LOCK_TIMEOUT_MS. + */ + timeoutMs?: number; + /** Base poll interval (ms); jittered. Default 250. */ + pollMs?: number; + /** Called once when we start waiting on a live holder. */ + onWaitStart?: (holder: LockRecord) => void; +} + +export class IndexLockTimeoutError extends Error { + readonly holder: LockRecord; + /** + * Whether `holder` carries a real, identifiable owner. False on the socket + * backend (and the file backend's malformed/vanished-lock timeouts), where the + * holder is a placeholder (`pid -1`) — the OS socket lock exposes no owner + * metadata (#2658 review M3). Consumers must not present `holder.pid` as a real + * pid when this is false. + */ + readonly holderKnown: boolean; + constructor(holder: LockRecord, waitedMs: number, holderKnown = true) { + super( + holderKnown + ? `Timed out after ${waitedMs}ms waiting for another gitnexus analyze ` + + `(pid ${holder.pid} on ${holder.hostname}, invocation ${holder.invocationId}) ` + + `to release the index lock.` + : `Timed out after ${waitedMs}ms waiting for another gitnexus analyze ` + + `(holder identity unknown) to release the index lock.`, + ); + this.name = 'IndexLockTimeoutError'; + this.holder = holder; + this.holderKnown = holderKnown; + } +} + +const HOSTNAME = os.hostname(); + +/** Linux: field 22 of /proc//stat (starttime). null elsewhere / on error. */ +const readProcStartTime = (pid: number): string | null => { + if (process.platform !== 'linux') return null; + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); + // comm (field 2) is parenthesized and may contain spaces/')' — split after + // the last ')' so the remaining fields align to their documented numbers. + const afterComm = stat + .slice(stat.lastIndexOf(') ') + 2) + .trim() + .split(' '); + // afterComm[0] is field 3 (state); starttime is field 22 → index 19. + return afterComm[19] ?? null; + } catch { + return null; + } +}; + +/** true if the pid exists (signal 0). EPERM means it exists but isn't ours. */ +const pidAlive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === 'EPERM'; + } +}; + +const buildRecord = (): LockRecord => ({ + v: LOCK_RECORD_VERSION, + pid: process.pid, + hostname: HOSTNAME, + startTime: readProcStartTime(process.pid), + token: randomBytes(16).toString('hex'), + invocationId: randomUUID(), + acquiredAt: new Date().toISOString(), +}); + +const readRecord = (lockPath: string): LockRecord | null => { + try { + const raw = readFileSync(lockPath, 'utf8'); + const parsed = JSON.parse(raw) as Partial; + // `typeof NaN === 'number'`, so a bare number check lets NaN/0/-1/Infinity/ + // fractional pids reach process.kill (#2658 review L4): a garbled or crafted + // lock file with `{"pid":0}` reads as a live holder and wedges a real analyze + // for the full wait timeout. A real pid is a positive integer. + if (!Number.isInteger(parsed.pid) || (parsed.pid as number) <= 0) return null; + if (typeof parsed.token !== 'string') return null; + return parsed as LockRecord; + } catch { + // Missing (won the race, file gone) or malformed/half-written → treat as + // "no readable holder"; the caller retries the O_EXCL create. + return null; + } +}; + +/** + * A same-host holder is stale iff its process is gone, or (Linux) its pid is + * alive but was reused — a different start time. A live holder is never stolen + * on age alone (a large repo legitimately analyzes for many minutes), and a + * foreign-host holder is never stale (its liveness is unknowable here). Where + * start-time verification is unavailable (non-Linux), a reused pid cannot be + * distinguished from a genuine live holder, so it is NOT stolen — the finite + * acquire timeout is what bounds that case instead (see AcquireOptions). + */ +const isStale = (holder: LockRecord): boolean => { + if (holder.hostname !== HOSTNAME) return false; + if (!pidAlive(holder.pid)) return true; + const now = readProcStartTime(holder.pid); + if (holder.startTime && now && holder.startTime !== now) return true; // pid reused + return false; +}; + +/** + * Reclaim a lock file we judged reclaimable — a dead holder (`expected` = its + * record) or a malformed/unreadable crash-orphan (`expected` = null) — moving + * the exact inode aside in ONE `rename` syscall to a token-unique name so two + * waiters reclaiming the same orphan can't both win (the loser's rename ENOENTs). + * + * CRITICAL (#2658 review): the reclaim must not act on a STALE judgment. The + * staleness decision (`isStale` / malformed-grace) happened a few syscalls ago; + * a live writer may have O_EXCL-created its own lock at `lockPath` since. Blindly + * renaming that live lock aside would delete it and admit a SECOND writer — the + * exact double-writer this lock exists to prevent (reproduced: ~18%/round under + * 4-way reclaim contention on the file backend). So: + * 1. re-read `lockPath` immediately BEFORE the rename and confirm it still holds + * exactly what we judged (same token, or still-unreadable) — shrinking the + * window to the single gap between this read and the rename; + * 2. after the rename, confirm what we ACTUALLY moved matches the judgment; if a + * live lock slipped into that residual gap, RESTORE it (rename back) so its + * holder is never displaced, and lose the reclaim. + * A concurrent creator whose fresh lock the restore overwrites is caught by the + * acquire loop's post-write read-back verify (see acquireViaFile), so it backs + * off rather than proceeding as a second writer. + * + * Returns true if we won the reclaim (caller retries the create), false if we + * lost the race or the judgment went stale (caller re-loops and re-reads). + */ +const matchesJudgment = (record: LockRecord | null, expected: LockRecord | null): boolean => + expected === null ? record === null : record?.token === expected.token; + +const stealLock = (lockPath: string, me: LockRecord, expected: LockRecord | null): boolean => { + // (1) Re-verify the judgment still holds right before we move anything. + if (!matchesJudgment(readRecord(lockPath), expected)) return false; + if (expected === null && !existsSync(lockPath)) return false; // malformed → but now vanished + + const aside = `${lockPath}.dead.${me.token}`; + try { + renameSync(lockPath, aside); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; // another stealer won + throw err; + } + // (2) Confirm what we moved is what we judged; if a live lock slipped into the + // read→rename gap, put it back — a live holder must never be displaced. + if (!matchesJudgment(readRecord(aside), expected)) { + try { + renameSync(aside, lockPath); // restore; an overwritten concurrent creator's read-back backs it off + } catch { + /* slot re-taken between our move and restore — leave it; we lost the reclaim */ + } + return false; + } + try { + unlinkSync(aside); // uniquely ours by token → safe; best-effort + } catch { + /* leftover .dead. is inert (not analyze.lock, not swept) — harmless */ + } + return true; +}; + +/** + * Placeholder holder for an {@link IndexLockTimeoutError} thrown while the lock + * file exists but no valid record can be read (malformed/partial), or it keeps + * vanishing — there is no real holder to name, but the error still needs one so + * the CLI's `err.holder.pid` stays defined. This path is a rare backstop: + * malformed files are reclaimed within {@link MALFORMED_GRACE_MS}. + */ +const unknownHolder = (): LockRecord => ({ + v: LOCK_RECORD_VERSION, + pid: -1, + hostname: HOSTNAME, + startTime: null, + token: '', + invocationId: '', + acquiredAt: '', +}); + +/** + * Filesystem-create error codes we tolerate by proceeding lock-free: a + * read-only mount (EROFS) or a denied create (EACCES/EPERM). Such a filesystem + * rejects every index WRITE in the same directory too, so no concurrent writer + * can exist and the lock is moot — an already-indexed repo on a `:ro` mount + * must still reach its `alreadyUpToDate` fast path (#2658). A genuinely-needed + * write fails later exactly as it would have without the lock. + */ +export const LOCK_UNWRITABLE_CODES: ReadonlySet = new Set(['EROFS', 'EACCES', 'EPERM']); +export const isLockUnwritableCode = (code: string | undefined): boolean => + code !== undefined && LOCK_UNWRITABLE_CODES.has(code); + +/** A lock handle that owns nothing — returned when the filesystem refuses to + * create the lock file (see {@link LOCK_UNWRITABLE_CODES}). Release is a no-op. */ +const noopHandle = (record: LockRecord): IndexLockHandle => ({ record, release: () => {} }); + +/** + * Delete orphaned build/staging artifacts left in the lock directory by a + * crashed prior writer. Safe precisely because we hold the exclusive lock: no + * other writer can be creating these here right now, so anything present is a + * crash orphan. Matches this slot's staging files ONLY — never `lbug` itself, + * never `lbug.wal`/`lbug.shadow` (the LIVE index's own sidecars), and never a + * `branches//` sub-slot (which owns its own lock + sweep). Non-recursive. + */ +export const sweepStagingArtifacts = (lockDir: string, log?: (msg: string) => void): void => { + // Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.`, `lbug.staging..wal`, … + // Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`. + const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/; + let removed = 0; + let entries: string[]; + try { + entries = readdirSync(lockDir); + } catch { + return; + } + for (const name of entries) { + if (!stagingRe.test(name)) continue; + try { + unlinkSync(path.join(lockDir, name)); + removed++; + } catch { + /* best-effort */ + } + } + if (removed > 0) { + log?.(`Cleared ${removed} orphaned index-staging file(s) from a prior interrupted analyze.`); + } +}; + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Poll delay with jitter (avoids two waiters lock-stepping), clamped so it + * never overshoots the remaining timeout budget. Callers guarantee + * `waited < timeoutMs`, so the result is ≥ 1. */ +const jitteredDelay = (pollMs: number, timeoutMs: number, waited: number): number => { + const jitter = Math.floor(Math.random() * pollMs); + const remaining = timeoutMs - waited; + return Math.max(1, Math.min(pollMs + jitter, remaining)); +}; + +/** + * Resolve the wait ceiling. Explicit `opt` wins; else + * GITNEXUS_INDEX_LOCK_TIMEOUT_MS; else {@link DEFAULT_TIMEOUT_MS}. A value ≤ 0 + * (from either source) means unbounded. + */ +const resolveTimeoutMs = (opt?: number): number => { + const raw = + typeof opt === 'number' + ? opt + : (() => { + const env = process.env.GITNEXUS_INDEX_LOCK_TIMEOUT_MS; + if (env === undefined || env === '') return DEFAULT_TIMEOUT_MS; + const n = Number(env); + return Number.isFinite(n) ? n : DEFAULT_TIMEOUT_MS; + })(); + return raw <= 0 ? Number.POSITIVE_INFINITY : raw; +}; + +/** + * Acquire the exclusive write lock for `lockDir` (the resolved index slot + * directory, e.g. `/.gitnexus` or `/.gitnexus/branches/`). + * + * Blocks until the lock is held (waiting only on live holders, stealing dead + * ones immediately), then sweeps orphaned staging files under the lock and + * returns a handle. Rejects with `IndexLockTimeoutError` if `timeoutMs` is + * exceeded while a live holder still holds the lock. + */ +/** + * File-based (O_EXCL pidfile) backend. The portable fallback used on platforms + * without the socket backend (macOS/BSD) or when the OS socket lock is + * unavailable. Carries the pid-liveness staleness, atomic rename-steal reclaim, + * bounded malformed-file handling, and read-only tolerance. Its stale-takeover + * has an irreducible (narrow) race — see the module header — which is why the + * socket backend is preferred where available. + */ +const acquireViaFile = async ( + lockDir: string, + me: LockRecord, + opts: AcquireOptions, +): Promise => { + try { + mkdirSync(lockDir, { recursive: true }); + } catch (err) { + // Read-only / denied filesystem → proceed lock-free (see LOCK_UNWRITABLE_CODES). + if (isLockUnwritableCode((err as NodeJS.ErrnoException).code)) return noopHandle(me); + throw err; + } + const lockPath = path.join(lockDir, LOCK_FILENAME); + const pollMs = opts.pollMs ?? DEFAULT_POLL_MS; + const timeoutMs = resolveTimeoutMs(opts.timeoutMs); + const startedAt = Date.now(); + let announcedWait = false; + let lastDiagnosticAt = 0; + // When the lock file exists but has no readable record, the timestamp we + // first observed it unreadable — used to reclaim a crash-orphan after a grace. + let malformedSince: number | null = null; + + for (;;) { + try { + // O_WRONLY | O_CREAT | O_EXCL — the atomic arbiter of ownership. + const fd = openSync(lockPath, 'wx'); + try { + writeSync(fd, JSON.stringify(me)); + } finally { + closeSync(fd); + } + // Read-back verify (#2658 review L5): if this process stalled (a >graceMs + // GC pause) between the O_EXCL create of the *empty* file and the write + // above, a waiter could have reclaimed the empty file (renamed it aside) + // and O_EXCL-created its own lock at `lockPath`. Our write then landed on + // the renamed-aside inode, not `lockPath`. Confirm `lockPath` still carries + // our token before claiming ownership; if it was stolen, contend normally. + const confirmed = readRecord(lockPath); + if (!confirmed || confirmed.token !== me.token) continue; + return { + record: me, + release: () => { + const current = readRecord(lockPath); + if (current && current.token !== me.token) return; // no longer ours + try { + unlinkSync(lockPath); + } catch { + /* already gone */ + } + }, + }; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'EEXIST') { + // fall through to holder inspection / wait / reclaim below + } else if (isLockUnwritableCode(code)) { + return noopHandle(me); // read-only / denied → proceed lock-free + } else { + throw err; + } + } + + const holder = readRecord(lockPath); + const waited = Date.now() - startedAt; + + if (holder) { + malformedSince = null; + if (isStale(holder)) { + opts.log?.( + `Reclaiming stale index lock from dead analyze (pid ${holder.pid}, ` + + `invocation ${holder.invocationId}).`, + ); + stealLock(lockPath, me, holder); // reclaim ONLY this dead record; live locks are never stolen + continue; + } + // Live holder → wait. + if (!announcedWait) { + announcedWait = true; + opts.onWaitStart?.(holder); + opts.log?.( + `Another gitnexus analyze (pid ${holder.pid} on ${holder.hostname}) is ` + + `refreshing this index — waiting for it to finish.`, + ); + } + if (waited >= timeoutMs) throw new IndexLockTimeoutError(holder, waited); + if (Date.now() - lastDiagnosticAt >= DIAGNOSTIC_INTERVAL_MS) { + lastDiagnosticAt = Date.now(); + if (waited >= DIAGNOSTIC_INTERVAL_MS) { + opts.log?.( + `Still waiting for analyze pid ${holder.pid} (${Math.round(waited / 1000)}s elapsed).`, + ); + } + } + await sleep(jitteredDelay(pollMs, timeoutMs, waited)); + continue; + } + + // holder === null: the lock file is either gone (vanished between the failed + // create and our read) or present-but-unreadable (a crash between the + // O_EXCL create and the record write, or a partial write). NEVER hot-loop + // here — both branches are bounded by sleep + timeout. + if (!existsSync(lockPath)) { + malformedSince = null; // genuinely vanished → the next create likely wins + if (waited >= timeoutMs) throw new IndexLockTimeoutError(unknownHolder(), waited, false); + await sleep(jitteredDelay(pollMs, timeoutMs, waited)); + continue; + } + // Malformed orphan present. Reclaim only after a grace, so a live owner's + // microsecond create→write window is never mistaken for a crash. + if (malformedSince === null) malformedSince = Date.now(); + if (Date.now() - malformedSince >= malformedGraceMs(pollMs)) { + opts.log?.('Reclaiming a malformed/partial index lock file (no readable owner record).'); + stealLock(lockPath, me, null); // reclaim ONLY while still unreadable; a live lock written since is left + malformedSince = null; + continue; + } + if (waited >= timeoutMs) throw new IndexLockTimeoutError(unknownHolder(), waited, false); + await sleep(jitteredDelay(pollMs, timeoutMs, waited)); + } +}; + +/** Signals that the OS socket backend can't be used here (e.g. abstract + * namespace disabled, sandbox, or an unexpected bind error) so the caller + * should fall back to the file backend. NOT thrown for EADDRINUSE (that is a + * live holder → wait) or timeouts (those propagate as IndexLockTimeoutError). */ +class SocketLockUnavailable extends Error { + constructor(readonly cause: NodeJS.ErrnoException) { + super(`OS socket lock unavailable: ${cause.code ?? cause.message}`); + this.name = 'SocketLockUnavailable'; + } +} + +/** + * Canonicalize a path to its real filesystem identity so lexical aliases of the + * same directory (a symlink, a bind-mount path, a Windows junction, a `\\?\` + * prefix) map to ONE name (#2658 review H1). `lockDir` (the index slot) often + * does not exist yet, so `realpathSync` the deepest existing ancestor and + * re-append the not-yet-created remainder. A path with no symlink components + * realpaths to itself, so the common (non-aliased) case is unchanged — a holder + * that used the old resolved name is never orphaned. + */ +const canonicalizeDir = (p: string): string => { + const resolved = path.resolve(p); + const tail: string[] = []; + let dir = resolved; + for (;;) { + try { + const real = realpathSync(dir); + return tail.length ? path.join(real, ...tail.reverse()) : real; + } catch { + const parent = path.dirname(dir); + if (parent === dir) return resolved; // reached the root with nothing to resolve + tail.push(path.basename(dir)); + dir = parent; + } + } +}; + +/** + * Stable OS-IPC endpoint name for an index directory. The name is derived from + * the REAL path (case-folded on Windows), so two processes targeting the same + * physical slot — even via different lexical aliases — collide, and separate + * worktrees/branches never do. The endpoint lives OUTSIDE the index directory + * (abstract namespace / pipe namespace), so the lock needs no filesystem write + * and is unaffected by a read-only index mount. + */ +const socketLockName = (lockDir: string): string => { + const resolved = canonicalizeDir(lockDir); + const key = createHash('sha256') + .update(process.platform === 'win32' ? resolved.toLowerCase() : resolved) + .digest('hex') + .slice(0, 32); + return process.platform === 'win32' + ? `\\\\.\\pipe\\gitnexus-idx-${key}` + : `\0gitnexus-idx-${key}`; // Linux abstract socket (no filesystem entry) +}; + +/** Attempt to listen; resolve to null on success or the error on failure. */ +const tryListen = (server: net.Server, name: string): Promise => + new Promise((resolve) => { + const onError = (err: NodeJS.ErrnoException): void => { + server.removeListener('listening', onListening); + resolve(err); + }; + const onListening = (): void => { + server.removeListener('error', onError); + resolve(null); + }; + server.once('error', onError); + server.once('listening', onListening); + server.listen(name); + }); + +/** + * OS-owned socket/pipe backend (Windows named pipe, Linux abstract socket). + * Holding the lock = holding a listening endpoint the kernel binds to this + * process; `EADDRINUSE` therefore means a *live* holder, and the kernel drops + * the binding the instant the holder exits (clean exit, crash, OOM, SIGKILL) — + * so there is no stale detection, no reclaim, and no takeover race. See the + * module header for why this is preferred over the file backend. + */ +const acquireViaSocket = async ( + lockDir: string, + me: LockRecord, + opts: AcquireOptions, +): Promise => { + const name = socketLockName(lockDir); + const pollMs = opts.pollMs ?? DEFAULT_POLL_MS; + const timeoutMs = resolveTimeoutMs(opts.timeoutMs); + const startedAt = Date.now(); + let announcedWait = false; + let lastDiagnosticAt = 0; + + for (;;) { + const server = net.createServer(); + // Never keep the process alive on the lock's account, and never hold an + // incoming connection (nothing should connect; drop any stray peer). + server.unref(); + server.on('connection', (sock) => sock.destroy()); + const listenErr = await tryListen(server, name); + + if (!listenErr) { + return { + record: me, + release: () => { + try { + server.close(); + } catch { + /* already closed / releasing on exit */ + } + }, + }; + } + + // This server never bound (listen failed); release its handle before the + // next poll or the fallback, so a long contended wait doesn't churn one + // unclosed net.Server per iteration (#2658 review L3). + try { + server.close(); + } catch { + /* never listened */ + } + + // Only EADDRINUSE means "held by a live holder → wait". Anything else means + // this environment can't use the socket backend → fall back to the file one. + if (listenErr.code !== 'EADDRINUSE') throw new SocketLockUnavailable(listenErr); + + if (!announcedWait) { + announcedWait = true; + opts.onWaitStart?.(me); + opts.log?.('Another gitnexus analyze is refreshing this index — waiting for it to finish.'); + } + const waited = Date.now() - startedAt; + // Socket backend exposes no owner metadata → holder identity is unknown (M3). + if (waited >= timeoutMs) throw new IndexLockTimeoutError(unknownHolder(), waited, false); + if (Date.now() - lastDiagnosticAt >= DIAGNOSTIC_INTERVAL_MS) { + lastDiagnosticAt = Date.now(); + if (waited >= DIAGNOSTIC_INTERVAL_MS) { + opts.log?.(`Still waiting for another analyze (${Math.round(waited / 1000)}s elapsed).`); + } + } + await sleep(jitteredDelay(pollMs, timeoutMs, waited)); + } +}; + +/** Platforms whose OS IPC namespace gives a clean, auto-releasing lock via + * `net`: Windows named pipes and Linux abstract sockets. Elsewhere (macOS/BSD) + * the file backend is used (no abstract namespace; filesystem sockets don't + * release cleanly on death). Override for tests via GITNEXUS_INDEX_LOCK_BACKEND + * = 'socket' | 'file'. + * + * Scope caveat: the socket backend's mutual-exclusion domain is NOT uniform. + * Windows `\\.\pipe\` names are machine-wide (all sessions); Linux abstract + * sockets are network-namespace-scoped (network_namespaces(7)). So two writers + * that share a bind-mounted index dir but sit in separate netns (e.g. two + * containers, Docker's default) do NOT collide on Linux — "single-host" is + * really "single-netns" here. That cross-netns-shared-mount case is the one + * the file backend (shared-filesystem O_EXCL) would cover; set + * GITNEXUS_INDEX_LOCK_BACKEND=file there. The motivating case (local hook- + * driven re-index) is single-netns, so the default socket backend covers it. */ +const selectBackend = (): 'socket' | 'file' => { + const override = process.env.GITNEXUS_INDEX_LOCK_BACKEND; + if (override === 'socket' || override === 'file') return override; + return process.platform === 'win32' || process.platform === 'linux' ? 'socket' : 'file'; +}; + +/** + * Acquire the exclusive write lock for `lockDir` (the resolved index slot + * directory). Uses the OS socket/pipe backend where available (Windows/Linux), + * falling back to the file backend otherwise or if the socket backend is + * unusable in this environment. After acquiring, sweeps orphaned staging files + * under the lock (best-effort; a no-op on a read-only mount). Rejects with + * `IndexLockTimeoutError` if `timeoutMs` elapses while another live holder holds + * the lock. + */ +export const acquireIndexLock = async ( + lockDir: string, + opts: AcquireOptions = {}, +): Promise => { + const me = buildRecord(); + let handle: IndexLockHandle; + if (selectBackend() === 'socket') { + try { + handle = await acquireViaSocket(lockDir, me, opts); + } catch (err) { + if (!(err instanceof SocketLockUnavailable)) throw err; // timeout etc. propagate + opts.log?.('Index lock: OS socket lock unavailable here — using the file lock.'); + handle = await acquireViaFile(lockDir, me, opts); + } + } else { + handle = await acquireViaFile(lockDir, me, opts); + } + // Reclaim crashed-build staging orphans while we hold the lock. Best-effort: + // a read-only mount (no orphans reachable) just no-ops. + try { + sweepStagingArtifacts(lockDir, opts.log); + } catch { + /* best-effort */ + } + return handle; +}; diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 64dacd51f..8f8c7cc22 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -55,13 +55,22 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // the main thread (the #1983 OOM). Because the two stores share this version, // any future change to the `ParsedFile` serialization shape MUST bump // SCHEMA_BUMP so both invalidate in lockstep. +// v22: `const X = ` emits one `Function` node +// instead of a `Function` plus an edgeless `Const` twin (#2687). Cached worker +// results are replayed verbatim — including across `--force` — so without this +// bump a warm cache keeps serving the old two-node set. +// v21: Java/Kotlin Spring DI facts persist constructor, field/property, and +// method injection sites plus bean-name and @Primary provider metadata. // v20: Java/Kotlin capture side-channels persist package and class-annotation // facts for shared Spring Bean resolution. +// v21: Java local class/enum/record/interface captures use javac-compatible, +// source-type-relative JLS 13.1 identities and declaration-to-block scopes +// (#2562). // v19: Java enum constant bodies emit E$N Class nodes; anonymous naming uses // JLS 13.1 immediate-host chains (#2555). // v18: Worker$N anonymous bodies. v17: callable-value-flow operand identity. // v16: direct callee identity. -const SCHEMA_BUMP = 20; +const SCHEMA_BUMP = 22; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 770293065..3cb62992d 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -461,19 +461,51 @@ export interface RepoMeta { * incremental write set only covers changed files, so a top-up against a * pre-v11 index would keep silently missing these CALLS edges for every * unchanged Rust trait file; force a full re-analyze instead. - * v12: main-aptos merge. The Move lineage's v9 — `attributesJson` column added - * to the Move node tables (Function, Struct, Enum, EnumVariant, Module) with - * full attribute payloads — is re-numbered past main's v9–v11: a persisted - * stamp of 9 is ambiguous between the two lineages, and a pre-merge Move - * index also predates main's v9–v11 rebuild reasons. Any pre-v12 stamp fails - * the strict-equality reuse gate; force a full re-analyze (same contract as v3). - * v15: generic Type nodes and the Move EnumVariant→Property / field - * USES_TYPE endpoint pairs became persistable. This Move-lineage change is - * numbered past upstream main's v13 (Java local-class identities) and v14 so - * indexes from either release channel cannot pass the strict-equality reuse - * gate under the other schema. Older indexes require a full rebuild. + * — Between v11 and v16 the version history forks: the aptos release channel + * (main-aptos) and upstream main each allocated stamps 12–15 independently, + * so a persisted stamp in that range is ambiguous between the two lineages. + * Upstream main's 12–15: + * v12: Rust range-binding stopped restoring ambiguous duplicate type names + * (#2514): a function/struct name defined three or more times used to + * re-resolve to the last-scanned file (a presence toggle), so odd duplicate + * counts emitted a wrong cross-file CALLS edge. Same v7/v11 contract: the + * incremental write set only covers changed files, so a top-up against a + * pre-v12 index would keep these spurious CALLS edges on every unchanged Rust + * file. v12 also changes edges in the other direction: range-binding now + * RESOLVES import-disambiguated duplicate names (`for item in make()` / + * `let Struct { f } = ..` where a `use` or `use x::*` import pins one of several + * same-named definitions) to the imported definition's type. Both the removed + * spurious edges and these new resolved edges are cross-file, so a pre-v12 + * top-up would leave unchanged Rust files stale either way; force a full + * re-analyze instead. + * v13: Java local classes, enums, records, and interfaces use + * source-type-relative JLS 13.1 identities (`Outer$1Local`). Number allocation + * matches javac: one sequence per (enclosing type, local simple name), with a + * separate sequence for anonymous types. Existing type/member ids, lexical + * bindings, and ownership edges must not be mixed with newly named unchanged + * Java files; force a full re-analyze. + * v14: C# and Kotlin free-call fallback now rejects same-file methods whose + * instance owner is outside the caller's enclosing class/MRO (#2563). The + * incremental write set would otherwise retain those stale CALLS edges on + * every unchanged C# and Kotlin file; force a full re-analyze instead. + * v15: `const X = ` no longer emits an edgeless + * `Const::X` twin beside its `Function` node (#2687). The incremental + * write set only covers changed files, so every unchanged TS/JS file would + * keep its twin and `impact`/`context` would stay ambiguous on those names; + * force a full re-analyze instead. + * The aptos lineage's 12–15: + * v12: first main-aptos merge — the Move lineage's v9 (`attributesJson` column + * on the Move node tables) re-numbered past main's v9–v11. + * v13/v15: generic Type nodes and the Move EnumVariant→Property / field + * USES_TYPE endpoint pairs became persistable (v13 in PR #2682, renumbered to + * 15 to clear main's then-current v14). + * v16: second main-aptos merge unifies the fork. Any stamp 12–15 is ambiguous + * between the lineages: an aptos-15 index predates main's v12–v15 rebuild + * reasons and a main-15 index has no Move/Type tables. Any pre-v16 stamp + * fails the strict-equality reuse gate; force a full re-analyze (same + * contract as the v12 merge). */ -export const INCREMENTAL_SCHEMA_VERSION = 15; +export const INCREMENTAL_SCHEMA_VERSION = 16; export interface IndexedRepo { repoPath: string; diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index 3cf8fa1dd..c0bc729aa 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -4,6 +4,7 @@ import { ProcessDetectionResult } from '../core/ingestion/process-processor.js'; import type { StandaloneIngestOutput } from '../core/ingestion/pipeline-phases/standalone-ingest.js'; import type { ResolutionOutcome } from '../core/ingestion/scope-resolution/resolution-outcome.js'; import type { PdgEmitManifest } from '../core/lbug/pdg-emit-sink.js'; +import type { GraphEmitManifest } from '../core/lbug/graph-emit-sink.js'; // CLI-specific: in-memory result with graph + detection results export interface PipelineResult< @@ -45,4 +46,12 @@ export interface PipelineResult< * register a richer phase may narrow this output to their phase contract. */ standaloneIngest: TStandaloneIngest; + /** + * Streamed structural-emit COPY manifest (#2680). Present only when + * `streamGraphEmit` was active (full rebuild + enabled): the per-pair CSVs of + * relationships that never entered the in-memory graph, for `loadGraphToLbug` + * to COPY ALONGSIDE the whole-graph CSVs (their pair keys overlap, so they are + * additional COPY jobs, not map entries). + */ + graphEmitManifest?: GraphEmitManifest; } diff --git a/gitnexus/test/fixtures/csharp-captures-golden/expected-captures.json b/gitnexus/test/fixtures/csharp-captures-golden/expected-captures.json index dc05e7cdf..38ac8dd71 100644 --- a/gitnexus/test/fixtures/csharp-captures-golden/expected-captures.json +++ b/gitnexus/test/fixtures/csharp-captures-golden/expected-captures.json @@ -659,6 +659,14 @@ "captureGroups": 12, "digest": "b6f9dd906e1309338f21633d71e663cdfd95a8707d38b9a8bf74813415ee5d13" }, + "csharp-using-static/App/NamespaceOwnerCollision.cs": { + "captureGroups": 16, + "digest": "d78082d240d14417ad3f502ef1e96e8e3575dd4e9cec00cfc15c7230c596ca53" + }, + "csharp-using-static/App/SameFileCases.cs": { + "captureGroups": 67, + "digest": "50f41faf386131ecbf6cf9c22b3594cbafee8291f385252aad7c10202881d27e" + }, "csharp-using-static/Helpers/MathUtils.cs": { "captureGroups": 8, "digest": "32c174cbaade4e2d6e0aa7e95a2b5addd138441deb43ced286c9cf5cd30750aa" diff --git a/gitnexus/test/fixtures/index-lock-child.mjs b/gitnexus/test/fixtures/index-lock-child.mjs new file mode 100644 index 000000000..9e64cca46 --- /dev/null +++ b/gitnexus/test/fixtures/index-lock-child.mjs @@ -0,0 +1,58 @@ +/** + * Child process for the cross-process index-lock tests (#2658), using the BUILT + * module (LOCK_MODULE). Two modes: + * + * - default (HOLD): acquire the lock on LOCK_DIR, write MARKER once held, then + * hold until killed. Proves real cross-process exclusion and SIGKILL + * kill-recovery against a parent that uses the source module. + * + * - MODE=EXCLUSIVE (SENTINEL set): acquire, then enter a critical section + * guarded by an O_EXCL sentinel create — if the sentinel already exists, + * another process holds the lock at the same time, which is the exact + * single-writer violation the test hunts. Hold briefly, remove the sentinel, + * release, exit 0. Exit 3 if the sentinel was already present (overlap). + * Used by the multi-reclaimer test where ≥2 children reclaim one dead holder. + */ +import { writeFileSync, openSync, closeSync, unlinkSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +// LOCK_MODULE is an absolute path. On Windows `import('C:\\…')` throws +// ERR_UNSUPPORTED_ESM_URL_SCHEME (a bare drive path is read as a URL scheme), so +// convert to a file:// URL — required on Windows, harmless on POSIX. +const { acquireIndexLock } = await import(pathToFileURL(process.env.LOCK_MODULE).href); + +if (process.env.MODE === 'EXCLUSIVE') { + const lock = await acquireIndexLock(process.env.LOCK_DIR, { timeoutMs: 30_000, pollMs: 25 }); + try { + // O_EXCL create fails if any other process is simultaneously in its own + // critical section — that is a broken single-writer invariant. + let fd; + try { + fd = openSync(process.env.SENTINEL, 'wx'); + } catch { + process.exit(3); // overlap detected: two holders at once + } + closeSync(fd); + // Hold the section briefly so concurrent reclaimers would collide here. + await new Promise((r) => setTimeout(r, 150)); + unlinkSync(process.env.SENTINEL); + } finally { + lock.release(); + } + process.exit(0); +} else { + const lock = await acquireIndexLock(process.env.LOCK_DIR, { timeoutMs: 30_000, pollMs: 25 }); + writeFileSync(process.env.MARKER, String(process.pid)); + // Hold the lock until the parent kills us. + setInterval(() => {}, 1000); + // Release on a graceful signal (the SIGKILL path in the test never reaches this). + const release = () => { + try { + lock.release(); + } finally { + process.exit(0); + } + }; + process.on('SIGTERM', release); + process.on('SIGINT', release); +} diff --git a/gitnexus/test/fixtures/lang-resolution/csharp-using-static/App/NamespaceOwnerCollision.cs b/gitnexus/test/fixtures/lang-resolution/csharp-using-static/App/NamespaceOwnerCollision.cs new file mode 100644 index 000000000..a99850906 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/csharp-using-static/App/NamespaceOwnerCollision.cs @@ -0,0 +1,18 @@ +namespace First +{ + public class NamespaceTwin + { + public void RejectOtherNamespaceOwner() + { + NamespaceCollision(); + } + } +} + +namespace Second +{ + public class NamespaceTwin + { + public void NamespaceCollision() { } + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/csharp-using-static/App/SameFileCases.cs b/gitnexus/test/fixtures/lang-resolution/csharp-using-static/App/SameFileCases.cs new file mode 100644 index 000000000..01293ff8e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/csharp-using-static/App/SameFileCases.cs @@ -0,0 +1,68 @@ +using System; +using static App.SameFileStatics; + +namespace App; + +public static class SameFileStatics +{ + public static void ImportedOnly() { } + + public static string Select(string value, int count) + { + return value; + } + + public static int Select(int value) + { + return value; + } +} + +public class SameFileIntruder +{ + public void LeakedOnly() { } + + public string Select(string value, int count) + { + return value; + } +} + +public class SameFileBase +{ + protected void InheritedOnly() { } +} + +public class SameFileConsumer : SameFileBase +{ + private void OwnOnly() { } + + public void Exercise() + { + LeakedOnly(); + ImportedOnly(); + OwnOnly(); + InheritedOnly(); + Select("value", 1); + + int LocalOnly(int value) + { + return value; + } + + Func lambda = () => LocalOnly(2); + } +} + +public partial class SameFilePartial +{ + public void CallAcrossFragment() + { + AcrossFragment(); + } +} + +public partial class SameFilePartial +{ + private void AcrossFragment() { } +} diff --git a/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Compact.java b/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Compact.java new file mode 100644 index 000000000..9a7702728 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Compact.java @@ -0,0 +1,12 @@ +record Compact(int value) { + Compact { + class Local { + void inner() {} + } + + new Local().inner(); + new Runnable() { + public void run() {} + }; + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Outer.java b/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Outer.java new file mode 100644 index 000000000..4b152a7ed --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Outer.java @@ -0,0 +1,123 @@ +class Outer { + class Cyclic { + void member() {} + } + + class MemberHost { + void make() { + class Local { + void ordinaryMemberHit() {} + } + + new Local().ordinaryMemberHit(); + } + } + + void first() { + class Local { + void inner() { + new Runnable() { + public void run() {} + }; + } + } + + class CtorHost { + CtorHost() { + class Local { + void inner() {} + } + new Local().inner(); + } + } + + class NestedHost { + class Member { + void make() { + class Local {} + } + } + } + + new Local().inner(); + new Runnable() { + public void run() {} + }; + } + + void second() { + new Runnable() { + public void run() {} + }; + + class Local { + void inner() {} + } + + new Local().inner(); + } + + void declarationOrder() { + new Cyclic().member(); + + class Cyclic { + void local() {} + } + + new Cyclic().local(); + } + + void blocks() { + { + class Local { + void firstBlock() {} + } + + new Local().firstBlock(); + } + + { + class Local { + void secondBlock() {} + } + + new Local().secondBlock(); + } + } + + static { + class StaticLocal { + void staticHit() {} + } + + new StaticLocal().staticHit(); + } + + { + class InstanceLocal { + void instanceHit() {} + } + + new InstanceLocal().instanceHit(); + } + + Runnable task = () -> { + class LambdaLocal { + void lambdaHit() {} + } + + new LambdaLocal().lambdaHit(); + }; + + Runnable anonymousTask = new Runnable() { + { + class Local { + void anonymousHit() {} + } + + new Local().anonymousHit(); + } + + public void run() {} + }; +} diff --git a/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Types.java b/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Types.java new file mode 100644 index 000000000..0ff98c36e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/java-local-class-naming/src/Types.java @@ -0,0 +1,24 @@ +class Types { + void types() { + enum E { + A; + + void enumHit() {} + } + + record R(int x) { + void recordHit() {} + } + + interface I { + void run(); + } + + E.A.enumHit(); + new R(1).recordHit(); + I implementation = new I() { + public void run() {} + }; + implementation.run(); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/kotlin-instance-ownership/src/App.kt b/gitnexus/test/fixtures/lang-resolution/kotlin-instance-ownership/src/App.kt new file mode 100644 index 000000000..f60657757 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/kotlin-instance-ownership/src/App.kt @@ -0,0 +1,43 @@ +open class Base { + fun inherited() {} +} + +class Owner : Base() { + fun own() {} + + fun callOwn() { + own() + } + + fun callInherited() { + inherited() + } +} + +class Unrelated { + fun collide() {} +} + +class Caller { + fun run() { + collide() + } +} + +class Outer { + fun outerMethod() {} + + inner class Inner { + fun callOuter() { + outerMethod() + } + } +} + +val handler = object { + fun sibling() {} + + fun callSibling() { + sibling() + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/c_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/c_a.rs new file mode 100644 index 000000000..97e1a626c --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/c_a.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbA } +pub struct DbA; +impl DbA { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/c_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/c_b.rs new file mode 100644 index 000000000..6eba40993 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/c_b.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbB } +pub struct DbB; +impl DbB { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/main.rs new file mode 100644 index 000000000..787aa4e42 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-2/src/main.rs @@ -0,0 +1,8 @@ +mod c_a; +mod c_b; +pub fn load() -> u8 { 0 } +fn use_it() { + let Config { db } = load(); + db.run(); +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_a.rs new file mode 100644 index 000000000..97e1a626c --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_a.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbA } +pub struct DbA; +impl DbA { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_b.rs new file mode 100644 index 000000000..6eba40993 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_b.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbB } +pub struct DbB; +impl DbB { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_c.rs new file mode 100644 index 000000000..e90cb83a2 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/c_c.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbC } +pub struct DbC; +impl DbC { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/main.rs new file mode 100644 index 000000000..92fbad3d4 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-fields-3/src/main.rs @@ -0,0 +1,9 @@ +mod c_a; +mod c_b; +mod c_c; +pub fn load() -> u8 { 0 } +fn use_it() { + let Config { db } = load(); + db.run(); +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/main.rs new file mode 100644 index 000000000..9d224de54 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/main.rs @@ -0,0 +1,8 @@ +mod t_a; +mod t_b; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-2/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/a_task.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/a_task.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/a_task.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/m_repo.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/m_repo.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/m_repo.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/main.rs new file mode 100644 index 000000000..764062061 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/main.rs @@ -0,0 +1,9 @@ +mod z_user; +mod m_repo; +mod a_task; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/z_user.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/z_user.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3-reordered/src/z_user.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/main.rs new file mode 100644 index 000000000..7c2c54f02 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/main.rs @@ -0,0 +1,9 @@ +mod t_a; +mod t_b; +mod t_c; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_c.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-dup-return-3/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/main.rs new file mode 100644 index 000000000..d5efc716d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/main.rs @@ -0,0 +1,10 @@ +mod t_a; +mod t_b; +mod t_c; +use crate::t_b::make as mk; +fn drive() { + for item in mk() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_c.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-alias-return/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/main.rs new file mode 100644 index 000000000..f099c1c96 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/main.rs @@ -0,0 +1,10 @@ +mod t_a; +mod t_b; +mod t_c; +use crate::t_b::Config; +pub fn load() -> u8 { 0 } +fn use_it() { + let Config { db } = load(); + db.run(); +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_a.rs new file mode 100644 index 000000000..97e1a626c --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbA } +pub struct DbA; +impl DbA { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_b.rs new file mode 100644 index 000000000..6eba40993 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbB } +pub struct DbB; +impl DbB { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_c.rs new file mode 100644 index 000000000..e90cb83a2 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-fields/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Config { pub db: DbC } +pub struct DbC; +impl DbC { pub fn run(&self) {} } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/main.rs new file mode 100644 index 000000000..6645a70ca --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/main.rs @@ -0,0 +1,10 @@ +mod t_a; +mod t_b; +mod t_c; +use crate::t_b::make; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_c.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-dup-return/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/main.rs new file mode 100644 index 000000000..72d2374e4 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/main.rs @@ -0,0 +1,11 @@ +mod t_a; +mod t_b; +mod t_c; +use crate::t_b::*; +use crate::t_c::*; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_c.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-ambiguous/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/main.rs new file mode 100644 index 000000000..d83ef15a8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/main.rs @@ -0,0 +1,15 @@ +mod t_a; +mod t_b; +mod t_c; +use crate::t_b::*; +pub struct Local; +impl Local { + pub fn save(&self) {} +} +pub fn make() -> Vec { vec![] } +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_c.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-local-shadows/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/main.rs new file mode 100644 index 000000000..add8f4b35 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/main.rs @@ -0,0 +1,10 @@ +mod t_a; +mod t_b; +mod t_c; +use crate::t_b::*; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_b.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_b.rs new file mode 100644 index 000000000..80c240632 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_b.rs @@ -0,0 +1,3 @@ +pub struct Repo { pub name: String } +impl Repo { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_c.rs b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_c.rs new file mode 100644 index 000000000..ac105f126 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-import-glob-return/src/t_c.rs @@ -0,0 +1,3 @@ +pub struct Task { pub name: String } +impl Task { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/lang-resolution/rust-unique-return/src/main.rs b/gitnexus/test/fixtures/lang-resolution/rust-unique-return/src/main.rs new file mode 100644 index 000000000..cedb46e85 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-unique-return/src/main.rs @@ -0,0 +1,7 @@ +mod t_a; +fn drive() { + for item in make() { + item.save(); + } +} +fn main() {} diff --git a/gitnexus/test/fixtures/lang-resolution/rust-unique-return/src/t_a.rs b/gitnexus/test/fixtures/lang-resolution/rust-unique-return/src/t_a.rs new file mode 100644 index 000000000..b62a1679d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/rust-unique-return/src/t_a.rs @@ -0,0 +1,3 @@ +pub struct User { pub name: String } +impl User { pub fn save(&self) {} } +pub fn make() -> Vec { vec![] } diff --git a/gitnexus/test/fixtures/rust-captures-golden/expected-captures.json b/gitnexus/test/fixtures/rust-captures-golden/expected-captures.json index f632f78aa..755630759 100644 --- a/gitnexus/test/fixtures/rust-captures-golden/expected-captures.json +++ b/gitnexus/test/fixtures/rust-captures-golden/expected-captures.json @@ -171,6 +171,78 @@ "captureGroups": 22, "digest": "c53db401a81fde2ffd5665393acb9cd605a62ec51c015c3aafb3f41c0897471f" }, + "rust-dup-fields-2/src/c_a.rs": { + "captureGroups": 11, + "digest": "4c7844b039d3b2c618e5e1978e0bed50a2d8de0a7a92a87fef4791c91fd2d0d2" + }, + "rust-dup-fields-2/src/c_b.rs": { + "captureGroups": 11, + "digest": "cd02dd0f2b74d8e9495634f3a33477c2b20d5f2877612733b66402eae6fe8426" + }, + "rust-dup-fields-2/src/main.rs": { + "captureGroups": 16, + "digest": "e96013ac801f874a1ad902c7bd2be277202b38fbf12030c4ebbb46edd8c0fe79" + }, + "rust-dup-fields-3/src/c_a.rs": { + "captureGroups": 11, + "digest": "4c7844b039d3b2c618e5e1978e0bed50a2d8de0a7a92a87fef4791c91fd2d0d2" + }, + "rust-dup-fields-3/src/c_b.rs": { + "captureGroups": 11, + "digest": "cd02dd0f2b74d8e9495634f3a33477c2b20d5f2877612733b66402eae6fe8426" + }, + "rust-dup-fields-3/src/c_c.rs": { + "captureGroups": 11, + "digest": "a5263afa5bc9cb6b499d3c5394cc8b0a942d9a1fe388a2366a3baca963ead634" + }, + "rust-dup-fields-3/src/main.rs": { + "captureGroups": 17, + "digest": "fad21b046b2f34b8a6fd98ffc3719446f176f7207864b3fdd9ba2d9bb07d607f" + }, + "rust-dup-return-2/src/main.rs": { + "captureGroups": 14, + "digest": "a4d637dc57a09e56dce75102c70ca6a45998f498990289855d4953bf4ed5461f" + }, + "rust-dup-return-2/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-dup-return-2/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-dup-return-3-reordered/src/a_task.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, + "rust-dup-return-3-reordered/src/m_repo.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-dup-return-3-reordered/src/main.rs": { + "captureGroups": 15, + "digest": "0f68357dbffb22af2c36ca5a935c3eea4025c119afa1a7c1d258ab9a74fe96e8" + }, + "rust-dup-return-3-reordered/src/z_user.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-dup-return-3/src/main.rs": { + "captureGroups": 15, + "digest": "a93ca0874eeaedd21dba987143fa389281d8b738612ca49c314ab91dd73e4065" + }, + "rust-dup-return-3/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-dup-return-3/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-dup-return-3/src/t_c.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, "rust-dyn-trait-object/src/lib.rs": { "captureGroups": 23, "digest": "720618dff6a43ab8e5b59aa354c0c448b9057dd6f2f7b3b22b13b82d53745943" @@ -267,6 +339,102 @@ "captureGroups": 20, "digest": "d8c1eb57431b915dd5c9055d8451054a454e340c4842628e38e4d46f69471abd" }, + "rust-import-alias-return/src/main.rs": { + "captureGroups": 16, + "digest": "c14fd2abf932f09fe19821951e73afdf03839bdb761a7d0ad347c9926a0d5542" + }, + "rust-import-alias-return/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-import-alias-return/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-import-alias-return/src/t_c.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, + "rust-import-dup-fields/src/main.rs": { + "captureGroups": 18, + "digest": "82b48d00fe2e4b5a2f52185d5d2501fdef408faf148b509fdc5f428171b9a784" + }, + "rust-import-dup-fields/src/t_a.rs": { + "captureGroups": 11, + "digest": "4c7844b039d3b2c618e5e1978e0bed50a2d8de0a7a92a87fef4791c91fd2d0d2" + }, + "rust-import-dup-fields/src/t_b.rs": { + "captureGroups": 11, + "digest": "cd02dd0f2b74d8e9495634f3a33477c2b20d5f2877612733b66402eae6fe8426" + }, + "rust-import-dup-fields/src/t_c.rs": { + "captureGroups": 11, + "digest": "a5263afa5bc9cb6b499d3c5394cc8b0a942d9a1fe388a2366a3baca963ead634" + }, + "rust-import-dup-return/src/main.rs": { + "captureGroups": 16, + "digest": "e70780bcc2ccbfca9fe2099887e30187f70c48b68a11948592778997d1e2bd13" + }, + "rust-import-dup-return/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-import-dup-return/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-import-dup-return/src/t_c.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, + "rust-import-glob-ambiguous/src/main.rs": { + "captureGroups": 17, + "digest": "31f683937b9d9208e7f2c6bd2d1092c59bcf6ab16a6a6923b48bba6fcab2a1a6" + }, + "rust-import-glob-ambiguous/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-import-glob-ambiguous/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-import-glob-ambiguous/src/t_c.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, + "rust-import-glob-local-shadows/src/main.rs": { + "captureGroups": 28, + "digest": "5242d2ee3c9bdb8c679ddc55726e423fc48997db2bcfc4f79dca3645289d1149" + }, + "rust-import-glob-local-shadows/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-import-glob-local-shadows/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-import-glob-local-shadows/src/t_c.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, + "rust-import-glob-return/src/main.rs": { + "captureGroups": 16, + "digest": "a157af0c6d7b8f9861712c1822bb5bab8e7fadbcdd58c08d4d034343117053cb" + }, + "rust-import-glob-return/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, + "rust-import-glob-return/src/t_b.rs": { + "captureGroups": 14, + "digest": "cb5f21ac23b71efdf24b121268f783d5b224e7d498ffa7a54555c97ab0f904bf" + }, + "rust-import-glob-return/src/t_c.rs": { + "captureGroups": 14, + "digest": "67e8de06340dd3b5c5c4bcc88836d1ef89b47f4a740697664dfa26cd36cf5ac5" + }, "rust-iter-for-loop/src/main.rs": { "captureGroups": 30, "digest": "529fda7f9f188814ce6044e2c99d6b005ec4a9b98835fe9609530897998b9f32" @@ -507,6 +675,14 @@ "captureGroups": 10, "digest": "e2a6fb9eab259b8c7104f1530b96b8c1f42ab32fe1d71d6bdca04d68263507f2" }, + "rust-unique-return/src/main.rs": { + "captureGroups": 13, + "digest": "72cc5728b40f51fae75359c2443986ce73f5d5450ebf8a367a2d860c87c84ed0" + }, + "rust-unique-return/src/t_a.rs": { + "captureGroups": 14, + "digest": "d531e0aec84d7da0e2c6818e445735b4cb1ff15e08d14586093069e5eeab41c9" + }, "rust-write-access/models.rs": { "captureGroups": 9, "digest": "660f755fd70cd1796f9da02ad7d65f599dea8029665ee45ecd18cd27919741f3" diff --git a/gitnexus/test/integration/analyze-atomic-swap.test.ts b/gitnexus/test/integration/analyze-atomic-swap.test.ts index 871acc428..8bee4aba0 100644 --- a/gitnexus/test/integration/analyze-atomic-swap.test.ts +++ b/gitnexus/test/integration/analyze-atomic-swap.test.ts @@ -1,9 +1,10 @@ /** * Integration test for the #2 atomic full-rebuild swap. * - * A full rebuild builds the fresh index at `.new` and swaps it over - * the live index in one atomic rename (POSIX). Two invariants: - * - success publishes a single valid `lbug` with no `.new` temp left behind, + * A full rebuild builds the fresh index at a per-run `.staging.` + * (#2658) and swaps it over the live index in one atomic rename (POSIX). Two + * invariants: + * - success publishes a single valid `lbug` with no staging temp left behind, * and a repeat rebuild replaces the inode (proving the swap, not an in-place * edit); and * - a failure BEFORE the swap leaves the previous index byte-for-byte intact @@ -49,7 +50,10 @@ const identity = async (p: string): Promise => { const lingeringTemp = async (lbugPath: string): Promise => { const base = path.basename(lbugPath); const entries = await fs.readdir(path.dirname(lbugPath)); - return entries.filter((e) => e.startsWith(`${base}.new`)); + // Staging temps are the legacy fixed `${base}.new*` and the current per-run + // `${base}.staging.*` (#2658). Match both so this leftover-temp guard + // still catches a failed swap under the new naming. + return entries.filter((e) => e.startsWith(`${base}.new`) || e.startsWith(`${base}.staging.`)); }; describe.skipIf(isWin)('atomic full-rebuild swap (#2)', () => { diff --git a/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts b/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts index ea4d995f4..2176c7a76 100644 --- a/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts +++ b/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts @@ -18,6 +18,9 @@ const runAnalyzeWithForcedOom = (cwd: string, gitnexusHome: string) => stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, + // This suite EXERCISES the heap respawn; the suite-wide + // GITNEXUS_MEMORY=off opt-out (vitest.config.ts) must not apply here. + GITNEXUS_MEMORY: '1', GITNEXUS_HOME: gitnexusHome, NODE_OPTIONS: '', GITNEXUS_TEST_RESPAWN_HEAP_MB: '32', diff --git a/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts b/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts new file mode 100644 index 000000000..d85744674 --- /dev/null +++ b/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts @@ -0,0 +1,173 @@ +/** + * Real cross-process tests for the index write lock (#2658): child processes + * contend for the lock on the same directory as this process. + * + * - Test 1 exercises the DEFAULT backend (the OS socket/pipe lock on + * Linux/Windows): while the child holds it, our acquire blocks and times out; + * after the child is SIGKILLed the kernel drops the binding and our next + * acquire succeeds — the kernel-auto-release guarantee, no stale handling. + * - Test 2 pins the FILE backend and races several children reclaiming one dead + * holder, asserting the atomic rename-steal never lets two into the critical + * section at once. + * + * The child imports the BUILT module (dist/) and this process imports the + * source, proving the guarantee is a genuine cross-process one (and, for the + * socket backend, that both derive the same endpoint name for a given dir). + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { acquireIndexLock, IndexLockTimeoutError } from '../../src/storage/index-lock.js'; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(testDir, '../..'); +const lockModule = path.join(repoRoot, 'dist', 'storage', 'index-lock.js'); +const childScript = path.resolve(testDir, '..', 'fixtures', 'index-lock-child.mjs'); + +let dir: string; +let marker: string; +let child: ChildProcess | undefined; + +const waitFor = async (predicate: () => boolean, timeoutMs: number): Promise => { + const start = Date.now(); + for (;;) { + if (predicate()) return; + if (Date.now() - start > timeoutMs) throw new Error('condition not met within timeout'); + await new Promise((r) => setTimeout(r, 25)); + } +}; + +const waitForExit = (proc: ChildProcess, timeoutMs: number): Promise => + new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('child did not exit')), timeoutMs); + proc.once('exit', () => { + clearTimeout(timer); + resolve(); + }); + }); + +beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), 'gnx-lock-xp-')); + marker = path.join(dir, 'held.marker'); +}); +afterEach(() => { + if (child && child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + rmSync(dir, { recursive: true, force: true }); +}); + +describe('index lock across processes (#2658)', () => { + it('excludes a second writer while held, then recovers after the holder is killed', async () => { + if (!existsSync(lockModule)) { + throw new Error( + `dist/storage/index-lock.js missing — run \`npm run build\` first ` + + `(or use \`npm run test:integration\`, which builds via pretest:integration).`, + ); + } + + child = spawn(process.execPath, [childScript], { + env: { ...process.env, LOCK_MODULE: lockModule, LOCK_DIR: dir, MARKER: marker }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + // Generous marker wait: Windows process startup is ~5x slower and the + // platform-sensitive shard runs heavy suites in parallel, so a child spawn + // can be badly delayed under load — the wait must tolerate that, not race it. + await waitFor(() => existsSync(marker), 40_000); + const holderPid = Number(readFileSync(marker, 'utf8')); + expect(holderPid).toBeGreaterThan(0); + + // Mutual exclusion: the live holder is waited on, then we time out. + await expect(acquireIndexLock(dir, { timeoutMs: 500, pollMs: 25 })).rejects.toBeInstanceOf( + IndexLockTimeoutError, + ); + + // Kill recovery: with the holder gone, its lock becomes reclaimable. + child.kill('SIGKILL'); + await waitForExit(child, 30_000); + const lock = await acquireIndexLock(dir, { timeoutMs: 15_000, pollMs: 25 }); + expect(lock.record.pid).toBe(process.pid); + lock.release(); + }, 90_000); + + // The FILE backend is the DEFAULT only on macOS/BSD; Windows and Linux default + // to the race-free kernel lock (named pipe / abstract socket). This case FORCES + // the file backend to stress its rename-steal reclaim, so it runs where that + // backend is actually production (macOS — where the double-admit bug this + // guards lived and is now fixed) plus Linux. It is skipped on Windows, where + // the file backend is never the default; Windows' real lock (the named pipe) is + // covered by index-lock.test.ts on the Windows matrix and by the + // default-backend cross-process case above. + it.skipIf(process.platform === 'win32')( + 'lets multiple waiters reclaim one dead holder without ever admitting two writers', + async () => { + if (!existsSync(lockModule)) { + throw new Error( + `dist/storage/index-lock.js missing — run \`npm run build\` first ` + + `(or use \`npm run test:integration\`, which builds via pretest:integration).`, + ); + } + // This case targets the FILE backend's reclaim path specifically (the socket + // backend has no stale file to reclaim). Seed a stale lock owned by a dead, + // same-host holder — every child must reclaim it, and the reclaim must let + // exactly one at a time win so no two children are ever in their O_EXCL + // sentinel section together. + // + // The reclaim's rename-steal must NOT act on a stale staleness judgment: a + // waiter that judged the dead record must re-verify the file still holds it + // before renaming, or it will rename a live winner's freshly-created lock + // aside and admit a second writer (#2658 review — this reproduced at ~18% per + // round of 4-way contention before the judgment-verified steal). One round + // catches that regression only ~1-in-6 of the time, so loop several rounds to + // make it a reliable guard; with the fix every round is clean. + const sentinel = path.join(dir, 'critical.sentinel'); + const seedDeadHolder = (): void => { + writeFileSync( + path.join(dir, 'analyze.lock'), + JSON.stringify({ + v: 1, + pid: 999_999_999, + hostname: os.hostname(), + startTime: null, + token: 'dead-holder-token', + invocationId: 'dead-holder', + acquiredAt: new Date(0).toISOString(), + }), + ); + }; + + const runChild = (): Promise<{ code: number | null; signal: NodeJS.Signals | null }> => + new Promise((resolve) => { + const c = spawn(process.execPath, [childScript], { + env: { + ...process.env, + LOCK_MODULE: lockModule, + LOCK_DIR: dir, + SENTINEL: sentinel, + MODE: 'EXCLUSIVE', + GITNEXUS_INDEX_LOCK_BACKEND: 'file', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + c.once('exit', (code, signal) => resolve({ code, signal })); + }); + + const ROUNDS = 8; + const KIDS = 5; + for (let round = 0; round < ROUNDS; round++) { + seedDeadHolder(); // the previous round's winner released (unlinked) the lock + const results = await Promise.all(Array.from({ length: KIDS }, () => runChild())); + // Every child acquired, ran its exclusive section, and exited cleanly (0). + // Exit 3 = it found the sentinel already present = two holders at once. + for (const r of results) { + expect(r.signal).toBeNull(); + expect(r.code).toBe(0); + } + // No leftover sentinel — the last holder cleaned up. + expect(existsSync(sentinel)).toBe(false); + } + }, + 60_000, + ); +}); diff --git a/gitnexus/test/integration/analyze-wal-checkpoint-failure.test.ts b/gitnexus/test/integration/analyze-wal-checkpoint-failure.test.ts index 1bae0bb4e..fdcd6ecae 100644 --- a/gitnexus/test/integration/analyze-wal-checkpoint-failure.test.ts +++ b/gitnexus/test/integration/analyze-wal-checkpoint-failure.test.ts @@ -72,45 +72,73 @@ afterAll(() => { if (suiteGitnexusHome) cleanupTempDirSync(suiteGitnexusHome); }); +const runAnalyze = () => + spawnSync(process.execPath, [...CLI_SPAWN_PREFIX, 'analyze', '--skip-skills'], { + cwd: repoPath, + encoding: 'utf8', + // Generous timeout: the test does real CSV/COPY work before the + // first failing checkpoint, and CI runners are slow. + timeout: process.env.CI ? 120_000 : 60_000, + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + GITNEXUS_HOME: suiteGitnexusHome, + // Skip ensureHeap re-exec (which drops the tsx loader). + NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --max-old-space-size=8192`.trim(), + // Tiny threshold forces auto-checkpoint on every write so the + // first write into the WAL trips the planted rename blocker. + GITNEXUS_WAL_CHECKPOINT_THRESHOLD: '1', + CI: '1', + }, + }); + describe('analyze WAL auto-checkpoint rename failure (real lbug, no mocks)', () => { it('surfaces the --wal-checkpoint-threshold recovery hint when the rename target is blocked', () => { - // Plant a non-empty directory at the path Ladybug's auto-checkpoint - // will try to rename `.wal` over. `fs.rename` cannot overwrite a - // non-empty directory, and the adapter's orphan-sidecar cleanup uses - // `fs.unlink` (which fails on directories) — so the blocker persists - // through `doInitLbug` and trips the very first auto-checkpoint that - // a `GITNEXUS_WAL_CHECKPOINT_THRESHOLD=1` setting forces. + // The checkpoint rename target must be a PREDICTABLE path so the blocker + // can be pre-planted. A full rebuild builds into a per-run + // `lbug.staging.` and checkpoints `lbug.staging..wal.checkpoint` + // (#2658) — an unknowable name. An INCREMENTAL run instead writes the live + // index in place, so its auto-checkpoint targets the fixed + // `lbug.wal.checkpoint`. So: first do a clean full analyze to create the + // index, then plant the blocker and drive an incremental analyze into it. const storageDir = path.join(repoPath, '.gitnexus'); - fs.mkdirSync(storageDir, { recursive: true }); - // A full rebuild now builds into `lbug.new` and swaps atomically (POSIX), so - // its auto-checkpoint targets `lbug.new.wal.checkpoint`; on the in-place / - // Windows path it targets `lbug.wal.checkpoint`. Block BOTH so the planted - // rename blocker trips the first checkpoint whichever path analyze takes. - for (const name of ['lbug.wal.checkpoint', 'lbug.new.wal.checkpoint']) { - const blockerDir = path.join(storageDir, name); - fs.mkdirSync(blockerDir, { recursive: true }); - fs.writeFileSync(path.join(blockerDir, 'blocker'), 'cannot-be-renamed-over'); - } - const result = spawnSync(process.execPath, [...CLI_SPAWN_PREFIX, 'analyze', '--skip-skills'], { + // 1) Clean full analyze (no blocker) — builds the index into staging and + // swaps it in. Must succeed; the staging checkpoint name is unblocked. + const first = runAnalyze(); + expect(first.status === null ? 'timeout' : first.status).toBe(0); + + // 2) Change a tracked source file and commit, so the next analyze is an + // incremental writeback (in-place), not a full rebuild. + const churnFile = path.join(repoPath, 'src', 'logger.ts'); + fs.appendFileSync(churnFile, `\nexport const walChurnMarker = ${Date.now()};\n`); + const gitEnv = { + ...process.env, + GIT_AUTHOR_NAME: 'test', + GIT_AUTHOR_EMAIL: 'test@test', + GIT_COMMITTER_NAME: 'test', + GIT_COMMITTER_EMAIL: 'test@test', + }; + spawnSync('git', ['add', '-A'], { cwd: repoPath, stdio: 'pipe' }); + spawnSync('git', ['commit', '-m', 'churn for incremental'], { cwd: repoPath, - encoding: 'utf8', - // Generous timeout: the test does real CSV/COPY work before the - // first failing checkpoint, and CI runners are slow. - timeout: process.env.CI ? 120_000 : 60_000, - stdio: ['pipe', 'pipe', 'pipe'], - env: { - ...process.env, - GITNEXUS_HOME: suiteGitnexusHome, - // Skip ensureHeap re-exec (which drops the tsx loader). - NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --max-old-space-size=8192`.trim(), - // Tiny threshold forces auto-checkpoint on every write so the - // first write into the WAL trips the planted rename blocker. - GITNEXUS_WAL_CHECKPOINT_THRESHOLD: '1', - CI: '1', - }, + stdio: 'pipe', + env: gitEnv, }); + // 3) Plant a non-empty directory at `lbug.wal.checkpoint`, the fixed rename + // target of the in-place checkpoint. `fs.rename` cannot overwrite a + // non-empty directory, and the adapter's orphan-sidecar cleanup uses + // `fs.unlink` (which fails on a directory) — so the blocker persists through + // `doInitLbug` and trips the auto-checkpoint the incremental writeback + // forces at `GITNEXUS_WAL_CHECKPOINT_THRESHOLD=1`. + const blockerDir = path.join(storageDir, 'lbug.wal.checkpoint'); + fs.rmSync(blockerDir, { recursive: true, force: true }); + fs.mkdirSync(blockerDir, { recursive: true }); + fs.writeFileSync(path.join(blockerDir, 'blocker'), 'cannot-be-renamed-over'); + + // 4) Incremental analyze into the blocked checkpoint target. + const result = runAnalyze(); const combined = `${result.stderr}\n${result.stdout}`; // The CLI must exit non-zero. status === null means the timeout fired diff --git a/gitnexus/test/integration/cfg/__snapshots__/pipeline-pdg.test.ts.snap b/gitnexus/test/integration/cfg/__snapshots__/pipeline-pdg.test.ts.snap index f1da88949..d8337e88c 100644 --- a/gitnexus/test/integration/cfg/__snapshots__/pipeline-pdg.test.ts.snap +++ b/gitnexus/test/integration/cfg/__snapshots__/pipeline-pdg.test.ts.snap @@ -27,15 +27,18 @@ exports[`U7 — C-family worker-mode --pdg pipeline > C#: --pdg off is byte-iden exports[`U7 — C-family worker-mode --pdg pipeline > C++: --pdg off is byte-identical (zero PDG nodes/edges, stable golden digest) 1`] = ` { "byRelType": { - "DEFINES": 6, + "CALLS": 1, + "DEFINES": 7, + "MEMBER_OF": 2, }, "byType": { + "Community": 1, "File": 1, - "Function": 6, + "Function": 7, }, - "edgeDigest": "4e8cfcfe7cbde0d0a858e2f5db8af82713fbb42527d23e6fabf704382d8088df", - "relationships": 6, - "symbols": 7, + "edgeDigest": "5b6f4ec33d30b5d95c06529dfe7bf7a279cfffa73638efc55432363365896328", + "relationships": 10, + "symbols": 9, } `; diff --git a/gitnexus/test/integration/closure-binding-labels.test.ts b/gitnexus/test/integration/closure-binding-labels.test.ts new file mode 100644 index 000000000..e37362f10 --- /dev/null +++ b/gitnexus/test/integration/closure-binding-labels.test.ts @@ -0,0 +1,189 @@ +/** + * #2687 follow-up — a closure bound to a name emits ONE `Function` node in + * every language, not `Variable` in some and `Property` in others. + * + * `const f = () => {}` already produced a `Function` in TS/JS (that is what the + * #2687 twin fix preserved), but the same construct produced a `Variable` in + * Go/Python/Dart/C++ and a `Property` in Kotlin/Swift. The graph schema states + * "Function: Functions and arrow functions", and every syntactic tagger the + * convention was checked against (tree-sitter tags, universal-ctags) labels the + * binding a function — so the callable label is the consistent one. + * + * Each language's value capture still matches the same declaration node, so + * these rely on the #2687 pre-scan collapsing the pair; a regression there + * would surface here as a twin rather than a wrong label. + * + * The label alone does not make `f()` resolve — free-call resolution runs off + * the per-language scope-resolution queries. Go, Python and C++ now also carry a + * `@declaration.function` capture anchored on the inner closure literal, so + * calls resolve there too (asserted in the second describe). Kotlin, Swift and + * Dart still lack a `@scope.function` whose range matches the closure literal — + * Kotlin deliberately scopes `lambda_literal` as a BLOCK (#1757) — and an + * unaligned declaration anchor mis-attributes callers, so those three keep the + * label fix only. + */ +import { describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; +import { DIST_WORKER_URL, distWorkerExists } from '../helpers/worker-parse.js'; +import { parseFilesWithWorkers } from '../helpers/worker-parse.js'; + +const labelsFor = async (path: string, content: string, name: string): Promise => { + const { graph } = await parseFilesWithWorkers([{ path, content }]); + return graph.nodes + .filter((node) => node.properties.name === name) + .map((node) => node.label) + .sort(); +}; + +describe('closure bindings emit a single Function node in every language', () => { + it('Go: var f = func(){}', async () => { + expect( + await labelsFor( + 'src/handler.go', + 'package main\n\nvar Handler = func(x int) int { return x }\n', + 'Handler', + ), + ).toEqual(['Function']); + }); + + it('Python: f = lambda x: x', async () => { + expect(await labelsFor('src/handler.py', 'handler = lambda x: x\n', 'handler')).toEqual([ + 'Function', + ]); + }); + + it('Kotlin: val f = { x -> x }', async () => { + expect(await labelsFor('src/Handler.kt', 'val handler = { x: Int -> x }\n', 'handler')).toEqual( + ['Function'], + ); + }); + + it('Swift: let f = { ... }', async () => { + expect( + await labelsFor( + 'src/Handler.swift', + 'let handler = { (x: Int) -> Int in return x }\n', + 'handler', + ), + ).toEqual(['Function']); + }); + + it('C++: auto f = [](int x){ ... }', async () => { + expect( + await labelsFor('src/handler.cpp', 'auto handler = [](int x) { return x; };\n', 'handler'), + ).toEqual(['Function']); + }); + + it('Dart: var f = (int x) => x', async () => { + expect(await labelsFor('src/handler.dart', 'var handler = (int x) => x;\n', 'handler')).toEqual( + ['Function'], + ); + }); + + // The suppression must key on an actual closure value, never on the + // declaration keyword — otherwise ordinary constants would vanish. One `it` + // per language: each spins its own worker pool and four in a single test + // exceeds the default timeout. + + it('Go: leaves a genuine const alone', async () => { + expect( + await labelsFor('src/consts.go', 'package main\n\nconst MaxSize = 10\n', 'MaxSize'), + ).toEqual(['Const']); + }); + + it('Python: leaves a genuine assignment alone', async () => { + expect(await labelsFor('src/consts.py', 'MAX_SIZE = 10\n', 'MAX_SIZE')).toEqual(['Variable']); + }); + + it('Kotlin: leaves a genuine property alone', async () => { + expect(await labelsFor('src/Consts.kt', 'val maxSize = 10\n', 'maxSize')).toEqual(['Property']); + }); + + it('C++: leaves a genuine variable alone', async () => { + expect(await labelsFor('src/consts.cpp', 'auto maxSize = 10;\n', 'maxSize')).toEqual([ + 'Variable', + ]); + }); + + it('Python: an annotated attribute stays a Property, not a Variable', async () => { + // Regression guard. Python matches BOTH `@definition.property` (annotated) + // and `@definition.variable` (bare assignment) on the same statement at the + // same byte offset. Ranking `Property` level with the value labels made the + // winner depend on match order, which silently turned every typed attribute + // — including dataclass fields — into a file-level `Variable`. + expect(await labelsFor('src/model.py', 'class C:\n name: str = "x"\n', 'name')).toEqual([ + 'Property', + ]); + }); + + it('Python: an annotated attribute keeps its owning HAS_PROPERTY edge', async () => { + // The label regression above also detached the attribute from its class: + // the node became `Variable::name` reached by `File -DEFINES->` + // instead of `Property::C.name` reached by `Class -HAS_PROPERTY->`. + const { graph } = await parseFilesWithWorkers([ + { path: 'src/owned.py', content: 'class C:\n name: str = "x"\n' }, + ]); + + expect( + graph.relationships + .filter((rel) => rel.type === 'HAS_PROPERTY') + .map((rel) => `${rel.sourceId} -> ${rel.targetId}`), + ).toEqual(['Class:src/owned.py:C -> Property:src/owned.py:C.name']); + }); +}); + +const describeIfWorkerBuilt = distWorkerExists() ? describe : describe.skip; + +/** Call targets resolved in a one-file repo, for the closure-call assertions. */ +const callTargetsFor = async (filename: string, source: string): Promise => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-closure-calls-')); + try { + fs.writeFileSync(path.join(dir, filename), source, 'utf-8'); + const result = await runPipelineFromRepo(dir, () => {}, { + workerPoolSize: 1, + workerUrlForTest: DIST_WORKER_URL, + }); + return result.graph.relationships + .filter((rel) => rel.type === 'CALLS') + .map((rel) => rel.targetId) + .sort(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}; + +describeIfWorkerBuilt('calls to a closure binding resolve to its Function node', () => { + // The label change alone is not enough: each language also needs a + // `@declaration.function` anchored on the inner closure literal, so the def is + // owned by the closure's own scope and free-call resolution can find it. + + it('Go: Handler(1) resolves', async () => { + const targets = await callTargetsFor( + 'main.go', + 'package main\n\nvar Handler = func(x int) int { return x }\n\nfunc Caller() int { return Handler(1) }\n', + ); + + expect(targets).toContain('Function:main.go:Handler'); + }); + + it('Python: handler(1) resolves', async () => { + const targets = await callTargetsFor( + 'app.py', + 'handler = lambda x: x\n\ndef caller():\n return handler(1)\n', + ); + + expect(targets).toContain('Function:app.py:handler'); + }); + + it('C++: handler(1) resolves', async () => { + const targets = await callTargetsFor( + 'main.cpp', + 'auto handler = [](int x) { return x; };\n\nint caller() { return handler(1); }\n', + ); + + expect(targets).toContain('Function:main.cpp:handler'); + }); +}); diff --git a/gitnexus/test/integration/const-function-twin.test.ts b/gitnexus/test/integration/const-function-twin.test.ts new file mode 100644 index 000000000..506b07ef8 --- /dev/null +++ b/gitnexus/test/integration/const-function-twin.test.ts @@ -0,0 +1,148 @@ +/** + * #2687 — `const X = ` must emit exactly ONE graph + * node: the `Function` node that carries the CALLS edges. Before the fix it also + * emitted an edgeless `Const::X` twin at the same line, which made every + * `impact`/`context` call on that name come back `status: "ambiguous"` with a + * top-level `impactedCount: 0` — indistinguishable from a real "nothing depends + * on this". + * + * Root cause: the parse-worker duplicate suppression is order-dependent. Only + * the value branch (`Const`/`Static`/`Variable`) consults + * `processedDefinitionNodes`; function-like labels merely register into it. And + * tree-sitter yields the `@definition.const` match BEFORE `@definition.function` + * for the same `lexical_declaration` (the const pattern completes at `@name`, + * the function pattern needs the trailing arrow/function-expression value), so + * the twin was emitted first and never suppressed. + * + * The over-suppression guards below matter as much as the twin assertions: a + * genuine non-callable `const`, an object-literal service (#1718), a `var` + * binding, and the non-function initializers must all keep their value nodes. + * + * Mirrors the sibling suppression case in `c-cpp-typedef-legacy-parse.test.ts`. + */ +import { describe, expect, it } from 'vitest'; +import { parseFilesWithWorkers } from '../helpers/worker-parse.js'; + +const parseNodes = async (path: string, content: string) => { + const { graph } = await parseFilesWithWorkers([{ path, content }]); + return graph.nodes; +}; + +type ParsedNode = Awaited>[number]; + +/** Sorted labels of every node carrying `name` — length doubles as the node count. */ +const labelsOf = (nodes: readonly ParsedNode[], name: string): string[] => + nodes + .filter((node) => node.properties.name === name) + .map((node) => node.label) + .sort(); + +describe('#2687 export-const function twin', () => { + it('emits one Function node for a bare const arrow', async () => { + const nodes = await parseNodes('src/bare.ts', 'const Bare = () => 1;\n'); + + expect(labelsOf(nodes, 'Bare')).toEqual(['Function']); + // The twin was `Const::Bare` at the same line — assert the id is gone. + expect(nodes.filter((node) => node.id === 'Const:src/bare.ts:Bare')).toHaveLength(0); + }); + + it('emits one Function node for an exported const arrow', async () => { + const nodes = await parseNodes('src/exported.ts', 'export const Exported = () => 2;\n'); + + expect(labelsOf(nodes, 'Exported')).toEqual(['Function']); + }); + + it('emits one Function node for a bare const function-expression', async () => { + const nodes = await parseNodes( + 'src/bare-fn.ts', + 'const BareFnExpr = function () {\n return 3;\n};\n', + ); + + expect(labelsOf(nodes, 'BareFnExpr')).toEqual(['Function']); + }); + + it('emits one Function node for an exported const function-expression', async () => { + const nodes = await parseNodes( + 'src/exported-fn.ts', + 'export const ExportedFnExpr = function () {\n return 4;\n};\n', + ); + + expect(labelsOf(nodes, 'ExportedFnExpr')).toEqual(['Function']); + }); + + it('emits one Function node for an exported component in a .tsx file', async () => { + // The reporter's shape: `export const Button = (props) => …` in a React file. + const nodes = await parseNodes( + 'src/ui/button.tsx', + 'export const Button = (props: { label: string }) => {\n return props.label;\n};\n', + ); + + expect(labelsOf(nodes, 'Button')).toEqual(['Function']); + }); + + it('emits one Function node for a let-bound arrow', async () => { + // `let` shares the `lexical_declaration` pattern, so it twinned too. + const nodes = await parseNodes('src/let.ts', 'let mutable = () => 1;\n'); + + expect(labelsOf(nodes, 'mutable')).toEqual(['Function']); + }); + + it('keeps the Const node for a non-callable const', async () => { + const nodes = await parseNodes('src/config.ts', 'export const CONFIG = { a: 1 };\n'); + + expect(labelsOf(nodes, 'CONFIG')).toEqual(['Const']); + }); + + it('keeps the Const node for an object-literal service (#1718)', async () => { + // `receiver-bound-calls.ts` Case 5 bridges `fooService.getUser()` through + // this exact `Const::fooService` node id. + const nodes = await parseNodes( + 'src/service.ts', + 'export const fooService = {\n getUser(id: string) {\n return id;\n },\n};\n', + ); + + expect(labelsOf(nodes, 'fooService')).toEqual(['Const']); + }); + + it('keeps the Const node for a non-function initializer', async () => { + const nodes = await parseNodes( + 'src/ternary.ts', + 'function A() {\n return 1;\n}\nfunction B() {\n return 2;\n}\nconst ternary = A ?? B;\n', + ); + + expect(labelsOf(nodes, 'ternary')).toEqual(['Const']); + }); + + it('keeps the Variable node for a var-bound function-expression', async () => { + // `var` has no matching `@definition.function` pattern, so nothing claims + // the name and the value node must survive untouched. + const nodes = await parseNodes('src/var.ts', 'var legacy = function () {\n return 3;\n};\n'); + + expect(labelsOf(nodes, 'legacy')).toEqual(['Variable']); + }); + + it('suppresses only the callable name in a multi-name declaration', async () => { + // Both declarators share ONE `lexical_declaration`, so a suppression keyed + // by definition-node start index alone would wrongly delete `a`. + const nodes = await parseNodes('src/multi.ts', 'const a = 1,\n b = () => {};\n'); + + expect(labelsOf(nodes, 'a')).toEqual(['Const']); + expect(labelsOf(nodes, 'b')).toEqual(['Function']); + }); + + it('keeps multi-name siblings when the callable is declared FIRST', async () => { + // Mirror of the case above. The callable's claim on the shared definition + // node used to be recorded under a bare `startIndex`, which swallowed every + // LATER sibling on that declaration — so `SIB_A`/`SIB_B` vanished entirely + // (no node, no symbol). Both claims are name-scoped now, so declarator + // order cannot decide whether a sibling exists. + const nodes = await parseNodes( + 'src/multi-first.ts', + 'export const cb = () => 1,\n SIB_A = 2,\n SIB_B = 3;\n', + ); + + expect(labelsOf(nodes, 'cb')).toEqual(['Function']); + expect(labelsOf(nodes, 'SIB_A')).toEqual(['Const']); + expect(labelsOf(nodes, 'SIB_B')).toEqual(['Const']); + }); +}); diff --git a/gitnexus/test/integration/graph-emit-streaming-roundtrip.test.ts b/gitnexus/test/integration/graph-emit-streaming-roundtrip.test.ts new file mode 100644 index 000000000..df8dbf1df --- /dev/null +++ b/gitnexus/test/integration/graph-emit-streaming-roundtrip.test.ts @@ -0,0 +1,181 @@ +/** + * Streamed structural emit — differential set-identity (issue #2680). + * + * The acceptance property: for the same node/edge set, the rows that reach the + * bulk COPY must be IDENTICAL whether streaming is on or off. With streaming + * on those rows arrive from two places — the residual in-memory graph (via + * `streamAllCSVsToDisk`) plus the sink's per-pair CSVs — and their union has to + * equal the single whole-graph emit. + * + * Modelled on `pdg-emit-streaming-roundtrip.test.ts`, which likewise drives the + * sink directly rather than running `analyze`: the guarantee under test is + * about emitted rows, and going through the worker pool would add a large + * amount of unrelated machinery without strengthening the assertion. + * + * Guarantee is set-level, not byte-level: streamed rows are written in emit + * order and are not re-sorted, so file bytes may differ while the row SET (and + * therefore the loaded graph) does not. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { streamAllCSVsToDisk } from '../../src/core/lbug/csv-generator.js'; +import { GraphEmitSink } from '../../src/core/lbug/graph-emit-sink.js'; +import type { KnowledgeGraph } from '../../src/core/graph/types.js'; +import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; + +const FILE_PATH = 'src/mod.ts'; + +const fileNode = (): GraphNode => ({ + id: `File:${FILE_PATH}`, + label: 'File', + properties: { name: 'mod.ts', filePath: FILE_PATH }, +}); + +const fnNode = (n: number): GraphNode => ({ + id: `Function:${FILE_PATH}:fn${n}`, + label: 'Function', + properties: { + name: `fn${n}`, + filePath: FILE_PATH, + startLine: n, + endLine: n + 1, + isExported: false, + }, +}); + +const classNode = (n: number): GraphNode => ({ + id: `Class:${FILE_PATH}:Cls${n}`, + label: 'Class', + properties: { name: `Cls${n}`, filePath: FILE_PATH, startLine: n, endLine: n + 5 }, +}); + +const edge = ( + type: GraphRelationship['type'], + sourceId: string, + targetId: string, +): GraphRelationship => ({ + id: `${type}:${sourceId}->${targetId}`, + sourceId, + targetId, + type, + confidence: 1, + reason: 'test', +}); + +/** A mix deliberately spanning both sides of RETAINED_REL_TYPES, plus a + * duplicate id and a self-edge — the cases where a naive sink diverges. */ +const buildFixture = ( + graph: KnowledgeGraph, +): { nodes: GraphNode[]; relationships: GraphRelationship[] } => { + const nodes: GraphNode[] = [fileNode(), classNode(1), classNode(2)]; + for (let i = 0; i < 12; i++) nodes.push(fnNode(i)); + + const relationships: GraphRelationship[] = []; + for (const n of nodes) relationships.push(edge('DEFINES', `File:${FILE_PATH}`, n.id)); // retained + for (let i = 0; i < 11; i++) { + relationships.push( + edge('CALLS', `Function:${FILE_PATH}:fn${i}`, `Function:${FILE_PATH}:fn${i + 1}`), + ); // streamed + relationships.push(edge('ACCESSES', `Function:${FILE_PATH}:fn${i}`, `Class:${FILE_PATH}:Cls1`)); // streamed + } + relationships.push(edge('EXTENDS', `Class:${FILE_PATH}:Cls2`, `Class:${FILE_PATH}:Cls1`)); // retained + relationships.push(edge('IMPORTS', `File:${FILE_PATH}`, `Class:${FILE_PATH}:Cls1`)); // streamed + // Self-edge and an exact duplicate id — both must appear exactly once. + relationships.push(edge('CALLS', `Function:${FILE_PATH}:fn0`, `Function:${FILE_PATH}:fn0`)); + relationships.push(edge('CALLS', `Function:${FILE_PATH}:fn0`, `Function:${FILE_PATH}:fn1`)); + + for (const n of nodes) graph.addNode(n); + for (const r of relationships) graph.addRelationship(r); + return { nodes, relationships }; +}; + +/** Every relationship row emitted for a graph, as a sorted `pairKey\0row` set. */ +const relRowsFromCsvDir = async (csvDir: string): Promise => { + const out: string[] = []; + for (const name of await fsp.readdir(csvDir)) { + if (!name.startsWith('rel_') || !name.endsWith('.csv')) continue; + const pairKey = name.slice('rel_'.length, -'.csv'.length); + const text = await fsp.readFile(path.join(csvDir, name), 'utf8'); + for (const line of text.split('\n').slice(1)) { + if (line.length > 0) out.push(`${pairKey}\u0000${line}`); + } + } + return out.sort(); +}; + +let tmpRoot: string; + +beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'graph-emit-roundtrip-')); + fs.mkdirSync(path.join(tmpRoot, 'repo'), { recursive: true }); + fs.writeFileSync(path.join(tmpRoot, 'repo', 'src-placeholder'), ''); +}); + +afterEach(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); +}); + +describe('streamed structural emit is set-identical to the whole-graph emit', () => { + it('emits the same relationship row set with the flag on and off', async () => { + const repoPath = path.join(tmpRoot, 'repo'); + + // ── Arm A: streaming OFF — one whole-graph emit over everything. + const graphOff = createKnowledgeGraph(); + buildFixture(graphOff); + const csvDirOff = path.join(tmpRoot, 'csv-off'); + await streamAllCSVsToDisk(graphOff, repoPath, csvDirOff); + const rowsOff = await relRowsFromCsvDir(csvDirOff); + + // ── Arm B: streaming ON — retained edges stay in the graph and are emitted + // by streamAllCSVsToDisk; the rest were streamed by the sink. + const realOn = createKnowledgeGraph(); + const sinkCsvDir = path.join(tmpRoot, 'csv-sink'); + const sink = new GraphEmitSink(realOn, sinkCsvDir); + sink.beginStreaming(); + buildFixture(sink); + const manifest = sink.finalize(); + + const csvDirOn = path.join(tmpRoot, 'csv-on'); + await streamAllCSVsToDisk(realOn, repoPath, csvDirOn); + + const rowsOn = [ + ...(await relRowsFromCsvDir(csvDirOn)), + ...(await relRowsFromCsvDir(sinkCsvDir)), + ].sort(); + + // The union of (residual graph emit + streamed CSVs) is the whole-graph emit. + expect(rowsOn).toEqual(rowsOff); + + // And the split is real — this is what buys the memory, so assert it rather + // than let a sink that streamed nothing pass the equality above. + expect(manifest.totalRows).toBeGreaterThan(0); + expect(realOn.relationshipCount).toBeGreaterThan(0); + expect(realOn.relationshipCount).toBeLessThan(graphOff.relationshipCount); + expect(realOn.relationshipCount + manifest.totalRows).toBe(graphOff.relationshipCount); + }); + + it('emits an identical node row set — nodes are never streamed', async () => { + const repoPath = path.join(tmpRoot, 'repo'); + + const graphOff = createKnowledgeGraph(); + buildFixture(graphOff); + const csvDirOff = path.join(tmpRoot, 'csv-off'); + const resultOff = await streamAllCSVsToDisk(graphOff, repoPath, csvDirOff); + + const realOn = createKnowledgeGraph(); + const sink = new GraphEmitSink(realOn, path.join(tmpRoot, 'csv-sink')); + sink.beginStreaming(); + buildFixture(sink); + sink.finalize(); + const csvDirOn = path.join(tmpRoot, 'csv-on'); + const resultOn = await streamAllCSVsToDisk(realOn, repoPath, csvDirOn); + + expect(realOn.nodeCount).toBe(graphOff.nodeCount); + expect([...resultOn.nodeFiles.keys()].sort()).toEqual([...resultOff.nodeFiles.keys()].sort()); + }); +}); diff --git a/gitnexus/test/integration/impact-ambiguous-blast-radius.test.ts b/gitnexus/test/integration/impact-ambiguous-blast-radius.test.ts index 0e5ee9667..0172f9606 100644 --- a/gitnexus/test/integration/impact-ambiguous-blast-radius.test.ts +++ b/gitnexus/test/integration/impact-ambiguous-blast-radius.test.ts @@ -41,6 +41,20 @@ const SEED = [ `MATCH (a:Function {id:'Function:src/actions.ts:syncContent'}), (b:Function {id:'${SYNC_LOGIC_ID}'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'direct', step:0}]->(b)`, `MATCH (a:Function {id:'Function:src/actions.ts:scheduleSync'}), (b:Function {id:'${SYNC_LOGIC_ID}'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'direct', step:0}]->(b)`, `MATCH (a:Function {id:'Function:src/ui-helpers.ts:renderCard'}), (b:Function {id:'${UI_HELPERS_ID}'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'direct', step:0}]->(b)`, + + // Two same-named non-callable consts — an ambiguity that survives the #2687 + // twin fix, used to pin that a value candidate reports a real `kind`. + `CREATE (k1:Const {id: 'Const:src/config-a.ts:APP_CONFIG', name: 'APP_CONFIG', filePath: 'src/config-a.ts', startLine: 1, endLine: 1, content: '', description: ''})`, + `CREATE (k2:Const {id: 'Const:src/config-b.ts:APP_CONFIG', name: 'APP_CONFIG', filePath: 'src/config-b.ts', startLine: 1, endLine: 1, content: '', description: ''})`, + + // A class and a same-named value binding in another file — the #480 + // Class/Constructor collapse must still fold onto the Class. Before the + // enrichment widening these value candidates carried `type: ''`, which is + // what kept the collapse gate open. + `CREATE (rc:Class {id: 'Class:src/registry.ts:Registry', name: 'Registry', filePath: 'src/registry.ts', startLine: 1, endLine: 9, isExported: true, content: '', description: ''})`, + `CREATE (rv:Const {id: 'Const:test/registry.test.ts:Registry', name: 'Registry', filePath: 'test/registry.test.ts', startLine: 3, endLine: 3, content: '', description: ''})`, + `CREATE (ru:Function {id: 'Function:src/boot.ts:boot', name: 'boot', filePath: 'src/boot.ts', startLine: 1, endLine: 5, isExported: true, content: '', description: ''})`, + `MATCH (a:Function {id:'Function:src/boot.ts:boot'}), (b:Class {id:'Class:src/registry.ts:Registry'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'direct', step:0}]->(b)`, ]; withTestLbugDB( @@ -85,6 +99,76 @@ withTestLbugDB( ); }); + it('reports an undetermined impactedCount, never a numeric zero (#2687)', async () => { + const result = await backend.callTool('impact', { + target: 'classifyCard', + direction: 'upstream', + }); + + // #2129 hoisted maxImpactedCount so a real caller could not hide behind + // the ambiguous zero — but the zero itself was still byte-identical to a + // genuine "nothing depends on this". A consumer testing + // `impactedCount === 0` got a confident all-clear without ever reading + // `candidates[]`. `null` is undetermined and cannot be misread that way. + expect(result).toMatchObject({ status: 'ambiguous', impactedCount: null, risk: 'UNKNOWN' }); + expect(typeof result.impactedCount).not.toBe('number'); + + // The truthful signal is still present and still non-zero. + expect(result.maxImpactedCount).toBeGreaterThanOrEqual(2); + }); + + it('reports a real kind for an ambiguous value candidate (#2687)', async () => { + // `labels(n)[0]` comes back empty for these node types, and the label + // enrichment UNION used to cover only Class/Interface/Function/Method/ + // Constructor — so a value candidate surfaced as `kind: ""`, which reads + // as "unknown kind" and leaves the `kind` disambiguation hint unable to + // filter it out. + const result = await backend.callTool('impact', { + target: 'APP_CONFIG', + direction: 'upstream', + }); + + expect(result.status).toBe('ambiguous'); + expect(result.candidates.map((c: { kind: string }) => c.kind)).toEqual(['Const', 'Const']); + }); + + it('still collapses a Class against a same-named value binding (#480)', async () => { + // Regression guard for the enrichment widening: the collapse gate keys on + // "some candidate has an indeterminate kind". Value candidates used to + // qualify by carrying `type: ''`; now that enrichment fills them in they + // must be named explicitly, or this resolves to `ambiguous` and every + // resolver-backed tool loses a previously confident answer. + const result = await backend.callTool('impact', { + target: 'Registry', + direction: 'upstream', + }); + + expect(result.status).not.toBe('ambiguous'); + expect(result.target).toMatchObject({ + id: 'Class:src/registry.ts:Registry', + type: 'Class', + }); + expect(result.impactedCount).toBeGreaterThanOrEqual(1); + }); + + it('reports an undetermined impactedCount for an ambiguous pdg target (#2687)', async () => { + // The pdg branch has no per-candidate fan-out, so it carries no + // maxImpactedCount at all — a numeric zero here is even less correctable. + const result = await backend.callTool('impact', { + target: 'classifyCard', + direction: 'upstream', + mode: 'pdg', + }); + + expect(result).toMatchObject({ + status: 'ambiguous', + mode: 'pdg', + impactedCount: null, + risk: 'UNKNOWN', + }); + expect(typeof result.impactedCount).not.toBe('number'); + }); + it('disambiguation by uid returns the exact dropped caller (BFS unchanged)', async () => { const result = await backend.callTool('impact', { target: 'classifyCard', diff --git a/gitnexus/test/integration/instance-ownership-pipeline-benchmark.test.ts b/gitnexus/test/integration/instance-ownership-pipeline-benchmark.test.ts new file mode 100644 index 000000000..456b92858 --- /dev/null +++ b/gitnexus/test/integration/instance-ownership-pipeline-benchmark.test.ts @@ -0,0 +1,129 @@ +/** + * Instance-ownership free-call gate benchmark. + * + * Generates C# and Kotlin projects where every file contains a caller and an + * unrelated class method with the same receiver-less call name. Repeated calls + * stress the ownership gate that prevents the same-file fallback from linking + * those unrelated methods. + * + * Run: + * cd gitnexus && GITNEXUS_BENCH=1 npx vitest run test/integration/instance-ownership-pipeline-benchmark.test.ts + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; + +const BENCH_ENABLED = process.env.GITNEXUS_BENCH === '1'; +const CALLS_PER_FILE = 24; +// time growth divided by file growth: quadratic work reaches 2 on a doubling. +const NORMALIZED_SCALING_LIMIT = 2; + +interface BenchResult { + fileCount: number; + callCount: number; + elapsedMs: number; + peakHeapMB: number; +} + +interface LanguageCase { + readonly label: string; + readonly extension: string; + source(fileIndex: number): string; +} + +const LANGUAGES: readonly LanguageCase[] = [ + { + label: 'C#', + extension: 'cs', + source: (fileIndex) => `namespace Bench${fileIndex}; + +public class Caller${fileIndex} +{ +${Array.from( + { length: CALLS_PER_FILE }, + (_, callIndex) => ` public void Run${callIndex}() { Foreign(); }`, +).join('\n')} +} + +public class Unrelated${fileIndex} +{ + public void Foreign() {} +} +`, + }, + { + label: 'Kotlin', + extension: 'kt', + source: (fileIndex) => `package bench${fileIndex} + +class Caller${fileIndex} { +${Array.from( + { length: CALLS_PER_FILE }, + (_, callIndex) => ` fun run${callIndex}() { foreign() }`, +).join('\n')} +} + +class Unrelated${fileIndex} { + fun foreign() {} +} +`, + }, +]; + +function generateFixture(language: LanguageCase, fileCount: number): string { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), `instance-ownership-${language.extension}-${fileCount}-`), + ); + for (let i = 0; i < fileCount; i++) { + fs.writeFileSync(path.join(dir, `Case${i}.${language.extension}`), language.source(i)); + } + return dir; +} + +async function runBenchmark(language: LanguageCase, fileCount: number): Promise { + const dir = generateFixture(language, fileCount); + let peakHeapMB = 0; + const heapSampler = setInterval(() => { + peakHeapMB = Math.max(peakHeapMB, process.memoryUsage().heapUsed / 1024 / 1024); + }, 25); + + try { + const start = performance.now(); + await runPipelineFromRepo(dir, () => {}, { skipGraphPhases: true }); + return { + fileCount, + callCount: fileCount * CALLS_PER_FILE, + elapsedMs: Math.round(performance.now() - start), + peakHeapMB: Math.round(peakHeapMB), + }; + } finally { + clearInterval(heapSampler); + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe.skipIf(!BENCH_ENABLED)('instance-ownership free-call gate benchmark', () => { + for (const language of LANGUAGES) { + it(`${language.label} scales sub-quadratically with ownership-gated calls`, async () => { + // Wide enough steps to expose quadratic growth without making this + // opt-in benchmark impractical on contributor machines. + let previous: BenchResult | undefined; + for (const fileCount of [100, 250, 500]) { + const result = await runBenchmark(language, fileCount); + console.log( + `${language.label}: ${result.fileCount} files / ${result.callCount} calls: ` + + `${result.elapsedMs}ms, ${result.peakHeapMB}MB heap`, + ); + + if (previous !== undefined) { + const fileRatio = result.fileCount / previous.fileCount; + const timeRatio = result.elapsedMs / previous.elapsedMs; + expect(timeRatio / fileRatio).toBeLessThan(NORMALIZED_SCALING_LIMIT); + } + previous = result; + } + }, 600_000); + } +}); diff --git a/gitnexus/test/integration/lbug-orphan-sidecar-recovery.test.ts b/gitnexus/test/integration/lbug-orphan-sidecar-recovery.test.ts index e74cd5cf7..97e71cfb2 100644 --- a/gitnexus/test/integration/lbug-orphan-sidecar-recovery.test.ts +++ b/gitnexus/test/integration/lbug-orphan-sidecar-recovery.test.ts @@ -328,3 +328,140 @@ describe('init lock — single-process ownership contract', () => { } }); }); + +// --------------------------------------------------------------------------- +// Missing-shadow WAL quarantine reclaim (issue #2637) +// --------------------------------------------------------------------------- + +const plantMissingShadowQuarantine = async (dbPath: string): Promise => { + const quarantinePath = `${dbPath}.wal.missing-shadow.${Date.now()}-${Math.random() + .toString(36) + .slice(2)}`; + await fs.writeFile(quarantinePath, 'stale-quarantined-wal-bytes'); + return quarantinePath; +}; + +describe('missing-shadow quarantine reclaim — native integration (issue #2637)', () => { + itLbugReopen( + 'reclaims a pre-existing missing-shadow WAL quarantine file on write-path init when the main DB is present', + async () => { + const tmp = await createTempDir('gitnexus-lbug-quarantine-'); + const dbPath = path.join(tmp.dbPath, 'lbug'); + + try { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + // Create a real DB first, then close it. + await adapter.initLbug(dbPath); + await adapter.closeLbug(); + + // Plant a quarantine file left over from an earlier crash. + const quarantinePath = await plantMissingShadowQuarantine(dbPath); + await expect(fs.access(quarantinePath)).resolves.toBeUndefined(); + + // Re-init with the main DB present — reclaim must fire unconditionally. + await adapter.initLbug(dbPath); + + const rows = await adapter.executeQuery('RETURN 1 AS ok'); + expect(rows).toEqual([{ ok: 1 }]); + + await expect(fs.access(quarantinePath)).rejects.toThrow(); + + await adapter.closeLbug(); + } finally { + await tmp.cleanup(); + } + }, + ); + + itLbugReopen( + 'reclaims a pre-existing missing-shadow WAL quarantine file when the main DB is ALSO missing (crash-recovery path)', + async () => { + const tmp = await createTempDir('gitnexus-lbug-quarantine-'); + const dbPath = path.join(tmp.dbPath, 'lbug'); + const shadowPath = `${dbPath}.shadow`; + const walCheckpointPath = `${dbPath}.wal.checkpoint`; + + try { + // No main DB file — plant the quarantine file alongside the #1618 + // orphan sidecars to prove both cleanup blocks coexist correctly. + const quarantinePath = await plantMissingShadowQuarantine(dbPath); + await fs.writeFile(shadowPath, 'stale-shadow-data'); + await fs.writeFile(walCheckpointPath, 'stale-wal-checkpoint-data'); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.initLbug(dbPath); + + const rows = await adapter.executeQuery('RETURN 1 AS ok'); + expect(rows).toEqual([{ ok: 1 }]); + + await expect(fs.access(quarantinePath)).rejects.toThrow(); + await expect(fs.access(shadowPath)).rejects.toThrow(); + await expect(fs.access(walCheckpointPath)).rejects.toThrow(); + + await adapter.closeLbug(); + } finally { + await tmp.cleanup(); + } + }, + ); + + itLbugReopen( + 'leaves a missing-shadow quarantine file untouched when GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT=1', + async () => { + const tmp = await createTempDir('gitnexus-lbug-quarantine-'); + const dbPath = path.join(tmp.dbPath, 'lbug'); + const previousEnv = process.env.GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT; + + try { + const quarantinePath = await plantMissingShadowQuarantine(dbPath); + + process.env.GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT = '1'; + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.initLbug(dbPath); + + const rows = await adapter.executeQuery('RETURN 1 AS ok'); + expect(rows).toEqual([{ ok: 1 }]); + + // Reclaim was suppressed — the quarantine file survives. + await expect(fs.access(quarantinePath)).resolves.toBeUndefined(); + + await adapter.closeLbug(); + } finally { + if (previousEnv === undefined) { + delete process.env.GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT; + } else { + process.env.GITNEXUS_DISABLE_LBUG_SIDECAR_PREFLIGHT = previousEnv; + } + await tmp.cleanup(); + } + }, + ); + + itLbugReopen( + 'does not touch a .dirty-recovery parked sidecar (isolation from the missing-shadow family)', + async () => { + const tmp = await createTempDir('gitnexus-lbug-quarantine-'); + const dbPath = path.join(tmp.dbPath, 'lbug'); + const dirtyRecoveryPath = `${dbPath}.wal.dirty-recovery`; + + try { + await fs.writeFile(dirtyRecoveryPath, 'parked-from-an-interrupted-dirty-recovery-rebuild'); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.initLbug(dbPath); + + const rows = await adapter.executeQuery('RETURN 1 AS ok'); + expect(rows).toEqual([{ ok: 1 }]); + + // Different sidecar family, different lifecycle — must survive. + await expect(fs.access(dirtyRecoveryPath)).resolves.toBeUndefined(); + + await adapter.closeLbug(); + } finally { + await tmp.cleanup(); + } + }, + ); +}); diff --git a/gitnexus/test/integration/local-symbol-pruner-pipeline.test.ts b/gitnexus/test/integration/local-symbol-pruner-pipeline.test.ts index 454904660..129f44889 100644 --- a/gitnexus/test/integration/local-symbol-pruner-pipeline.test.ts +++ b/gitnexus/test/integration/local-symbol-pruner-pipeline.test.ts @@ -57,9 +57,16 @@ class Client { expect(findNode(result, 'Const', 'handler')).toBeUndefined(); expect(findNode(result, 'Const', 'MODULE_CONST')).toBeDefined(); - expect(findNode(result, 'Const', 'exportedHandler')).toBeDefined(); expect(findNode(result, 'Function', 'handler')).toBeDefined(); + // #2687: `export const exportedHandler = () => …` emits ONE node — the + // Function that carries the CALLS edges — not a Function plus an edgeless + // Const twin. This makes the module-scoped arrow consistent with the + // block-scoped `const handler = () => boring` asserted above, which has + // never had a surviving Const node. + expect(findNode(result, 'Const', 'exportedHandler')).toBeUndefined(); + expect(findNode(result, 'Function', 'exportedHandler')).toBeDefined(); + const keepsResolvedClientCall = result.graph.relationships.some((rel) => { if (rel.type !== 'CALLS') return false; const source = result.graph.getNode(rel.sourceId); diff --git a/gitnexus/test/integration/resolvers/csharp.test.ts b/gitnexus/test/integration/resolvers/csharp.test.ts index b4e49af0d..91945cc92 100644 --- a/gitnexus/test/integration/resolvers/csharp.test.ts +++ b/gitnexus/test/integration/resolvers/csharp.test.ts @@ -241,6 +241,54 @@ describe('C# using static member injection', () => { expect(sqCall!.targetFilePath).toBe('Helpers/MathUtils.cs'); expect(['import-resolved', 'global']).toContain(sqCall!.rel.reason); }); + + it("does not resolve an unrelated same-file class's bare method", () => { + const calls = getRelationships(result, 'CALLS'); + expect(calls.find((c) => c.source === 'Exercise' && c.target === 'LeakedOnly')).toBeUndefined(); + expect( + calls.find( + (c) => c.source === 'RejectOtherNamespaceOwner' && c.target === 'NamespaceCollision', + ), + ).toBeUndefined(); + }); + + it('preserves same-file using-static visibility when finalize masks its provenance', () => { + const calls = getRelationships(result, 'CALLS'); + const imported = calls.find((c) => c.source === 'Exercise' && c.target === 'ImportedOnly'); + expect(imported).toBeDefined(); + expect(imported!.rel.targetId).toContain('SameFileStatics.ImportedOnly'); + }); + + it('preserves own-instance and inherited bare calls', () => { + const calls = getRelationships(result, 'CALLS'); + expect(calls.find((c) => c.source === 'Exercise' && c.target === 'OwnOnly')).toBeDefined(); + expect( + calls.find((c) => c.source === 'Exercise' && c.target === 'InheritedOnly'), + ).toBeDefined(); + }); + + it('preserves bare calls across same-file partial-class fragments', () => { + const calls = getRelationships(result, 'CALLS'); + expect( + calls.find((c) => c.source === 'CallAcrossFragment' && c.target === 'AcrossFragment'), + ).toBeDefined(); + }); + + it('resolves a local function called from a lambda body', () => { + const calls = getRelationships(result, 'CALLS'); + expect(calls.find((c) => c.source === 'Exercise' && c.target === 'LocalOnly')).toBeDefined(); + }); + + it('narrows static-import overloads after rejecting a leaked same-file method', () => { + const calls = getRelationships(result, 'CALLS'); + const selectCalls = calls.filter((c) => c.source === 'Exercise' && c.target === 'Select'); + expect(selectCalls).toHaveLength(1); + expect(selectCalls[0]!.rel.targetId).toContain('SameFileStatics.Select'); + expect(result.graph.getNode(selectCalls[0]!.rel.targetId)?.properties.parameterTypes).toEqual([ + 'string', + 'int', + ]); + }); }); // --------------------------------------------------------------------------- diff --git a/gitnexus/test/integration/resolvers/java-javac-local-types.test.ts b/gitnexus/test/integration/resolvers/java-javac-local-types.test.ts new file mode 100644 index 000000000..363fb49ce --- /dev/null +++ b/gitnexus/test/integration/resolvers/java-javac-local-types.test.ts @@ -0,0 +1,59 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { FIXTURES } from './helpers.js'; + +const javacAvailable = spawnSync('javac', ['-version'], { stdio: 'ignore' }).status === 0; + +describe('Java local-type names emitted by javac', () => { + it.runIf(javacAvailable)('matches the identities asserted by the resolver fixture', () => { + const temp = mkdtempSync(path.join(tmpdir(), 'gitnexus-javac-local-types-')); + const output = path.join(temp, 'classes'); + mkdirSync(output); + + try { + const sourceDir = path.join(FIXTURES, 'java-local-class-naming', 'src'); + const sources = readdirSync(sourceDir) + .filter((name) => name.endsWith('.java')) + .map((name) => path.join(sourceDir, name)); + execFileSync('javac', ['-d', output, ...sources]); + + expect(readdirSync(output).sort()).toEqual([ + 'Compact$1.class', + 'Compact$1Local.class', + 'Compact.class', + 'Outer$1.class', + 'Outer$1CtorHost$1Local.class', + 'Outer$1CtorHost.class', + 'Outer$1Cyclic.class', + 'Outer$1InstanceLocal.class', + 'Outer$1LambdaLocal.class', + 'Outer$1Local$1.class', + 'Outer$1Local.class', + 'Outer$1NestedHost$Member$1Local.class', + 'Outer$1NestedHost$Member.class', + 'Outer$1NestedHost.class', + 'Outer$1StaticLocal.class', + 'Outer$2.class', + 'Outer$2Local.class', + 'Outer$3$1Local.class', + 'Outer$3.class', + 'Outer$3Local.class', + 'Outer$4Local.class', + 'Outer$Cyclic.class', + 'Outer$MemberHost$1Local.class', + 'Outer$MemberHost.class', + 'Outer.class', + 'Types$1.class', + 'Types$1E.class', + 'Types$1I.class', + 'Types$1R.class', + 'Types.class', + ]); + } finally { + rmSync(temp, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/integration/resolvers/java.test.ts b/gitnexus/test/integration/resolvers/java.test.ts index bfaa00196..2fc63a0ea 100644 --- a/gitnexus/test/integration/resolvers/java.test.ts +++ b/gitnexus/test/integration/resolvers/java.test.ts @@ -2885,6 +2885,118 @@ describe('Java instance-ownership free-call gate (#2550)', () => { }, 60000); }); +describe('Java local-type identity and lexical scope (#2562)', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'java-local-class-naming'), () => {}); + }, 60000); + + it('matches javac local-name and anonymous-name sequences per immediate host', () => { + const classes = getNodesByLabel(result, 'Class'); + expect(classes).toContain('Outer$1Local'); + expect(classes).toContain('Outer$1CtorHost'); + expect(classes).toContain('Outer$1NestedHost'); + expect(classes).toContain('Outer$1'); + expect(classes).toContain('Outer$2'); + expect(classes).toContain('Outer$2Local'); + expect(classes).toContain('Outer$1Local$1'); + expect(classes).toContain('Outer$1CtorHost$1Local'); + expect(classes).toContain('Outer$1NestedHost$Member$1Local'); + expect(classes).toContain('Outer$MemberHost$1Local'); + expect(classes).toContain('Outer$1StaticLocal'); + expect(classes).toContain('Outer$1InstanceLocal'); + expect(classes).toContain('Outer$1LambdaLocal'); + expect(classes).toContain('Outer$3$1Local'); + expect(classes).toContain('Compact$1Local'); + expect(classes).toContain('Compact$1'); + expect(classes).not.toContain('Local'); + }); + + it('emits the correct graph label and owner for every local type kind', () => { + expect(getNodesByLabel(result, 'Enum')).toContain('Types$1E'); + expect(getNodesByLabel(result, 'Record')).toContain('Types$1R'); + expect(getNodesByLabel(result, 'Interface')).toContain('Types$1I'); + expect(getNodesByLabel(result, 'Class')).not.toContain('Types$1E'); + expect(getNodesByLabel(result, 'Class')).not.toContain('Types$1R'); + expect(getNodesByLabel(result, 'Class')).not.toContain('Types$1I'); + + const hasMethod = getRelationships(result, 'HAS_METHOD'); + for (const [label, owner, method] of [ + ['Class', 'Outer$1Local', 'inner'], + ['Class', 'Outer$2Local', 'inner'], + ['Class', 'Outer$1CtorHost$1Local', 'inner'], + ['Class', 'Outer$1NestedHost$Member', 'make'], + ['Enum', 'Types$1E', 'enumHit'], + ['Record', 'Types$1R', 'recordHit'], + ['Interface', 'Types$1I', 'run'], + ]) { + expect( + hasMethod.some( + (edge) => + edge.rel.sourceId === + `${label}:src/${owner.startsWith('Types') ? 'Types' : 'Outer'}.java:${owner}` && + edge.rel.targetId === + `Method:src/${owner.startsWith('Types') ? 'Types' : 'Outer'}.java:${owner}.${method}#0`, + ), + ).toBe(true); + } + }); + + it('keeps source-level construction dispatch bound to each local identity', () => { + const calls = getRelationships(result, 'CALLS'); + expect(calls.find((c) => c.source === 'first' && c.target === 'inner')?.rel.targetId).toBe( + 'Method:src/Outer.java:Outer$1Local.inner#0', + ); + expect(calls.find((c) => c.source === 'second' && c.target === 'inner')?.rel.targetId).toBe( + 'Method:src/Outer.java:Outer$2Local.inner#0', + ); + expect(calls.find((c) => c.source === 'CtorHost' && c.target === 'inner')?.rel.targetId).toBe( + 'Method:src/Outer.java:Outer$1CtorHost$1Local.inner#0', + ); + for (const targetId of [ + 'Method:src/Outer.java:Outer$1StaticLocal.staticHit#0', + 'Method:src/Outer.java:Outer$1InstanceLocal.instanceHit#0', + 'Method:src/Outer.java:Outer$1LambdaLocal.lambdaHit#0', + 'Method:src/Outer.java:Outer$3$1Local.anonymousHit#0', + 'Method:src/Outer.java:Outer$MemberHost$1Local.ordinaryMemberHit#0', + 'Method:src/Compact.java:Compact$1Local.inner#0', + 'Method:src/Types.java:Types$1E.enumHit#0', + 'Method:src/Types.java:Types$1R.recordHit#0', + 'Method:src/Types.java:Types$1.run#0', + ]) { + expect( + calls.some((call) => call.rel.targetId === targetId), + targetId, + ).toBe(true); + } + }); + + it('respects declaration-order visibility against a same-named member type', () => { + const calls = getRelationships(result, 'CALLS').filter( + (call) => call.source === 'declarationOrder', + ); + + expect(calls.find((call) => call.target === 'member')?.rel.targetId).toBe( + 'Method:src/Outer.java:Cyclic.member#0', + ); + expect(calls.find((call) => call.target === 'local')?.rel.targetId).toBe( + 'Method:src/Outer.java:Outer$1Cyclic.local#0', + ); + }); + + it('keeps same-named local types isolated to their disjoint blocks', () => { + const calls = getRelationships(result, 'CALLS').filter((call) => call.source === 'blocks'); + + expect(calls.find((call) => call.target === 'firstBlock')?.rel.targetId).toBe( + 'Method:src/Outer.java:Outer$3Local.firstBlock#0', + ); + expect(calls.find((call) => call.target === 'secondBlock')?.rel.targetId).toBe( + 'Method:src/Outer.java:Outer$4Local.secondBlock#0', + ); + }); +}); + // --------------------------------------------------------------------------- // #2550 review hardening: (a) an anonymous class inherits from its // constructed type, so bare calls to inherited methods INSIDE the diff --git a/gitnexus/test/integration/resolvers/kotlin.test.ts b/gitnexus/test/integration/resolvers/kotlin.test.ts index 942bc374d..a0aa3cd90 100644 --- a/gitnexus/test/integration/resolvers/kotlin.test.ts +++ b/gitnexus/test/integration/resolvers/kotlin.test.ts @@ -2958,3 +2958,32 @@ describe('Kotlin anonymous object-expression method scoping (#2545)', () => { expect(getNodesByLabel(result, 'Method')).toContain('println'); }); }); + +describe('Kotlin instance-ownership free-call gate (#2563)', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'kotlin-instance-ownership'), () => {}); + }, 60000); + + it("does not resolve a bare call to an unrelated same-file class's method", () => { + const leaked = getRelationships(result, 'CALLS').find( + (call) => call.source === 'run' && call.target === 'collide', + ); + expect(leaked).toBeUndefined(); + }); + + it('preserves own, inherited, outer-instance, and anonymous-object sibling calls', () => { + const calls = getRelationships(result, 'CALLS'); + expect(calls.find((call) => call.source === 'callOwn' && call.target === 'own')).toBeDefined(); + expect( + calls.find((call) => call.source === 'callInherited' && call.target === 'inherited'), + ).toBeDefined(); + expect( + calls.find((call) => call.source === 'callSibling' && call.target === 'sibling'), + ).toBeDefined(); + expect( + calls.find((call) => call.source === 'callOuter' && call.target === 'outerMethod'), + ).toBeDefined(); + }); +}); diff --git a/gitnexus/test/integration/resolvers/rust.test.ts b/gitnexus/test/integration/resolvers/rust.test.ts index fe970b777..2e4a400c4 100644 --- a/gitnexus/test/integration/resolvers/rust.test.ts +++ b/gitnexus/test/integration/resolvers/rust.test.ts @@ -13,6 +13,7 @@ import { edgeSet, runPipelineFromRepo, type PipelineResult, + type RelEdge, } from './helpers.js'; // --------------------------------------------------------------------------- @@ -2350,3 +2351,211 @@ describe('Rust macro resolution (issue #1934 F72)', () => { expect(calls.every((e) => e.targetLabel !== 'Macro')).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// #2514: duplicate type names must stay ambiguous regardless of duplicate +// count or file order. The range-binding prepass used Map presence as an +// ambiguity toggle (has→delete / else→set), so a 3rd same-named definition +// re-inserted a resolvable — and wrong — cross-file type (the last-scanned +// file's). The fix latches ambiguity in a separate Set: once a name has two +// definitions it never resolves again. +// +// Observable: for-loop `for item in make() { item.save(); }` where each +// `make()` (or each `Config` field) lives in its own file with no `use` +// import, so the receiver type can only come from the global range-binding +// map. A cross-file `save`/`run` CALLS edge means the name resolved. +// --------------------------------------------------------------------------- + +describe('Rust duplicate-name ambiguity latch (#2514)', () => { + // Cross-file receiver-method CALLS edges emitted from the fixture driver fn. + const receiverCalls = (result: PipelineResult, source: string, method: string): RelEdge[] => + getRelationships(result, 'CALLS').filter((c) => c.source === source && c.target === method); + + // --- return-type registry (allReturnTypes) --- + + describe('two same-named fns with different return types', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-dup-return-2'), () => {}); + }, 60000); + + it('suppresses cross-file return-type inference — item.save() does not resolve', () => { + expect(receiverCalls(result, 'drive', 'save')).toEqual([]); + }); + }); + + describe('three same-named fns with different return types', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-dup-return-3'), () => {}); + }, 60000); + + it('still suppresses inference — the 3rd duplicate does not restore a binding', () => { + expect(receiverCalls(result, 'drive', 'save')).toEqual([]); + }); + }); + + describe('three same-named fns, permuted input file order', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'rust-dup-return-3-reordered'), + () => {}, + ); + }, 60000); + + it('resolution is independent of file order — still no edge', () => { + expect(receiverCalls(result, 'drive', 'save')).toEqual([]); + }); + }); + + describe('unique fn still infers normally (over-suppression guard)', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-unique-return'), () => {}); + }, 60000); + + it('resolves item.save() to User#save via cross-file return type', () => { + const edges = receiverCalls(result, 'drive', 'save'); + expect(edges.length).toBe(1); + expect(edges[0]).toMatchObject({ source: 'drive', target: 'save', targetLabel: 'Function' }); + expect(edges[0].targetFilePath).toContain('t_a.rs'); + }); + }); + + // --- field-type registry (allFieldTypes) via struct destructuring --- + + describe('two same-named structs with conflicting field types', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-dup-fields-2'), () => {}); + }, 60000); + + it('suppresses global field-type inference — db.run() does not resolve', () => { + expect(receiverCalls(result, 'use_it', 'run')).toEqual([]); + }); + }); + + describe('three same-named structs with conflicting field types', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-dup-fields-3'), () => {}); + }, 60000); + + it('still suppresses field inference — the 3rd duplicate does not restore', () => { + expect(receiverCalls(result, 'use_it', 'run')).toEqual([]); + }); + }); +}); + +// --------------------------------------------------------------------------- +// #2514 follow-up: when a `use` import disambiguates one of several same-named +// definitions, the type must resolve to THAT definition (like the compiler), +// not stay ambiguous. The bare-name map is ambiguous, but the call site's +// import pins a single defining file, so range-binding reads that definition's +// FULL return/field type — recovering generic element types the bare-name map +// would have lost. Genuinely-ambiguous (no-import) duplicates still stay +// unresolved (covered by the #2514 block above). +// --------------------------------------------------------------------------- + +describe('Rust import-disambiguated duplicate resolution (#2514 follow-up)', () => { + describe('for-loop over an imported generic-returning duplicate fn', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-import-dup-return'), () => {}); + }, 60000); + + it('resolves item.save() to the imported definition in t_b (Repo), not ambiguous', () => { + const edges = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'drive' && c.target === 'save', + ); + expect(edges.length).toBe(1); + expect(edges[0]).toMatchObject({ source: 'drive', target: 'save', targetLabel: 'Function' }); + expect(edges[0].targetFilePath).toContain('t_b.rs'); + }); + }); + + describe('struct destructuring of an imported duplicate struct', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-import-dup-fields'), () => {}); + }, 60000); + + it('resolves db.run() to the imported definition in t_b (DbB) via its field type', () => { + const edges = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'use_it' && c.target === 'run', + ); + expect(edges.length).toBe(1); + expect(edges[0]).toMatchObject({ source: 'use_it', target: 'run', targetLabel: 'Function' }); + expect(edges[0].targetFilePath).toContain('t_b.rs'); + }); + }); + + describe('aliased import (`use t_b::make as mk`) still resolves the definition', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-import-alias-return'), () => {}); + }, 60000); + + it('keys on the definition name, not the alias — item.save() resolves to t_b (Repo)', () => { + const edges = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'drive' && c.target === 'save', + ); + expect(edges.length).toBe(1); + expect(edges[0]).toMatchObject({ source: 'drive', target: 'save', targetLabel: 'Function' }); + expect(edges[0].targetFilePath).toContain('t_b.rs'); + }); + }); + + describe('single glob import (`use t_b::*`) resolves the one globbed definition', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'rust-import-glob-return'), () => {}); + }, 60000); + + it('resolves item.save() to t_b (Repo) via the one glob-target that defines it', () => { + const edges = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'drive' && c.target === 'save', + ); + expect(edges.length).toBe(1); + expect(edges[0]).toMatchObject({ source: 'drive', target: 'save', targetLabel: 'Function' }); + expect(edges[0].targetFilePath).toContain('t_b.rs'); + }); + }); + + describe('two glob imports that both export the name stay ambiguous', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'rust-import-glob-ambiguous'), + () => {}, + ); + }, 60000); + + it('leaves item.save() unresolved when two `use x::*` both define make', () => { + const edges = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'drive' && c.target === 'save', + ); + expect(edges).toEqual([]); + }); + }); + + describe('a local definition shadows a glob import', () => { + let result: PipelineResult; + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'rust-import-glob-local-shadows'), + () => {}, + ); + }, 60000); + + it('resolves item.save() to the local make in main.rs, not the glob target', () => { + const edges = getRelationships(result, 'CALLS').filter( + (c) => c.source === 'drive' && c.target === 'save', + ); + expect(edges.length).toBe(1); + expect(edges[0]).toMatchObject({ source: 'drive', target: 'save', targetLabel: 'Function' }); + expect(edges[0].targetFilePath).toContain('main.rs'); + }); + }); +}); diff --git a/gitnexus/test/integration/spring-di-benchmark.test.ts b/gitnexus/test/integration/spring-di-benchmark.test.ts new file mode 100644 index 000000000..cdd08e086 --- /dev/null +++ b/gitnexus/test/integration/spring-di-benchmark.test.ts @@ -0,0 +1,284 @@ +/** + * Spring standard-DI scaling benchmark (#2414 / PR #2632 review). + * + * Guards the two hot paths introduced by standard Spring injection: + * + * 1. Java and Kotlin capture emission collect DI facts from their existing + * scope-query traversals instead of recursively walking the AST root a + * second time. + * 2. Post-resolution metadata attachment finds captured fields through the + * owning class scope's bindings instead of scanning every HAS_PROPERTY + * relationship in the graph. + * + * The normal-CI tripwires use dense Java/Kotlin files to catch a capture + * re-regression. The gated suites measure Java and Kotlin capture plus + * full-pipeline scaling: + * + * GITNEXUS_BENCH=1 npx vitest run test/integration/spring-di-benchmark.test.ts + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { emitJavaScopeCaptures } from '../../src/core/ingestion/languages/java/captures.js'; +import { collectJavaCaptureSideChannel } from '../../src/core/ingestion/languages/java/capture-side-channel.js'; +import { emitKotlinScopeCaptures } from '../../src/core/ingestion/languages/kotlin/captures.js'; +import { collectKotlinCaptureSideChannel } from '../../src/core/ingestion/languages/kotlin/capture-side-channel.js'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; + +const BENCH_ENABLED = process.env.GITNEXUS_BENCH === '1'; + +function denseSpringSource(consumerCount: number): string { + const consumers = Array.from( + { length: consumerCount }, + (_, index) => ` +@Service +class Consumer${index} { + @Autowired private Gateway field${index}; + + Consumer${index}(@Qualifier("gatewayImpl") Gateway gateway) {} + + @Inject void setGateway(Gateway gateway) {} +} +`, + ).join('\n'); + + return `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import jakarta.inject.Inject; + +interface Gateway {} + +@Service +class GatewayImpl implements Gateway {} + +${consumers} +`; +} + +interface CaptureBenchResult { + consumers: number; + elapsedMs: number; + captureCount: number; + factCount: number; +} + +function runCaptureBenchmark(consumerCount: number, run: number): CaptureBenchResult { + const filePath = `src/SpringDiBench${consumerCount}_${run}.java`; + const start = performance.now(); + const captures = emitJavaScopeCaptures(denseSpringSource(consumerCount), filePath); + const elapsedMs = performance.now() - start; + const facts = collectJavaCaptureSideChannel(filePath)?.springDiFacts ?? []; + return { + consumers: consumerCount, + elapsedMs, + captureCount: captures.length, + factCount: facts.length, + }; +} + +function denseKotlinSpringSource(consumerCount: number): string { + const consumers = Array.from( + { length: consumerCount }, + (_, index) => ` +@Service +class Consumer${index} @Autowired constructor( + @param:Qualifier("gatewayImpl") gateway: Gateway, +) { + @field:Autowired lateinit var field${index}: Gateway + @Inject fun setGateway(gateway: Gateway) {} +} +`, + ).join('\n'); + + return `package com.example +import org.springframework.stereotype.Service +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import jakarta.inject.Inject + +interface Gateway + +@Service +class GatewayImpl : Gateway + +${consumers} +`; +} + +function runKotlinCaptureBenchmark(consumerCount: number, run: number): CaptureBenchResult { + const filePath = `src/SpringDiBench${consumerCount}_${run}.kt`; + const start = performance.now(); + const captures = emitKotlinScopeCaptures(denseKotlinSpringSource(consumerCount), filePath); + const elapsedMs = performance.now() - start; + const facts = collectKotlinCaptureSideChannel(filePath)?.springDiFacts ?? []; + return { + consumers: consumerCount, + elapsedMs, + captureCount: captures.length, + factCount: facts.length, + }; +} + +describe('Spring DI capture O(n²) regression tripwire (#2414)', () => { + it('captures a dense 400-consumer file within a coarse linear-time budget', () => { + const consumers = 400; + const budgetMs = 10_000; + + runCaptureBenchmark(4, 0); + const result = runCaptureBenchmark(consumers, 1); + + expect(result.factCount).toBe(consumers + 1); + expect(result.captureCount).toBeGreaterThan(consumers * 10); + expect(result.elapsedMs).toBeLessThan(budgetMs); + }, 30_000); + + it('captures a dense 400-consumer Kotlin file within a coarse linear-time budget', () => { + const consumers = 400; + const budgetMs = 10_000; + + runKotlinCaptureBenchmark(4, 0); + const result = runKotlinCaptureBenchmark(consumers, 1); + + expect(result.factCount).toBe(consumers + 1); + expect(result.captureCount).toBeGreaterThan(consumers * 8); + expect(result.elapsedMs).toBeLessThan(budgetMs); + }, 30_000); +}); + +describe.skipIf(!BENCH_ENABLED)('Spring DI capture scaling benchmark (#2414)', () => { + it('scales sub-quadratically as classes and injection sites grow together', () => { + const scales = [100, 200, 400]; + const repetitions = 4; + const results: CaptureBenchResult[] = []; + + runCaptureBenchmark(8, 0); + for (const consumers of scales) { + let elapsedMs = 0; + let captureCount = 0; + let factCount = 0; + for (let run = 0; run < repetitions; run++) { + const current = runCaptureBenchmark(consumers, run + 1); + elapsedMs += current.elapsedMs; + captureCount = current.captureCount; + factCount = current.factCount; + } + results.push({ consumers, elapsedMs, captureCount, factCount }); + console.log( + ` capture n=${consumers} ×${repetitions}: ${elapsedMs.toFixed(1)}ms ` + + `(${factCount} facts, ${captureCount} captures/run)`, + ); + } + + const first = results[0]; + const last = results[results.length - 1]; + const sizeRatio = last.consumers / first.consumers; + if (first.elapsedMs >= 20) { + const wallRatio = last.elapsedMs / first.elapsedMs; + expect(wallRatio).toBeLessThan(Math.pow(sizeRatio, 1.5)); + } else { + expect(last.elapsedMs).toBeLessThan(10_000); + } + expect(last.factCount).toBe(last.consumers + 1); + }, 120_000); +}); + +describe.skipIf(!BENCH_ENABLED)('Kotlin Spring DI capture scaling benchmark (#2414)', () => { + it('scales sub-quadratically as classes and injection sites grow together', () => { + const scales = [100, 200, 400]; + const repetitions = 4; + const results: CaptureBenchResult[] = []; + + runKotlinCaptureBenchmark(8, 0); + for (const consumers of scales) { + let elapsedMs = 0; + let captureCount = 0; + let factCount = 0; + for (let run = 0; run < repetitions; run++) { + const current = runKotlinCaptureBenchmark(consumers, run + 1); + elapsedMs += current.elapsedMs; + captureCount = current.captureCount; + factCount = current.factCount; + } + results.push({ consumers, elapsedMs, captureCount, factCount }); + console.log( + ` kotlin capture n=${consumers} ×${repetitions}: ${elapsedMs.toFixed(1)}ms ` + + `(${factCount} facts, ${captureCount} captures/run)`, + ); + } + + const first = results[0]; + const last = results[results.length - 1]; + const sizeRatio = last.consumers / first.consumers; + if (first.elapsedMs >= 20) { + const wallRatio = last.elapsedMs / first.elapsedMs; + expect(wallRatio).toBeLessThan(Math.pow(sizeRatio, 1.5)); + } else { + expect(last.elapsedMs).toBeLessThan(10_000); + } + expect(last.factCount).toBe(last.consumers + 1); + }, 120_000); +}); + +function writeSpringDiRepo(consumerCount: number): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `spring-di-bench-${consumerCount}-`)); + fs.writeFileSync( + path.join(dir, 'Gateway.java'), + `package com.example; +public interface Gateway {} +`, + ); + fs.writeFileSync( + path.join(dir, 'GatewayImpl.java'), + `package com.example; +import org.springframework.stereotype.Service; +@Service +public class GatewayImpl implements Gateway {} +`, + ); + for (let index = 0; index < consumerCount; index++) { + fs.writeFileSync( + path.join(dir, `Consumer${index}.java`), + `package com.example; +import org.springframework.stereotype.Service; +@Service +public class Consumer${index} { + public Consumer${index}(Gateway gateway) {} +} +`, + ); + } + return dir; +} + +describe.skipIf(!BENCH_ENABLED)('Spring DI end-to-end scaling benchmark (#2414)', () => { + it('keeps full-pipeline injection resolution sub-quadratic across file counts', async () => { + const scales = [25, 50, 100]; + const results: Array<{ consumers: number; elapsedMs: number; injects: number }> = []; + + for (const consumers of scales) { + const dir = writeSpringDiRepo(consumers); + try { + const start = performance.now(); + const result = await runPipelineFromRepo(dir, () => {}, {}); + const elapsedMs = performance.now() - start; + const injects = [...result.graph.iterRelationshipsByType('INJECTS')].length; + results.push({ consumers, elapsedMs, injects }); + console.log( + ` pipeline n=${consumers}: ${elapsedMs.toFixed(1)}ms (${injects} INJECTS edges)`, + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + } + + for (const result of results) expect(result.injects).toBe(result.consumers); + const first = results[0]; + const last = results[results.length - 1]; + const sizeRatio = last.consumers / first.consumers; + const wallRatio = last.elapsedMs / first.elapsedMs; + expect(wallRatio).toBeLessThan(Math.pow(sizeRatio, 1.5)); + }, 300_000); +}); diff --git a/gitnexus/test/integration/spring-di-pipeline.test.ts b/gitnexus/test/integration/spring-di-pipeline.test.ts index 1b18efcc8..a355ad5f2 100644 --- a/gitnexus/test/integration/spring-di-pipeline.test.ts +++ b/gitnexus/test/integration/spring-di-pipeline.test.ts @@ -41,6 +41,15 @@ public class Consumer { } `; +const WILDCARD_CONSUMER = `package com.example; +import java.util.*; +import org.springframework.beans.factory.annotation.*; + +public class WildcardConsumer { + @Autowired private List foos; +} +`; + /** A consumer whose collection fields carry NO injection annotation. */ const PLAIN_CONSUMER = `package com.example; import java.util.List; @@ -69,6 +78,19 @@ function injectsPairs(result: PipelineResult): string[] { .sort(); } +function injectsDetails(result: PipelineResult) { + const nameById = new Map(); + result.graph.forEachNode((node) => nameById.set(node.id, String(node.properties.name))); + return result.graph.relationships + .filter((relationship) => relationship.type === 'INJECTS') + .map((relationship) => ({ + pair: `${nameById.get(relationship.sourceId)}->${nameById.get(relationship.targetId)}`, + confidence: relationship.confidence, + reason: relationship.reason, + })) + .sort((left, right) => left.pair.localeCompare(right.pair)); +} + describe('Spring DI collection-injection pipeline (#2200)', () => { let dir: string; let result: PipelineResult; @@ -117,6 +139,28 @@ describe('Spring DI collection-injection pipeline (#2200)', () => { }); }); +describe('Spring DI wildcard-import collection fallback (#2200, #2414)', () => { + let dir: string; + let result: PipelineResult; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-spring-di-wildcard-')); + fs.writeFileSync(path.join(dir, 'IFoo.java'), IFOO); + fs.writeFileSync(path.join(dir, 'FooA.java'), FOO_A); + fs.writeFileSync(path.join(dir, 'FooB.java'), FOO_B); + fs.writeFileSync(path.join(dir, 'WildcardConsumer.java'), WILDCARD_CONSUMER); + result = await runPipelineFromRepo(dir, () => {}, {}); + }, 60_000); + + afterAll(() => { + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('preserves collection edges when multiple wildcard imports prevent annotation FQN resolution', () => { + expect(injectsPairs(result)).toEqual(['WildcardConsumer->FooA', 'WildcardConsumer->FooB']); + }); +}); + describe('Spring DI pipeline negative control: no injection annotations anywhere (#2200)', () => { let dir: string; let result: PipelineResult; @@ -140,3 +184,395 @@ describe('Spring DI pipeline negative control: no injection annotations anywhere expect(injectsPairs(result)).toEqual([]); }); }); + +describe('Spring standard injection pipeline (#2414)', () => { + let dir: string; + let result: PipelineResult; + + const sources: Record = { + 'PaymentGateway.java': `package com.example; +public interface PaymentGateway {} +`, + 'FastGateway.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.context.annotation.Primary; +@Service @Primary +public class FastGateway implements PaymentGateway {} +`, + 'SlowGateway.java': `package com.example; +import org.springframework.stereotype.Service; +@Service("slowGateway") +public class SlowGateway implements PaymentGateway {} +`, + 'ConcreteRepo.java': `package com.example; +import org.springframework.stereotype.Repository; +@Repository +public class ConcreteRepo {} +`, + 'S3Client.java': `package com.example; +import org.springframework.stereotype.Service; +@Service +public class S3Client {} +`, + 'DigitBeanNameConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Qualifier; +@Service +public class DigitBeanNameConsumer { + public DigitBeanNameConsumer(@Qualifier("s3Client") S3Client client) {} +} +`, + 'EmptyParenService.java': `package com.example; +import org.springframework.stereotype.Service; +@Service() +public class EmptyParenService {} +`, + 'EmptyParenConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Qualifier; +@Service +public class EmptyParenConsumer { + public EmptyParenConsumer( + @Qualifier("emptyParenService") EmptyParenService service + ) {} +} +`, + 'ConstructorConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +@Service +public class ConstructorConsumer { + public ConstructorConsumer(PaymentGateway gateway, ConcreteRepo repo) {} +} +`, + 'ExplicitConstructorConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +@Service +public class ExplicitConstructorConsumer { + public ExplicitConstructorConsumer() {} + @Autowired public ExplicitConstructorConsumer(ConcreteRepo repo) {} +} +`, + 'QualifiedConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Qualifier; +@Service +public class QualifiedConsumer { + public QualifiedConsumer(@Qualifier("slowGateway") PaymentGateway gateway) {} +} +`, + 'DynamicQualifierConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Qualifier; +@Service +public class DynamicQualifierConsumer { + private static final String GATEWAY = "slowGateway"; + public DynamicQualifierConsumer(@Qualifier(GATEWAY) PaymentGateway gateway) {} +} +`, + 'DynamicCollectionQualifierConsumer.java': `package com.example; +import java.util.List; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +@Service +public class DynamicCollectionQualifierConsumer { + private static final String GATEWAY = "slowGateway"; + @Autowired @Qualifier(GATEWAY) private List gateways; +} +`, + 'PlainConstructorConsumer.java': `package com.example; +public class PlainConstructorConsumer { + public PlainConstructorConsumer(PaymentGateway gateway) {} +} +`, + 'FieldConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +@Service +public class FieldConsumer { + @Autowired private PaymentGateway gateway; +} +`, + 'QualifiedCollectionConsumer.java': `package com.example; +import java.util.List; +import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +@Service +public class QualifiedCollectionConsumer { + @Autowired @Qualifier("slowGateway") private List gateways; +} +`, + 'SetterConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +import jakarta.inject.Inject; +@Service +public class SetterConsumer { + @Inject public void setRepo(ConcreteRepo repo) {} +} +`, + 'Formatter.java': `package com.example; +public interface Formatter {} +`, + 'JsonFormatter.java': `package com.example; +import org.springframework.stereotype.Service; +@Service +public class JsonFormatter implements Formatter {} +`, + 'XmlFormatter.java': `package com.example; +import org.springframework.stereotype.Service; +@Service +public class XmlFormatter implements Formatter {} +`, + 'AmbiguousConsumer.java': `package com.example; +import org.springframework.stereotype.Service; +@Service +public class AmbiguousConsumer { + public AmbiguousConsumer(Formatter formatter) {} +} +`, + }; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-spring-standard-di-')); + for (const [fileName, source] of Object.entries(sources)) { + fs.writeFileSync(path.join(dir, fileName), source); + } + result = await runPipelineFromRepo(dir, () => {}, {}); + }, 60_000); + + afterAll(() => { + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('resolves implicit constructor, concrete, field, setter, qualifier, and primary injection', () => { + const details = injectsDetails(result); + expect(details.map((detail) => detail.pair)).toEqual([ + 'AmbiguousConsumer->JsonFormatter', + 'AmbiguousConsumer->XmlFormatter', + 'ConstructorConsumer->ConcreteRepo', + 'ConstructorConsumer->FastGateway', + 'DigitBeanNameConsumer->S3Client', + 'EmptyParenConsumer->EmptyParenService', + 'ExplicitConstructorConsumer->ConcreteRepo', + 'FieldConsumer->FastGateway', + 'QualifiedCollectionConsumer->SlowGateway', + 'QualifiedConsumer->SlowGateway', + 'SetterConsumer->ConcreteRepo', + ]); + + expect( + details.find((detail) => detail.pair === 'ConstructorConsumer->FastGateway'), + ).toMatchObject({ confidence: 0.95, reason: expect.stringContaining('selected @Primary') }); + expect( + details.find((detail) => detail.pair === 'QualifiedConsumer->SlowGateway'), + ).toMatchObject({ + confidence: 0.95, + reason: expect.stringContaining('qualifier "slowGateway"'), + }); + expect( + details.find((detail) => detail.pair === 'SetterConsumer->ConcreteRepo')?.reason, + ).toContain('@Inject method'); + }); + + it('surfaces unresolved single-bean ambiguity as multiple low-confidence candidates', () => { + const ambiguous = injectsDetails(result).filter((detail) => + detail.pair.startsWith('AmbiguousConsumer->'), + ); + expect(ambiguous).toHaveLength(2); + expect(ambiguous.every((detail) => detail.confidence === 0.5)).toBe(true); + expect(ambiguous.every((detail) => detail.reason.includes('ambiguous candidates'))).toBe(true); + }); + + it('fails closed for unmanaged implicit constructors and unresolved dynamic qualifiers', () => { + const pairs = injectsDetails(result).map((detail) => detail.pair); + expect(pairs.some((pair) => pair.startsWith('PlainConstructorConsumer->'))).toBe(false); + expect(pairs.some((pair) => pair.startsWith('DynamicQualifierConsumer->'))).toBe(false); + expect(pairs.some((pair) => pair.startsWith('DynamicCollectionQualifierConsumer->'))).toBe( + false, + ); + }); +}); + +describe('Kotlin Spring standard injection pipeline (#2414)', () => { + let dir: string; + let result: PipelineResult; + + const sources: Record = { + 'PaymentGateway.kt': `package com.example +interface PaymentGateway +`, + 'FastGateway.kt': `package com.example +import org.springframework.context.annotation.Primary +import org.springframework.stereotype.Service +@Service @Primary +class FastGateway : PaymentGateway +`, + 'SlowGateway.kt': `package com.example +import org.springframework.stereotype.Service +@Service("slowGateway") +class SlowGateway : PaymentGateway +`, + 'ConcreteRepo.kt': `package com.example +import org.springframework.stereotype.Repository +@Repository +class ConcreteRepo +`, + 'ConstructorConsumer.kt': `package com.example +import org.springframework.stereotype.Service +@Service +class ConstructorConsumer( + val gateway: PaymentGateway, + repo: ConcreteRepo?, +) +`, + 'ExplicitConstructorConsumer.kt': `package com.example +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +@Service +class ExplicitConstructorConsumer() { + @Autowired constructor(repo: ConcreteRepo) : this() +} +`, + 'QualifiedConsumer.kt': `package com.example +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.stereotype.Service +@Service +class QualifiedConsumer( + @param:Qualifier("slowGateway") gateway: PaymentGateway, +) +`, + 'NamedConsumer.kt': `package com.example +import jakarta.inject.Named +import org.springframework.stereotype.Service +@Service +class NamedConsumer(@Named("slowGateway") gateway: PaymentGateway) +`, + 'FieldConsumer.kt': `package com.example +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +@Service +class FieldConsumer { + @field:Autowired + lateinit var gateway: PaymentGateway +} +`, + 'QualifiedFieldConsumer.kt': `package com.example +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.stereotype.Service +@Service +class QualifiedFieldConsumer { + @field:Autowired + @field:Qualifier("slowGateway") + lateinit var gateway: PaymentGateway +} +`, + 'SetterPropertyConsumer.kt': `package com.example +import jakarta.inject.Inject +import org.springframework.stereotype.Service +@Service +class SetterPropertyConsumer { + @set:Inject + var repo: ConcreteRepo? = null +} +`, + 'MethodConsumer.kt': `package com.example +import javax.inject.Inject +import org.springframework.stereotype.Service +@Service +class MethodConsumer { + @Inject fun setRepo(repo: ConcreteRepo) {} +} +`, + 'CollectionConsumer.kt': `package com.example +import org.springframework.stereotype.Service +@Service +class CollectionConsumer(val gateways: List) +`, + 'MutableCollectionConsumer.kt': `package com.example +import org.springframework.stereotype.Service +@Service +class MutableCollectionConsumer(val gateways: MutableList?) +`, + 'PlainConstructorConsumer.kt': `package com.example +class PlainConstructorConsumer(gateway: PaymentGateway) +`, + 'MultipleConstructorConsumer.kt': `package com.example +import org.springframework.stereotype.Service +@Service +class MultipleConstructorConsumer(gateway: PaymentGateway) { + constructor(repo: ConcreteRepo) : this(FastGateway()) +} +`, + 'GetterTargetConsumer.kt': `package com.example +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +@Service +class GetterTargetConsumer { + @get:Autowired + var gateway: PaymentGateway? = null +} +`, + 'DynamicQualifierConsumer.kt': `package com.example +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.stereotype.Service +const val GATEWAY = "slowGateway" +@Service +class DynamicQualifierConsumer(@Qualifier(GATEWAY) gateway: PaymentGateway) +`, + }; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-kotlin-spring-standard-di-')); + for (const [fileName, source] of Object.entries(sources)) { + fs.writeFileSync(path.join(dir, fileName), source); + } + result = await runPipelineFromRepo(dir, () => {}, {}); + }, 60_000); + + afterAll(() => { + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('resolves Kotlin primary/secondary constructors, properties, methods, qualifiers, primary, nullable types, and projections', () => { + const details = injectsDetails(result); + expect(details.map((detail) => detail.pair)).toEqual([ + 'CollectionConsumer->FastGateway', + 'CollectionConsumer->SlowGateway', + 'ConstructorConsumer->ConcreteRepo', + 'ConstructorConsumer->FastGateway', + 'ExplicitConstructorConsumer->ConcreteRepo', + 'FieldConsumer->FastGateway', + 'MethodConsumer->ConcreteRepo', + 'MutableCollectionConsumer->FastGateway', + 'MutableCollectionConsumer->SlowGateway', + 'NamedConsumer->SlowGateway', + 'QualifiedConsumer->SlowGateway', + 'QualifiedFieldConsumer->SlowGateway', + 'SetterPropertyConsumer->ConcreteRepo', + ]); + + expect( + details.find((detail) => detail.pair === 'ConstructorConsumer->FastGateway'), + ).toMatchObject({ confidence: 0.95, reason: expect.stringContaining('selected @Primary') }); + expect( + details.find((detail) => detail.pair === 'QualifiedConsumer->SlowGateway'), + ).toMatchObject({ + confidence: 0.95, + reason: expect.stringContaining('qualifier "slowGateway"'), + }); + expect( + details.find((detail) => detail.pair === 'SetterPropertyConsumer->ConcreteRepo')?.reason, + ).toContain('@Inject property'); + }); + + it('fails closed for unmanaged or ambiguous constructors, unsupported getter targets, and dynamic qualifiers', () => { + const pairs = injectsPairs(result); + expect(pairs.some((pair) => pair.startsWith('PlainConstructorConsumer->'))).toBe(false); + expect(pairs.some((pair) => pair.startsWith('MultipleConstructorConsumer->'))).toBe(false); + expect(pairs.some((pair) => pair.startsWith('GetterTargetConsumer->'))).toBe(false); + expect(pairs.some((pair) => pair.startsWith('DynamicQualifierConsumer->'))).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/ai-context.test.ts b/gitnexus/test/unit/ai-context.test.ts index ecf801d2e..cbe3bf457 100644 --- a/gitnexus/test/unit/ai-context.test.ts +++ b/gitnexus/test/unit/ai-context.test.ts @@ -429,6 +429,181 @@ Old content here. } }); + it('mirrors standard skills to .agents/skills/ when .agents/ exists', async () => { + // Some agents prefer repo-local .agents/skills over the global + // ~/.agents/skills install. When the repo contains an .agents/ directory, + // installSkills() must mirror the same SKILL.md files there so those agents + // serve up-to-date repo-specific skills instead of stale global copies. + const agentsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ai-ctx-agents-')); + const agentsStorage = path.join(agentsDir, '.gitnexus'); + await fs.mkdir(agentsStorage, { recursive: true }); + // Opt-in: create the repo-local .agents/ directory. + await fs.mkdir(path.join(agentsDir, '.agents'), { recursive: true }); + try { + const stats = { nodes: 50, edges: 100, processes: 5 }; + const result = await generateAIContextFiles(agentsDir, agentsStorage, 'TestProject', stats); + + // Canonical .claude copy is always written (flat layout, #2434). + expect(result.files).toContain('.claude/skills/gitnexus-*/ (6 skills)'); + // Mirror is reported. + expect(result.files).toContain('.agents/skills/gitnexus-*/ (6 skills mirrored for .agents)'); + + const claudeSkill = await fs.readFile( + path.join(agentsDir, '.claude', 'skills', 'gitnexus-cli', 'SKILL.md'), + 'utf-8', + ); + const agentsSkill = await fs.readFile( + path.join(agentsDir, '.agents', 'skills', 'gitnexus-cli', 'SKILL.md'), + 'utf-8', + ); + expect(agentsSkill).toBe(claudeSkill); + expect(agentsSkill.length).toBeGreaterThan(0); + } finally { + await fs.rm(agentsDir, { recursive: true, force: true }); + } + }); + + it('does not mirror skills to .agents/ when the directory is absent', async () => { + // Without an .agents/ opt-in, only the canonical .claude/skills/ copy is + // written — no .agents/ tree should be created. + const noAgentsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ai-ctx-no-agents-')); + const noAgentsStorage = path.join(noAgentsDir, '.gitnexus'); + await fs.mkdir(noAgentsStorage, { recursive: true }); + try { + const stats = { nodes: 50, edges: 100, processes: 5 }; + const result = await generateAIContextFiles( + noAgentsDir, + noAgentsStorage, + 'TestProject', + stats, + ); + + expect(result.files).toContain('.claude/skills/gitnexus-*/ (6 skills)'); + expect(result.files.some((f) => f.startsWith('.agents/skills/'))).toBe(false); + await expect(fs.access(path.join(noAgentsDir, '.agents'))).rejects.toThrow(); + } finally { + await fs.rm(noAgentsDir, { recursive: true, force: true }); + } + }); + + it('keeps canonical skills intact when .agents/skills is a file (mirror mkdir fails)', async () => { + // MEDIUM 1 (standard-skill half): when .agents/skills exists as a regular + // file, the per-skill mirror mkdir fails. The failure must be warned per + // skill and canonical .claude/skills/ must still hold all 6 skills. + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const agentsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ai-ctx-agents-file-')); + const agentsStorage = path.join(agentsDir, '.gitnexus'); + await fs.mkdir(agentsStorage, { recursive: true }); + await fs.mkdir(path.join(agentsDir, '.agents'), { recursive: true }); + // .agents/skills is a file — per-skill mirror mkdir will EEXIST. + await fs.writeFile(path.join(agentsDir, '.agents', 'skills'), 'not a directory'); + try { + const stats = { nodes: 50, edges: 100, processes: 5 }; + const result = await generateAIContextFiles(agentsDir, agentsStorage, 'TestProject', stats); + + // Canonical 6 skills are all present. + expect(result.files).toContain('.claude/skills/gitnexus-*/ (6 skills)'); + for (const name of [ + 'gitnexus-exploring', + 'gitnexus-debugging', + 'gitnexus-impact-analysis', + 'gitnexus-refactoring', + 'gitnexus-guide', + 'gitnexus-cli', + ]) { + await expect( + fs.readFile(path.join(agentsDir, '.claude', 'skills', name, 'SKILL.md'), 'utf-8'), + ).resolves.toHaveProperty('length'); + } + // Mirror failures were warned, not thrown. + const warned = cap.records().some((r) => r.level === 40); // pino warn level + expect(warned).toBe(true); + } finally { + cap.restore(); + await fs.rm(agentsDir, { recursive: true, force: true }); + } + }); + + it('does not mirror and does not create .agents/ when .agents is a file (gate is false)', async () => { + // The gate checks isDirectory(); a file at .agents must NOT trigger + // mirroring and must not throw. + const fileDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ai-ctx-agents-filegate-')); + const fileStorage = path.join(fileDir, '.gitnexus'); + await fs.mkdir(fileStorage, { recursive: true }); + await fs.writeFile(path.join(fileDir, '.agents'), 'not a directory'); + try { + const stats = { nodes: 50, edges: 100, processes: 5 }; + const result = await generateAIContextFiles(fileDir, fileStorage, 'TestProject', stats); + + expect(result.files).toContain('.claude/skills/gitnexus-*/ (6 skills)'); + expect(result.files.some((f) => f.startsWith('.agents/skills/'))).toBe(false); + } finally { + await fs.rm(fileDir, { recursive: true, force: true }); + } + }); + + it('is idempotent across repeated runs (no duplicates, stable mirror content)', async () => { + const agentsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ai-ctx-idem-')); + const agentsStorage = path.join(agentsDir, '.gitnexus'); + await fs.mkdir(agentsStorage, { recursive: true }); + await fs.mkdir(path.join(agentsDir, '.agents'), { recursive: true }); + try { + const stats = { nodes: 50, edges: 100, processes: 5 }; + await generateAIContextFiles(agentsDir, agentsStorage, 'TestProject', stats); + const first = await fs.readFile( + path.join(agentsDir, '.agents', 'skills', 'gitnexus-cli', 'SKILL.md'), + 'utf-8', + ); + + // Second run — must not duplicate or corrupt. + await generateAIContextFiles(agentsDir, agentsStorage, 'TestProject', stats); + const second = await fs.readFile( + path.join(agentsDir, '.agents', 'skills', 'gitnexus-cli', 'SKILL.md'), + 'utf-8', + ); + expect(second).toBe(first); + + // Mirror tree has exactly one dir per standard skill (no duplicates). + const entries = await fs.readdir(path.join(agentsDir, '.agents', 'skills'), { + withFileTypes: true, + }); + const skillDirs = entries.filter((e) => e.isDirectory()).map((e) => e.name); + expect(skillDirs).toContain('gitnexus-cli'); + expect(skillDirs.filter((n) => n === 'gitnexus-cli')).toHaveLength(1); + } finally { + await fs.rm(agentsDir, { recursive: true, force: true }); + } + }); + + it('does not mirror standard skills when --skip-skills is set', async () => { + const skipDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-ai-ctx-skip-mirror-')); + const skipStorage = path.join(skipDir, '.gitnexus'); + await fs.mkdir(skipStorage, { recursive: true }); + await fs.mkdir(path.join(skipDir, '.agents'), { recursive: true }); + try { + const stats = { nodes: 50, edges: 100, processes: 5 }; + const result = await generateAIContextFiles( + skipDir, + skipStorage, + 'TestProject', + stats, + undefined, + { + skipSkills: true, + }, + ); + + expect(result.files).toContain('.claude/skills/gitnexus-*/ (skipped via --skip-skills)'); + expect(result.files.some((f) => f.startsWith('.agents/skills/'))).toBe(false); + await expect( + fs.access(path.join(skipDir, '.agents', 'skills', 'gitnexus-cli')), + ).rejects.toThrow(); + } finally { + await fs.rm(skipDir, { recursive: true, force: true }); + } + }); + it('writes nothing when both skipAgentsMd and skipSkills are true (--index-only, #742)', async () => { // Regression guard for #742. analyzeCommand() resolves --index-only // into BOTH skipAgentsMd=true and skipSkills=true. This test pins diff --git a/gitnexus/test/unit/analyze-heap-respawn.test.ts b/gitnexus/test/unit/analyze-heap-respawn.test.ts index e384e460c..53b684f9b 100644 --- a/gitnexus/test/unit/analyze-heap-respawn.test.ts +++ b/gitnexus/test/unit/analyze-heap-respawn.test.ts @@ -15,8 +15,9 @@ vi.mock('v8', () => ({ }, })); -// Pin physical RAM to 16GB so the RAM-aware auto-cap (0.75 x RAM, clamped -// >= 16384) resolves deterministically to 16384 regardless of the host machine. +// Pin physical RAM to 16GB so the RAM-aware auto-cap (floor raised to 16384 +// but capped at 0.80 x RAM, #2649) resolves deterministically to 13107 +// regardless of the host machine. vi.mock('os', async () => { const actual = await vi.importActual('os'); const mocked = { ...actual, totalmem: () => 16 * 1024 * 1024 * 1024 }; @@ -75,6 +76,7 @@ describe('analyzeCommand heap respawn', () => { beforeEach(() => { initialNodeOptions = process.env.NODE_OPTIONS; + delete process.env.GITNEXUS_MEMORY; vi.resetModules(); spawnMock.mockReset(); getHeapStatisticsMock.mockReset(); @@ -114,9 +116,9 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).toHaveBeenCalledTimes(1); const [, args, opts] = spawnMock.mock.calls[0]; - expect(args).toContain('--max-old-space-size=16384'); + expect(args).toContain('--max-old-space-size=13107'); expect(args).toContain('--max-semi-space-size=128'); - expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=16384'); + expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=13107'); expect(opts.env.NODE_OPTIONS).toContain('--max-semi-space-size=128'); expect(opts.env.GITNEXUS_RESPAWN_PROGRESS_TTY).toBe('1'); }); @@ -146,6 +148,129 @@ describe('analyzeCommand heap respawn', () => { expect(spawnMock).not.toHaveBeenCalled(); }); + it('re-execs with the auto cap when ambient NODE_OPTIONS pins a smaller heap (#2649)', async () => { + process.env.NODE_OPTIONS = '--max-old-space-size=4096'; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 4096 * 1024 * 1024 }); + mockSpawnExit(); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + cap.restore(); + + expect(spawnMock).toHaveBeenCalledTimes(1); + const [, args, opts] = spawnMock.mock.calls[0]; + expect(args).toContain('--max-old-space-size=13107'); + // The auto flag is appended after the ambient value, so V8's + // later-flag-wins semantics resolve to the larger cap. + expect(opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=13107')).toBeGreaterThan( + opts.env.NODE_OPTIONS.indexOf('--max-old-space-size=4096'), + ); + const warn = cap.records().find((r) => r.msg.includes('pins the heap to 4096MB')); + expect(warn?.msg).toContain('Re-running analyze with the larger auto-sized cap'); + }); + + it('honors GITNEXUS_MEMORY=off: keeps the small ambient heap, silently (#2649)', async () => { + process.env.NODE_OPTIONS = '--max-old-space-size=4096'; + process.env.GITNEXUS_MEMORY = 'off'; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 4096 * 1024 * 1024 }); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand('/__gitnexus_nonexistent__', {}); + cap.restore(); + + expect(spawnMock).not.toHaveBeenCalled(); + // Explicit opt-out stays quiet: stderr-sensitive consumers (e2e + // harnesses, scripts) rely on no extra warning here. + const warns = cap.records().filter((r) => r.msg.includes('pins the heap')); + expect(warns).toEqual([]); + }); + + it('preserves parent execArgv (e.g. a tsx loader) in the respawned child argv (#2649)', async () => { + delete process.env.NODE_OPTIONS; + restoreStderrIsTTY = setStreamIsTTY(process.stderr, true); + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + mockSpawnExit(); + + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + expect(spawnMock).toHaveBeenCalledTimes(1); + const [, args] = spawnMock.mock.calls[0]; + // The child argv must start with the parent's node flags so + // loader-launched CLIs (node --import tsx src/cli/index.ts) survive the + // respawn; our heap flags follow and win via later-flag-wins. + expect(args.slice(0, process.execArgv.length)).toEqual(process.execArgv); + }); + + it('parseMaxOldSpaceMb: last occurrence wins, absent and malformed values are null', async () => { + const { parseMaxOldSpaceMb } = await import('../../src/cli/analyze.js'); + expect(parseMaxOldSpaceMb('--max-old-space-size=4096 --max-old-space-size=8192')).toBe(8192); + expect(parseMaxOldSpaceMb('--max-semi-space-size=128')).toBeNull(); + expect(parseMaxOldSpaceMb('')).toBeNull(); + expect(parseMaxOldSpaceMb('--max-old-space-size=0')).toBeNull(); + // V8 treats - and _ interchangeably in flag names, and Node accepts a + // space-separated value in NODE_OPTIONS; every spelling of the pin must + // be honored instead of silently overridden. + expect(parseMaxOldSpaceMb('--max_old_space_size=4096')).toBe(4096); + expect(parseMaxOldSpaceMb('--max-old-space-size 4096')).toBe(4096); + expect(parseMaxOldSpaceMb('--max-old-space-size --other-flag')).toBeNull(); + }); + + it('GITNEXUS_MEMORY=off also disables the default (unpinned) respawn (#2649 review)', async () => { + delete process.env.NODE_OPTIONS; + process.env.GITNEXUS_MEMORY = 'off'; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand('/__gitnexus_nonexistent__', {}); + + expect(spawnMock).not.toHaveBeenCalled(); + }); + + it('an explicit per-invocation execArgv heap flag always wins (no respawn)', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + const execArgvDesc = Object.getOwnPropertyDescriptor(process, 'execArgv'); + Object.defineProperty(process, 'execArgv', { + configurable: true, + value: ['--max-old-space-size=2048'], + }); + try { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand('/__gitnexus_nonexistent__', {}); + expect(spawnMock).not.toHaveBeenCalled(); + } finally { + if (execArgvDesc) Object.defineProperty(process, 'execArgv', execArgvDesc); + } + }); + + it('does not replay --inspect flags into the respawned child (debug-port clash)', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + mockSpawnExit(); + const execArgvDesc = Object.getOwnPropertyDescriptor(process, 'execArgv'); + Object.defineProperty(process, 'execArgv', { + configurable: true, + value: ['--inspect', '--inspect-brk=9230', '--enable-source-maps'], + }); + try { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + expect(spawnMock).toHaveBeenCalledTimes(1); + const [, args] = spawnMock.mock.calls[0]; + expect({ + inspectFlags: args.filter((a: string) => a.startsWith('--inspect')), + keepsOtherFlags: args.includes('--enable-source-maps'), + }).toEqual({ inspectFlags: [], keepsOtherFlags: true }); + } finally { + if (execArgvDesc) Object.defineProperty(process, 'execArgv', execArgvDesc); + } + }); + it('prints heap guidance when respawned analyze exits with likely OOM', async () => { delete process.env.NODE_OPTIONS; getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); @@ -164,7 +289,7 @@ describe('analyzeCommand heap respawn', () => { .find((r) => r.msg.includes('Analysis likely ran out of memory')); expect(oomGuidance).toBeDefined(); const msg = oomGuidance?.msg ?? ''; - expect(msg).toContain('auto-sized to 16384MB'); + expect(msg).toContain('auto-sized to 13107MB'); expect(msg).toContain('NODE_OPTIONS="--max-old-space-size="'); expect(msg).toContain('[your-args]'); expect(msg).toContain('native crash unrelated to heap size'); @@ -289,10 +414,22 @@ describe('computeHeapCapMb (RAM-aware auto heap cap)', () => { expect(computeHeapCapMb(31 * GB, null)).toBe(23808); }); - it('clamps to the 16384 floor on small boxes', async () => { + it('keeps the cap below RAM on small boxes instead of the old >=RAM floor (#2649)', async () => { const { computeHeapCapMb } = await import('../../src/cli/analyze.js'); - // 8GB -> 0.75 * 8192 = 6144 -> clamped to 16384 - expect(computeHeapCapMb(8 * GB, null)).toBe(16384); + // 8GB -> floor wins the max (16384) but is capped to 0.80 * 8192 = 6553 + expect(computeHeapCapMb(8 * GB, null)).toBe(6553); + }); + + it('caps a 16GB box at 0.80x RAM, below physical memory (#2649)', async () => { + const { computeHeapCapMb } = await import('../../src/cli/analyze.js'); + // 16GB -> max(16384, 12288) = 16384 -> min(16384, floor(0.80 * 16384)) = 13107 + expect(computeHeapCapMb(16 * GB, null)).toBe(13107); + }); + + it('lets the 0.75x rule win once RAM clears the floor region', async () => { + const { computeHeapCapMb } = await import('../../src/cli/analyze.js'); + // 24GB -> max(16384, 18432) = 18432 -> min(18432, 19660) = 18432 + expect(computeHeapCapMb(24 * GB, null)).toBe(18432); }); it('ignores the unconstrained sentinel from constrainedMemory()', async () => { @@ -303,8 +440,16 @@ describe('computeHeapCapMb (RAM-aware auto heap cap)', () => { it('honors a real cgroup cap smaller than physical RAM', async () => { const { computeHeapCapMb } = await import('../../src/cli/analyze.js'); - // min(31, 12) = 12GB -> 0.75 * 12288 = 9216 -> clamped to 16384 - expect(computeHeapCapMb(31 * GB, 12 * GB)).toBe(16384); + // min(31, 12) = 12GB effective -> capped to 0.80 * 12288 = 9830, not the 16384 floor + expect(computeHeapCapMb(31 * GB, 12 * GB)).toBe(9830); + }); + + it('never returns a cap at or above effective RAM', async () => { + const { computeHeapCapMb } = await import('../../src/cli/analyze.js'); + const ramsGb = [4, 8, 12, 16, 20, 24, 32, 48, 64]; + const caps = ramsGb.map((gb) => computeHeapCapMb(gb * GB, null)); + const belowRam = caps.map((cap, i) => cap < ramsGb[i] * 1024); + expect(belowRam).toEqual(ramsGb.map(() => true)); }); it('uses a large cgroup cap when it exceeds the floor', async () => { diff --git a/gitnexus/test/unit/analyze-self-commit-bridge.test.ts b/gitnexus/test/unit/analyze-self-commit-bridge.test.ts new file mode 100644 index 000000000..d8c69dcb9 --- /dev/null +++ b/gitnexus/test/unit/analyze-self-commit-bridge.test.ts @@ -0,0 +1,203 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + runFullAnalysisMock, + generateAIContextFilesMock, + generateSkillFilesMock, + cliErrorMock, + selfCommitContextFilesMock, + snapshotSelfCommitSafetyMock, +} = vi.hoisted(() => { + const runFullAnalysisMock = vi.fn(); + const generateAIContextFilesMock = vi.fn(async () => ({ files: [] as string[] })); + const generateSkillFilesMock = vi.fn(async () => ({ + skills: [{ name: 'c', label: 'Community', symbolCount: 1, fileCount: 1 }], + outputPath: '/repo/.claude/skills', + })); + const cliErrorMock = vi.fn(); + const selfCommitContextFilesMock = vi.fn(); + const snapshotSelfCommitSafetyMock = vi.fn( + () => + new Map([ + ['AGENTS.md', true], + ['CLAUDE.md', true], + ]), + ); + return { + runFullAnalysisMock, + generateAIContextFilesMock, + generateSkillFilesMock, + cliErrorMock, + selfCommitContextFilesMock, + snapshotSelfCommitSafetyMock, + }; +}); + +vi.mock('../../src/core/run-analyze.js', () => ({ + runFullAnalysis: runFullAnalysisMock, +})); + +vi.mock('../../src/cli/ai-context.js', () => ({ + generateAIContextFiles: generateAIContextFilesMock, +})); + +vi.mock('../../src/cli/skill-gen.js', () => ({ + generateSkillFiles: generateSkillFilesMock, +})); + +vi.mock('../../src/cli/cli-message.js', () => ({ + cliError: cliErrorMock, +})); + +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + closeLbug: vi.fn(async () => undefined), + closeLbugBeforeExit: vi.fn(async () => undefined), + isLbugReady: vi.fn(() => false), +})); + +vi.mock('../../src/storage/repo-manager.js', () => ({ + getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + getGlobalRegistryPath: vi.fn(() => 'registry.json'), + RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, + AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, + assertAnalysisFinalized: vi.fn(async () => undefined), +})); + +vi.mock('../../src/storage/git.js', () => ({ + getGitRoot: vi.fn(() => '/repo'), + hasGitDir: vi.fn(() => true), + getDefaultBranch: vi.fn(() => null), + selfCommitContextFiles: selfCommitContextFilesMock, + snapshotSelfCommitSafety: snapshotSelfCommitSafetyMock, +})); + +vi.mock('../../src/core/ingestion/utils/max-file-size.js', () => ({ + getMaxFileSizeBannerMessage: vi.fn(() => null), +})); + +describe('analyzeCommand --self-commit bridge (#2639)', () => { + beforeEach(() => { + vi.resetModules(); + runFullAnalysisMock.mockReset(); + runFullAnalysisMock.mockResolvedValue({ + repoName: 'repo', + repoPath: '/repo', + stats: {}, + alreadyUpToDate: true, + }); + generateAIContextFilesMock.mockReset(); + generateAIContextFilesMock.mockResolvedValue({ files: [] }); + generateSkillFilesMock.mockReset(); + generateSkillFilesMock.mockResolvedValue({ + skills: [{ name: 'c', label: 'Community', symbolCount: 1, fileCount: 1 }], + outputPath: '/repo/.claude/skills', + }); + cliErrorMock.mockReset(); + selfCommitContextFilesMock.mockReset(); + snapshotSelfCommitSafetyMock.mockClear(); + process.exitCode = undefined; + process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=8192`.trim(); + }); + + it('does not call selfCommitContextFiles when --self-commit is omitted (default off)', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, {}); + + expect(selfCommitContextFilesMock).not.toHaveBeenCalled(); + }); + + it('does not call selfCommitContextFiles when --self-commit is explicitly false', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, { selfCommit: false }); + + expect(selfCommitContextFilesMock).not.toHaveBeenCalled(); + }); + + it('calls selfCommitContextFiles scoped to AGENTS.md/CLAUDE.md on the already-up-to-date fast path', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, { selfCommit: true }); + + expect(selfCommitContextFilesMock).toHaveBeenCalledTimes(1); + expect(selfCommitContextFilesMock).toHaveBeenCalledWith( + '/repo', + ['AGENTS.md', 'CLAUDE.md'], + expect.any(Map), + ); + }); + + it('calls selfCommitContextFiles on the primary (non-fast-path) analyze run', async () => { + runFullAnalysisMock.mockResolvedValueOnce({ + repoName: 'repo', + repoPath: '/repo', + stats: { + files: 1, + nodes: 10, + edges: 20, + communities: 0, + processes: 5, + }, + alreadyUpToDate: false, + pipelineResult: { communityResult: undefined }, + }); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + try { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, { selfCommit: true }); + + expect(selfCommitContextFilesMock).toHaveBeenCalledTimes(1); + expect(selfCommitContextFilesMock).toHaveBeenCalledWith( + '/repo', + ['AGENTS.md', 'CLAUDE.md'], + expect.any(Map), + ); + } finally { + exitSpy.mockRestore(); + } + }); + + it('does not call selfCommitContextFiles on the primary run when --self-commit is omitted', async () => { + runFullAnalysisMock.mockResolvedValueOnce({ + repoName: 'repo', + repoPath: '/repo', + stats: { + files: 1, + nodes: 10, + edges: 20, + communities: 0, + processes: 5, + }, + alreadyUpToDate: false, + pipelineResult: { communityResult: undefined }, + }); + + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + try { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, {}); + + expect(selfCommitContextFilesMock).not.toHaveBeenCalled(); + } finally { + exitSpy.mockRestore(); + } + }); + + it('composes with --no-stats (both flags threaded independently)', async () => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, { selfCommit: true, stats: false }); + + const opts = runFullAnalysisMock.mock.calls[0][1]; + expect(opts.noStats).toBe(true); + expect(selfCommitContextFilesMock).toHaveBeenCalledWith( + '/repo', + ['AGENTS.md', 'CLAUDE.md'], + expect.any(Map), + ); + }); +}); diff --git a/gitnexus/test/unit/analyze-worker-core.test.ts b/gitnexus/test/unit/analyze-worker-core.test.ts index f63ad08da..06e3acd35 100644 --- a/gitnexus/test/unit/analyze-worker-core.test.ts +++ b/gitnexus/test/unit/analyze-worker-core.test.ts @@ -20,6 +20,7 @@ import { import type { AnalyzeResult } from '../../src/core/run-analyze.js'; import type { WorkerMessage } from '../../src/server/analyze-worker.js'; import type { AnalyzerRunnerIdentity } from '../../src/storage/repo-manager.js'; +import { IndexLockTimeoutError, type LockRecord } from '../../src/storage/index-lock.js'; const baseResult: AnalyzeResult = { repoName: 'repo', @@ -121,6 +122,37 @@ describe('runWorkerAnalysis — finalize guard (#2264 P2)', () => { expect(send).toHaveBeenCalledWith({ type: 'error', message: 'boom' }); expect(finalize).not.toHaveBeenCalled(); }); + + it('tags an index-lock timeout as a retryable index-lock-timeout error (#2658 review M2)', async () => { + const send = vi.fn<(msg: WorkerMessage) => void>(); + const holder: LockRecord = { + v: 1, + pid: -1, + hostname: 'host', + startTime: null, + token: '', + invocationId: 'unknown', + acquiredAt: '', + }; + const lockContended: WorkerAnalysisDeps['runFullAnalysis'] = vi.fn(async () => { + throw new IndexLockTimeoutError(holder, 600_000, false); + }); + + await runWorkerAnalysis( + '/repo', + {}, + { + runFullAnalysis: lockContended, + assertAnalysisFinalized: okFinalize, + send, + claimTerminal: alwaysClaim, + }, + ); + + expect(send).toHaveBeenCalledWith( + expect.objectContaining({ type: 'error', code: 'index-lock-timeout', retryable: true }), + ); + }); }); describe('runWorkerAnalysis — terminal-claim coordination (#2264 P3)', () => { diff --git a/gitnexus/test/unit/analyzer-identity-is-inside.test.ts b/gitnexus/test/unit/analyzer-identity-is-inside.test.ts new file mode 100644 index 000000000..6bb3492a5 --- /dev/null +++ b/gitnexus/test/unit/analyzer-identity-is-inside.test.ts @@ -0,0 +1,56 @@ +/** + * `isInside` containment guard — cross-drive Windows correctness. + * + * `path.relative` cannot express a relative path between two Windows drives, so + * it returns the absolute target. Without an `isAbsolute` rejection the `..` + * checks alone classify an unrelated drive as *inside* the parent, which in this + * module meant `resolveInvokedArtifact` adopting an out-of-tree file as the + * invoked analyzer artifact — that file is then absent from the validated build + * and identity resolution throws, so `analyze`/`status` fail outright on a + * multi-drive Windows install. + * + * The `pathApi` argument makes the win32 semantics testable from a POSIX runner, + * so these assertions are meaningful on every CI platform (no fixture, no fs). + */ +import { describe, it, expect } from 'vitest'; +import path from 'node:path'; +import { _isInsideForTests as isInside } from '../../src/core/analyzer-identity.js'; + +describe('isInside — Windows cross-drive containment', () => { + it('rejects a candidate on a different drive', () => { + // Regression: path.win32.relative returns 'D:\\...' here, which does not + // start with '..', so the pre-fix guard reported this as inside. + expect(path.win32.relative('C:\\parent\\src', 'D:\\other\\file.js')).toBe('D:\\other\\file.js'); + expect(isInside('C:\\parent\\src', 'D:\\other\\file.js', path.win32)).toBe(false); + }); + + it('still accepts real containment on the same drive', () => { + expect(isInside('C:\\parent\\src', 'C:\\parent\\src', path.win32)).toBe(true); + expect(isInside('C:\\parent\\src', 'C:\\parent\\src\\core\\a.js', path.win32)).toBe(true); + }); + + it('still rejects a sibling escape on the same drive', () => { + expect(isInside('C:\\parent\\src', 'C:\\parent\\other\\a.js', path.win32)).toBe(false); + expect(isInside('C:\\parent\\src', 'C:\\parent', path.win32)).toBe(false); + }); + + it('is case- and separator-tolerant for a genuine child (win32 semantics)', () => { + // win32 path.relative is case-insensitive on the drive letter. + expect(isInside('C:\\parent', 'c:\\parent\\child.js', path.win32)).toBe(true); + }); + + it('keeps POSIX behavior unchanged', () => { + expect(isInside('/parent/src', '/parent/src/core/a.js', path.posix)).toBe(true); + expect(isInside('/parent/src', '/parent/src', path.posix)).toBe(true); + expect(isInside('/parent/src', '/parent/other/a.js', path.posix)).toBe(false); + expect(isInside('/parent/src', '/parent', path.posix)).toBe(false); + // POSIX has no drive concept, so an unrelated root is expressed with '..'. + expect(isInside('/parent/src', '/elsewhere/file.js', path.posix)).toBe(false); + }); + + it('defaults to the platform-bound path module', () => { + const parent = path.resolve('parent'); + expect(isInside(parent, path.join(parent, 'child.js'))).toBe(true); + expect(isInside(parent, path.resolve('sibling', 'child.js'))).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/analyzer-identity-path-normalization.test.ts b/gitnexus/test/unit/analyzer-identity-path-normalization.test.ts new file mode 100644 index 000000000..d88f3d4c1 --- /dev/null +++ b/gitnexus/test/unit/analyzer-identity-path-normalization.test.ts @@ -0,0 +1,70 @@ +/** + * #2668 path-normalization guard — split out of `analyzer-identity.test.ts` so it + * can run on the Windows/macOS matrix. + * + * `normalizeAnalyzerRootPath` is a POSIX no-op, so these assertions only bite on + * windows-latest; the file is registered in `scripts/cross-platform-tests.ts` for + * exactly that reason. It is deliberately separate from `analyzer-identity.test.ts`, + * and holds ONLY pure-function assertions that pass an explicit `platform` argument + * — no fixture, no filesystem. That restriction is load-bearing: the fixture-based + * identity tests cannot run on this matrix, for two independent reasons observed in + * CI on this PR — + * - macOS: they compare identity fields against the raw temp-dir path while the + * identity realpaths it, so `/var/...` comes back as `/private/var/...`; + * - Windows: the GH runner puts the repo on `D:` and temp fixtures on `C:`, and + * `isInside()` misjudges cross-drive paths (`path.win32.relative` returns the + * absolute target, which does not start with `..`), so `resolveInvokedArtifact` + * picks the vitest fork worker and identity resolution throws. + * Keep this file fixture-free so it stays green on every runner. + */ +import { describe, it, expect } from 'vitest'; +import { normalizeAnalyzerRootPath } from '../../src/core/analyzer-identity.js'; + +// #2668: `status` reported a freshly-analyzed, untouched repo as stale on +// Windows because `build.rootPath` (and the other compared identity path +// fields) carried the drive-letter case that `realpathSync.native` did not +// normalize — so `analyze` and `status`, launched under different casing, +// stamped vs recomputed unequal identities. `normalizeAnalyzerRootPath` +// collapses that variance at the single upstream source (resolveBuildRoot). +describe('normalizeAnalyzerRootPath (#2668)', () => { + it('uppercases a Windows drive letter so case-variant roots collapse', () => { + expect(normalizeAnalyzerRootPath('c:\\gitnexus\\dist', 'win32')).toBe('C:\\gitnexus\\dist'); + expect(normalizeAnalyzerRootPath('c:\\gitnexus\\dist', 'win32')).toBe( + normalizeAnalyzerRootPath('C:\\gitnexus\\dist', 'win32'), + ); + // Forward-slash drive form (as some resolvers emit) is normalized too. + expect(normalizeAnalyzerRootPath('d:/build', 'win32')).toBe('D:/build'); + }); + + it('is idempotent and leaves an already-uppercase drive unchanged', () => { + expect(normalizeAnalyzerRootPath('C:\\build', 'win32')).toBe('C:\\build'); + expect( + normalizeAnalyzerRootPath(normalizeAnalyzerRootPath('c:\\build', 'win32'), 'win32'), + ).toBe('C:\\build'); + }); + + it('only touches a leading drive letter, not other path bytes', () => { + // A UNC path has no drive letter; interior case is preserved. + expect(normalizeAnalyzerRootPath('\\\\server\\Share\\Repo', 'win32')).toBe( + '\\\\server\\Share\\Repo', + ); + expect(normalizeAnalyzerRootPath('C:\\Repo\\subDir', 'win32')).toBe('C:\\Repo\\subDir'); + }); + + it('normalizes the drive under an extended-length \\\\?\\ prefix, preserving the prefix', () => { + expect(normalizeAnalyzerRootPath('\\\\?\\c:\\gitnexus\\dist', 'win32')).toBe( + '\\\\?\\C:\\gitnexus\\dist', + ); + // Extended UNC form has no drive letter — left untouched. + expect(normalizeAnalyzerRootPath('\\\\?\\UNC\\server\\Share', 'win32')).toBe( + '\\\\?\\UNC\\server\\Share', + ); + }); + + it('is a no-op on POSIX (case-sensitive paths must not be mutated)', () => { + expect(normalizeAnalyzerRootPath('/home/user/gitnexus/dist', 'linux')).toBe( + '/home/user/gitnexus/dist', + ); + expect(normalizeAnalyzerRootPath('/Home/User/Dist', 'darwin')).toBe('/Home/User/Dist'); + }); +}); diff --git a/gitnexus/test/unit/analyzer-identity.test.ts b/gitnexus/test/unit/analyzer-identity.test.ts index 3ce5ff0a1..98be37acb 100644 --- a/gitnexus/test/unit/analyzer-identity.test.ts +++ b/gitnexus/test/unit/analyzer-identity.test.ts @@ -11,6 +11,7 @@ import { analyzerRunnerIdentitiesEqual, captureAnalyzerIdentityBeforeLoad, finalizeAnalyzerRunnerIdentity, + normalizeAnalyzerRootPath, normalizeAnalyzerRunnerIdentityForComparison, resolveAnalyzerRunnerIdentity, } from '../../src/core/analyzer-identity.js'; @@ -1508,3 +1509,41 @@ describe('analyzer runner identity', () => { } }, 300_000); }); + +// #2668 threading guard: the produced identity's path fields must already be +// normalizer-stable, i.e. resolveBuildRoot/resolveRuntimeVariant actually route +// build.rootPath and runtime.executablePath through normalizeAnalyzerRootPath. +// Ubuntu-only by necessity: this needs a real fixture identity, and the fixture +// harness cannot run on the Windows matrix (the runner's repo is on D: while temp +// is on C:, and isInside() misjudges cross-drive paths so resolveInvokedArtifact +// picks the vitest fork worker). The pure-transform assertions that DO run on +// windows-latest live in analyzer-identity-path-normalization.test.ts. +describe('analyzer identity path threading (#2668)', () => { + it('produces identity path fields that are already normalizer-stable', async () => { + const fixture = await createTempDir(); + try { + const sourceRoot = path.join(fixture.dbPath, 'src'); + const modulePath = path.join(sourceRoot, 'core', 'analyzer.ts'); + await mkdir(path.dirname(modulePath), { recursive: true }); + await writeFile( + path.join(fixture.dbPath, 'package.json'), + '{"name":"fixture-analyzer","version":"1.0.0"}\n', + ); + await writeFile(path.join(fixture.dbPath, 'package-lock.json'), '{"lockfileVersion":3}\n'); + await writeFile(modulePath, 'export const analyzer = 1;\n'); + + const identity = resolveAnalyzerRunnerIdentity(pathToFileURL(modulePath).href, { + cacheDirectory: path.join(fixture.dbPath, 'identity-cache'), + }); + + expect(identity.build.rootPath).toBe( + normalizeAnalyzerRootPath(identity.build.rootPath, process.platform), + ); + expect(identity.runtime.executablePath).toBe( + normalizeAnalyzerRootPath(identity.runtime.executablePath, process.platform), + ); + } finally { + await fixture.cleanup(); + } + }); +}); diff --git a/gitnexus/test/unit/call-summary-schema-version.test.ts b/gitnexus/test/unit/call-summary-schema-version.test.ts index 90e252c66..ae4ac2765 100644 --- a/gitnexus/test/unit/call-summary-schema-version.test.ts +++ b/gitnexus/test/unit/call-summary-schema-version.test.ts @@ -73,8 +73,8 @@ describe('CALL_SUMMARY relation-type exclusion (U-C1)', () => { }); describe('incremental schema reuse gate', () => { - it('INCREMENTAL_SCHEMA_VERSION is bumped to 15 (persisted Move Type/field graph)', () => { - expect(INCREMENTAL_SCHEMA_VERSION).toBe(15); + it('INCREMENTAL_SCHEMA_VERSION is bumped to 16 (main-aptos sync unifying the forked 12–15 range)', () => { + expect(INCREMENTAL_SCHEMA_VERSION).toBe(16); }); it('a pre-current stamp fails the `=== INCREMENTAL_SCHEMA_VERSION` reuse gate → forces full re-analyze', () => { @@ -116,18 +116,18 @@ describe('incremental schema reuse gate', () => { // (#2604) — abstract trait methods would keep being uncaptured (no // ownerId/CALLS resolution) on unchanged Rust trait files → must NOT reuse. expect(passesReuseGate(10)).toBe(false); - // A pre-v12 (v11) index predates the main-aptos merge — it lacks the Move - // `attributesJson` node-table column, so the bulk COPY referencing it - // would fail on a top-up → must NOT reuse. + // Stamps 11–15 span the forked main/main-aptos range: on either lineage + // they predate rebuild reasons from the other (aptos: Move attributesJson + // column, generic Type nodes, EnumVariant/field USES_TYPE; main: #2514 + // Rust range-binding, #2562 Java local-type identities, #2563 C#/Kotlin + // instance-ownership gate, #2687 const-arrow twin removal), and a stamp + // in 12–15 is ambiguous between the lineages → must NOT reuse. expect(passesReuseGate(11)).toBe(false); - // A pre-v15 (v12) index silently dropped generic Type nodes and Move - // EnumVariant→Property / field USES_TYPE relationships during CSV routing. expect(passesReuseGate(12)).toBe(false); - // Upstream main uses v13/v14 for a different schema lineage. Neither may - // be mistaken for this build's persisted Move Type/field graph. expect(passesReuseGate(13)).toBe(false); expect(passesReuseGate(14)).toBe(false); + expect(passesReuseGate(15)).toBe(false); // A current-version stamp passes the gate (incremental top-up eligible). - expect(passesReuseGate(15)).toBe(true); + expect(passesReuseGate(16)).toBe(true); }); }); diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index 18206ff80..1c55e11cc 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -8,6 +8,7 @@ * the dispatch and error handling logic in isolation. */ import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; +import type { StalenessInfo } from '../../src/core/git-staleness.js'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; import fsPromises from 'fs/promises'; import os from 'os'; @@ -1091,7 +1092,9 @@ describe('LocalBackend.callTool', () => { expect(result.status).toBe('ambiguous'); expect(result.candidates).toHaveLength(2); - expect(result.impactedCount).toBe(0); + // #2687: undetermined, NOT a numeric zero — a measured 0 is indistinguishable + // from a genuine "nothing depends on this". + expect(result.impactedCount).toBeNull(); expect(result.risk).toBe('UNKNOWN'); expect(result.target.name).toBe('login'); for (const c of result.candidates) { @@ -2515,7 +2518,9 @@ describe('LocalBackend impact mode (KTD1/KTD5/KTD12)', () => { expect(result.status).toBe('ambiguous'); expect(result.mode).toBe('pdg'); expect(result.candidates).toHaveLength(2); - expect(result.impactedCount).toBe(0); + // #2687: undetermined, NOT a numeric zero. This branch runs no per-candidate + // fan-out, so it carries no maxImpactedCount to correct a zero against. + expect(result.impactedCount).toBeNull(); expect(result.risk).toBe('UNKNOWN'); // The callgraph per-candidate probe fan-out MUST NOT run under pdg. expect(bfsSpy).not.toHaveBeenCalled(); @@ -3893,3 +3898,296 @@ describe('LocalBackend.resolveRepo branch scope (#2106)', () => { expect(closedPaths.some((p) => p.includes(path.join('.gitnexus', 'branches')))).toBe(true); }); }); + +// #2655 review: the per-index tool-staleness cache must key by lbugPath, not +// repoPath — flat and branch handles for one repo share a repoPath but carry +// different lastCommit values, so a repoPath key would serve one handle's +// freshness for the other within the TTL window. +describe('LocalBackend tool-staleness cache keying (#2655 review)', () => { + let backend: LocalBackend; + + beforeEach(async () => { + vi.clearAllMocks(); + backend = new LocalBackend(); + setupSingleRepo(); + await backend.init(); + }); + + it('does not share a staleness entry between flat and branch handles of one repo', async () => { + const flat = { + id: 'r', + name: 'r', + repoPath: '/r', + storagePath: '/r/.gitnexus', + lbugPath: '/r/.gitnexus/lbug', + indexedAt: '', + lastCommit: 'FLATSHA', + }; + const branch = { + ...flat, + lbugPath: `/r/.gitnexus/${path.join('branches', 'x', 'lbug')}`, + lastCommit: 'BRANCHSHA', + }; + vi.spyOn(backend, 'resolveRepo') + .mockResolvedValueOnce(flat as any) + .mockResolvedValueOnce(branch as any); + // The tool itself returns a plain (staleness-carryable) object. + vi.spyOn(backend as any, 'query').mockResolvedValue({ ok: true }); + + const { checkStalenessAsync } = await import('../../src/core/git-staleness.js'); + (checkStalenessAsync as any).mockImplementation((_repoPath: string, lastCommit: string) => + Promise.resolve( + lastCommit === 'FLATSHA' + ? { isStale: true, commitsBehind: 5, hint: '5 behind' } + : { isStale: false, commitsBehind: 0 }, + ), + ); + + const flatRes = await backend.callTool('query', { search_query: 'x', repo: 'r' }); + const branchRes = await backend.callTool('query', { + search_query: 'x', + repo: 'r', + branch: 'x', + }); + + // Flat index (lastCommit=FLATSHA) is 5 behind -> field present. + expect(flatRes).toMatchObject({ staleness: { commitsBehind: 5 } }); + // Branch index (different lbugPath + lastCommit) is current; it must NOT + // inherit the flat handle's cached staleness (the pre-fix repoPath-keyed bug). + expect(branchRes).not.toHaveProperty('staleness'); + }); +}); + +// #2655 review F1–F4: the staleness signal wired into query/cypher/context/impact +// must degrade gracefully on a rejecting freshness check, attach on every wrapped +// tool (not just query), leave the adjacent read tools alone, and dedupe/expire +// its per-index cache. +describe('LocalBackend tool-staleness signal (#2655 review)', () => { + let backend: LocalBackend; + + beforeEach(async () => { + vi.clearAllMocks(); + backend = new LocalBackend(); + setupSingleRepo(); + await backend.init(); + }); + + const handle = { + id: 'r', + name: 'r', + repoPath: '/r', + storagePath: '/r/.gitnexus', + lbugPath: '/r/.gitnexus/lbug', + indexedAt: '', + lastCommit: 'HEADSHA', + }; + + const stubResolve = () => vi.spyOn(backend, 'resolveRepo').mockResolvedValue(handle as any); + + const stubStale = async () => { + const { checkStalenessAsync } = await import('../../src/core/git-staleness.js'); + (checkStalenessAsync as any).mockResolvedValue({ + isStale: true, + commitsBehind: 3, + hint: '3 behind', + }); + return checkStalenessAsync as unknown as ReturnType; + }; + + // F1: a rejecting checkStalenessAsync must never fail the tool nor poison the + // 5s cache entry — the result comes back without a staleness field, and a + // later call (after the poisoned entry is evicted) still works. + it('degrades to no-staleness when the freshness check rejects, then recovers', async () => { + stubResolve(); + vi.spyOn(backend as any, 'query').mockResolvedValue({ ok: true }); + const { checkStalenessAsync } = await import('../../src/core/git-staleness.js'); + (checkStalenessAsync as any) + .mockRejectedValueOnce(new Error('git blew up')) + .mockResolvedValue({ isStale: true, commitsBehind: 2, hint: '2 behind' }); + + const rejected = await backend.callTool('query', { search_query: 'x', repo: 'r' }); + expect(rejected).toMatchObject({ ok: true }); + expect(rejected).not.toHaveProperty('staleness'); + + // The rejected entry must not be cached — the next call re-runs and attaches. + const recovered = await backend.callTool('query', { search_query: 'x', repo: 'r' }); + expect(recovered).toMatchObject({ ok: true, staleness: { commitsBehind: 2 } }); + }); + + // F2: every wrapped tool attaches the field on a carryable object result. + it('attaches staleness on query, context, and impact object results', async () => { + stubResolve(); + await stubStale(); + vi.spyOn(backend as any, 'query').mockResolvedValue({ ok: true }); + vi.spyOn(backend as any, 'context').mockResolvedValue({ symbol: 'x' }); + vi.spyOn(backend as any, 'impact').mockResolvedValue({ impactedCount: 0 }); + + expect(await backend.callTool('query', { search_query: 'x', repo: 'r' })).toMatchObject({ + staleness: { commitsBehind: 3, hint: '3 behind' }, + }); + expect(await backend.callTool('context', { name: 'x', repo: 'r' })).toMatchObject({ + staleness: { commitsBehind: 3 }, + }); + expect(await backend.callTool('impact', { target: 'x', repo: 'r' })).toMatchObject({ + staleness: { commitsBehind: 3 }, + }); + }); + + // F2: cypher's tabular {markdown,row_count} object gets the field; a raw-array + // (non-tabular) result keeps its shape untouched so Array.isArray consumers work. + it('attaches staleness to the cypher table object but never to a raw-array result', async () => { + stubResolve(); + await stubStale(); + + // Non-empty array of keyed objects -> formatCypherAsMarkdown returns {markdown,row_count}. + lbugMocks.executeParameterized.mockResolvedValueOnce([{ a: 1 }]); + const tabular = await backend.callTool('cypher', { + statement: 'MATCH (n) RETURN n', + repo: 'r', + }); + expect(tabular).toMatchObject({ row_count: 1, staleness: { commitsBehind: 3 } }); + + // Empty result -> formatCypherAsMarkdown passes the raw array through unchanged. + lbugMocks.executeParameterized.mockResolvedValueOnce([]); + const raw = await backend.callTool('cypher', { statement: 'MATCH (n) RETURN n', repo: 'r' }); + expect(Array.isArray(raw)).toBe(true); + expect(raw).toHaveLength(0); + }); + + // F3: drift guard — exactly the four read tools route through stalenessForTool; + // the adjacent read-ish tools must not, so a future tool added without staleness + // (or one dropped) is caught. + it('routes only query/cypher/context/impact through the freshness check', async () => { + stubResolve(); + await stubStale(); + const spy = vi.spyOn(backend as any, 'stalenessForTool'); + // Stub each tool to a benign object so dispatch reaches withToolStaleness. + for (const m of [ + 'query', + 'context', + 'impact', + 'explain', + 'pdgQuery', + 'detectChanges', + 'check', + ]) { + vi.spyOn(backend as any, m).mockResolvedValue({ ok: true }); + } + // cypher runs its real path; a keyed-object row makes formatCypherAsMarkdown + // return a carryable {markdown,row_count} so the freshness check is reached. + lbugMocks.executeParameterized.mockResolvedValue([{ a: 1 }]); + + await backend.callTool('query', { search_query: 'x', repo: 'r' }); + await backend.callTool('cypher', { statement: 'RETURN 1', repo: 'r' }); + await backend.callTool('context', { name: 'x', repo: 'r' }); + await backend.callTool('impact', { target: 'x', repo: 'r' }); + const wrappedCalls = spy.mock.calls.length; + + await backend.callTool('explain', { target: 'x', repo: 'r' }); + await backend.callTool('pdg_query', { anchor: 'x', repo: 'r' }); + await backend.callTool('detect_changes', { scope: 'unstaged', repo: 'r' }); + await backend.callTool('check', { cycles: true, repo: 'r' }); + + expect(wrappedCalls).toBe(4); + expect(spy.mock.calls.length).toBe(4); + }); + + // F4: the per-index freshness result is deduped within TOOL_STALENESS_TTL_MS and + // recomputed once the window elapses. Drive time via Date.now (not fake timers, + // which would entangle the awaited async dispatch with the microtask queue). + it('dedupes the freshness check within the TTL and recomputes after it expires', async () => { + stubResolve(); + vi.spyOn(backend as any, 'query').mockResolvedValue({ ok: true }); + const check = await stubStale(); + check.mockClear(); + + const dateSpy = vi.spyOn(Date, 'now').mockReturnValue(1000); + await backend.callTool('query', { search_query: 'x', repo: 'r' }); + await backend.callTool('query', { search_query: 'x', repo: 'r' }); + expect(check).toHaveBeenCalledTimes(1); // deduped within the window + + dateSpy.mockReturnValue(1000 + 5000 + 1); // past TOOL_STALENESS_TTL_MS + await backend.callTool('query', { search_query: 'x', repo: 'r' }); + expect(check).toHaveBeenCalledTimes(2); // recomputed after expiry + + dateSpy.mockRestore(); + }); + + // @group-routed calls forward to callToolAtGroupRepo BEFORE the wrapping + // switch, so they deliberately never get the staleness signal (multi-repo, + // single-commit staleness is ill-defined). Pin that so it can't silently flip. + it('does not attach staleness to an @group-routed call', async () => { + const groupSpy = vi + .spyOn(backend as any, 'callToolAtGroupRepo') + .mockResolvedValue({ ok: true }); + const freshSpy = vi.spyOn(backend as any, 'stalenessForTool'); + await stubStale(); // stale — but @group must skip the signal regardless + + const res = await backend.callTool('query', { search_query: 'x', repo: '@grp' }); + + expect(groupSpy).toHaveBeenCalledOnce(); + expect(freshSpy).not.toHaveBeenCalled(); + expect(res).not.toHaveProperty('staleness'); + }); + + // The freshness check is deduped by sharing the IN-FLIGHT promise, not merely + // by reusing an already-resolved value: two calls that arrive before the first + // `checkStalenessAsync` settles must still spawn only one. + it('shares one in-flight freshness check across truly concurrent calls', async () => { + stubResolve(); + vi.spyOn(backend as any, 'query').mockResolvedValue({ ok: true }); + const dateSpy = vi.spyOn(Date, 'now').mockReturnValue(2000); + const { checkStalenessAsync } = await import('../../src/core/git-staleness.js'); + let settle: (v: StalenessInfo) => void = () => {}; + const pending = new Promise((res) => { + settle = res; + }); + (checkStalenessAsync as any).mockClear(); + (checkStalenessAsync as any).mockReturnValue(pending); + + // Both dispatched before the check resolves — they must share the entry. + const p1 = backend.callTool('query', { search_query: 'x', repo: 'r' }); + const p2 = backend.callTool('query', { search_query: 'x', repo: 'r' }); + await new Promise((r) => setTimeout(r, 0)); // let both reach stalenessForTool + settle({ isStale: true, commitsBehind: 1, hint: '1 behind' }); + const [r1, r2] = await Promise.all([p1, p2]); + + expect(checkStalenessAsync).toHaveBeenCalledTimes(1); // one spawn, shared + expect(r1).toMatchObject({ staleness: { commitsBehind: 1 } }); + expect(r2).toMatchObject({ staleness: { commitsBehind: 1 } }); + dateSpy.mockRestore(); + }); + + // The evict-on-reject is guarded by object identity (=== entry), so a LATE + // rejection from a superseded entry must not drop the newer entry that + // replaced it after the TTL rolled over. + it('a late rejection does not evict the newer cache entry', async () => { + stubResolve(); + vi.spyOn(backend as any, 'query').mockResolvedValue({ ok: true }); + const dateSpy = vi.spyOn(Date, 'now').mockReturnValue(1000); + const { checkStalenessAsync } = await import('../../src/core/git-staleness.js'); + let rejectFirst: (e: unknown) => void = () => {}; + const first = new Promise((_res, rej) => { + rejectFirst = rej; + }); + (checkStalenessAsync as any) + .mockReturnValueOnce(first) // entry 1 — held open, will reject late + .mockResolvedValue({ isStale: true, commitsBehind: 7, hint: '7 behind' }); // entry 2+ + + const p1 = backend.callTool('query', { search_query: 'x', repo: 'r' }); // installs entry1 @1000 + await new Promise((r) => setTimeout(r, 0)); // entry1 installed, awaiting `first` + + dateSpy.mockReturnValue(1000 + 5000 + 1); // past TTL → next call installs entry2 + const r2 = await backend.callTool('query', { search_query: 'x', repo: 'r' }); + expect(r2).toMatchObject({ staleness: { commitsBehind: 7 } }); + + rejectFirst(new Error('late git failure')); // entry1's guarded catch must NOT evict entry2 + await p1.catch(() => {}); // p1 degrades to no-staleness + + const callsBefore = (checkStalenessAsync as any).mock.calls.length; + const r3 = await backend.callTool('query', { search_query: 'x', repo: 'r' }); // still within entry2 TTL + expect((checkStalenessAsync as any).mock.calls.length).toBe(callsBefore); // cache hit → entry2 survived + expect(r3).toMatchObject({ staleness: { commitsBehind: 7 } }); + dateSpy.mockRestore(); + }); +}); diff --git a/gitnexus/test/unit/community-processor.test.ts b/gitnexus/test/unit/community-processor.test.ts index c0e5ff14c..2e207a22a 100644 --- a/gitnexus/test/unit/community-processor.test.ts +++ b/gitnexus/test/unit/community-processor.test.ts @@ -1,4 +1,8 @@ import { EventEmitter } from 'node:events'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Worker } from 'node:worker_threads'; import { describe, it, expect, vi } from 'vitest'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import type { GraphNode, GraphRelationship } from '../../src/core/graph/types.js'; @@ -7,6 +11,7 @@ import { COMMUNITY_COLORS, buildCommunityCsr, buildCommunityProjection, + buildIcebugWorkerSource, processCommunities, resolveCommunityDetectionEngine, } from '../../src/core/ingestion/community-processor.js'; @@ -145,6 +150,8 @@ describe('community-processor', () => { }); describe('processCommunities engine fallback', () => { + let terminateCalls = 0; + it('falls back to graphology when explicit icebug engine is unavailable', async () => { const graph = createKnowledgeGraph(); graph.addNode(makeNode('fn:a', 'a', 'Function', '/src/group/a.ts')); @@ -164,6 +171,28 @@ describe('community-processor', () => { expect(result.memberships).toHaveLength(2); }); + it('announces the experimental engine on request, before any fallback', async () => { + const graph = createKnowledgeGraph(); + graph.addNode(makeNode('fn:a', 'a', 'Function', '/src/group/a.ts')); + graph.addNode(makeNode('fn:b', 'b', 'Function', '/src/group/b.ts')); + graph.addRelationship(makeRel('rel:ab', 'fn:a', 'fn:b')); + + const experimental: string[] = []; + await processCommunities(graph, (message) => experimental.push(message), { engine: 'auto' }); + const notice = experimental.findIndex((message) => message.startsWith('Experimental auto')); + const fallback = experimental.findIndex((message) => + message.includes('falling back to Graphology'), + ); + + expect(experimental[notice]).toContain('will not match the Graphology default'); + expect(notice).toBeLessThan(fallback); + + const defaultEngine: string[] = []; + await processCommunities(graph, (message) => defaultEngine.push(message)); + + expect(defaultEngine.some((message) => message.startsWith('Experimental'))).toBe(false); + }); + it('falls back to graphology when icebug returns invalid modularity', async () => { vi.resetModules(); vi.doMock('node:worker_threads', () => { @@ -176,8 +205,11 @@ describe('community-processor', () => { } terminate(): Promise { + terminateCalls++; return Promise.resolve(0); } + + unref(): void {} } return { Worker: MockWorker }; @@ -204,6 +236,10 @@ describe('community-processor', () => { expect(progress.some((message) => message.includes('falling back to Graphology'))).toBe( true, ); + // GUARDRAILS non-negotiable 6 (#2432): the icebug worker spends its whole + // life inside N-API, so terminating it aborts the process instead of + // falling back. It ends after one postMessage and exits on its own. + expect(terminateCalls).toBe(0); } finally { vi.doUnmock('node:worker_threads'); vi.resetModules(); @@ -231,4 +267,188 @@ describe('community-processor', () => { expect(randomizeResult.stats.fallbackReason).toContain('randomize=false'); }); }); + + describe('icebug worker source', () => { + // Executes the real worker source against a stub shaped like + // @ladybugmem/icebug, so the package name, class names, constructor + // argument order and getPartition() shape are all pinned. The native + // package itself cannot run in CI (its prebuilds need system Arrow 24, + // libomp and glibc >= 2.38). + const STUB = ` +'use strict'; +const fs = require('node:fs'); +const calls = []; +const log = () => fs.writeFileSync(process.env.ICEBUG_STUB_LOG, JSON.stringify(calls)); + +class GraphR { + constructor(n, directed, outIndices, outIndptr) { + calls.push(['GraphR', n, directed, Array.from(outIndices, Number), Array.from(outIndptr, Number)]); + } +} + +class Leiden { + constructor(graph, iterations, randomize, gamma) { + calls.push(['Leiden', graph instanceof GraphR, iterations, randomize, gamma]); + } + run() { + calls.push(['run']); + log(); + } + getPartition() { + return { membership: Float64Array.from([7, 7, 3]), count: 2 }; + } + modularity() { + return 0.25; + } +} + +module.exports = { + GraphR, + Leiden, + setNumberOfThreads: (n) => calls.push(['setNumberOfThreads', n]), + setSeed: (seed, useThreadId) => calls.push(['setSeed', seed, useThreadId]), +}; +`; + + const runWorkerAgainstStub = async (stubSource: string) => { + const dir = mkdtempSync(join(tmpdir(), 'icebug-stub-')); + const stubPath = join(dir, 'stub.cjs'); + const logPath = join(dir, 'calls.json'); + writeFileSync(stubPath, stubSource); + + const worker = new Worker(buildIcebugWorkerSource(stubPath), { + eval: true, + env: { ...process.env, ICEBUG_STUB_LOG: logPath }, + workerData: { + nodeCount: 3, + indices: BigUint64Array.from([1n, 0n, 2n, 1n]), + indptr: BigUint64Array.from([0n, 1n, 3n, 4n]), + threads: 1, + seed: 49374, + iterations: 4, + gamma: 1.0, + randomize: false, + }, + }); + + try { + const message = await new Promise>((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); + // Absent when the worker bailed before run() — an empty call log. + const calls: unknown[] = existsSync(logPath) + ? JSON.parse(readFileSync(logPath, 'utf8')) + : []; + return { message, calls }; + } finally { + await worker.terminate(); + rmSync(dir, { recursive: true, force: true }); + } + }; + + it('drives GraphR + Leiden in the order the published API expects', async () => { + const { message, calls } = await runWorkerAgainstStub(STUB); + + expect(calls).toEqual([ + ['setNumberOfThreads', 1], + ['setSeed', 49374, false], + ['GraphR', 3, false, [1, 0, 2, 1], [0, 1, 3, 4]], + // (graph, iterations, randomize, gamma) — randomize precedes gamma. + ['Leiden', true, 4, false, 1.0], + ['run'], + ]); + expect(message).toMatchObject({ ok: true, modularity: 0.25 }); + expect(Array.from(message.partition as Float64Array)).toEqual([7, 7, 3]); + }); + + it('refuses a build without the deterministic thread and seed controls', async () => { + const { message } = await runWorkerAgainstStub( + STUB.replace("setNumberOfThreads: (n) => calls.push(['setNumberOfThreads', n]),", ''), + ); + + expect(message).toMatchObject({ ok: false }); + expect(message.error).toContain('deterministic thread/seed controls'); + }); + }); + + describe('vendored Leiden partitioning', () => { + // Golden values for the seeded graph below, captured from the vendored + // implementation. They pin the partition, not just its shape. + const GOLDEN_COMMUNITY_COUNT = 99; + const GOLDEN_NODES_PROCESSED = 1199; + const GOLDEN_MODULARITY = 0.7032803125; + + // Guards the mergeNodesSubset scratch-buffer change in vendor/leiden/utils.cjs + // (#2337): the pre-merge snapshot must still hold each subset node's + // externalEdgeWeightPerCommunity from *before* the merge loop. Getting the + // snapshot wrong shifts the partition, which these golden values catch. + // Seeded planted partition with cross-community noise. Unlike clean cliques, + // the noisy edges make the outcome sensitive to the merge-phase bookkeeping + // that `microDegrees` feeds, so a wrong snapshot shifts the golden values. + const buildPlantedGraph = (nodeCount: number, edgeCount: number, groupCount: number) => { + let state = 0x1234_5678; + const random = () => { + state = (state + 0x6d2b79f5) >>> 0; + let mixed = Math.imul(state ^ (state >>> 15), 1 | state); + mixed = (mixed + Math.imul(mixed ^ (mixed >>> 7), 61 | mixed)) ^ mixed; + return ((mixed ^ (mixed >>> 14)) >>> 0) / 4294967296; + }; + + const graph = createKnowledgeGraph(); + const groups: number[][] = Array.from({ length: groupCount }, () => []); + + for (let node = 0; node < nodeCount; node++) { + const group = Math.floor(random() * groupCount); + groups[group].push(node); + graph.addNode(makeNode(`fn:${node}`, `f${node}`, 'Function', `/src/g${group}/f${node}.ts`)); + } + + const seen = new Set(); + let added = 0; + let guard = edgeCount * 50; + + while (added < edgeCount && guard-- > 0) { + const group = groups[Math.floor(random() * groupCount)]; + const intraCommunity = random() < 0.85 && group.length >= 2; + const source = intraCommunity + ? group[Math.floor(random() * group.length)] + : Math.floor(random() * nodeCount); + const target = intraCommunity + ? group[Math.floor(random() * group.length)] + : Math.floor(random() * nodeCount); + const low = Math.min(source, target); + const high = Math.max(source, target); + const key = `${low}:${high}`; + + if (low === high || seen.has(key)) continue; + + seen.add(key); + graph.addRelationship(makeRel(`rel:${key}`, `fn:${low}`, `fn:${high}`)); + added++; + } + + return graph; + }; + + it('recovers the planted partition with the expected golden quality', async () => { + const result = await processCommunities(buildPlantedGraph(1200, 4000, 60)); + + expect(result.stats).toMatchObject({ + engine: 'graphology', + totalCommunities: GOLDEN_COMMUNITY_COUNT, + nodesProcessed: GOLDEN_NODES_PROCESSED, + }); + expect(result.stats.modularity).toBeCloseTo(GOLDEN_MODULARITY, 6); + }); + + it('produces an identical partition across repeated runs', async () => { + const graph = buildPlantedGraph(600, 2000, 30); + const first = await processCommunities(graph); + const second = await processCommunities(graph); + + expect(second.memberships).toEqual(first.memberships); + expect(second.stats.modularity).toBe(first.stats.modularity); + }); + }); }); diff --git a/gitnexus/test/unit/doctor-format.test.ts b/gitnexus/test/unit/doctor-format.test.ts index 2d06f5124..33a70dd0e 100644 --- a/gitnexus/test/unit/doctor-format.test.ts +++ b/gitnexus/test/unit/doctor-format.test.ts @@ -4,8 +4,11 @@ import { doctorCommand, localEmbeddingDoctorStatus, padDisplayEnd, + nativeStatusLine, pageSizeDoctorLines, + poolSizeDoctorLine, } from '../../src/cli/doctor.js'; +import type { NativeCheckResult } from '../../src/core/lbug/native-check.js'; describe('doctor output formatting', () => { it('keeps ASCII padding equivalent to String.padEnd', () => { @@ -164,6 +167,58 @@ describe('doctor page-size lines (#1231, #2424 review)', () => { }); }); +describe('doctor pool-size line (#2631)', () => { + const MiB = 1024 * 1024; + + it('prints the hintless pool in MiB with no env note when the env var is unset', () => { + expect(poolSizeDoctorLine(2048 * MiB, undefined)).toBe( + ` ${padDisplayEnd('pool size', 10)}2048 MiB`, + ); + }); + + it('marks an operator-supplied absolute value as an env override, with no scaling suffix', () => { + expect(poolSizeDoctorLine(4096 * MiB, String(4096 * MiB))).toBe( + ` ${padDisplayEnd('pool size', 10)}4096 MiB (env override)`, + ); + }); + + it('labels the 0 sentinel as the native default instead of "0 MiB"', () => { + expect(poolSizeDoctorLine(0, '0')).toBe( + ` ${padDisplayEnd('pool size', 10)}native 80% of RAM (env override)`, + ); + }); +}); + +// #2672: every failed check used to print "lbugjs.node missing", including the +// glibc case where the binary is present and merely unloadable — contradicting +// the detail printed directly beneath it and sending users to reinstall a file +// they already had. +describe('doctor native status line (#2672)', () => { + const nativeStatusCases: ReadonlyArray = [ + ['a loaded binary', { ok: true, binaryPath: '/x/lbugjs.node' }, '✓ lbugjs.node loaded'], + [ + 'an uninstalled package', + { ok: false, kind: 'package_missing', message: 'x' }, + '✗ @ladybugdb/core not installed', + ], + [ + 'an absent binary', + { ok: false, kind: 'binary_missing', binaryPath: '/x/lbugjs.node', message: 'x' }, + '✗ lbugjs.node missing', + ], + [ + 'a present-but-unloadable binary (glibc too old, truncated download)', + { ok: false, kind: 'load_failed', binaryPath: '/x/lbugjs.node', message: 'x' }, + '✗ lbugjs.node present but failed to load', + ], + ['a failure with no kind recorded', { ok: false, message: 'x' }, '✗ lbugjs.node missing'], + ]; + + it.each(nativeStatusCases)('reports %s', (_name, check, expected) => { + expect(nativeStatusLine(check)).toBe(` ${padDisplayEnd('native', 10)}${expected}`); + }); +}); + describe('doctor survives a malformed GITNEXUS_EMBEDDING_DIMS (#2385)', () => { const ENV_KEYS = [ 'GITNEXUS_EMBEDDING_URL', diff --git a/gitnexus/test/unit/extension-load-error.test.ts b/gitnexus/test/unit/extension-load-error.test.ts index 71a9c7d42..4cbba9233 100644 --- a/gitnexus/test/unit/extension-load-error.test.ts +++ b/gitnexus/test/unit/extension-load-error.test.ts @@ -154,6 +154,11 @@ describe('classifyExtensionLoadError', () => { expect(remedy).toMatch(/will NOT help/); // Must not resurrect the old, wrong "retry the network install" instruction. expect(remedy).not.toMatch(/Retry with network access/i); + // #2669: the zero-install path — Git for Windows already ships those DLLs. + expect(remedy).toMatch(/Git Bash/); + expect(remedy).toMatch(/mingw64/); + // Never a user-profile path: remedy text reaches /api/search unredacted. + expect(remedy).not.toMatch(/C:\\Users\\/); }); it('hedged fallback remedy points at the OS error and offers both branches (language-independent)', () => { @@ -318,6 +323,9 @@ describe('diagnoseExtensionLoad (structural, language-independent)', () => { const { kind, remedy } = diagnoseExtensionLoad(reason); expect(kind).toBe('missing_dependency'); expect(remedy).toMatch(/vc_redist\.x64\.exe/); + // #2669: the structural remedy carries the same zero-install hint. + expect(remedy).toMatch(/Git Bash/); + expect(remedy).not.toMatch(/C:\\Users\\/); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/gitnexus/test/unit/git-utils.test.ts b/gitnexus/test/unit/git-utils.test.ts index d04524d5c..b1fc8f7bd 100644 --- a/gitnexus/test/unit/git-utils.test.ts +++ b/gitnexus/test/unit/git-utils.test.ts @@ -341,3 +341,516 @@ describe('getCanonicalRepoRoot', () => { } }); }); + +// ─── selfCommitContextFiles (#2639) ──────────────────────────────────────── + +describe('selfCommitContextFiles', () => { + const initRepo = (): string => { + const repoDir = makeIsolatedTempDir('gitnexus-self-commit-'); + execFileSync(gitExecutable, ['init', '-q'], { cwd: repoDir, stdio: 'ignore' }); + execSync('git config user.email "test@example.com"', { cwd: repoDir }); + execSync('git config user.name "Test"', { cwd: repoDir }); + return repoDir; + }; + + const lastCommitMessage = (repoDir: string): string => + execSync('git log -1 --format=%s', { cwd: repoDir, encoding: 'utf8' }).trim(); + + const commitCount = (repoDir: string): number => + Number(execSync('git rev-list --count HEAD', { cwd: repoDir, encoding: 'utf8' }).trim()); + + const stagedFiles = (repoDir: string): string[] => + execSync('git diff --cached --name-only', { cwd: repoDir, encoding: 'utf8' }) + .split('\n') + .map((s) => s.trim()) + .filter(Boolean); + + // Most tests below aren't exercising snapshotSelfCommitSafety itself (that + // has its own describe block); they just need "everything is safe to + // commit," matching a normal run where nothing was dirty beforehand. + const allSafe = (names: string[]): Map => + new Map(names.map((name) => [name, true])); + + it('commits only the changed candidate file, scoped by name (never git add -A)', async () => { + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const repoDir = initRepo(); + try { + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v1\n'); + execSync('git add AGENTS.md', { cwd: repoDir }); + execSync('git commit -q -m "initial"', { cwd: repoDir }); + + // Dirty AGENTS.md (candidate) plus an unrelated untracked file that + // must never be swept in. + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v2\n'); + fs.writeFileSync(path.join(repoDir, 'unrelated.txt'), 'should stay untouched\n'); + + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ); + + expect(commitCount(repoDir)).toBe(2); + expect(lastCommitMessage(repoDir)).toBe('chore(gitnexus): refresh index stats [skip ci]'); + const status = execSync('git status --porcelain', { cwd: repoDir, encoding: 'utf8' }); + // unrelated.txt is still untracked/dirty — proves the commit was scoped. + expect(status).toContain('unrelated.txt'); + expect(status).not.toContain('AGENTS.md'); + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('no-ops (no new commit) when neither candidate file changed', async () => { + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const repoDir = initRepo(); + try { + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v1\n'); + execSync('git add AGENTS.md', { cwd: repoDir }); + execSync('git commit -q -m "initial"', { cwd: repoDir }); + + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ); + + expect(commitCount(repoDir)).toBe(1); + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('commits newly-created (untracked) candidate files, not just modified ones', async () => { + // Regression guard: a first-time `analyze --self-commit` run creates + // AGENTS.md/CLAUDE.md fresh — they are untracked, not modified. An + // implementation based on `git diff --quiet` misses untracked files + // entirely and would silently skip this case. + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const repoDir = initRepo(); + try { + execSync('git commit -q --allow-empty -m "initial"', { cwd: repoDir }); + + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'fresh from analyze\n'); + + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ); + + expect(commitCount(repoDir)).toBe(2); + expect(lastCommitMessage(repoDir)).toBe('chore(gitnexus): refresh index stats [skip ci]'); + const status = execSync('git status --porcelain', { cwd: repoDir, encoding: 'utf8' }); + expect(status.trim()).toBe(''); + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('no-ops when neither candidate file exists on disk', async () => { + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const repoDir = initRepo(); + try { + execSync('git commit -q --allow-empty -m "initial"', { cwd: repoDir }); + + expect(() => + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ), + ).not.toThrow(); + expect(commitCount(repoDir)).toBe(1); + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('never throws when repoPath is not a git repository', async () => { + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const tmpDir = makeIsolatedTempDir('gitnexus-self-commit-nongit-'); + try { + fs.writeFileSync(path.join(tmpDir, 'AGENTS.md'), 'not a git repo\n'); + expect(() => + selfCommitContextFiles( + tmpDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ), + ).not.toThrow(); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('commits both files when both changed, still scoped (no -A)', async () => { + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const repoDir = initRepo(); + try { + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v1\n'); + fs.writeFileSync(path.join(repoDir, 'CLAUDE.md'), 'v1\n'); + execSync('git add AGENTS.md CLAUDE.md', { cwd: repoDir }); + execSync('git commit -q -m "initial"', { cwd: repoDir }); + + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v2\n'); + fs.writeFileSync(path.join(repoDir, 'CLAUDE.md'), 'v2\n'); + + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ); + + expect(commitCount(repoDir)).toBe(2); + const status = execSync('git status --porcelain', { cwd: repoDir, encoding: 'utf8' }); + expect(status.trim()).toBe(''); + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('logs a warning (never throws) when the commit step fails, e.g. no git identity', async () => { + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const { _captureLogger } = await import('../../src/core/logger.js'); + const repoDir = initRepo(); + try { + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v1\n'); + execSync('git add AGENTS.md', { cwd: repoDir }); + execSync('git commit -q -m "initial"', { cwd: repoDir }); + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v2\n'); + + // useConfigOnly forces git to error on a missing identity instead of + // guessing from OS user/hostname; HOME/XDG_CONFIG_HOME are redirected + // and GIT_CONFIG_NOSYSTEM disables the system config, so no ambient + // global identity on the CI runner can leak in and make git succeed + // anyway. Together these deterministically reproduce "no git identity + // configured" regardless of the machine running the test. + execSync('git config user.useConfigOnly true', { cwd: repoDir }); + execSync('git config --unset user.name', { cwd: repoDir }); + execSync('git config --unset user.email', { cwd: repoDir }); + + const savedHome = process.env.HOME; + const savedXdg = process.env.XDG_CONFIG_HOME; + const savedNoSystem = process.env.GIT_CONFIG_NOSYSTEM; + process.env.HOME = makeIsolatedTempDir('gitnexus-self-commit-noidentity-home-'); + process.env.XDG_CONFIG_HOME = process.env.HOME; + process.env.GIT_CONFIG_NOSYSTEM = '1'; + + const cap = _captureLogger(); + try { + expect(() => + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ), + ).not.toThrow(); + } finally { + if (savedHome === undefined) delete process.env.HOME; + else process.env.HOME = savedHome; + if (savedXdg === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = savedXdg; + if (savedNoSystem === undefined) delete process.env.GIT_CONFIG_NOSYSTEM; + else process.env.GIT_CONFIG_NOSYSTEM = savedNoSystem; + } + const warning = cap + .records() + .find((r) => r.msg.includes('--self-commit failed to commit context files')); + cap.restore(); + + expect(warning).toBeDefined(); + expect(commitCount(repoDir)).toBe(1); // commit never landed + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('skips (and logs) a candidate that already had an uncommitted edit before this run, never sweeping it into the generated commit', async () => { + // Regression for #2640 review round 1/2: without snapshotSelfCommitSafety, + // a pre-existing unstaged user edit in AGENTS.md and this run's stats + // refresh are indistinguishable — both just show up as "AGENTS.md is + // dirty" — so the old implementation silently committed both together. + const { selfCommitContextFiles, snapshotSelfCommitSafety } = + await import('../../src/storage/git.js'); + const { _captureLogger } = await import('../../src/core/logger.js'); + const repoDir = initRepo(); + try { + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'v1\n'); + fs.writeFileSync(path.join(repoDir, 'CLAUDE.md'), 'v1\n'); + execSync('git add AGENTS.md CLAUDE.md', { cwd: repoDir }); + execSync('git commit -q -m "initial"', { cwd: repoDir }); + + // A user edit lands in AGENTS.md BEFORE analyze/self-commit ever runs. + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'user note\n'); + + // The real call sequence: snapshot safety first (this is what + // analyze.ts does before writing), THEN simulate analyze's own write + // on top of the user's pre-existing edit, for both candidates. + const safety = snapshotSelfCommitSafety(repoDir, ['AGENTS.md', 'CLAUDE.md']); + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'user note\ngenerated stats refresh\n'); + fs.writeFileSync(path.join(repoDir, 'CLAUDE.md'), 'generated stats refresh\n'); + + const cap = _captureLogger(); + selfCommitContextFiles(repoDir, ['AGENTS.md', 'CLAUDE.md'], safety); + const warning = cap + .records() + .find((r) => r.msg.includes('skipping file(s) with uncommitted changes')); + cap.restore(); + + expect(warning).toBeDefined(); + // CLAUDE.md was clean pre-run (safe) and got committed; AGENTS.md was + // already dirty pre-run (unsafe) and must stay out of the commit and + // out of the index entirely — proving its edit wasn't swept in. + expect(commitCount(repoDir)).toBe(2); + expect(lastCommitMessage(repoDir)).toBe('chore(gitnexus): refresh index stats [skip ci]'); + const diffTreeFiles = execSync('git diff-tree --no-commit-id --name-only -r HEAD', { + cwd: repoDir, + encoding: 'utf8', + }) + .split('\n') + .map((s) => s.trim()) + .filter(Boolean); + expect(diffTreeFiles).toEqual(['CLAUDE.md']); + const status = execSync('git status --porcelain -- AGENTS.md', { + cwd: repoDir, + encoding: 'utf8', + }); + expect(status.trim()).not.toBe(''); // AGENTS.md's edit is still there, untouched + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); + + it('restores the index for exactly the staged files when commit fails after git add (no leftover staged state)', async () => { + // Regression for #2640 review round 2: `git add` runs before `git commit`; + // if commit then fails (e.g. missing identity), the old implementation + // left the candidate staged even though it reported nothing happened — + // silently mutating the user's index on a run that "did nothing." + const { selfCommitContextFiles } = await import('../../src/storage/git.js'); + const repoDir = initRepo(); + try { + execSync('git commit -q --allow-empty -m "initial"', { cwd: repoDir }); + fs.writeFileSync(path.join(repoDir, 'AGENTS.md'), 'fresh from analyze\n'); + + execSync('git config user.useConfigOnly true', { cwd: repoDir }); + execSync('git config --unset user.name', { cwd: repoDir }); + execSync('git config --unset user.email', { cwd: repoDir }); + + const savedHome = process.env.HOME; + const savedXdg = process.env.XDG_CONFIG_HOME; + const savedNoSystem = process.env.GIT_CONFIG_NOSYSTEM; + process.env.HOME = makeIsolatedTempDir('gitnexus-self-commit-noidentity-home2-'); + process.env.XDG_CONFIG_HOME = process.env.HOME; + process.env.GIT_CONFIG_NOSYSTEM = '1'; + try { + selfCommitContextFiles( + repoDir, + ['AGENTS.md', 'CLAUDE.md'], + allSafe(['AGENTS.md', 'CLAUDE.md']), + ); + } finally { + if (savedHome === undefined) delete process.env.HOME; + else process.env.HOME = savedHome; + if (savedXdg === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = savedXdg; + if (savedNoSystem === undefined) delete process.env.GIT_CONFIG_NOSYSTEM; + else process.env.GIT_CONFIG_NOSYSTEM = savedNoSystem; + } + + expect(commitCount(repoDir)).toBe(1); // commit never landed + expect(stagedFiles(repoDir)).toEqual([]); // and nothing was left staged + // The file itself is still there, unstaged, exactly as analyze left it. + const status = execSync('git status --porcelain -- AGENTS.md', { + cwd: repoDir, + encoding: 'utf8', + }); + expect(status.trim()).toBe('?? AGENTS.md'); + } finally { + fs.rmSync(repoDir, { recursive: true, force: true }); + } + }); +}); + +// ─── isWorkingTreeDirty ─────────────────────────────────────────────────── +// +// analyze's fast-path gate. GitNexus writes to .gitnexus/, .claude/, .cursor/, +// AGENTS.md, CLAUDE.md, and the repo-local .agents/ skill mirror during a run; +// those writes must never count as "dirty" or the up-to-date fast path is +// defeated on every re-run. Real temporary git repos exercise the actual +// `git status --porcelain` pathspec exclude list. + +/** Create a fresh git repo in an isolated temp dir and return its path. */ +function makeIsolatedGitRepo(): string { + const dir = makeIsolatedTempDir('gn-dirty-'); + execSync('git init -q', { cwd: dir, stdio: 'ignore' }); + // Set a stable identity so commit doesn't fail on environments without + // global git config (CI containers, fresh sandboxes). + execSync('git config user.email t@t', { cwd: dir, stdio: 'ignore' }); + execSync('git config user.name t', { cwd: dir, stdio: 'ignore' }); + return dir; +} + +describe('isWorkingTreeDirty', () => { + it('returns false for a clean tree with only GitNexus-managed paths written', async () => { + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + // Initial commit so the tree has a HEAD. + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + // Simulate GitNexus writing its managed outputs. + fs.mkdirSync(path.join(repo, '.gitnexus'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.gitnexus', 'meta.json'), '{}'); + fs.mkdirSync(path.join(repo, '.claude', 'skills', 'gitnexus-cli'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.claude', 'skills', 'gitnexus-cli', 'SKILL.md'), 'x'); + fs.mkdirSync(path.join(repo, '.agents', 'skills', 'gitnexus-area-auth'), { + recursive: true, + }); + fs.writeFileSync(path.join(repo, '.agents', 'skills', 'gitnexus-area-auth', 'SKILL.md'), 'x'); + fs.writeFileSync(path.join(repo, 'AGENTS.md'), 'x'); + fs.writeFileSync(path.join(repo, 'CLAUDE.md'), 'x'); + + expect(isWorkingTreeDirty(repo)).toBe(false); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('returns true when a real source file changes (regression: excludes must not mask real edits)', async () => { + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + // A real business-file edit alongside GitNexus writes. + fs.mkdirSync(path.join(repo, 'src'), { recursive: true }); + fs.writeFileSync(path.join(repo, 'src', 'foo.ts'), 'export const x = 2;'); + fs.mkdirSync(path.join(repo, '.agents', 'skills', 'x'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.agents', 'skills', 'x', 'SKILL.md'), 'x'); + + expect(isWorkingTreeDirty(repo)).toBe(true); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('treats the entire .agents/ tree as excluded (root file, nested skills, deep paths)', async () => { + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + // Root-level file under .agents/. + fs.mkdirSync(path.join(repo, '.agents'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.agents', 'foo.txt'), 'x'); + // Deep nested mirror path. + fs.mkdirSync(path.join(repo, '.agents', 'skills', 'gitnexus-area-auth'), { + recursive: true, + }); + fs.writeFileSync(path.join(repo, '.agents', 'skills', 'gitnexus-area-auth', 'SKILL.md'), 'x'); + + expect(isWorkingTreeDirty(repo)).toBe(false); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('does not error when .agents/ does not exist (no pathspec failure)', async () => { + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + expect(isWorkingTreeDirty(repo)).toBe(false); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('does not error when .agents is a file rather than a directory', async () => { + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + // .agents exists as a regular file (e.g. user created it by mistake). + fs.writeFileSync(path.join(repo, '.agents'), 'not a directory'); + + // Must not throw; the tree is otherwise clean so it is not dirty. + expect(isWorkingTreeDirty(repo)).toBe(false); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('does NOT exclude prefix-colliding names like .agentsrc or .claudefoo', async () => { + // pathspec `:(exclude).agents` must not swallow `.agentsrc` (no path + // separator). A change to such a colliding name still counts as dirty. + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + fs.writeFileSync(path.join(repo, '.agentsrc'), 'x'); + fs.writeFileSync(path.join(repo, '.claudefoo'), 'x'); + + expect(isWorkingTreeDirty(repo)).toBe(true); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('does NOT exclude a nested .agents/ inside a subdirectory (root-relative pathspec)', async () => { + // `:(exclude).agents` is relative to the repo root; a subdirectory's + // .agents/ is unrelated and must still count as dirty. + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + + fs.mkdirSync(path.join(repo, 'subdir', '.agents'), { recursive: true }); + fs.writeFileSync(path.join(repo, 'subdir', '.agents', 'x'), 'x'); + + expect(isWorkingTreeDirty(repo)).toBe(true); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); + + it('returns true (conservative) when called outside a git repository', async () => { + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const dir = makeIsolatedTempDir('gn-nongit-'); + try { + // No git init — git status fails, and the gate must fail closed (dirty). + expect(isWorkingTreeDirty(dir)).toBe(true); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('returns true (conservative) when git is not on PATH', async () => { + // PATH cleared so `git` cannot be found. The catch block must return true + // (fail closed) rather than silently treating the tree as clean — a clean + // false-positive would skip re-indexing of a genuinely-changed repo. + const { isWorkingTreeDirty } = await import('../../src/storage/git.js'); + const repo = makeIsolatedGitRepo(); + const savedPath = process.env.PATH; + try { + fs.writeFileSync(path.join(repo, 'README.md'), 'hi'); + execSync('git add -A && git commit -q -m init', { cwd: repo, stdio: 'ignore' }); + process.env.PATH = ''; + expect(isWorkingTreeDirty(repo)).toBe(true); + } finally { + process.env.PATH = savedPath; + fs.rmSync(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/index-lock.test.ts b/gitnexus/test/unit/index-lock.test.ts new file mode 100644 index 000000000..1a6846cc0 --- /dev/null +++ b/gitnexus/test/unit/index-lock.test.ts @@ -0,0 +1,381 @@ +/** + * Unit tests for the cross-process index write lock (#2658). + * + * These exercise the lock's decision logic deterministically by pre-seeding + * `analyze.lock` records and asserting acquire/steal/release/sweep behavior — + * including the kill-recovery mechanism (a dead holder's lock is reclaimed) and + * mutual exclusion (a live holder is waited on, never stolen). A real + * two-process exclusion + SIGKILL-recovery test lives in + * test/integration/analyze-index-lock-concurrency.test.ts. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + mkdtempSync, + rmSync, + writeFileSync, + readFileSync, + existsSync, + chmodSync, + symlinkSync, +} from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + acquireIndexLock, + sweepStagingArtifacts, + isLockUnwritableCode, + IndexLockTimeoutError, + type LockRecord, +} from '../../src/storage/index-lock.js'; +import { classifyFtsBuildError, ftsFailureIsFatal } from '../../src/core/search/fts-indexes.js'; + +let dir: string; +const lockPath = () => path.join(dir, 'analyze.lock'); + +const seedLock = (overrides: Partial): void => { + const record: LockRecord = { + v: 1, + pid: 999999999, // implausible pid → dead by default + hostname: os.hostname(), + startTime: null, + token: 'seed-token', + invocationId: 'seed-invocation', + acquiredAt: new Date().toISOString(), + ...overrides, + }; + writeFileSync(lockPath(), JSON.stringify(record)); +}; + +beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), 'gnx-lock-')); + // These suites exercise the file (O_EXCL pidfile) backend directly. On Linux + // the default is the socket backend, so pin the file backend explicitly. + process.env.GITNEXUS_INDEX_LOCK_BACKEND = 'file'; +}); +afterEach(() => { + delete process.env.GITNEXUS_INDEX_LOCK_BACKEND; + rmSync(dir, { recursive: true, force: true }); +}); + +describe('acquireIndexLock', () => { + it('acquires a free directory and writes a record carrying our pid', async () => { + const lock = await acquireIndexLock(dir); + expect(existsSync(lockPath())).toBe(true); + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk).toMatchObject({ v: 1, pid: process.pid, hostname: os.hostname() }); + expect(lock.record.token).toBe(onDisk.token); + lock.release(); + expect(existsSync(lockPath())).toBe(false); + }); + + it('reclaims a stale lock left by a dead process (kill recovery)', async () => { + seedLock({ pid: 999999999, token: 'dead-holder' }); + const lock = await acquireIndexLock(dir, { timeoutMs: 2000 }); + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.pid).toBe(process.pid); + expect(onDisk.token).not.toBe('dead-holder'); + lock.release(); + }); + + it('waits on a live holder and times out instead of stealing (mutual exclusion)', async () => { + // A live pid (our own) with a different token — never stale, so acquire + // must block and then time out rather than clobber the holder. + seedLock({ pid: process.pid, startTime: null, token: 'live-holder' }); + await expect(acquireIndexLock(dir, { timeoutMs: 300, pollMs: 20 })).rejects.toBeInstanceOf( + IndexLockTimeoutError, + ); + // The live holder's record is untouched. + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.token).toBe('live-holder'); + }); + + it('surfaces the holder identity on timeout', async () => { + seedLock({ pid: process.pid, startTime: null, token: 'live-holder', invocationId: 'held-run' }); + await expect(acquireIndexLock(dir, { timeoutMs: 200, pollMs: 20 })).rejects.toMatchObject({ + holder: { invocationId: 'held-run', pid: process.pid }, + }); + }); + + it.skipIf(process.platform !== 'linux')( + 'treats a reused pid (live pid, different start time) as stale', + async () => { + // Our pid is alive but the seeded start time cannot match it → reused. + seedLock({ pid: process.pid, startTime: '1', token: 'reused-pid' }); + const lock = await acquireIndexLock(dir, { timeoutMs: 2000 }); + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.token).not.toBe('reused-pid'); + lock.release(); + }, + ); + + it('reclaims an empty lock file (crash between O_EXCL create and record write) without hanging', async () => { + // Pre-fix, readRecord→null hot-looped forever here. Post-fix it reclaims + // the malformed orphan after the grace and acquires. + writeFileSync(lockPath(), ''); + const lock = await acquireIndexLock(dir, { timeoutMs: 5000, pollMs: 20 }); + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.pid).toBe(process.pid); + lock.release(); + }); + + it('reclaims a partial/malformed record (valid JSON, missing token) without hanging', async () => { + writeFileSync(lockPath(), '{"pid":123}'); + const lock = await acquireIndexLock(dir, { timeoutMs: 5000, pollMs: 20 }); + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.pid).toBe(process.pid); + expect(onDisk.token.length).toBeGreaterThan(0); + lock.release(); + }); + + it('treats a non-positive/NaN pid as no readable holder and reclaims (never wedges on process.kill) (#2658 review L4)', async () => { + // `{"pid":0}` pre-fix: typeof 0 === 'number' passed readRecord, then + // process.kill(0,0) reported the process group "alive" → treated as a live + // holder → the acquire wedged until the full timeout. Post-fix a pid that is + // not a positive integer makes readRecord return null, so the file is a + // malformed orphan that is reclaimed after the grace. + seedLock({ pid: 0, token: 'zero-pid' }); + const lock = await acquireIndexLock(dir, { timeoutMs: 5000, pollMs: 20 }); + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.pid).toBe(process.pid); + expect(onDisk.token).not.toBe('zero-pid'); + lock.release(); + }); + + it('honors GITNEXUS_INDEX_LOCK_TIMEOUT_MS as the wait ceiling (bounds pid-reuse hangs)', async () => { + // A live holder we cannot steal (own pid, no start-time recorded). Without a + // finite ceiling this would hang; the env var must bound it (#2658). + seedLock({ pid: process.pid, startTime: null, token: 'live-holder' }); + const prev = process.env.GITNEXUS_INDEX_LOCK_TIMEOUT_MS; + process.env.GITNEXUS_INDEX_LOCK_TIMEOUT_MS = '150'; + try { + // No explicit timeoutMs → the env ceiling applies (not the 10-min default). + await expect(acquireIndexLock(dir, { pollMs: 20 })).rejects.toBeInstanceOf( + IndexLockTimeoutError, + ); + } finally { + if (prev === undefined) delete process.env.GITNEXUS_INDEX_LOCK_TIMEOUT_MS; + else process.env.GITNEXUS_INDEX_LOCK_TIMEOUT_MS = prev; + } + }); +}); + +describe('release', () => { + it('does not remove a lock that has been re-taken by another owner', async () => { + const lock = await acquireIndexLock(dir); + // Simulate the file being replaced by a different owner after we acquired. + seedLock({ pid: process.pid, token: 'someone-else' }); + lock.release(); + expect(existsSync(lockPath())).toBe(true); // not ours → left intact + const onDisk = JSON.parse(readFileSync(lockPath(), 'utf8')) as LockRecord; + expect(onDisk.token).toBe('someone-else'); + }); + + it('is idempotent', async () => { + const lock = await acquireIndexLock(dir); + lock.release(); + expect(() => lock.release()).not.toThrow(); + }); +}); + +describe('sweepStagingArtifacts', () => { + it('removes only staging files, never the live index or its sidecars', () => { + const files = [ + 'lbug', + 'lbug.wal', + 'lbug.shadow', + 'lbug.new', + 'lbug.new.wal', + 'lbug.new.wal.checkpoint', + 'lbug.staging.abc-123', + 'lbug.staging.abc-123.wal', + 'lbug.staging.abc-123.shadow', + 'gitnexus.json', + ]; + for (const f of files) writeFileSync(path.join(dir, f), 'x'); + + sweepStagingArtifacts(dir); + + const survives = (f: string) => existsSync(path.join(dir, f)); + expect(survives('lbug')).toBe(true); + expect(survives('lbug.wal')).toBe(true); + expect(survives('lbug.shadow')).toBe(true); + expect(survives('gitnexus.json')).toBe(true); + expect(survives('lbug.new')).toBe(false); + expect(survives('lbug.new.wal')).toBe(false); + expect(survives('lbug.new.wal.checkpoint')).toBe(false); + expect(survives('lbug.staging.abc-123')).toBe(false); + expect(survives('lbug.staging.abc-123.wal')).toBe(false); + expect(survives('lbug.staging.abc-123.shadow')).toBe(false); + }); + + it('runs the sweep automatically on acquire', async () => { + writeFileSync(path.join(dir, 'lbug.staging.orphan'), 'x'); + writeFileSync(path.join(dir, 'lbug'), 'x'); + const lock = await acquireIndexLock(dir); + expect(existsSync(path.join(dir, 'lbug.staging.orphan'))).toBe(false); + expect(existsSync(path.join(dir, 'lbug'))).toBe(true); + lock.release(); + }); +}); + +describe('classifyFtsBuildError', () => { + it('classifies IO/rename/checkpoint/corruption failures as integrity', () => { + expect( + classifyFtsBuildError( + 'IO exception: Error renaming file lbug.new.wal to lbug.new.wal.checkpoint. ErrorMessage: No such file or directory', + ), + ).toBe('integrity'); + expect(classifyFtsBuildError('checkpoint failed')).toBe('integrity'); + expect(classifyFtsBuildError('database file is corrupt')).toBe('integrity'); + expect(classifyFtsBuildError('write failed: no space left on device (ENOSPC)')).toBe( + 'integrity', + ); + }); + + it('classifies row-level tokenizer failures as capability (degrade)', () => { + expect(classifyFtsBuildError('Failed calling LOWER: Invalid UTF-8')).toBe('capability'); + expect(classifyFtsBuildError('tokenizer error on row 5')).toBe('capability'); + }); + + it('defaults unknown failures to capability so runs are not newly failed', () => { + expect(classifyFtsBuildError('some unrecognised message')).toBe('capability'); + expect(classifyFtsBuildError('missing indexes after build: File.name_fts')).toBe('capability'); + }); + + it('keeps a bare ENOENT / bad-fd as capability so it degrades, not aborts (#2658 review L1)', () => { + // A missing extension asset / closed handle reports a generic OS error; those + // must NOT escalate to an abort on the atomic-swap path. Only a specific + // write/rename/checkpoint failure is integrity. + expect(classifyFtsBuildError('ENOENT: no such file or directory, open fts.ext')).toBe( + 'capability', + ); + expect(classifyFtsBuildError('read failed: bad file descriptor (EBADF)')).toBe('capability'); + // The genuine build-broke rename race is still integrity via 'error renaming'. + expect( + classifyFtsBuildError('Error renaming lbug.new.wal to checkpoint: No such file or directory'), + ).toBe('integrity'); + }); + + it('lets a row-level tokenizer error win even if it mentions an integrity word', () => { + // A tokenizer error is a bad row, not a broken build — must still degrade. + expect(classifyFtsBuildError('Invalid UTF-8 during io exception path')).toBe('capability'); + }); +}); + +describe('ftsFailureIsFatal (#2658)', () => { + it('is fatal ONLY for an integrity failure on the atomic-swap path', () => { + // Atomic swap: staging DB, previous index intact → integrity may abort. + expect(ftsFailureIsFatal('integrity', true)).toBe(true); + // In-place: live DB already mutated, nothing to roll back → degrade. + expect(ftsFailureIsFatal('integrity', false)).toBe(false); + // Capability never aborts, either path. + expect(ftsFailureIsFatal('capability', true)).toBe(false); + expect(ftsFailureIsFatal('capability', false)).toBe(false); + // Missing class (ok result, or no classification) never aborts. + expect(ftsFailureIsFatal(undefined, true)).toBe(false); + }); +}); + +// The OS socket/pipe backend is only meaningful where `net` gives a clean, +// auto-releasing namespace: Linux abstract sockets and Windows named pipes. +describe.skipIf(process.platform !== 'linux' && process.platform !== 'win32')( + 'OS socket lock backend (#2658)', + () => { + // Override the file-backend pin from the outer beforeEach. + beforeEach(() => { + process.env.GITNEXUS_INDEX_LOCK_BACKEND = 'socket'; + }); + + it('holds no filesystem lock file (works on a read-only index dir)', async () => { + const lock = await acquireIndexLock(dir, { timeoutMs: 2000 }); + expect(existsSync(lockPath())).toBe(false); // endpoint is outside the dir + lock.release(); + }); + + it('excludes a second acquire on the same dir, then frees it on release', async () => { + const first = await acquireIndexLock(dir, { timeoutMs: 2000 }); + // A second acquire on the SAME slot is refused by the kernel (EADDRINUSE) + // and waits, then times out — the live holder is never displaced. + await expect(acquireIndexLock(dir, { timeoutMs: 300, pollMs: 20 })).rejects.toBeInstanceOf( + IndexLockTimeoutError, + ); + first.release(); + // Once released, the endpoint is free again. + const second = await acquireIndexLock(dir, { timeoutMs: 2000 }); + second.release(); + }); + + it('reports the holder as unknown on timeout — never a bogus "pid -1" (#2658 review M3)', async () => { + // The OS socket lock exposes no owner metadata, so a contended-wait timeout + // must not surface the unknownHolder() placeholder pid (-1) as if it were a + // real process the operator can look up. + const first = await acquireIndexLock(dir, { timeoutMs: 2000 }); + try { + const err = await acquireIndexLock(dir, { timeoutMs: 200, pollMs: 20 }).catch((e) => e); + expect(err).toBeInstanceOf(IndexLockTimeoutError); + expect((err as IndexLockTimeoutError).holderKnown).toBe(false); + expect((err as IndexLockTimeoutError).message).not.toContain('pid -1'); + } finally { + first.release(); + } + }); + + it('excludes an acquire reaching the same physical dir via a symlink alias (#2658 review H1)', async () => { + // Pre-fix the endpoint name hashed the LEXICAL path, so `alias` (a symlink + // to `dir`) produced a different name and BOTH acquired — a double-writer. + // Post-fix both canonicalize to `dir`'s real path → one name → excluded. + const alias = mkdtempSync(path.join(os.tmpdir(), 'gnx-lock-aliasparent-')); + const aliasLink = path.join(alias, 'link'); + symlinkSync(dir, aliasLink); + try { + const first = await acquireIndexLock(dir, { timeoutMs: 2000 }); + await expect( + acquireIndexLock(aliasLink, { timeoutMs: 300, pollMs: 20 }), + ).rejects.toBeInstanceOf(IndexLockTimeoutError); + first.release(); + } finally { + rmSync(alias, { recursive: true, force: true }); + } + }); + + it('gives independent locks to different index dirs', async () => { + const other = mkdtempSync(path.join(os.tmpdir(), 'gnx-lock-other-')); + try { + const a = await acquireIndexLock(dir, { timeoutMs: 2000 }); + const b = await acquireIndexLock(other, { timeoutMs: 2000 }); // distinct name → no contention + a.release(); + b.release(); + } finally { + rmSync(other, { recursive: true, force: true }); + } + }); + }, +); + +describe('read-only / permission-denied filesystem (#2658)', () => { + it('classifies EROFS/EACCES/EPERM as tolerable, others not', () => { + expect(isLockUnwritableCode('EROFS')).toBe(true); + expect(isLockUnwritableCode('EACCES')).toBe(true); + expect(isLockUnwritableCode('EPERM')).toBe(true); + expect(isLockUnwritableCode('EEXIST')).toBe(false); + expect(isLockUnwritableCode('ENOENT')).toBe(false); + expect(isLockUnwritableCode(undefined)).toBe(false); + }); + + // Mode bits are bypassed for uid 0, so the denied-create path only reproduces + // as non-root. The predicate test above is the always-on guard. + it.skipIf(!process.getuid || process.getuid() === 0)( + 'returns a no-op handle instead of throwing when the lock dir cannot be written', + async () => { + chmodSync(dir, 0o555); + try { + const lock = await acquireIndexLock(dir, { timeoutMs: 2000 }); + expect(typeof lock.release).toBe('function'); + expect(() => lock.release()).not.toThrow(); + expect(existsSync(lockPath())).toBe(false); // lock file was never created + } finally { + chmodSync(dir, 0o755); + } + }, + ); +}); diff --git a/gitnexus/test/unit/ingestion/di.test.ts b/gitnexus/test/unit/ingestion/di.test.ts index b5b1990f5..4bed5966e 100644 --- a/gitnexus/test/unit/ingestion/di.test.ts +++ b/gitnexus/test/unit/ingestion/di.test.ts @@ -17,6 +17,7 @@ import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { diPhase } from '../../../src/core/ingestion/pipeline-phases/di.js'; import { parseSpringCollectionType, + SPRING_DI_INJECTION_SITES_PROPERTY, springDiFieldMatcher, } from '../../../src/core/ingestion/di-extractors/spring.js'; import { generateId } from '../../../src/lib/utils.js'; @@ -728,6 +729,81 @@ describe('di phase', () => { ambiguousSkipped: 1, }); }); + + it('falls back to structural providers when no implementation is a known bean', async () => { + const graph = createKnowledgeGraph(); + + addInterface(graph, 'Port'); + addClass(graph, 'FirstPort', 'java'); + addClass(graph, 'SecondPort', 'java'); + addImplements(graph, 'FirstPort', 'Port'); + addImplements(graph, 'SecondPort', 'Port'); + addClass(graph, 'Consumer', 'java', 'Class', { + [SPRING_DI_INJECTION_SITES_PROPERTY]: [ + { + targetTypeName: 'Port', + cardinality: 'single', + reason: 'Spring DI: test constructor', + }, + ], + }); + + const output = await diPhase.execute(makeCtx(graph), new Map()); + + expect(injectsEdges(graph)).toHaveLength(2); + expect(injectsEdges(graph).every((edge) => edge.confidence === 0.5)).toBe(true); + expect(output).toMatchObject({ injectsEdges: 2, ambiguousInjections: 1 }); + }); + + it('fails closed when one injection type name denotes both a class and an interface', async () => { + const graph = createKnowledgeGraph(); + + addClass(graph, 'Port', 'java'); + addInterface(graph, 'Port'); + addClass(graph, 'PortImpl', 'java'); + addImplements(graph, 'PortImpl', 'Port'); + addClass(graph, 'Consumer', 'java', 'Class', { + [SPRING_DI_INJECTION_SITES_PROPERTY]: [ + { + targetTypeName: 'Port', + cardinality: 'single', + reason: 'Spring DI: test constructor', + }, + ], + }); + + const output = await diPhase.execute(makeCtx(graph), new Map()); + + expect(injectsEdges(graph)).toHaveLength(0); + expect(output).toMatchObject({ injectsEdges: 0, ambiguousSkipped: 1 }); + }); + + it('documents the legacy collection behavior change for a Class/Interface name collision', async () => { + const graph = createKnowledgeGraph(); + + addClass(graph, 'Port', 'java'); + addInterface(graph, 'Port'); + addClass(graph, 'PortImpl', 'java'); + addImplements(graph, 'PortImpl', 'Port'); + addClass(graph, 'Consumer', 'java', 'Class', { + [SPRING_DI_INJECTION_SITES_PROPERTY]: [ + { + targetTypeName: 'Port', + cardinality: 'collection', + reason: 'Spring DI: @Autowired List', + }, + ], + }); + + const output = await diPhase.execute(makeCtx(graph), new Map()); + + // Before concrete-class lookup was added, the interface alone won and + // collection injection fanned out to PortImpl. The graph-only resolver + // cannot disambiguate the colliding Java types, so the new behavior is an + // intentional fail-closed skip rather than a simple-name guess. + expect(injectsEdges(graph)).toHaveLength(0); + expect(output).toMatchObject({ injectsEdges: 0, ambiguousSkipped: 1 }); + }); }); // --------------------------------------------------------------------------- diff --git a/gitnexus/test/unit/ingestion/non-value-definition-keys.test.ts b/gitnexus/test/unit/ingestion/non-value-definition-keys.test.ts new file mode 100644 index 000000000..ca6e35bc0 --- /dev/null +++ b/gitnexus/test/unit/ingestion/non-value-definition-keys.test.ts @@ -0,0 +1,128 @@ +/** + * #2687 — unit coverage for `buildNonValueDefinitionNameKeys`, the pre-scan that + * makes the parse-worker's duplicate suppression order-independent. + * + * The parse-worker consults these keys from its value-label branch, so what this + * pre-scan registers decides which `Const`/`Static`/`Variable` nodes get dropped. + * The two guards that matter most: keys are name-qualified (a multi-name + * declaration shares one definition node), and a match resolving to a value label + * registers nothing (so a match can never suppress itself). + */ +import { describe, expect, it } from 'vitest'; +import type { LanguageProvider } from '../../../src/core/ingestion/language-provider.js'; +import { + buildDefinitionPreScan, + type SyntaxNode, +} from '../../../src/core/ingestion/utils/ast-helpers.js'; + +/** Minimal stub — the pre-scan only reads `startIndex` and `text`. */ +const node = (startIndex: number, text: string): SyntaxNode => + ({ startIndex, text }) as unknown as SyntaxNode; + +const match = (captures: Record) => ({ + captures: Object.entries(captures).map(([name, syntaxNode]) => ({ name, node: syntaxNode })), +}); + +/** `getLabelFromCaptures` only reaches for `labelOverride`; nothing else. */ +const PROVIDER = {} as unknown as LanguageProvider; + +/** The non-value claim set — what `Const`/`Static`/`Variable` consult. */ +const nonValueOf = ( + matches: Parameters[0], + provider: LanguageProvider, +): ReadonlySet => buildDefinitionPreScan(matches, provider).nonValue; + +describe('buildDefinitionPreScan', () => { + it('registers a function capture under its startIndex and name', () => { + const keys = nonValueOf( + [match({ 'definition.function': node(0, 'const Bare = () => 1;'), name: node(6, 'Bare') })], + PROVIDER, + ); + + expect([...keys]).toEqual(['0:Bare']); + }); + + it('registers nothing for a value capture', () => { + const keys = nonValueOf( + [match({ 'definition.const': node(0, 'const CONFIG = {};'), name: node(6, 'CONFIG') })], + PROVIDER, + ); + + expect([...keys]).toEqual([]); + }); + + it('registers nothing for a match with no name capture', () => { + const keys = nonValueOf([match({ 'definition.function': node(0, '() => 1') })], PROVIDER); + + expect([...keys]).toEqual([]); + }); + + it('registers nothing for a match with no definition capture', () => { + const keys = nonValueOf([match({ name: node(0, 'orphan') })], PROVIDER); + + expect([...keys]).toEqual([]); + }); + + it('keys by name so a shared definition node does not over-suppress', () => { + // `const a = 1, b = () => {}` — both declarators share ONE definition node, + // so only `b`'s name may be claimed. + const declaration = node(0, 'const a = 1, b = () => {}'); + const keys = nonValueOf( + [ + match({ 'definition.const': declaration, name: node(6, 'a') }), + match({ 'definition.function': declaration, name: node(13, 'b') }), + ], + PROVIDER, + ); + + expect([...keys]).toEqual(['0:b']); + }); + + it('registers nothing when a provider reclassifies a function capture to a value label', () => { + // Guards against self-suppression: if the pre-scan keyed off capture names + // rather than the resolved label, this match would register a key and then + // the main loop's value branch would drop its own node. + const provider = { + labelOverride: () => 'Const', + } as unknown as LanguageProvider; + + const keys = nonValueOf( + [match({ 'definition.function': node(0, 'val x = {}'), name: node(4, 'x') })], + provider, + ); + + expect([...keys]).toEqual([]); + }); + + it('returns an empty set for no matches', () => { + expect([...nonValueOf([], PROVIDER)]).toEqual([]); + }); + + it('ranks a property claim as non-value but NOT callable', () => { + // The rank split is what keeps an annotated Python attribute ahead of its + // bare-assignment `Variable` twin while still letting a callable collapse a + // Kotlin/Swift closure property. A property in `callable` would make a + // property suppress itself. + const claims = buildDefinitionPreScan( + [match({ 'definition.property': node(0, 'name: str = "x"'), name: node(0, 'name') })], + PROVIDER, + ); + + expect({ nonValue: [...claims.nonValue], callable: [...claims.callable] }).toEqual({ + nonValue: ['0:name'], + callable: [], + }); + }); + + it('ranks a callable claim into both sets', () => { + const claims = buildDefinitionPreScan( + [match({ 'definition.function': node(0, 'val f = { }'), name: node(4, 'f') })], + PROVIDER, + ); + + expect({ nonValue: [...claims.nonValue], callable: [...claims.callable] }).toEqual({ + nonValue: ['0:f'], + callable: ['0:f'], + }); + }); +}); diff --git a/gitnexus/test/unit/lazy-action.test.ts b/gitnexus/test/unit/lazy-action.test.ts index 9afe7bc27..973d728ca 100644 --- a/gitnexus/test/unit/lazy-action.test.ts +++ b/gitnexus/test/unit/lazy-action.test.ts @@ -90,4 +90,46 @@ describe('createAnalyzerLbugLazyAction', () => { expect(events).toEqual(['identity-module', 'receipt-captured', 'analyzer-module']); expect(run).toHaveBeenCalledWith(receipt, 'repo', { force: true }); }); + + it('sets exit code 1 and skips the analyzer import when native load fails', async () => { + // Regression guard for #2441: a LadybugDB native-load failure must fail + // closed — no analyzer import, no index write, non-zero exit — not the + // pre-fix "print help then exit 0" silent success. Mirrors the + // createLbugLazyAction failure test above for the analyze-only wrapper. + checkLbugNativeMock.mockReturnValueOnce({ + ok: false, + message: + 'LadybugDB native binary (lbugjs.node) exists but failed to load:\n' + ' dlopen failed', + }); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + process.exitCode = undefined; + const run = vi.fn(async () => undefined); + const analyzerLoader = vi.fn(async () => ({ run })); + const identityLoader = vi.fn(async () => ({ + captureAnalyzerIdentityBeforeLoad: async (_url: string, loader: () => Promise) => { + const loaded = await loader(); + return { runnerIdentity: { schemaVersion: 4 }, loaded }; + }, + })); + const action = createAnalyzerLbugLazyAction( + identityLoader as never, + analyzerLoader, + 'run', + 'file:///fixture/dist/cli/index.js', + ); + + try { + await expect(action('repo', { force: true })).resolves.toBeUndefined(); + + expect(analyzerLoader).not.toHaveBeenCalled(); + expect(run).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + expect(stderrSpy).toHaveBeenCalledWith( + expect.stringContaining('LadybugDB native binary (lbugjs.node) exists but failed to load:'), + ); + } finally { + stderrSpy.mockRestore(); + process.exitCode = undefined; + } + }); }); diff --git a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts index ebd956228..3243c1cfd 100644 --- a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts +++ b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts @@ -44,6 +44,7 @@ function makeFsMock(dbPath: string) { rename: vi.fn(async () => {}), mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, }; } @@ -586,6 +587,7 @@ function makeFsMockWithWalSize( rename: vi.fn(async () => {}), mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, }; } diff --git a/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts b/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts index 286c30a99..e06e7445c 100644 --- a/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts +++ b/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts @@ -45,6 +45,7 @@ const mockFsForInit = (dbPath: string) => { unlink: vi.fn(async () => {}), mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); }; @@ -85,6 +86,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { unlink: unlinkMock, mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ @@ -156,6 +158,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { unlink: unlinkMock, mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ @@ -220,6 +223,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { unlink: unlinkMock, mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ @@ -285,6 +289,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { unlink: unlinkMock, mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ @@ -345,6 +350,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { unlink: unlinkMock, mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ @@ -415,6 +421,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { unlink: unlinkMock, mkdir: vi.fn(async () => {}), open: makeOpenMock(), + readdir: vi.fn(async () => []), }, })); const openLbugConnectionMock = vi.fn(async () => ({ db, conn })); diff --git a/gitnexus/test/unit/lbug-config-pagesize.test.ts b/gitnexus/test/unit/lbug-config-pagesize.test.ts index f2108d5f8..0acd9e990 100644 --- a/gitnexus/test/unit/lbug-config-pagesize.test.ts +++ b/gitnexus/test/unit/lbug-config-pagesize.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { execFileSync } from 'child_process'; import { - _resetOsPageSizeCacheForTest, + _setOsPageSizeForTests, getOsPageSize, isLbugPageSizeFrameError, isPageSizeAwareLadybug, @@ -92,7 +92,7 @@ describe('isPageSizeAwareLadybug', () => { describe('getOsPageSize', () => { afterEach(() => { - _resetOsPageSizeCacheForTest(); + _setOsPageSizeForTests(undefined); execFileSyncSpy.mockClear(); }); @@ -155,7 +155,7 @@ describe('getOsPageSize', () => { it.skipIf(onWindows)('probes at most once per process (cached)', () => { expect(getOsPageSize()).toBe(getOsPageSize()); expect(execFileSyncSpy).toHaveBeenCalledTimes(1); - _resetOsPageSizeCacheForTest(); + _setOsPageSizeForTests(undefined); getOsPageSize(); expect(execFileSyncSpy).toHaveBeenCalledTimes(2); }); diff --git a/gitnexus/test/unit/lbug-config-wal.test.ts b/gitnexus/test/unit/lbug-config-wal.test.ts index 1b56f6145..8806f2aef 100644 --- a/gitnexus/test/unit/lbug-config-wal.test.ts +++ b/gitnexus/test/unit/lbug-config-wal.test.ts @@ -1,11 +1,13 @@ import os from 'os'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createLbugDatabase, estimateBufferPool, isLbugCheckpointIoError, isWalCorruptionError, setBufferPoolSizeHint, + _setOsPageSizeForTests, + bufferPoolExhaustionRemedy, } from '../../src/core/lbug/lbug-config.js'; import { _captureLogger } from '../../src/core/logger.js'; @@ -166,6 +168,12 @@ describe('createLbugDatabase WAL replay option', () => { describe('createLbugDatabase buffer pool size (#2557)', () => { const GiB = 1024 * 1024 * 1024; + // Pin a 4 KiB page so every expectation below is host-independent — on a + // 16 KiB-page Apple Silicon runner the #2631 granule scaling would + // otherwise multiply them by 4. + beforeEach(() => _setOsPageSizeForTests(4096)); + afterEach(() => _setOsPageSizeForTests(undefined)); + const bufferPoolArg = (Database: ReturnType): unknown => Database.mock.calls[0][1]; it.each([ @@ -259,7 +267,11 @@ describe('adaptive buffer pool hint', () => { const MiB = 1024 * 1024; const bufferPoolArg = (Database: ReturnType): unknown => Database.mock.calls[0][1]; - afterEach(() => setBufferPoolSizeHint(undefined)); + beforeEach(() => _setOsPageSizeForTests(4096)); + afterEach(() => { + setBufferPoolSizeHint(undefined); + _setOsPageSizeForTests(undefined); + }); describe('estimateBufferPool', () => { it.each([ @@ -355,3 +367,143 @@ describe('isLbugCheckpointIoError', () => { expect(isLbugCheckpointIoError(undefined)).toBe(false); }); }); + +// ─── #2631: page-size-scaled pool sizing (granule accounting) ─────────────── +describe('page-size-scaled buffer pool sizing (#2631)', () => { + const MiB = 1024 * 1024; + const GiB = 1024 * MiB; + const bufferPoolArg = (Database: ReturnType): unknown => Database.mock.calls[0][1]; + + afterEach(() => { + setBufferPoolSizeHint(undefined); + _setOsPageSizeForTests(undefined); + vi.unstubAllEnvs(); + }); + + // main-aptos: ADAPTIVE_POOL_FLOOR is 512 MiB here, not main's 256 MiB — the + // Move node tables' extra columns deterministically OOM a 256 MiB pool + // during COPY. Expectations below use the 512 MiB base, and the estimate + // cases use element counts whose estimate clears the larger floor. Keep the + // 512 MiB-based values when syncing from main. + it.each([ + ['64 KiB pages scale the floor ×16', 65536, 41, 16 * 512 * MiB], + [ + '64 KiB pages scale the estimate ×16 (150k × 4 KiB × 16 = 9.8 GB)', + 65536, + 150_000, + 150_000 * 4 * 1024 * 16, + ], + ['16 KiB pages (Apple Silicon) scale the floor ×4', 16384, 41, 4 * 512 * MiB], + ['4 KiB pages are byte-identical to the unscaled behavior', 4096, 150_000, 150_000 * 4 * 1024], + ])('%s', (_label, pageSize, elements, expected) => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + _setOsPageSizeForTests(pageSize); + expect(estimateBufferPool(elements)).toBe(expected); + } finally { + totalmemSpy.mockRestore(); + } + }); + + it('the scaled cap is still bounded by 80% of RAM (64 KiB pages, huge graph)', () => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + _setOsPageSizeForTests(65536); + // min(2 GiB × 16, 0.8 × 32 GiB) = min(32 GiB, 25.6 GiB) = 25.6 GiB + expect(estimateBufferPool(100_000_000)).toBe(Math.floor(0.8 * 32 * GiB)); + } finally { + totalmemSpy.mockRestore(); + } + }); + + it('an undetectable page size behaves exactly like 4 KiB (ratio 1)', () => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + _setOsPageSizeForTests(null); + expect(estimateBufferPool(150_000)).toBe(150_000 * 4 * 1024); + } finally { + totalmemSpy.mockRestore(); + } + }); + + it('the hintless default passed to the Database ctor stays at the unscaled #2557 cap on 64 KiB hosts', () => { + // The guard for the #2557 OOM protection: MCP serve / doctor / any open + // without a per-run hint must NOT inherit the page-size-scaled budget — + // the pool is an eager allocation at DB open. + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + _setOsPageSizeForTests(65536); + const Database = vi.fn(function (this: any) {}); + createLbugDatabase({ Database } as any, '/tmp/lbug-pool-64k'); + expect(bufferPoolArg(Database)).toBe(2 * GiB); + } finally { + totalmemSpy.mockRestore(); + } + }); + + it('the analyze hint path DOES scale on 64 KiB hosts (scaled floor, bounded by 80% RAM)', () => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + _setOsPageSizeForTests(65536); + setBufferPoolSizeHint(estimateBufferPool(41)); + const Database = vi.fn(function (this: any) {}); + createLbugDatabase({ Database } as any, '/tmp/lbug-pool-64k-hint'); + // 41 elements → below the scaled COPY floor → 16 × 512 MiB = 8 GiB + expect(bufferPoolArg(Database)).toBe(16 * 512 * MiB); + } finally { + totalmemSpy.mockRestore(); + } + }); + + it('GITNEXUS_LBUG_BUFFER_POOL_SIZE stays absolute on 64 KiB hosts (incl. 0 = native default)', () => { + _setOsPageSizeForTests(65536); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', String(512 * MiB)); + const Database = vi.fn(function (this: any) {}); + createLbugDatabase({ Database } as any, '/tmp/lbug-pool-64k-env'); + expect(bufferPoolArg(Database)).toBe(512 * MiB); + }); +}); + +// ─── #2631: actionable pool-exhaustion remedy ─────────────────────────────── +describe('bufferPoolExhaustionRemedy (#2631)', () => { + afterEach(() => _setOsPageSizeForTests(undefined)); + + const EXHAUSTION = + 'Buffer manager exception: Unable to allocate memory! The buffer pool is full and no memory could be freed!'; + + it('names the override knob for the exhaustion error', () => { + _setOsPageSizeForTests(4096); + const remedy = bufferPoolExhaustionRemedy(EXHAUSTION); + expect(remedy).toContain('GITNEXUS_LBUG_BUFFER_POOL_SIZE'); + expect(remedy).toContain('buffer pool'); + // ratio 1 → no page-size amplification note + expect(remedy).not.toContain('OS page size'); + }); + + it('explains the granule amplification on a 64 KiB-page host', () => { + _setOsPageSizeForTests(65536); + const remedy = bufferPoolExhaustionRemedy(EXHAUSTION); + expect(remedy).toContain('64 KiB OS page size'); + expect(remedy).toContain('16×'); + expect(remedy).toContain('GITNEXUS_LBUG_BUFFER_POOL_SIZE'); + }); + + it('is silent for non-exhaustion errors', () => { + _setOsPageSizeForTests(65536); + expect( + bufferPoolExhaustionRemedy('Binder exception: Table CodeEmbedding does not exist.'), + ).toBeUndefined(); + }); + + it('labels the 0 sentinel as the native default instead of "0 MiB"', () => { + _setOsPageSizeForTests(4096); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', '0'); + try { + const remedy = bufferPoolExhaustionRemedy(EXHAUSTION); + expect(remedy).toContain('native 80%-of-RAM default'); + expect(remedy).not.toContain('(0 MiB)'); + } finally { + vi.unstubAllEnvs(); + } + }); +}); diff --git a/gitnexus/test/unit/lbug-native-check.test.ts b/gitnexus/test/unit/lbug-native-check.test.ts index c54b1635b..2bb9958a5 100644 --- a/gitnexus/test/unit/lbug-native-check.test.ts +++ b/gitnexus/test/unit/lbug-native-check.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect } from 'vitest'; import os from 'os'; import path from 'path'; import fs from 'fs/promises'; -import { checkLbugNative } from '../../src/core/lbug/native-check.js'; +import { checkLbugNative, glibcTooOldMessage } from '../../src/core/lbug/native-check.js'; describe('checkLbugNative', () => { it('returns ok:true when the real @ladybugdb/core binary is present', () => { @@ -20,6 +20,7 @@ describe('checkLbugNative', () => { const result = checkLbugNative(tmpDir); expect(result.ok).toBe(false); + expect(result.kind).toBe('binary_missing'); expect(result.message).toContain('missing'); expect(result.message).toContain('install.js'); expect(result.message).toContain('trustedDependencies'); @@ -43,10 +44,145 @@ describe('checkLbugNative', () => { const result = checkLbugNative(tmpDir); expect(result.ok).toBe(false); + // Present but unloadable — doctor must not call this "missing" (#2672). + expect(result.kind).toBe('load_failed'); expect(result.message).toContain('failed to load'); expect(result.message).toContain('install.js'); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); } }); + + it('returns ok:false when lbugjs.node is truncated (loader crashes with a signal)', async () => { + // A partially written .node (valid header, missing pages) SIGBUSes dlopen — a + // signal, not a catchable throw. The out-of-process probe must observe the + // crash and report it, instead of the whole process dying with exit 135 (#2441). + const realPath = checkLbugNative().binaryPath; + expect(realPath).toBeDefined(); + const truncated = (await fs.readFile(realPath!)).subarray(0, 300_000); + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'lbug-check-')); + try { + await fs.writeFile(path.join(tmpDir, 'install.js'), ''); + await fs.writeFile(path.join(tmpDir, 'lbugjs.node'), truncated); + + const result = checkLbugNative(tmpDir); + + expect(result.ok).toBe(false); + expect(result.message).toContain('failed to load'); + expect(result.message).toContain('install.js'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it('does not prescribe a reinstall when the host glibc is too old (#2672)', async () => { + // The generic failure text ("truncated / ABI mismatch / re-run install.js") + // is actively wrong for this class: the reinstall re-downloads the identical + // prebuilt binary and fails identically. Guard the whole assembled message, + // not just the helper, so the branch stays wired into checkLbugNative. + const message = glibcTooOldMessage( + "Error: /lib64/libc.so.6: version `GLIBC_2.34' not found (required by " + + '/usr/lib/node_modules/gitnexus/node_modules/@ladybugdb/core/lbugjs.node)', + ); + + expect(message).toContain('glibc 2.34 or newer'); + expect(message).toContain('will NOT help'); + expect(message).not.toContain('install.js'); + expect(message).not.toContain('trustedDependencies'); + expect(message).not.toContain('--allow-build'); + }); + + // POSIX-only: the fake "node" is a shebang script, which Windows cannot exec. + // The real Windows path has no glibc, so this class cannot occur there anyway. + it.skipIf(process.platform === 'win32')( + 'checkLbugNative routes a glibc load failure to that message, not the reinstall text', + async () => { + // Proves the branch is WIRED, not merely present: the probe child is + // replaced by a script that emits the loader error the #2672 reporter saw. + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'lbug-check-glibc-')); + const originalExecPath = process.execPath; + try { + await fs.writeFile(path.join(tmpDir, 'lbugjs.node'), Buffer.from('content-irrelevant')); + await fs.writeFile(path.join(tmpDir, 'install.js'), ''); + const fakeNode = path.join(tmpDir, 'fake-node'); + await fs.writeFile( + fakeNode, + '#!/bin/sh\n' + + 'echo "Error: /lib64/libc.so.6: version \\`GLIBC_2.34\' not found' + + ' (required by /x/@ladybugdb/core/lbugjs.node)" >&2\n' + + 'exit 1\n', + ); + await fs.chmod(fakeNode, 0o755); + process.execPath = fakeNode; + + const result = checkLbugNative(tmpDir); + + expect(result.ok).toBe(false); + expect(result.kind).toBe('load_failed'); + expect(result.message).toContain('glibc 2.34 or newer'); + expect(result.message).toContain('will NOT help'); + expect(result.message).not.toContain('install.js'); + } finally { + process.execPath = originalExecPath; + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }, + ); + + it('names this host glibc alongside the required one', () => { + const message = glibcTooOldMessage("version `GLIBC_2.34' not found"); + // Never pin the runner's own glibc — assert the line is populated either way. + expect(message).toMatch(/this host: (glibc \d+\.\d+|glibc version could not be determined)/); + }); + + const requiredVersionCases: ReadonlyArray = [ + ['reports the single required version', "version `GLIBC_2.34' not found", '2.34'], + [ + 'reports the highest of several required versions', + "version `GLIBC_2.29' not found\nversion `GLIBC_2.34' not found", + '2.34', + ], + [ + 'orders versions numerically, not lexically', + "version `GLIBC_2.9' not found\nversion `GLIBC_2.34' not found", + '2.34', + ], + ]; + + it.each(requiredVersionCases)('%s', (_name, stderr, expected) => { + expect(glibcTooOldMessage(stderr)).toContain(`glibc ${expected} or newer`); + }); + + const nonGlibcCases: ReadonlyArray = [ + ['an unrelated loader error', 'Error: invalid ELF header'], + ['empty stderr', ''], + ['a GLIBC token with no not-found line', 'linked against GLIBC_2.34 successfully'], + ]; + + it.each(nonGlibcCases)('returns null for %s, leaving the generic message', (_name, stderr) => { + expect(glibcTooOldMessage(stderr)).toBeNull(); + }); + + it('returns ok:true when the load probe cannot be spawned (inconclusive, not a broken binary)', async () => { + // The binary is present, but the child probe cannot launch — a sandbox that + // forbids subprocesses, or a non-Node execPath. We could not test the binary, + // so a healthy one must not be condemned; the command's own load stays + // authoritative. (Binary content is irrelevant here — the probe never runs.) + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'lbug-check-')); + const originalExecPath = process.execPath; + try { + await fs.writeFile(path.join(tmpDir, 'lbugjs.node'), Buffer.from('content-irrelevant')); + await fs.writeFile(path.join(tmpDir, 'install.js'), ''); + process.execPath = path.join(tmpDir, 'definitely-not-node'); + + const result = checkLbugNative(tmpDir); + + expect(result.ok).toBe(true); + expect(result.message).toBeUndefined(); + } finally { + process.execPath = originalExecPath; + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/lbug/graph-emit-sink.test.ts b/gitnexus/test/unit/lbug/graph-emit-sink.test.ts new file mode 100644 index 000000000..7f051ce53 --- /dev/null +++ b/gitnexus/test/unit/lbug/graph-emit-sink.test.ts @@ -0,0 +1,403 @@ +/** + * GraphEmitSink unit tests (issue #2680). + * + * Verifies the streaming structural emit sink: + * - routes non-retained relationships to bounded CSV-on-disk and never stores + * them, while retained types reach the real graph untouched; + * - dedups by relationship id (the whole-graph emit does, and COPY into a + * PK-bearing table would violate on a repeat) — PdgEmitSink relies on an + * upstream per-file guarantee that does NOT exist for structural edges; + * - refuses to silently forget a streamed edge on removeRelationship; + * - exposes the streamed-endpoint predicate the local-symbol pruner needs to + * avoid pruning a node that a streamed edge still references; + * - fails loudly rather than handing a truncated CSV to the bulk COPY. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; +import { + GraphEmitSink, + RETAINED_REL_TYPES, + StreamedRelationshipRemovalError, +} from '../../../src/core/lbug/graph-emit-sink.js'; +import type { GraphRelationship } from 'gitnexus-shared'; + +const fnId = (name: string): string => `Function:src/a.ts:${name}`; + +const rel = ( + type: GraphRelationship['type'], + from: string, + to: string, + suffix = '', +): GraphRelationship => ({ + id: `${type}:${fnId(from)}->${fnId(to)}${suffix}`, + sourceId: fnId(from), + targetId: fnId(to), + type, + confidence: 1, + reason: 'direct', +}); + +const dataRows = async (csvPath: string): Promise => { + const text = await fsp.readFile(csvPath, 'utf8'); + return text + .split('\n') + .filter((l) => l.length > 0) + .slice(1); // drop header +}; + +let tmpRoot: string; +let csvDir: string; + +beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'graph-emit-sink-')); + csvDir = path.join(tmpRoot, 'streamed'); +}); + +afterEach(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); +}); + +describe('GraphEmitSink routing', () => { + it('streams a non-retained type to CSV and keeps it out of the graph', async () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + sink.addRelationship(rel('CALLS', 'a', 'b')); + const manifest = sink.finalize(); + + expect(real.relationshipCount).toBe(0); + expect(manifest).toMatchObject({ totalRows: 1 }); + const pair = manifest.relsByPair.get('Function|Function'); + expect(pair).toMatchObject({ rows: 1 }); + expect(await dataRows(pair!.csvPath)).toHaveLength(1); + }); + + it('delegates every retained type to the real graph and writes no CSV', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + for (const type of RETAINED_REL_TYPES) { + sink.addRelationship(rel(type, 'a', 'b', `:${type}`)); + } + const manifest = sink.finalize(); + + expect(real.relationshipCount).toBe(RETAINED_REL_TYPES.size); + expect(manifest).toMatchObject({ totalRows: 0 }); + expect(manifest.relsByPair.size).toBe(0); + }); + + it('never streams nodes — they stay in the real graph', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + sink.addNode({ + id: fnId('a'), + label: 'Function', + properties: { name: 'a', filePath: 'src/a.ts', startLine: 1, endLine: 2 }, + }); + sink.finalize(); + + expect(real.nodeCount).toBe(1); + expect(fs.readdirSync(csvDir)).toEqual([]); + }); + + it('skips edges whose endpoint labels are not valid node tables', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + sink.addRelationship({ + id: 'CALLS:bogus->alsobogus', + sourceId: 'NotATable:src/a.ts:x', + targetId: 'NotATable:src/a.ts:y', + type: 'CALLS', + confidence: 1, + reason: 'direct', + }); + const manifest = sink.finalize(); + + expect(manifest).toMatchObject({ totalRows: 0 }); + expect(real.relationshipCount).toBe(0); + }); +}); + +describe('GraphEmitSink arming', () => { + it('retains everything in the graph until armed', () => { + // The pre-parse phases are not all write-only: mapCobolToGraph scans CALLS + // edges and removes the unresolved ones. If the sink streamed from + // construction, that scan would see nothing and COBOL cross-program calls + // would silently stop resolving. + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + + sink.addRelationship(rel('CALLS', 'a', 'b')); + + expect(real.relationshipCount).toBe(1); + expect(sink.finalize()).toMatchObject({ totalRows: 0 }); + }); + + it('removal of a pre-arm CALLS edge still works (the COBOL path)', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + const unresolved = rel('CALLS', 'a', 'b'); + sink.addRelationship(unresolved); + + expect(sink.removeRelationship(unresolved.id)).toBe(true); + expect(real.relationshipCount).toBe(0); + sink.finalize(); + }); +}); + +describe('GraphEmitSink dedup', () => { + it('writes a duplicate relationship id exactly once', async () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + const duplicated = rel('CALLS', 'a', 'b'); + sink.addRelationship(duplicated); + sink.addRelationship(duplicated); + sink.addRelationship({ ...duplicated }); + const manifest = sink.finalize(); + + // A second row would violate the relationship table's PK on COPY. + expect(manifest).toMatchObject({ totalRows: 1 }); + expect(await dataRows(manifest.relsByPair.get('Function|Function')!.csvPath)).toHaveLength(1); + }); +}); + +describe('GraphEmitSink removal safety', () => { + it('throws rather than silently forgetting an already-streamed edge', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + const streamed = rel('CALLS', 'a', 'b'); + sink.addRelationship(streamed); + + expect(() => sink.removeRelationship(streamed.id)).toThrow(StreamedRelationshipRemovalError); + sink.finalize(); + }); + + it('still removes a retained edge normally', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + const retained = rel('DEFINES', 'a', 'b'); + sink.addRelationship(retained); + + expect(sink.removeRelationship(retained.id)).toBe(true); + expect(real.relationshipCount).toBe(0); + sink.finalize(); + }); +}); + +describe('GraphEmitSink reads are complete', () => { + it('iterRelationships returns streamed edges alongside retained ones', () => { + // This is the property that lets streaming be the default: every consumer + // (communities, processes, taint, the pruner) reads through this and must + // see the whole graph, not just what stayed in memory. + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + sink.addRelationship(rel('DEFINES', 'file', 'fn')); // retained + sink.addRelationship(rel('CALLS', 'a', 'b')); // streamed + sink.addRelationship(rel('ACCESSES', 'b', 'c')); // streamed + + const seen = [...sink.iterRelationships()]; + expect(seen.map((r) => r.type).sort()).toEqual(['ACCESSES', 'CALLS', 'DEFINES']); + expect(sink.relationshipCount).toBe(3); + // The real graph still holds only the retained one — the saving is real. + expect(real.relationshipCount).toBe(1); + sink.finalize(); + }); + + it('preserves endpoints and confidence on a streamed edge', () => { + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + sink.addRelationship({ ...rel('CALLS', 'caller', 'callee'), confidence: 0.25 }); + + expect([...sink.iterRelationships()]).toMatchObject([ + { sourceId: fnId('caller'), targetId: fnId('callee'), type: 'CALLS', confidence: 0.25 }, + ]); + sink.finalize(); + }); + + it('iterRelationshipsByType finds a streamed type', () => { + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + sink.addRelationship(rel('CALLS', 'a', 'b')); + sink.addRelationship(rel('ACCESSES', 'a', 'c')); + + expect([...sink.iterRelationshipsByType('CALLS')]).toHaveLength(1); + expect([...sink.iterRelationshipsByType('ACCESSES')]).toHaveLength(1); + expect([...sink.iterRelationshipsByType('EXTENDS')]).toEqual([]); + sink.finalize(); + }); + + it('forEachRelationship visits streamed edges too', () => { + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + sink.addRelationship(rel('CALLS', 'a', 'b')); + + const visited: string[] = []; + sink.forEachRelationship((r) => visited.push(r.type)); + expect(visited).toEqual(['CALLS']); + sink.finalize(); + }); +}); + +describe('GraphEmitSink IO faults', () => { + it('surfaces a writer-open failure from finalize instead of a partial manifest', () => { + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + sink.addRelationship(rel('CALLS', 'a', 'b')); + + // Destroy the CSV dir so the next pair's writer cannot be opened, the way + // an out-of-fds (EMFILE) or disk-full run would fail mid-emit. + fs.rmSync(csvDir, { recursive: true, force: true }); + expect(() => + sink.addRelationship({ + id: 'CALLS:File:src/a.ts->Function:src/a.ts:b', + sourceId: 'File:src/a.ts', + targetId: fnId('b'), + type: 'CALLS', + confidence: 1, + reason: 'direct', + }), + ).toThrow(); + + expect(() => sink.finalize()).toThrow(/streamed CSV writer\(s\) hit an IO error/); + }); + + it('refuses a second finalize', () => { + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + sink.finalize(); + expect(() => sink.finalize()).toThrow(/called twice/); + }); +}); + +describe('dedup key exactness', () => { + const endpoints = { sourceId: fnId('f'), targetId: fnId('g') }; + const withId = (id: string): GraphRelationship => ({ + id, + ...endpoints, + type: 'CALLS', + confidence: 1, + reason: 'direct', + }); + + it('keeps two ids that differ only in how many tail segments they carry', () => { + // Regression: the dedup key packs the id's trailing numeric segments, and an + // absent second segment defaults to 0. Without the segment COUNT in the key, + // `:7` and `:7:0` collapse onto one key and the second edge is silently + // discarded — a lost relationship with no error. Distinct ids must never + // collapse; identical ones must (see the duplicate test above). + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + + sink.addRelationship(withId(`rel:CALLS:${endpoints.sourceId}->${endpoints.targetId}:7`)); + sink.addRelationship(withId(`rel:CALLS:${endpoints.sourceId}->${endpoints.targetId}:7:0`)); + + expect(sink.relationshipCount).toBe(2); + expect(sink.finalize()).toMatchObject({ totalRows: 2 }); + }); + + it('keeps two call sites between the same pair', () => { + // The `:line:col` case from emit-references — same endpoints and type, so + // identical CSV rows; only the id distinguishes them, and the whole-graph + // emit keeps both. + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + + sink.addRelationship(withId(`rel:CALLS:${endpoints.sourceId}->${endpoints.targetId}:10:4`)); + sink.addRelationship(withId(`rel:CALLS:${endpoints.sourceId}->${endpoints.targetId}:99:7`)); + + expect(sink.relationshipCount).toBe(2); + sink.finalize(); + }); + + it('still collapses a genuinely repeated id', () => { + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + const id = `rel:CALLS:${endpoints.sourceId}->${endpoints.targetId}:10:4`; + + sink.addRelationship(withId(id)); + sink.addRelationship(withId(id)); + + expect(sink.relationshipCount).toBe(1); + sink.finalize(); + }); + + it('falls back to the full id for a non-numeric tail', () => { + // `rel:imports:...:${localName}` has a textual tail; the compact form does + // not apply and the id must be stored verbatim rather than truncated. + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + + sink.addRelationship(withId(`rel:IMPORTS:${endpoints.sourceId}->${endpoints.targetId}:alpha`)); + sink.addRelationship(withId(`rel:IMPORTS:${endpoints.sourceId}->${endpoints.targetId}:beta`)); + + expect(sink.relationshipCount).toBe(2); + sink.finalize(); + }); +}); + +describe('removeRelationship contract divergence', () => { + it('throws for an absent id once streaming has begun, by design', () => { + // KnowledgeGraph.removeRelationship returns false for an id it does not + // hold. The sink cannot rebuild a compact dedup key from a bare id, so it + // refuses to answer "false" for something that might already be on disk and + // unrecallable. Pinned so the divergence stays deliberate. + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + sink.addRelationship(rel('CALLS', 'a', 'b')); + + expect(() => sink.removeRelationship('rel:CALLS:never:emitted')).toThrow( + StreamedRelationshipRemovalError, + ); + sink.finalize(); + }); + + it('returns false for an absent id before anything has streamed', () => { + const sink = new GraphEmitSink(createKnowledgeGraph(), csvDir); + sink.beginStreaming(); + + expect(sink.removeRelationship('rel:CALLS:never:emitted')).toBe(false); + sink.finalize(); + }); +}); + +describe('field scan matches the object scan', () => { + it('yields the same (source, target, type, confidence) tuples either way', () => { + // Guards the five whole-graph scans converted to forEachRelationshipFields: + // a divergence between the two forms would silently skew community + // detection, process extraction and the pruner. + const real = createKnowledgeGraph(); + const sink = new GraphEmitSink(real, csvDir); + sink.beginStreaming(); + sink.addRelationship(rel('DEFINES', 'file', 'fn')); + sink.addRelationship(rel('CALLS', 'a', 'b')); + sink.addRelationship({ ...rel('ACCESSES', 'b', 'c'), confidence: 0.5 }); + + const viaObjects = [...sink.iterRelationships()] + .map((r) => `${r.sourceId}|${r.targetId}|${r.type}|${r.confidence}`) + .sort(); + const viaFields: string[] = []; + sink.forEachRelationshipFields((s, t, ty, c) => viaFields.push(`${s}|${t}|${ty}|${c}`)); + + expect(viaFields.sort()).toEqual(viaObjects); + sink.finalize(); + }); +}); diff --git a/gitnexus/test/unit/parse-impl-heap-guard-pipeline.test.ts b/gitnexus/test/unit/parse-impl-heap-guard-pipeline.test.ts new file mode 100644 index 000000000..d2ecf9c72 --- /dev/null +++ b/gitnexus/test/unit/parse-impl-heap-guard-pipeline.test.ts @@ -0,0 +1,136 @@ +/** + * #2649 review — pipeline-level coverage for the parse-phase heap guardrails. + * + * The pure predicates are covered in parse-impl-heap-guard.test.ts; these + * tests pin the WIRING inside runChunkedParseAndResolve: the mid-loop abort + * actually rejects the parse with the remedy message (nothing en route may + * swallow or remap it — the #2441 exit-0 bug class), and the preflight + * projection is computed from PARSEABLE files, not total scanned files (the + * miscalibration fixed on this branch). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const getHeapStatisticsMock = vi.hoisted(() => vi.fn()); +vi.mock('node:v8', async () => { + const actual = await vi.importActual('node:v8'); + const mocked = { ...actual, getHeapStatistics: getHeapStatisticsMock }; + return { ...mocked, default: mocked }; +}); + +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { + projectParseHeapNeedBytes, + runChunkedParseAndResolve, +} from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; +import { _captureLogger } from '../../src/core/logger.js'; + +const MB = 1024 * 1024; + +let repoDir: string; +let workerStubPath: string; +let memoryUsageSpy: ReturnType | undefined; + +beforeEach(() => { + delete process.env.GITNEXUS_MEMORY; + repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-heap-guard-pipeline-')); + fs.mkdirSync(path.join(repoDir, 'src'), { recursive: true }); + // The pool validates the worker script's existence up front; the abort test + // never dispatches, and the preflight test parses one real file. + workerStubPath = path.join(repoDir, 'fake-worker.js'); + fs.writeFileSync(workerStubPath, '// worker stub for createWorkerPool'); +}); + +afterEach(() => { + memoryUsageSpy?.mockRestore(); + memoryUsageSpy = undefined; + delete process.env.GITNEXUS_MEMORY; + fs.rmSync(repoDir, { recursive: true, force: true }); +}); + +const writeFixture = (rel: string, content: string): { path: string; size: number } => { + const full = path.join(repoDir, rel); + fs.writeFileSync(full, content); + return { path: rel, size: fs.statSync(full).size }; +}; + +describe('#2649 heap guardrails wired into runChunkedParseAndResolve', () => { + it('mid-loop guard rejects the parse with the actionable remedy message', async () => { + const file = writeFixture('src/a.ts', 'export function a() { return 1; }\n'); + // 1GB limit with 95% "in use": above the 92% abort threshold. + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 1024 * MB }); + memoryUsageSpy = vi.spyOn(process, 'memoryUsage').mockReturnValue({ + rss: 0, + heapTotal: 1024 * MB, + heapUsed: 973 * MB, + external: 0, + arrayBuffers: 0, + }); + + const graph = createKnowledgeGraph(); + await expect( + runChunkedParseAndResolve(graph, [file], [file.path], 1, repoDir, Date.now(), () => {}, { + workerUrlForTest: pathToFileURL(workerStubPath), + workerPoolSize: 1, + }), + ).rejects.toThrow(/Analyze stopped before running out of memory/); + }); + + it('preflight warn projects from PARSEABLE files only and names the parseable count', async () => { + // Mock the heap limit relative to what ONE parseable file actually projects, + // so this stays a test of the WARN BEHAVIOUR rather than of the projection + // constant. Hard-coding 150_000 tied it to PROJECTED_HEAP_BYTES_PER_NODE = + // 2250; recalibrating that constant for streamed emit (#2680) dropped the + // projection to 0.80 of the limit and the warn silently stopped firing. + // Deriving the limit keeps the ratio at 0.90 — above the 0.85 threshold — + // whatever the constant becomes. + process.env.GITNEXUS_MEMORY = 'off'; + getHeapStatisticsMock.mockReturnValue({ + heap_size_limit: Math.floor(projectParseHeapNeedBytes(1) / 0.9), + }); + + const parseable = writeFixture('src/b.ts', 'export function b() { return 2; }\n'); + const unparseable = writeFixture('src/data.zzz9', 'not source code\n'); + + const cap = _captureLogger(); + const graph = createKnowledgeGraph(); + try { + await runChunkedParseAndResolve( + graph, + [parseable, unparseable], + [parseable.path, unparseable.path], + 2, + repoDir, + Date.now(), + () => {}, + { + workerUrlForTest: pathToFileURL( + path.resolve( + __dirname, + '..', + '..', + 'dist', + 'core', + 'ingestion', + 'workers', + 'parse-worker.js', + ), + ), + workerPoolSize: 1, + }, + ); + } finally { + cap.restore(); + } + + const warn = cap.records().find((r) => r.msg.includes('Large repository')); + // "analyzing 1 files" — the parseable count, not the 2 scanned files. + expect({ + fired: warn !== undefined, + parseableBasis: warn?.msg.includes('analyzing 1 files') ?? false, + }).toEqual({ fired: true, parseableBasis: true }); + }); +}); diff --git a/gitnexus/test/unit/parse-impl-heap-guard.test.ts b/gitnexus/test/unit/parse-impl-heap-guard.test.ts new file mode 100644 index 000000000..89a535d41 --- /dev/null +++ b/gitnexus/test/unit/parse-impl-heap-guard.test.ts @@ -0,0 +1,87 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Pin physical RAM to 32GB so the remedy branch (auto cap = 24GB) resolves +// deterministically regardless of the host machine. +vi.mock('os', async () => { + const actual = await vi.importActual('os'); + const mocked = { ...actual, totalmem: () => 32 * 1024 * 1024 * 1024 }; + return { ...mocked, default: mocked }; +}); + +import { + heapPressureRemedy, + projectParseHeapNeedBytes, + shouldAbortForHeapPressure, +} from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; + +const GB = 1024 * 1024 * 1024; + +const setConstrainedMemory = (value: number): (() => void) => { + const desc = Object.getOwnPropertyDescriptor(process, 'constrainedMemory'); + Object.defineProperty(process, 'constrainedMemory', { configurable: true, value: () => value }); + return () => { + if (desc) Object.defineProperty(process, 'constrainedMemory', desc); + else delete (process as { constrainedMemory?: unknown }).constrainedMemory; + }; +}; + +describe('#2649 parse-phase heap guardrails', () => { + let initialGuard: string | undefined; + let restoreConstrained: (() => void) | undefined; + + beforeEach(() => { + initialGuard = process.env.GITNEXUS_MEMORY; + delete process.env.GITNEXUS_MEMORY; + // Unconstrained by default so the mocked 32GB totalmem governs. + restoreConstrained = setConstrainedMemory(0); + }); + + afterEach(() => { + if (initialGuard === undefined) delete process.env.GITNEXUS_MEMORY; + else process.env.GITNEXUS_MEMORY = initialGuard; + restoreConstrained?.(); + restoreConstrained = undefined; + }); + + it('projects kernel-scale repos far past the 4GB default heap and small repos well under it', () => { + // 94,773 files x 55 nodes x 2250 bytes ≈ 11.7GB (the measured #2649 case); + // 2,000 files ≈ 236MB. + expect({ + kernelExceeds4Gb: projectParseHeapNeedBytes(94773) > 4 * GB, + smallRepoUnder1Gb: projectParseHeapNeedBytes(2000) < 1 * GB, + }).toEqual({ kernelExceeds4Gb: true, smallRepoUnder1Gb: true }); + }); + + it('aborts above 92% of the heap limit and not below it', () => { + const limit = 4 * GB; + expect([0.91, 0.93].map((f) => shouldAbortForHeapPressure(limit * f, limit))).toEqual([ + false, + true, + ]); + }); + + it('GITNEXUS_MEMORY=0 disables the abort entirely', () => { + process.env.GITNEXUS_MEMORY = 'off'; + const limit = 4 * GB; + expect(shouldAbortForHeapPressure(limit * 0.99, limit)).toBe(false); + }); + + it('points at the NODE_OPTIONS pin when the machine has more memory to give', () => { + // 4GB limit on a 32GB machine (auto cap 24GB): the pin is the problem. + expect(heapPressureRemedy(4 * GB)).toContain('re-run without the --max-old-space-size'); + }); + + it('points at scope or hardware when the machine is the ceiling', () => { + // 23GB limit on a 32GB machine (~auto cap): nothing more to unlock locally. + expect(heapPressureRemedy(23 * GB)).toContain('.gitnexusignore'); + }); + + it('remedy respects a real cgroup limit: a memory-limited container is never told to "drop the pin" (#2649 review)', () => { + // 8GB cgroup limit on the mocked 32GB host, heap already sized to the + // container (~6.5GB): raw totalmem would claim "more memory available"; + // the container is actually at its ceiling. + restoreConstrained?.(); + restoreConstrained = setConstrainedMemory(8 * GB); + expect(heapPressureRemedy(6.5 * GB)).toContain('.gitnexusignore'); + }); +}); diff --git a/gitnexus/test/unit/review-agent-workflow.test.ts b/gitnexus/test/unit/review-agent-workflow.test.ts index 35657e6ea..179698cb8 100644 --- a/gitnexus/test/unit/review-agent-workflow.test.ts +++ b/gitnexus/test/unit/review-agent-workflow.test.ts @@ -572,7 +572,7 @@ describe('gitnexus review-agent workflow security contract', () => { const expectedPins = [ 'actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0', 'actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3', - 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e', + 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020', 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a', 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c', 'anthropics/claude-code-action/base-action@3553f84341b92da26052e28acf1aa898f9511f32', diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 2a4d9c86a..89ef7f606 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -499,6 +499,8 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { ok: false, error: 'missing indexes after build: Function.function_fts', })), + ftsFailureIsFatal: (fc: 'capability' | 'integrity' | undefined, swap: boolean) => + fc === 'integrity' && swap, })); vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ @@ -526,6 +528,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { ); expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('build-failed'); // #2658 review L2 expect(logs.join('\n')).toMatch( /FTS index build failed.*missing indexes after build.*keyword search degraded this run/i, ); @@ -538,6 +541,78 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { } }); + it('ABORTS (throws before publish, leaves the previous index intact) on an FTS integrity failure on the atomic-swap path (#2658 review M1)', async () => { + // The single-writer lock rules out a concurrent-writer race, so an + // integrity-class FTS failure on the atomic-swap (--force) path is a real + // broken build: run-analyze must throw BEFORE swapping the staging DB in, + // leaving the previous live index untouched — not silently publish a + // search-less index as success. This end-to-end throw path was previously + // untested (only the ftsFailureIsFatal truth table was). + vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ + initLbug: vi.fn(async () => undefined), + loadGraphToLbug: vi.fn(async () => undefined), + getLbugStats: vi.fn(async () => ({ nodes: 0, edges: 0, communities: 0, processes: 0 })), + executeQuery: vi.fn(async () => []), + executeWithReusedStatement: vi.fn(async () => []), + closeLbug: vi.fn(async () => undefined), + wipeLbugDbFiles: vi.fn(async () => undefined), + loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), + deleteNodesForFile: vi.fn(async () => undefined), + deleteNodesForFiles: vi.fn(async () => undefined), + deleteAllCommunitiesAndProcesses: vi.fn(async () => undefined), + queryImporters: vi.fn(async () => []), + queryImportersBatch: vi.fn(async () => []), + loadFTSExtension: vi.fn(async () => true), + })); + // Import the REAL classifier/predicate (not a re-stub) so the test pins the + // actual fatal-decision logic, per the #2658 review. + vi.doMock('../../src/core/search/fts-indexes.js', async () => { + const actual = await vi.importActual( + '../../src/core/search/fts-indexes.js', + ); + return { + ...actual, + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + buildSearchIndexesOrDegrade: vi.fn(async () => ({ + ok: false, + failureClass: 'integrity' as const, + error: 'IO exception: Error renaming lbug.staging.wal to checkpoint', + })), + }; + }); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + standaloneIngest: emptyMoveIngestOutput(), + })), + })); + + const tmpRepo = await createTempDir('gitnexus-run-analyze-integrity-abort-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + // A pre-existing "previous index" that must survive the aborted rebuild. + await createPlaceholderGraphStore(lbugPath); + const before = await fs.readFile(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const message = await runFullAnalysis( + tmpRepo.dbPath, + { force: true }, + { onProgress: () => {}, onLog: () => {} }, + ).catch((e: unknown) => (e instanceof Error ? e.message : String(e))); + + expect(message).toMatch(/integrity error/i); + expect(message).toMatch(/aborted|previous index is\s+left intact/i); + // The previous index bytes are untouched (throw happened before the swap). + const after = await fs.readFile(lbugPath); + expect(after.equals(before)).toBe(true); + } finally { + await tmpRepo.cleanup(); + } + }); + it('full analyze degrades gracefully (no throw, warns, skips index creation) when FTS extension is unavailable', async () => { // Offline-first degradation: when loadFTSExtension() returns false, the // analyze path must NOT call createSearchFTSIndexes / verifySearchFTSIndexes @@ -598,6 +673,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { ); expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('extension-unavailable'); // #2658 review L2 expect(createSearchFTSIndexes).not.toHaveBeenCalled(); expect(verifySearchFTSIndexes).not.toHaveBeenCalled(); expect(logs.join('\n')).toMatch(/FTS extension unavailable; skipping search-index creation/i); @@ -680,6 +756,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { ); expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('extension-unavailable'); // #2658 review L2 const degradeLine = logs .filter((l) => l.includes('skipping search-index creation')) .join('\n'); @@ -1022,3 +1099,124 @@ describe('runFullAnalysis dirty-recovery parking failure fails fast (this shippi } }); }); + +describe('runFullAnalysis re-resolves git state under the lock (#2658 review H2)', () => { + afterEach(() => { + vi.doUnmock('../../src/storage/git.js'); + vi.doUnmock('../../src/core/ingestion/pipeline.js'); + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('re-reads HEAD after acquiring the lock, so a commit that lands during the wait is not missed', async () => { + // acquireIndexLock can wait up to the timeout ceiling; HEAD may advance + // during that wait. Pre-fix, resolveWriteTarget was called ONCE (before the + // lock) and its stale snapshot fed the freshness check — a waiter could + // return alreadyUpToDate against the OLD commit. Post-fix the wrapper + // re-resolves UNDER the lock, so getCurrentCommit is called again and the + // post-wait commit is what the pipeline uses. Simulate the advance by making + // getCurrentCommit return a new value on each call. + const commits = ['commit-before-wait', 'commit-after-wait']; + let call = 0; + const getCurrentCommit = vi.fn( + () => commits[call < commits.length ? call++ : commits.length - 1], + ); + vi.doMock('../../src/storage/git.js', async () => { + const actual = await vi.importActual( + '../../src/storage/git.js', + ); + return { + ...actual, + getCurrentCommit, + hasGitDir: () => true, + getCurrentBranch: () => 'main', + isWorkingTreeDirty: () => false, + }; + }); + // Stop the run right after the wrapper's two resolveWriteTarget calls so the + // test pins the re-resolve, not the full pipeline. + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async () => { + throw new Error('stop-after-resolve'); + }), + })); + + const tmpRepo = await createTempDir('gitnexus-h2-relock-'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis( + tmpRepo.dbPath, + { force: true }, + { onProgress: () => {}, onLog: () => {} }, + ).catch(() => undefined); + + // Pre-fix: exactly 1 (single pre-lock resolve). Post-fix: >= 2 (re-resolve + // under the lock), and the second call observed the post-wait commit. + expect(getCurrentCommit.mock.calls.length).toBeGreaterThanOrEqual(2); + expect(getCurrentCommit.mock.results[1]?.value).toBe('commit-after-wait'); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('releases the lock when the under-lock re-resolve throws (no leak) (#2658 review H2 self-review)', async () => { + // The re-resolve runs UNDER the held lock and can throw (e.g. a `--branch` + // that no longer matches a checkout switched during the wait). That throw + // must still release the lock — the loop lives inside the try/finally. + vi.doUnmock('../../src/storage/git.js'); + const release = vi.fn(); + vi.doMock('../../src/storage/index-lock.js', async () => { + const actual = await vi.importActual( + '../../src/storage/index-lock.js', + ); + return { + ...actual, + acquireIndexLock: vi.fn(async () => ({ + record: { + v: 1, + pid: 1, + hostname: 'h', + startTime: null, + token: 't', + invocationId: 'i', + acquiredAt: '', + }, + release, + })), + }; + }); + // getCurrentCommit succeeds on the pre-lock resolve, then throws on the + // under-lock re-resolve — the exact shape a mid-wait git change produces. + let call = 0; + vi.doMock('../../src/storage/git.js', async () => { + const actual = await vi.importActual( + '../../src/storage/git.js', + ); + return { + ...actual, + hasGitDir: () => true, + getCurrentBranch: () => 'main', + isWorkingTreeDirty: () => false, + getCurrentCommit: () => { + if (call++ === 0) return 'c1'; + throw new Error('git HEAD read failed mid-wait'); + }, + }; + }); + + const tmpRepo = await createTempDir('gitnexus-h2-leak-'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const err = await runFullAnalysis( + tmpRepo.dbPath, + { force: true }, + { onProgress: () => {}, onLog: () => {} }, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(Error); + expect(release).toHaveBeenCalledTimes(1); // lock freed despite the throw + } finally { + vi.doUnmock('../../src/storage/index-lock.js'); + await tmpRepo.cleanup(); + } + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts b/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts index 3a22dac63..74e4369c9 100644 --- a/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts +++ b/gitnexus/test/unit/scope-resolution/callable-value-flow-worklist.test.ts @@ -17,7 +17,10 @@ import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import type { ScopeResolutionIndexes } from '../../../src/core/ingestion/model/scope-resolution-indexes.js'; import { buildGraphNodeLookup } from '../../../src/core/ingestion/scope-resolution/graph-bridge/node-lookup.js'; import { createCalleeIdAccumulator } from '../../../src/core/ingestion/scope-resolution/graph-bridge/callee-id-sink.js'; -import { emitCallableValueFlow } from '../../../src/core/ingestion/scope-resolution/passes/callable-value-flow.js'; +import { + aggregateCallableValueFlowWarnings, + emitCallableValueFlow, +} from '../../../src/core/ingestion/scope-resolution/passes/callable-value-flow.js'; const FILE = 'chain.ts'; const MODULE = 'scope:module' as ScopeId; @@ -199,4 +202,39 @@ describe('callable-value-flow dependency worklist', () => { ), ).toEqual(['target']); }); + + it('groups repeated internal overflows by causal source context', () => { + expect( + aggregateCallableValueFlowWarnings([ + { + language: 'javascript', + context: 'site:bundle.js:4:1933', + candidateCount: 33, + cap: 32, + }, + { + language: 'javascript', + context: 'site:bundle.js:4:1933', + candidateCount: 40, + cap: 32, + }, + { + language: 'javascript', + context: 'copy:bundle.js', + candidateCount: 33, + cap: 32, + }, + ]), + ).toEqual([ + { + language: 'javascript', + context: 'site:bundle.js:4:1933', + candidateCount: 40, + cap: 32, + occurrences: 3, + distinctContexts: 2, + contextSamples: ['site:bundle.js:4:1933', 'copy:bundle.js'], + }, + ]); + }); }); diff --git a/gitnexus/test/unit/scope-resolution/java/java-captures.test.ts b/gitnexus/test/unit/scope-resolution/java/java-captures.test.ts index 36c88daca..4032b51c2 100644 --- a/gitnexus/test/unit/scope-resolution/java/java-captures.test.ts +++ b/gitnexus/test/unit/scope-resolution/java/java-captures.test.ts @@ -146,3 +146,130 @@ class C { expect(invokeFactsFor(src)).toBe(1); }); }); + +describe('emitJavaScopeCaptures — local-type identities (#2562)', () => { + it('uses the source-type-relative identity for the definition and the simple lexical binding', () => { + const matches = emitJavaScopeCaptures( + 'class Outer { void m() { class Local {} new Local(); } }', + 'Outer.java', + ); + const local = matches.find((m) => m['@declaration.name']?.text === 'Outer$1Local'); + + expect(local?.['@declaration.binding-name']?.text).toBe('Local'); + }); + + it('leaves non-local class declarations unchanged', () => { + const matches = emitJavaScopeCaptures('class Outer { class Member {} }', 'Outer.java'); + const member = matches.find((m) => m['@declaration.name']?.text === 'Member'); + + expect(member?.['@declaration.binding-name']).toBeUndefined(); + }); + + it('recognizes a local class inside a record compact constructor', () => { + const matches = emitJavaScopeCaptures( + 'record R(int x) { R { class Local {} new Runnable() {}; } }', + 'R.java', + ); + const names = matches.flatMap((m) => m['@declaration.name']?.text ?? []); + + expect(names).toContain('R$1Local'); + expect(names).toContain('R$1'); + }); + + it('uses javac-compatible independent sequences for anonymous and named local types', () => { + const matches = emitJavaScopeCaptures( + `class Outer { + void first() { + new Runnable() {}; + class Local {} + class Other {} + new Runnable() {}; + } + void second() { class Local {} } + }`, + 'Outer.java', + ); + const names = matches.flatMap((m) => m['@declaration.name']?.text ?? []); + + expect(names).toEqual( + expect.arrayContaining([ + 'Outer$1', + 'Outer$2', + 'Outer$1Local', + 'Outer$2Local', + 'Outer$1Other', + ]), + ); + }); + + it('synthesizes every legal local type kind with its lexical binding name', () => { + const matches = emitJavaScopeCaptures( + `class Outer { + void types() { + class C {} + enum E { A } + record R(int x) {} + interface I { void run(); } + } + }`, + 'Outer.java', + ); + + for (const [tag, identityName, bindingName] of [ + ['@declaration.class', 'Outer$1C', 'C'], + ['@declaration.enum', 'Outer$1E', 'E'], + ['@declaration.record', 'Outer$1R', 'R'], + ['@declaration.interface', 'Outer$1I', 'I'], + ] as const) { + const declaration = matches.find( + (match) => match[tag] !== undefined && match['@declaration.name']?.text === identityName, + ); + expect(declaration?.['@declaration.binding-name']?.text).toBe(bindingName); + } + }); + + it('detects local types from block position in initializers, lambdas, and anonymous bodies', () => { + const matches = emitJavaScopeCaptures( + `class Outer { + static { class StaticLocal {} } + { record InstanceLocal(int x) {} } + Runnable task = () -> { interface LambdaLocal {} }; + Runnable anon = new Runnable() { + { enum AnonymousLocal { A } } + public void run() {} + }; + }`, + 'Outer.java', + ); + const names = matches.flatMap((match) => match['@declaration.name']?.text ?? []); + + expect(names).toEqual( + expect.arrayContaining([ + 'Outer$1StaticLocal', + 'Outer$1InstanceLocal', + 'Outer$1LambdaLocal', + 'Outer$1$1AnonymousLocal', + ]), + ); + }); + + it('emits declaration-to-block visibility scopes for local types', () => { + const matches = emitJavaScopeCaptures( + `class Outer { + void blocks() { + new Local(); + class Local {} + new Local(); + } + }`, + 'Outer.java', + ); + const local = matches.find((match) => match['@declaration.name']?.text === 'Outer$1Local'); + const visibility = matches.find( + (match) => + match['@scope.block']?.range.startLine === local?.['@declaration.class']?.range.startLine, + ); + + expect(visibility?.['@scope.block']?.range.endLine).toBe(6); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/run-progress.test.ts b/gitnexus/test/unit/scope-resolution/run-progress.test.ts index 45bbfe9d7..17f8a79a8 100644 --- a/gitnexus/test/unit/scope-resolution/run-progress.test.ts +++ b/gitnexus/test/unit/scope-resolution/run-progress.test.ts @@ -1,12 +1,15 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import type { ParsedFile, ScopeId, Scope } from 'gitnexus-shared'; import { + formatScopeResolutionWarningContext, + MAX_PROGRESS_WARNING_CONTEXT_CHARS, runScopeResolution, type ScopeResolutionSubPhase, } from '../../../src/core/ingestion/scope-resolution/pipeline/run.js'; import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; import { createSemanticModel } from '../../../src/core/ingestion/model/semantic-model.js'; import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; +import { _captureLogger, warnRespectingProgressBar } from '../../../src/core/logger.js'; const mkScope = (id: ScopeId, filePath: string): Scope => ({ id, @@ -41,6 +44,37 @@ const stubProvider = { } as unknown as ScopeResolver; describe('runScopeResolution onProgress', () => { + it('escapes control bytes and bounds progress warning contexts', () => { + const formatted = formatScopeResolutionWarningContext(`binding\0${'x'.repeat(200)}`); + + expect(formatted).not.toContain('\0'); + expect(formatted).toContain('\\u0000'); + expect(formatted).toHaveLength(MAX_PROGRESS_WARNING_CONTEXT_CHARS); + expect(formatted.endsWith('...')).toBe(true); + }); + + it('routes warnings through the analyze progress logger instead of Pino', () => { + const previous = process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE; + const capture = _captureLogger(); + const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE = '1'; + warnRespectingProgressBar('progress-safe warning', { + fields: { language: 'move', occurrences: 2 }, + message: 'structured warning', + }); + + expect(consoleWarn).toHaveBeenCalledOnce(); + expect(consoleWarn).toHaveBeenCalledWith('progress-safe warning'); + expect(capture.records()).toEqual([]); + } finally { + consoleWarn.mockRestore(); + capture.restore(); + if (previous === undefined) delete process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE; + else process.env.GITNEXUS_ANALYZE_PROGRESS_ACTIVE = previous; + } + }); + it('emits sub-phases in order for a 3-file input', () => { const files = [ { path: 'a.py', content: '' }, diff --git a/gitnexus/test/unit/scope-resolution/walkers-augmentations.test.ts b/gitnexus/test/unit/scope-resolution/walkers-augmentations.test.ts index 21d48fc49..a4dbb0e5b 100644 --- a/gitnexus/test/unit/scope-resolution/walkers-augmentations.test.ts +++ b/gitnexus/test/unit/scope-resolution/walkers-augmentations.test.ts @@ -12,6 +12,7 @@ import { describe, it, expect } from 'vitest'; import { + findAllCallableBindingCandidatesInScope, findCallableBindingInScope, findClassBindingInScope, findExportedDefByName, @@ -215,6 +216,25 @@ describe('walker helpers read bindingAugmentations', () => { expect(findCallableBindingInScope(SCOPE, 'callMe', indexes)?.nodeId).toBe('callMe'); }); + it('preserves augmentation provenance masked by a finalized binding', () => { + const moduleScope = scope(SCOPE); + const callable = def('callMe'); + const finalized = { def: callable, origin: 'local' } as BindingRef; + const staticImport = { + def: callable, + origin: 'import', + visibility: 'static-member-import', + } as BindingRef; + const indexes = indexesForScopeLookup(moduleScope, new Map([['callMe', [staticImport]]])); + indexes.bindings.set(SCOPE, new Map([['callMe', [finalized]]])); + + const candidates = findAllCallableBindingCandidatesInScope(SCOPE, 'callMe', indexes); + + expect(candidates).toHaveLength(1); + expect(candidates[0]!.def).toBe(callable); + expect(candidates[0]!.bindings).toEqual([finalized, staticImport]); + }); + it('findExportedDefByName finds callable refs that exist only in augmentations', () => { const moduleScope = scope(SCOPE); const callableRef = { def: def('fromAugmentation'), origin: 'import' } as BindingRef; diff --git a/gitnexus/test/unit/skill-gen.test.ts b/gitnexus/test/unit/skill-gen.test.ts index 797f67c4c..0ffb60ec0 100644 --- a/gitnexus/test/unit/skill-gen.test.ts +++ b/gitnexus/test/unit/skill-gen.test.ts @@ -643,6 +643,266 @@ describe('generateSkillFiles — file output', () => { await expect(fs.access(path.join(skillsRoot, 'gitnexus-area-old'))).rejects.toThrow(); }); + /** + * When the repo contains an .agents/ directory, generated community skills + * must be mirrored to .agents/skills/ (flat gitnexus-area-* layout, #2434) + * so agents that prefer repo-local .agents/skills over the global + * ~/.agents/skills install serve the up-to-date set. The mirror content must + * match the .claude copy. + */ + it('mirrors generated skills to .agents/skills/ when .agents/ exists', async () => { + const { graph, communities, memberships } = twoCommSetup(); + await fs.mkdir(path.join(tmpDir, '.agents'), { recursive: true }); + + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ + graph, + repoPath: tmpDir, + communities, + memberships, + }), + ); + + const claudeAlpha = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ); + const agentsAlpha = await fs.readFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ); + const agentsBeta = await fs.readFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-beta', 'SKILL.md'), + 'utf-8', + ); + expect(agentsAlpha).toBe(claudeAlpha); + expect(agentsBeta.length).toBeGreaterThan(0); + }); + + /** + * Without an .agents/ opt-in, no .agents/skills/ tree should be created — + * only the canonical .claude/skills/ copy is written. + */ + it('does not mirror generated skills to .agents/ when the directory is absent', async () => { + const { graph, communities, memberships } = twoCommSetup(); + + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ + graph, + repoPath: tmpDir, + communities, + memberships, + }), + ); + + // Canonical copy exists, mirror does not. + const claudeAlpha = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ); + expect(claudeAlpha.length).toBeGreaterThan(0); + await expect(fs.access(path.join(tmpDir, '.agents'))).rejects.toThrow(); + }); + + /** + * MEDIUM 1 (reviewer repro): when `.agents/skills` exists as a regular file, + * the mirror root mkdir fails. Mirroring must degrade gracefully (warn + + * disable) and the canonical community skills under .claude/skills/ must + * still be written in full — never deleted-then-not-rewritten. + */ + it('keeps canonical skills intact when .agents/skills is a file (mirror root mkdir fails)', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const { graph, communities, memberships } = twoCommSetup(); + // .agents/ exists, but .agents/skills is a file — mkdir will EEXIST. + await fs.mkdir(path.join(tmpDir, '.agents'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.agents', 'skills'), 'not a directory'); + + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ graph, repoPath: tmpDir, communities, memberships }), + ); + + // Canonical skills are fully present. + const claudeAlpha = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ); + const claudeBeta = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-beta', 'SKILL.md'), + 'utf-8', + ); + expect(claudeAlpha.length).toBeGreaterThan(0); + expect(claudeBeta.length).toBeGreaterThan(0); + // Mirror was disabled with a warning, not a thrown error. + expect(logSpy).toHaveBeenCalled(); + }); + + /** + * MEDIUM 1 per-skill: the mirror root is writable, but an individual skill's + * mirror write fails. The failure must be warned and contained — other + * communities' canonical AND mirror writes still succeed. + */ + it('isolates a per-skill mirror write failure to that skill (best-effort)', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const { graph, communities, memberships } = twoCommSetup(); + await fs.mkdir(path.join(tmpDir, '.agents'), { recursive: true }); + + // Sabotage only the alpha mirror dir: make it a read-only file so the + // per-skill mkdir(agentsSkillDir) throws EEXIST (not a dir) and is caught. + await fs.mkdir(path.join(tmpDir, '.agents', 'skills'), { recursive: true }); + await fs.writeFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-alpha'), + 'file blocks dir', + ); + + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ graph, repoPath: tmpDir, communities, memberships }), + ); + + // Canonical for both communities is intact. + await expect( + fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ), + ).resolves.toHaveProperty('length'); + const claudeBeta = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-beta', 'SKILL.md'), + 'utf-8', + ); + expect(claudeBeta.length).toBeGreaterThan(0); + // Beta mirror still written (alpha failure did not abort the loop). + const agentsBeta = await fs.readFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-beta', 'SKILL.md'), + 'utf-8', + ); + expect(agentsBeta).toBe(claudeBeta); + expect(logSpy).toHaveBeenCalled(); + }); + + /** + * MEDIUM 1 delete-then-rewrite ordering: the canonical gitnexus-area-* + * cleanup runs before the mirror writes. A mirror failure after cleanup + * must not leave canonical missing — canonical is rewritten regardless. + */ + it('rewrites canonical skills after cleanup even when mirroring fails', async () => { + const { graph, communities, memberships } = twoCommSetup(); + await fs.mkdir(path.join(tmpDir, '.agents'), { recursive: true }); + + // First run: write canonical + mirror normally. + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ graph, repoPath: tmpDir, communities, memberships }), + ); + const firstAlpha = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ); + + // Second run with mirror broken: .agents/skills becomes a file. + await fs.rm(path.join(tmpDir, '.agents', 'skills'), { recursive: true, force: true }); + await fs.writeFile(path.join(tmpDir, '.agents', 'skills'), 'now a file'); + vi.spyOn(console, 'log').mockImplementation(() => {}); + + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ graph, repoPath: tmpDir, communities, memberships }), + ); + + // Canonical alpha is still present and content is stable (cleanup deleted + // the old dir, then canonical rewrote it — not lost). + const secondAlpha = await fs.readFile( + path.join(tmpDir, '.claude', 'skills', 'gitnexus-area-alpha', 'SKILL.md'), + 'utf-8', + ); + expect(secondAlpha).toBe(firstAlpha); + }); + + /** + * Mirror cleanup is namespace-scoped: only stale gitnexus-area-* mirror + * dirs are removed; mirrored standard skills and user-authored skills under + * .agents/skills/ survive a re-run. + */ + it('clears only stale gitnexus-area-* mirror dirs, preserving others', async () => { + const { graph, communities, memberships } = twoCommSetup(); + await fs.mkdir(path.join(tmpDir, '.agents'), { recursive: true }); + // Pre-existing non-community content that must survive. + await fs.mkdir(path.join(tmpDir, '.agents', 'skills', 'gitnexus-cli'), { recursive: true }); + await fs.writeFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-cli', 'SKILL.md'), + 'standard', + ); + await fs.mkdir(path.join(tmpDir, '.agents', 'skills', 'user-author'), { recursive: true }); + await fs.writeFile(path.join(tmpDir, '.agents', 'skills', 'user-author', 'SKILL.md'), 'mine'); + // Stale community mirror from a prior run. + await fs.mkdir(path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-old'), { + recursive: true, + }); + await fs.writeFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-old', 'SKILL.md'), + 'stale', + ); + + await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ graph, repoPath: tmpDir, communities, memberships }), + ); + + // Stale community mirror gone; non-community content preserved. + await expect( + fs.access(path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-old')), + ).rejects.toThrow(); + expect( + await fs.readFile( + path.join(tmpDir, '.agents', 'skills', 'gitnexus-cli', 'SKILL.md'), + 'utf-8', + ), + ).toBe('standard'); + expect( + await fs.readFile(path.join(tmpDir, '.agents', 'skills', 'user-author', 'SKILL.md'), 'utf-8'), + ).toBe('mine'); + // Fresh community mirrors written. + await expect( + fs.access(path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-alpha', 'SKILL.md')), + ).resolves.toBeUndefined(); + }); + + /** + * Empty edge case: no significant communities + .agents/ present must not + * write or mirror anything, and must not throw. + */ + it('writes nothing when no communities are significant, even with .agents/ present', async () => { + await fs.mkdir(path.join(tmpDir, '.agents'), { recursive: true }); + const graph = createKnowledgeGraph(); + // 2-symbol community — below the 3-symbol threshold. + for (let i = 0; i < 2; i++) { + graph.addNode(makeNode(`fn:n${i}`, `n${i}`, 'Function', `${tmpDir}/f${i}.ts`, 1, false)); + } + const communities = [makeCommunity('c1', 'Tiny', 2)]; + const memberships = [makeMembership('fn:n0', 'c1'), makeMembership('fn:n1', 'c1')]; + + const result = await generateSkillFiles( + tmpDir, + 'TestProject', + buildPipelineResult({ graph, repoPath: tmpDir, communities, memberships }), + ); + + expect(result.skills).toEqual([]); + await expect( + fs.access(path.join(tmpDir, '.agents', 'skills', 'gitnexus-area-tiny')), + ).rejects.toThrow(); + }); + /** * SKILL.md files should start with YAML frontmatter containing * name and description fields. diff --git a/gitnexus/test/unit/skip-git-cli.test.ts b/gitnexus/test/unit/skip-git-cli.test.ts index 0b3bed936..9163e8692 100644 --- a/gitnexus/test/unit/skip-git-cli.test.ts +++ b/gitnexus/test/unit/skip-git-cli.test.ts @@ -45,6 +45,7 @@ describe('--skip-git CLI flag', () => { expect(helpOutput).toContain('--skip-agents-md'); expect(helpOutput).toContain('--skip-skills'); expect(helpOutput).toContain('directly under .claude/skills/'); + expect(helpOutput).toContain('.agents/skills/'); expect(helpOutput).toContain('.claude/skills/gitnexus-area-*'); expect(helpOutput).toContain('--index-only'); expect(helpOutput).not.toContain('--no-git'); diff --git a/gitnexus/test/unit/spring-bean-extractor.test.ts b/gitnexus/test/unit/spring-bean-extractor.test.ts index bde4b8ee4..cbc5f5c4a 100644 --- a/gitnexus/test/unit/spring-bean-extractor.test.ts +++ b/gitnexus/test/unit/spring-bean-extractor.test.ts @@ -19,6 +19,12 @@ function captureClassAnnotations(code: string): JavaCaptureSideChannel['classAnn return collectJavaCaptureSideChannel(filePath)?.classAnnotations ?? []; } +function captureSpringDiFacts(code: string): NonNullable { + const filePath = 'src/Test.java'; + emitJavaScopeCaptures(code, filePath); + return collectJavaCaptureSideChannel(filePath)?.springDiFacts ?? []; +} + describe('Java class annotation capture', () => { it('collects annotation names during the existing scope-query traversal', () => { const facts = captureClassAnnotations(` @@ -51,12 +57,58 @@ describe('Java class annotation capture', () => { }); }); +describe('Java Spring injection syntax capture', () => { + it('preserves constructor, field, method, qualifier, and bean-name syntax in the side channel', () => { + const facts = captureSpringDiFacts(` + @Service("checkout") class Checkout { + Checkout(@Qualifier("fastGateway") Gateway gateway) {} + @Autowired Gateway fallback; + @Inject void setRepo(Repo repo) {} + } + `); + + expect(facts).toHaveLength(1); + expect(facts[0].classAnnotations).toEqual([{ name: 'Service', text: '@Service("checkout")' }]); + expect(facts[0].injectionSites).toMatchObject([ + { + kind: 'constructor', + implicitConstructor: true, + dependencies: [ + { + name: 'gateway', + rawType: 'Gateway', + annotations: [{ name: 'Qualifier', text: '@Qualifier("fastGateway")' }], + }, + ], + }, + { + kind: 'field', + memberName: 'fallback', + dependencies: [{ name: 'fallback', rawType: 'Gateway' }], + }, + { + kind: 'method', + memberName: 'setRepo', + dependencies: [{ name: 'repo', rawType: 'Repo' }], + }, + ]); + }); +}); + function captureKotlinClassAnnotations(code: string): KotlinCaptureSideChannel['classAnnotations'] { const filePath = 'src/Test.kt'; emitKotlinScopeCaptures(code, filePath); return collectKotlinCaptureSideChannel(filePath)?.classAnnotations ?? []; } +function captureKotlinSpringDiFacts( + code: string, +): NonNullable { + const filePath = 'src/Test.kt'; + emitKotlinScopeCaptures(code, filePath); + return collectKotlinCaptureSideChannel(filePath)?.springDiFacts ?? []; +} + describe('Kotlin class annotation capture', () => { it('captures supported class forms and excludes non-candidate declarations', () => { const facts = captureKotlinClassAnnotations(` @@ -92,6 +144,106 @@ describe('Kotlin class annotation capture', () => { }); }); +describe('Kotlin Spring injection syntax capture', () => { + it('preserves primary constructor, property, method, nullable type, projection, and use-site syntax', () => { + const facts = captureKotlinSpringDiFacts(` + @Service("checkout") @Primary + class Checkout @Autowired constructor( + @param:Qualifier("fastGateway") private val gateway: PaymentGateway, + @Named("repo") repo: Repo?, + val gateways: List, + ) { + @field:Autowired + @field:Qualifier("slowGateway") + lateinit var fallback: PaymentGateway + + @set:Inject + var optional: Repo? = null + + @Inject + fun setRepo(@Named("repo") repo: Repo) {} + } + `); + + expect(facts).toHaveLength(1); + expect(facts[0].classAnnotations).toEqual([ + { name: 'Service', text: '@Service("checkout")' }, + { name: 'Primary', text: '@Primary' }, + ]); + expect(facts[0].injectionSites).toMatchObject([ + { + kind: 'constructor', + implicitConstructor: false, + dependencies: [ + { + name: 'gateway', + rawType: 'PaymentGateway', + annotations: [ + { + name: 'Qualifier', + text: '@param:Qualifier("fastGateway")', + useSiteTarget: 'param', + }, + ], + }, + { + name: 'repo', + rawType: 'Repo?', + annotations: [{ name: 'Named', text: '@Named("repo")' }], + }, + { + name: 'gateways', + rawType: 'List', + }, + ], + }, + { + kind: 'property', + memberName: 'fallback', + annotations: [ + { name: 'Autowired', text: '@field:Autowired', useSiteTarget: 'field' }, + { + name: 'Qualifier', + text: '@field:Qualifier("slowGateway")', + useSiteTarget: 'field', + }, + ], + }, + { + kind: 'property', + memberName: 'optional', + annotations: [{ name: 'Inject', text: '@set:Inject', useSiteTarget: 'set' }], + }, + { + kind: 'method', + memberName: 'setRepo', + dependencies: [ + { + name: 'repo', + rawType: 'Repo', + annotations: [{ name: 'Named', text: '@Named("repo")' }], + }, + ], + }, + ]); + }); + + it('captures sole stereotype primary constructors as implicit injection sites', () => { + const facts = captureKotlinSpringDiFacts(` + @Service + class Checkout(private val gateway: PaymentGateway) + `); + + expect(facts[0].injectionSites).toMatchObject([ + { + kind: 'constructor', + implicitConstructor: true, + dependencies: [{ name: 'gateway', rawType: 'PaymentGateway' }], + }, + ]); + }); +}); + describe('deriveSpringBeanMetadata', () => { it('maps all supported canonical stereotypes to roles', () => { const cases = [ diff --git a/gitnexus/test/unit/stream-graph-emit-config.test.ts b/gitnexus/test/unit/stream-graph-emit-config.test.ts new file mode 100644 index 000000000..63972c97d --- /dev/null +++ b/gitnexus/test/unit/stream-graph-emit-config.test.ts @@ -0,0 +1,187 @@ +/** + * Streamed structural graph emit — config gate and pruner integration (#2680). + * + * The gate is a soundness boundary, not a preference: streaming is only valid + * on a full rebuild, because the incremental writeback reads relationships back + * out of the in-memory graph. + * + * The pruner cases are the sharp end of the feature. `pruneLocalValueSymbols` + * decides "is this block-local symbol referenced?" from an in-memory + * relationship scan; under streaming that scan cannot see edges already on + * disk, so without the predicate a referenced symbol is deleted and its + * streamed CSV row is left pointing at a node with no row. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; + +import { resolveStreamGraphEmit } from '../../src/core/run-analyze.js'; +import { buildPhaseList } from '../../src/core/ingestion/pipeline.js'; +import { RETAINED_REL_TYPES } from '../../src/core/lbug/graph-emit-sink.js'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import type { RelationshipType } from 'gitnexus-shared'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('resolveStreamGraphEmit', () => { + it('is ON by default on a full rebuild — no opt-in needed', () => { + expect(resolveStreamGraphEmit({ force: true })).toBe(true); + }); + + it('is turned off by an explicit falsy env value (the escape hatch)', () => { + vi.stubEnv('GITNEXUS_STREAM_GRAPH_EMIT', '0'); + expect(resolveStreamGraphEmit({ force: true })).toBe(false); + }); + + it('is turned off by an explicit option, which beats the env', () => { + vi.stubEnv('GITNEXUS_STREAM_GRAPH_EMIT', '1'); + expect(resolveStreamGraphEmit({ force: true, streamGraphEmit: false })).toBe(false); + }); + + it('honors the explicit option on a full rebuild', () => { + expect(resolveStreamGraphEmit({ force: true, streamGraphEmit: true })).toBe(true); + }); + + it('honors the env toggle on a full rebuild', () => { + vi.stubEnv('GITNEXUS_STREAM_GRAPH_EMIT', '1'); + expect(resolveStreamGraphEmit({ force: true })).toBe(true); + }); + + it('refuses an incremental run even when explicitly requested', () => { + // The incremental writeback reads relationships back out of the in-memory + // graph; streaming has already offloaded them. + expect(resolveStreamGraphEmit({ force: false, streamGraphEmit: true })).toBe(false); + expect(resolveStreamGraphEmit({ streamGraphEmit: true })).toBe(false); + }); + + it('refuses an incremental run even when the env toggle is set', () => { + vi.stubEnv('GITNEXUS_STREAM_GRAPH_EMIT', '1'); + expect(resolveStreamGraphEmit({ force: false })).toBe(false); + }); +}); + +const FILE_ID = 'File:src/a.ts'; +const LOCAL_ID = 'Const:src/a.ts:localValue'; + +const localConst = (): GraphNode => ({ + id: LOCAL_ID, + label: 'Const', + properties: { name: 'localValue', filePath: 'src/a.ts', scope: 'block' }, +}); + +/** Graph holding only the structural File->DEFINES->localConst edge, i.e. the + * shape the pruner sees when the symbol's only *semantic* reference streamed + * out to CSV. */ +const graphWithOnlyStructuralEdge = () => { + const graph = createKnowledgeGraph(); + graph.addNode({ + id: FILE_ID, + label: 'File', + properties: { name: 'a.ts', filePath: 'src/a.ts' }, + }); + graph.addNode(localConst()); + graph.addRelationship({ + id: `DEFINES:${FILE_ID}->${LOCAL_ID}`, + sourceId: FILE_ID, + targetId: LOCAL_ID, + type: 'DEFINES', + confidence: 1, + reason: 'structural', + }); + return graph; +}; + +describe('buildPhaseList under streamGraphEmit', () => { + const names = (o: Parameters[0]) => buildPhaseList(o).map((p) => p.name); + + it('keeps every CALLS-consuming phase enabled — nothing is traded away', () => { + // The sink answers a complete relationship read, so these phases work + // unchanged. If this ever regresses to filtering them out, streaming can no + // longer be the default. + const streamed = names({ streamGraphEmit: true, pdg: true, force: true }); + + expect(streamed).toContain('communities'); + expect(streamed).toContain('processes'); + expect(streamed).toContain('taintSummaries'); + expect(streamed).toContain('callSummaries'); + }); + + it('keeps mro and di, whose reads are all in the retained set', () => { + const streamed = names({ streamGraphEmit: true, pdg: true, force: true }); + + expect(streamed).toContain('mro'); + expect(streamed).toContain('di'); + expect(streamed).toContain('parse'); + expect(streamed).toContain('scopeResolution'); + expect(streamed).toContain('pruneLocalSymbols'); + }); + + it('leaves the phase list untouched when the flag is off', () => { + // Guards the default path: the gating predicates must not filter anything + // for existing (flag-off) users. + const withPdg = names({ pdg: true, force: true }); + + expect(withPdg).toContain('communities'); + expect(withPdg).toContain('processes'); + expect(withPdg).toContain('taintSummaries'); + expect(withPdg).toContain('callSummaries'); + }); + + it('still honours skipGraphPhases independently of the streaming flag', () => { + const skipped = names({ skipGraphPhases: true }); + + expect(skipped).not.toContain('communities'); + expect(skipped).not.toContain('processes'); + expect(skipped).toContain('pruneLocalSymbols'); + }); +}); + +describe('RETAINED_REL_TYPES tracks its readers', () => { + it('retains every relationship type any phase reads back mid-pipeline', async () => { + // The round-trip test CANNOT catch drift here: addRelationship partitions + // edges between the graph and the CSVs, and a partition's union is + // invariant under where the line falls — so it stays green for any + // partitioning, including a wrong one. Nothing else guards the invariant, + // and getting it wrong yields a silently incomplete edge set mid-pipeline + // rather than a crash. So derive the required set from the source and + // compare. + const { readdir, readFile } = await import('node:fs/promises'); + const { fileURLToPath } = await import('node:url'); + const path = await import('node:path'); + const srcDir = fileURLToPath(new URL('../../src/', import.meta.url)); + + // Every literal `iterRelationshipsByType('X')` reachable while streaming is + // armed. In-process scan over src/*.ts (a shelled-out grep is not reliably + // present or path-compatible on Windows); the sink itself is fine to + // include because its own fast-path check reads the constant, not an edge. + const readTypes = new Set(); + const entries = await readdir(srcDir, { recursive: true, withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.ts')) continue; + const content = await readFile(path.join(entry.parentPath, entry.name), 'utf8'); + for (const m of content.matchAll(/iterRelationshipsByType\('([A-Z_]+)'\)/g)) { + readTypes.add(m[1]); + } + } + + // CALLS is read by taintSummaries, which is exactly why the sink answers a + // COMPLETE read instead of retaining it — so it is a known exemption. + readTypes.delete('CALLS'); + + const missing = [...readTypes].filter((t) => !RETAINED_REL_TYPES.has(t as RelationshipType)); + expect(missing).toEqual([]); + }); +}); + +describe('streamGraphEmit without a CSV dir', () => { + it('throws instead of silently running without streaming', async () => { + // Streaming is on by default, so a programmatic host that builds its own + // PipelineOptions and forgets the directory must not get a successful run + // that quietly did no streaming. + const { runPipelineFromRepo } = await import('../../src/core/ingestion/pipeline.js'); + + await expect( + runPipelineFromRepo('/nonexistent-repo', () => {}, { streamGraphEmit: true }), + ).rejects.toThrow(/graphEmitCsvDir is missing/); + }); +}); diff --git a/gitnexus/test/unit/tool-staleness.test.ts b/gitnexus/test/unit/tool-staleness.test.ts new file mode 100644 index 000000000..e5fe6e8f0 --- /dev/null +++ b/gitnexus/test/unit/tool-staleness.test.ts @@ -0,0 +1,73 @@ +/** + * #2655: `query`/`context`/`impact`/`cypher` tool responses carry a non-blocking + * `staleness` signal when the index is behind HEAD, mirroring `list_repos`. + * + * These tests cover `attachToolStaleness` — the shape contract that guarantees + * the signal is only ever ADDED to an object result and never mutates an + * existing result's shape (so the CLI's `Array.isArray`-based `--limit` on + * raw-array cypher rows, and any consumer's shape assumptions, keep working). + */ +import { describe, it, expect } from 'vitest'; +import type { StalenessInfo } from '../../src/core/git-staleness.js'; +import { attachToolStaleness } from '../../src/mcp/local/local-backend.js'; + +const STALE: StalenessInfo = { + isStale: true, + commitsBehind: 3, + hint: '⚠️ Index is 3 commits behind HEAD. Run analyze tool to update.', +}; +const FRESH: StalenessInfo = { isStale: false, commitsBehind: 0 }; + +describe('attachToolStaleness (#2655)', () => { + it('adds a list_repos-shaped staleness field to an object result when stale', () => { + const out = attachToolStaleness({ processes: [], total: 0 }, STALE); + expect(out).toMatchObject({ + processes: [], + total: 0, + staleness: { commitsBehind: 3, hint: STALE.hint }, + }); + }); + + it('leaves the result untouched when the index is fresh', () => { + const result = { processes: [], total: 0 }; + expect(attachToolStaleness(result, FRESH)).toBe(result); + }); + + it('never changes the shape of a raw-array result (CLI --limit relies on Array.isArray)', () => { + const rows = [{ a: 1 }, { a: 2 }]; + const out = attachToolStaleness(rows, STALE); + expect(Array.isArray(out)).toBe(true); + expect(out).toBe(rows); + }); + + it('does not annotate an error envelope', () => { + const err = { error: 'LadybugDB not ready. Index may be corrupted.' }; + expect(attachToolStaleness(err, STALE)).toBe(err); + }); + + it('is idempotent — a result that already has staleness is left as-is', () => { + const already = { total: 1, staleness: { commitsBehind: 9, hint: 'x' } }; + expect(attachToolStaleness(already, STALE)).toBe(already); + }); + + it('leaves non-object results (null / primitives) unchanged', () => { + expect(attachToolStaleness(null, STALE)).toBeNull(); + expect(attachToolStaleness('markdown text', STALE)).toBe('markdown text'); + }); + + it('is null-safe — a missing staleness info never throws or mutates the result', () => { + const result = { total: 0 }; + expect(attachToolStaleness(result, undefined)).toBe(result); + }); + + it('carries hint through as-is (may be undefined on a stale-without-hint info)', () => { + const out = attachToolStaleness( + { ok: true }, + { + isStale: true, + commitsBehind: 1, + }, + ) as { staleness: { commitsBehind: number; hint?: string } }; + expect(out.staleness).toMatchObject({ commitsBehind: 1 }); + }); +}); diff --git a/gitnexus/test/unit/worker-pool-resource-limits.test.ts b/gitnexus/test/unit/worker-pool-resource-limits.test.ts new file mode 100644 index 000000000..94ea73346 --- /dev/null +++ b/gitnexus/test/unit/worker-pool-resource-limits.test.ts @@ -0,0 +1,166 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +// Pin physical RAM to 32GB so the half-of-RAM-per-worker formula resolves +// deterministically regardless of the host machine. +vi.mock('os', async () => { + const actual = await vi.importActual('os'); + const mocked = { ...actual, totalmem: () => 32 * 1024 * 1024 * 1024 }; + return { ...mocked, default: mocked }; +}); + +// Capture the exact options the pool's PRODUCTION factory passes to the +// Worker constructor — the formula alone doesn't prove the wiring, and a +// typo'd resourceLimits key would silently uncap workers again (#2649). +// vi.mock factories are hoisted above imports, so the capture array must be +// hoisted too and EventEmitter imported inside the factory. +const workerCtorOptions = vi.hoisted(() => [] as unknown[]); +vi.mock('node:worker_threads', async () => { + const actual = await vi.importActual('node:worker_threads'); + const { EventEmitter } = await import('node:events'); + class CapturingWorker extends EventEmitter { + private currentPaths: string[] = []; + constructor(_url: unknown, options: unknown) { + super(); + workerCtorOptions.push(options); + queueMicrotask(() => this.emit('message', { type: 'ready' })); + } + postMessage(msg: unknown): void { + if (msg === null || typeof msg !== 'object') return; + const type = (msg as { type?: unknown }).type; + if (type === 'sub-batch') { + const files = (msg as { files?: Array<{ path: string }> }).files ?? []; + this.currentPaths = files.map((file) => file.path); + queueMicrotask(() => { + this.emit('message', { type: 'progress', filesProcessed: this.currentPaths.length }); + this.emit('message', { type: 'sub-batch-done' }); + }); + return; + } + if (type === 'flush') { + const paths = this.currentPaths.slice(); + queueMicrotask(() => this.emit('message', { type: 'result', data: { paths } })); + } + } + async terminate(): Promise { + this.emit('exit', 0); + return 0; + } + unref(): void {} + } + return { ...actual, Worker: CapturingWorker }; +}); + +const setConstrainedMemory = (value: number): (() => void) => { + const desc = Object.getOwnPropertyDescriptor(process, 'constrainedMemory'); + Object.defineProperty(process, 'constrainedMemory', { configurable: true, value: () => value }); + return () => { + if (desc) Object.defineProperty(process, 'constrainedMemory', desc); + else delete (process as { constrainedMemory?: unknown }).constrainedMemory; + }; +}; + +describe('resolveWorkerHeapCapMb (#2649 per-worker heap cap)', () => { + let initialOverride: string | undefined; + let restoreConstrained: (() => void) | undefined; + + beforeEach(() => { + initialOverride = process.env.GITNEXUS_WORKER_HEAP_MB; + delete process.env.GITNEXUS_WORKER_HEAP_MB; + // Unconstrained by default so the mocked 32GB totalmem governs. + restoreConstrained = setConstrainedMemory(0); + workerCtorOptions.length = 0; + vi.resetModules(); + }); + + afterEach(() => { + if (initialOverride === undefined) delete process.env.GITNEXUS_WORKER_HEAP_MB; + else process.env.GITNEXUS_WORKER_HEAP_MB = initialOverride; + restoreConstrained?.(); + restoreConstrained = undefined; + }); + + it('splits half of RAM across the pool, clamped to the 4096 ceiling', async () => { + const { resolveWorkerHeapCapMb } = + await import('../../src/core/ingestion/workers/worker-pool.js'); + // 32GB -> half = 16384MB; /16 workers = 1024; /4 workers = 4096 (at ceiling); + // /2 workers = 8192 -> clamped to 4096. + expect([16, 4, 2].map((n) => resolveWorkerHeapCapMb(n))).toEqual([1024, 4096, 4096]); + }); + + it('never drops below the 512MB floor on small shares', async () => { + const { resolveWorkerHeapCapMb } = + await import('../../src/core/ingestion/workers/worker-pool.js'); + // 32GB half-share across 64 workers = 256 -> floored to 512. + expect(resolveWorkerHeapCapMb(64)).toBe(512); + }); + + it('GITNEXUS_WORKER_HEAP_MB overrides the formula', async () => { + process.env.GITNEXUS_WORKER_HEAP_MB = '768'; + const { resolveWorkerHeapCapMb } = + await import('../../src/core/ingestion/workers/worker-pool.js'); + expect([1, 16].map((n) => resolveWorkerHeapCapMb(n))).toEqual([768, 768]); + }); + + it('warns when a floored pool would overcommit a tiny container (#2649 review)', async () => { + // 2GB cgroup limit, pool of 8: every worker floors at 512MB, so the pool + // may commit 4096MB against a 2048MB container — the warn must name it. + restoreConstrained?.(); + restoreConstrained = setConstrainedMemory(2 * 1024 * 1024 * 1024); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-worker-overcommit-')); + const workerPath = path.join(tempDir, 'fake-worker.js'); + fs.writeFileSync(workerPath, '// fake worker path for createWorkerPool'); + try { + const { _captureLogger } = await import('../../src/core/logger.js'); + const { createWorkerPool } = await import('../../src/core/ingestion/workers/worker-pool.js'); + const cap = _captureLogger(); + const pool = createWorkerPool(pathToFileURL(workerPath) as URL, 8, { shutdownDrainMs: 25 }); + await pool.terminate(); + cap.restore(); + const warn = cap.records().find((r) => r.msg.includes('may overcommit memory')); + expect(warn?.msg).toContain('GITNEXUS_WORKER_POOL_SIZE'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('honors a real cgroup limit instead of host RAM (#2649 review — container overcommit)', async () => { + // 8GB cgroup limit on the mocked 32GB host, pool of 4: the cap must come + // from the container (8192/2/4 = 1024), not the host (32768/2/4 = 4096 — + // which would let one worker outgrow a quarter of the whole container). + restoreConstrained?.(); + restoreConstrained = setConstrainedMemory(8 * 1024 * 1024 * 1024); + const { resolveWorkerHeapCapMb } = + await import('../../src/core/ingestion/workers/worker-pool.js'); + expect(resolveWorkerHeapCapMb(4)).toBe(1024); + }); + + it('wires the cap into the production Worker resourceLimits (#2649 review)', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-worker-limits-')); + const workerPath = path.join(tempDir, 'fake-worker.js'); + fs.writeFileSync(workerPath, '// fake worker path for createWorkerPool'); + try { + const { createWorkerPool, resolveWorkerHeapCapMb } = + await import('../../src/core/ingestion/workers/worker-pool.js'); + const pool = createWorkerPool(pathToFileURL(workerPath) as URL, 2, { + shutdownDrainMs: 25, + }); + try { + await pool.dispatch<{ path: string; content: string }, { paths: string[] }>([ + { path: 'src/a.ts', content: 'const a = 1;' }, + ]); + } finally { + await pool.terminate(); + } + expect(workerCtorOptions.length).toBeGreaterThan(0); + expect(workerCtorOptions[0]).toMatchObject({ + resourceLimits: { stackSizeMb: 16, maxOldGenerationSizeMb: resolveWorkerHeapCapMb(2) }, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/unit/worker-pool-stall-credit.test.ts b/gitnexus/test/unit/worker-pool-stall-credit.test.ts new file mode 100644 index 000000000..b57b0a70c --- /dev/null +++ b/gitnexus/test/unit/worker-pool-stall-credit.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { EventEmitter } from 'node:events'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { createWorkerPool } from '../../src/core/ingestion/workers/worker-pool.js'; +import { _captureLogger } from '../../src/core/logger.js'; + +// First worker never answers its sub-batch (looks idle); every later worker +// completes normally so the dispatch still resolves after the retire path. +class StallThenHealthyWorker extends EventEmitter { + static instances: StallThenHealthyWorker[] = []; + + readonly id: number; + private currentPaths: string[] = []; + + constructor() { + super(); + this.id = StallThenHealthyWorker.instances.length; + StallThenHealthyWorker.instances.push(this); + queueMicrotask(() => this.emit('message', { type: 'ready' })); + } + + postMessage(msg: unknown): void { + if (msg === null || typeof msg !== 'object') return; + const type = (msg as { type?: unknown }).type; + if (type === 'sub-batch') { + const files = (msg as { files?: Array<{ path: string }> }).files ?? []; + this.currentPaths = files.map((file) => file.path); + if (this.id === 0) return; + queueMicrotask(() => { + this.emit('message', { type: 'progress', filesProcessed: this.currentPaths.length }); + this.emit('message', { type: 'sub-batch-done' }); + }); + return; + } + if (type === 'flush') { + const paths = this.currentPaths.slice(); + queueMicrotask(() => this.emit('message', { type: 'result', data: { paths } })); + } + } + + async terminate(): Promise { + this.emit('exit', 0); + return 0; + } + + unref(): void {} +} + +let tempDir: string; +let workerUrl: URL; + +beforeEach(() => { + StallThenHealthyWorker.instances = []; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-worker-stall-credit-')); + const workerPath = path.join(tempDir, 'fake-worker.js'); + fs.writeFileSync(workerPath, '// fake worker path for createWorkerPool'); + workerUrl = pathToFileURL(workerPath) as URL; +}); + +afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +const dispatchWithProbe = async (stallMsProbe: () => number) => { + const cap = _captureLogger(); + const pool = createWorkerPool(workerUrl, 1, { + subBatchIdleTimeoutMs: 30, + maxTimeoutRetries: 1, + timeoutBackoffFactor: 2, + shutdownDrainMs: 25, + stallMsProbe, + workerFactory: () => + new StallThenHealthyWorker() as unknown as import('node:worker_threads').Worker, + }); + try { + const results = await pool.dispatch<{ path: string; content: string }, { paths: string[] }>([ + { path: 'src/starved.ts', content: 'const x = 1;' }, + ]); + return { results, records: cap.records() }; + } finally { + cap.restore(); + await pool.terminate(); + } +}; + +describe('worker pool GC-stall credit (#2649)', () => { + it('credits a main-thread stall >= half the budget with one re-arm before retiring', async () => { + // Monotonic fake stall clock: every read advances 20ms, so each armed + // 30ms window observes ~tens of ms of "stall" — always above the 15ms + // credit threshold. Only ONE credit may be spent regardless. + let stall = 0; + const { results, records } = await dispatchWithProbe(() => { + stall += 20; + return stall; + }); + + expect(results).toEqual([{ paths: ['src/starved.ts'] }]); + const creditWarns = records.filter((r) => + r.msg.includes('overlapped a main-thread stall (GC pressure); re-arming once'), + ); + const timeoutWarns = records.filter((r) => r.msg.includes('parse job idle timeout')); + expect({ credits: creditWarns.length, timeoutsAtLeast: timeoutWarns.length >= 1 }).toEqual({ + credits: 1, + timeoutsAtLeast: true, + }); + }); + + it('does not credit when the main thread was responsive (probe reads zero stall)', async () => { + const { results, records } = await dispatchWithProbe(() => 0); + + expect(results).toEqual([{ paths: ['src/starved.ts'] }]); + const creditWarns = records.filter((r) => + r.msg.includes('overlapped a main-thread stall (GC pressure); re-arming once'), + ); + expect(creditWarns).toEqual([]); + }); +}); + +describe('startHeartbeatStallTracker (#2649 review — the production probe itself)', () => { + it('accumulates observed stalls, ignores on-time ticks, and freezes after stop()', async () => { + const { startHeartbeatStallTracker } = + await import('../../src/core/ingestion/workers/worker-pool.js'); + vi.useFakeTimers(); + try { + const tracker = startHeartbeatStallTracker(); + // Two on-time ticks: zero drift, nothing accumulates. + vi.advanceTimersByTime(500); + const afterOnTime = tracker.read(); + // Simulate a ~2s main-thread stall: jump the wall clock, then let the + // delayed tick observe the drift. + vi.setSystemTime(Date.now() + 2000); + vi.advanceTimersByTime(250); + const afterStall = tracker.read(); + tracker.stop(); + vi.setSystemTime(Date.now() + 2000); + vi.advanceTimersByTime(500); + expect({ + afterOnTime, + stallSeen: afterStall >= 1500, + frozenAfterStop: tracker.read() === afterStall, + }).toEqual({ afterOnTime: 0, stallSeen: true, frozenAfterStop: true }); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/gitnexus/vendor/leiden/utils.cjs b/gitnexus/vendor/leiden/utils.cjs index b6c6132da..a53248969 100644 --- a/gitnexus/vendor/leiden/utils.cjs +++ b/gitnexus/vendor/leiden/utils.cjs @@ -6,6 +6,21 @@ * * Vendored from: https://github.com/graphology/graphology/tree/master/src/communities-leiden * License: MIT + * + * LOCAL MODIFICATION (#2337) — do NOT re-vendor this file by copying upstream + * over it without re-applying the change below. + * + * `mergeNodesSubset` used to snapshot the pre-merge + * `externalEdgeWeightPerCommunity` with a full N-length `.slice()` on every + * macro-community — O(communities x N) copying, ~70% of Leiden runtime on + * large repos. It now writes only the current subset's entries into a + * scratch buffer allocated once (`this.microDegrees`). + * + * A revert is invisible to the test suite: the two versions produce + * bit-identical partitions, so every golden and determinism test still passes. + * The only symptom is the old timeout cliff returning — a 200k-symbol + * projection back over LEIDEN_TIMEOUT_MS, collapsing every symbol into one + * community. Search for "microDegrees" to find both edits. */ var SparseMap = require('mnemonist/sparse-map'); var createRandom = require('pandemonium/random').createRandom; @@ -52,6 +67,10 @@ function UndirectedLeidenAddenda(index, options) { this.belongings = new NodesPointerArray(order); this.neighboringCommunities = new SparseMap(WeightsArray, order); this.cumulativeIncrement = new Float64Array(order); + // Scratch buffer for mergeNodesSubset's pre-merge snapshot, allocated once. + // Upstream re-`.slice()`d the full N-length array per macro-community, which is + // O(communities x N) copying — 70% of Leiden runtime on large repos (#2337). + this.microDegrees = new WeightsArray(order); this.macroCommunities = null; } @@ -147,7 +166,15 @@ UndirectedLeidenAddenda.prototype.mergeNodesSubset = function (start, stop) { } } - var microDegrees = this.externalEdgeWeightPerCommunity.slice(); + // Only entries for nodes inside [start, stop) are ever read below (every `et` + // is filtered on `belongings[et] === currentMacroCommunity`), so snapshot just + // those instead of copying the whole N-length array. + var microDegrees = this.microDegrees; + + for (j = start; j < stop; j++) { + i = this.nodesSortedByCommunities[j]; + microDegrees[i] = this.externalEdgeWeightPerCommunity[i]; + } var s, ri, ci; var order = stop - start; diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 0a115a24c..0e26026ce 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -9,6 +9,12 @@ export default defineConfig({ pool: 'forks', globals: true, teardownTimeout: 3000, + // E2E harnesses pin a small NODE_OPTIONS heap so spawned CLI children + // stay light; without this opt-out the #2649 auto-heap override would + // respawn every such child with a RAM-sized cap. Children inherit it via + // the harnesses' `{ ...process.env }` spreads. Tests that exercise the + // respawn behavior itself delete GITNEXUS_MEMORY in their own setup. + env: { GITNEXUS_MEMORY: 'off' }, // N-API destructors can crash worker forks on macOS during process exit. // This is independent of the QueryResult lifetime fix in @ladybugdb/core 0.15.2 — // it's a vitest forks + native addon interaction where destructors run in