diff --git a/.claude/skills/gitnexus-guide/SKILL.md b/.claude/skills/gitnexus-guide/SKILL.md index e52560422..bf3c73948 100644 --- a/.claude/skills/gitnexus-guide/SKILL.md +++ b/.claude/skills/gitnexus-guide/SKILL.md @@ -83,15 +83,23 @@ Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). ### Inline staleness signal (`query` / `context` / `impact` / `cypher`) -These four hot read tools attach a non-blocking `staleness` field to their response when the index is behind the checkout's current HEAD — the same `{ commitsBehind, hint }` shape `list_repos` already reports — so a direct tool call surfaces a behind-HEAD index without a separate `list_repos` call: +These four hot read tools attach a non-blocking `staleness` field to their response when the index is not at the checkout's current HEAD — the same `{ status, commitsBehind?, hint? }` shape `list_repos` already reports — so a direct tool call surfaces a stale index without a separate `list_repos` call: ```jsonc { /* …the tool's normal result… */ - "staleness": { "commitsBehind": 3, "hint": "⚠️ Index is 3 commits behind HEAD. Run analyze tool to update." } + "staleness": { "status": "behind", "commitsBehind": 3, "hint": "⚠️ Index is 3 commits behind HEAD. Run analyze tool to update." } } ``` -The field is **absent when the index is current** (or when the freshness check can't run), so its presence is the signal. It is only ever added to object results — raw-array `cypher` output and error envelopes are returned unchanged. `@group`-targeted calls do not carry it (multi-repo staleness is ill-defined). When you see it, the graph may be behind the working tree — re-run `analyze` before trusting blast-radius or dependence answers. +`commitsBehind` is present only when git counted the gap. When git could not count it but HEAD still resolves to a commit other than the indexed one — usually because the indexed commit is no longer in the clone's history — the index is provably not at HEAD with no countable gap, so no number is reported: + +```jsonc +{ /* …the tool's normal result… */ + "staleness": { "status": "diverged", "hint": "⚠️ Index is not at HEAD and the commit gap could not be counted — the recorded commit may no longer be in this clone's history. Run analyze tool to update." } +} +``` + +The field is **absent when the index is current**, and these four tools also omit it when the freshness check could not run at all — that case is `status: "unknown"`, which only the `list_repos` listing reports. So its presence means the status is not `current`: read `status` before using `commitsBehind`. It is only ever added to object results — raw-array `cypher` output and error envelopes are returned unchanged. `@group`-targeted calls do not carry it (multi-repo staleness is ill-defined). When you see it, the graph may be behind the working tree — re-run `analyze` before trusting blast-radius or dependence answers. ### Taint findings (`explain`) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 01cfa24b6..641df1821 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -599,6 +599,26 @@ jobs: run: node --import tsx bench/parse-dispatch-rounds/measure.mjs --check working-directory: gitnexus + - name: Python workspace import-scan guards (#3254) + if: ${{ !cancelled() }} + # Build-free: same baseline approach as parse-dispatch-rounds — + # exact link/lookalike floors plus a fingerprint, then ratio timing + # only (scan scaling and from-token prefilter advantage). See + # bench/python-workspace-import-scan/measure.mjs. + run: node --import tsx bench/python-workspace-import-scan/measure.mjs --check + working-directory: gitnexus + + - name: MCP tools/list countRepos vs listRepos guards (#3259, #3184) + if: ${{ !cancelled() }} + # Build-free: exact registry cardinality + tool-roster + schema-flag + # floors, then ratio timing only (countRepos/listRepos and + # listTools/listRepos). No millisecond ceiling — this repo has + # already been bitten by a fixed ms budget. Isolated GITNEXUS_HOME; + # fixture is N real git repos so listRepos pays rev-list. See + # bench/mcp-tools-list/measure.mjs. + run: node --import tsx bench/mcp-tools-list/measure.mjs --check + working-directory: gitnexus + - name: C++ qualified-namespace resolution guards (#2788) if: ${{ !cancelled() }} # Build-free: asserts resolveCppQualifiedNamespaceMember resolves an @@ -718,6 +738,13 @@ jobs: run: node --import tsx bench/kotlin-import-target/measure.mjs --check working-directory: gitnexus + - name: Ruby gem-boundary correctness + scaling guards (#3096) + if: ${{ !cancelled() }} + # Includes real manifest loading; checks scoped resolution and scaling + # as sibling projects or declared gem counts grow independently. + run: node --import tsx bench/ruby-gem-resolution/measure.mjs --check + working-directory: gitnexus + - name: Receiver-resolution drop guards if: ${{ !cancelled() }} # NOT build-free: this one runs the real pipeline, so it needs dist/ @@ -774,6 +801,18 @@ jobs: run: node --import tsx bench/objective-c-resolution/measure.mjs --check working-directory: gitnexus + - name: Callable-value reference resolution guards (#3399) + if: ${{ !cancelled() }} + # Build-free: pins the resolved-target SET of `resolveValueRefTarget` + # (exact site/resolved/declined counts plus an order-independent + # fingerprint) and asserts its per-site cost stays independent of + # workspace size across a 4x file-count step. The pass resolves a + # qualified receiver through `scopes.qualifiedNames`, a workspace-wide + # index: keyed it is O(1) per site, scanned it is O(files) — a + # regression a fixture cannot see and a 257-file binding table can. + run: node --import tsx bench/value-ref-resolution/measure.mjs --check + working-directory: gitnexus + - name: CFG construction time / disk / memory guards (#2081 M1) if: ${{ !cancelled() }} # Build-free: asserts collectFunctionCfgs output is unchanged diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dac6a7398..0334531c1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -138,7 +138,7 @@ jobs: # Required for multi-platform (linux/arm64) emulation. - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c03a801c1..ca81460f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -831,7 +831,7 @@ jobs: fi - name: Create GitHub Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v2 with: tag_name: ${{ steps.vtag-gate.outputs.vtag }} name: >- diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a1d2589bb..1c6a99c36 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -24,7 +24,7 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). - **HTTP bridge:** `serve.ts` → Express (`api.ts`, `mcp-http.ts`) for web UI - **CLI direct:** `gitnexus query|context|impact|cypher` in `tool.ts` -4. **Staleness** — `staleness.ts` compares indexed `lastCommit` to `HEAD`, surfaces hints. +4. **Staleness** — `core/git-staleness.ts` compares indexed `lastCommit` to `HEAD` and classifies the result as `current`, `behind`, `diverged` (HEAD moved off the indexed commit, gap uncountable) or `unknown`; `core/staleness-status.ts` builds the one `staleness` payload that MCP `list_repos`, the read tools and the `serve` repo routes all emit. ## MCP tools diff --git a/README.md b/README.md index 95cfaf5f0..80f93f9d9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # GitNexus (Akon Labs) -**⚠️ Important Notice:** GitNexus has NO official cryptocurrency, token, or coin. Any token/coin using the GitNexus name on Pump.fun or any other platform is **not affiliated with, endorsed by, or created by** this project or its maintainers. Do not purchase any cryptocurrency claiming association with GitNexus. -
@@ -409,7 +407,7 @@ backoff. Invalid `.gitnexusrc` or ignore-file reloads pause ordinary refreshes until the control file is fixed. Stop the watcher with Ctrl+C. Watch mode accepts `--debounce`, `--workers`, `--worker-timeout`, -`--max-file-size`, `--branch`, `--pdg`, `--name`, `--allow-duplicate-name`, and +`--max-file-size`, `--branch`, `--pdg`, `--skip-fts`, `--name`, `--allow-duplicate-name`, and `--verbose`. Explicit one-shot options such as `--force`, `--repair-fts`, embedding flags, `--skills`, `--self-commit`, `--index-only`, and `--skip-git` are rejected. Unsupported defaults from `.gitnexusrc` are ignored with a @@ -443,6 +441,7 @@ The token may be set in the shell, `.env.local`, or `.env` in the working direct gitnexus analyze --force # Full graph + FTS rebuild (reuses unchanged parser output) gitnexus analyze --no-parse-cache # Full rebuild that re-parses every source file gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data +gitnexus analyze --skip-fts # Index graph/embeddings without loading FTS or building keyword indexes gitnexus analyze --skills # Generate repo-specific skill files from detected communities gitnexus analyze --skip-embeddings # Skip embedding generation (faster) gitnexus analyze --embeddings [limit] # Enable embedding generation (slower, better search) @@ -460,6 +459,8 @@ gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-check # (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) ``` +`--skip-fts` (or `GITNEXUS_SKIP_FTS=1`) disables FTS extension loading and keyword-index construction for this analysis. Graph queries, communities, processes, and existing embeddings remain available. Status and search report "FTS disabled for this index". Remove both the flag and environment setting and run `analyze` again to restore keyword search, even at the same commit. Only the exact environment value `1` enables the opt-out; the flag takes precedence. It cannot be combined with `--repair-fts`. Disabling an existing FTS index may require one graph-store rebuild to avoid unsafe writes through native indexes. + `--spring-actuator` is explicitly opt-in and accepts either a JSON bundle keyed by `mappings`, `beans`, `conditions`, `configprops`, and/or `env`, or a directory containing endpoint-named JSON files. It confirms matching static nodes and adds conservative runtime-only routes, beans, and property keys. The configured input is excluded from source scanning; only normalized repository-relative exclusions are retained for future scans, never absolute paths. Env/configprops values, origins, condition messages, and source names are never persisted or printed. Because snapshots are external runtime state, an enabled run always rebuilds; the first later run without the option rebuilds once to remove runtime evidence. The same path can be set as `springActuator` in `.gitnexusrc`. `--asyncapi-spec` is explicitly opt-in and accepts a directory of AsyncAPI documents or a single document; the path is resolved against the repository root, so a committed `docs/asyncapi` and an absolute cache written by something else both work. Each `operations[]` entry of an **AsyncAPI 3.x** document can contribute a `Destination` node keyed by broker and address, with `action: send` emitting `PUBLISHES_TO` and `action: receive` emitting `CONSUMES_FROM`, so a document and source code that name one address on one broker land on the same node. Edges start at the document, not at a callable — a document states that the service talks to an address, not which method does — and no address a document names is ever attached to an unresolved source site. @@ -585,6 +586,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | +| `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS` | unset | When truthy (`1`/`true`/`yes`), per-attempt HTTP embedding timeouts (`TimeoutError` on fetch or body read) go through the bounded `GITNEXUS_EMBEDDING_MAX_ATTEMPTS` retry loop instead of failing the job. Any other value leaves it off, so cloud/default timeouts remain terminal. | Local accelerators that drop a device lock when the client disconnects and succeed on the next request (observed with FastFlowLM on Ryzen AI). | | `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. | | `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. | | `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | @@ -596,6 +598,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget in milliseconds for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. | Slow or heavily loaded hosts where a full pool cold-starting concurrently needs more than 5s, and analyze aborts with "did not report ready within 5000ms". | | `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | +| `GITNEXUS_SKIP_FTS` | unset | When exactly `1`, skips FTS extension loading and keyword index creation during analyze. Equivalent to `--skip-fts`; a later analyze without either option restores FTS. | Graph-only consumers with their own retrieval, or short-lived indexes that do not need keyword search. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | | `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | | `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). Invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. | diff --git a/RUNBOOK.md b/RUNBOOK.md index d16ccd52d..2f2c06f14 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -187,6 +187,58 @@ If the error text is `"Only one write transaction at a time is allowed in the sy --- +## File acquisition/reclaim guard recovery + +The portable file-lock backend uses `analyze.lock.guard` beside `analyze.lock`. +Every acquisition, including an empty slot, exclusively creates the guard before +inspecting, reclaiming, creating, and verifying the main lock. It removes the +guard before returning a workload handle or waiting on a live workload holder. +Linux abstract-socket and Windows named-pipe locking are unchanged. + +A stalled or crashed guard owner blocks file acquisition even when its PID is +dead, its metadata is incomplete, or no main lock exists. **The guard is never +automatically stolen.** Guard contention times out after at most 30 seconds, +capped by the remaining acquisition timeout. This separate ceiling applies even +when `GITNEXUS_INDEX_LOCK_TIMEOUT_MS` is zero or negative (unbounded workload wait). +A guard-cleanup failure rejects acquisition; it must not start unprotected work. + +Manual recovery is an outage procedure, not an age/PID-based cleanup: + +1. Identify the exact lock directory named in the error. This shared primitive + also protects group sync and registry operations, not just repo analysis. +2. Stop **all relevant writers** and prevent restart: editor/agent hooks, watch + processes, scheduled jobs, services, and any containers sharing the directory. + Account for paused processes and every host with access. If quiescence cannot + be established, do not remove the guard. PID metadata is diagnostic only. +3. While restart remains disabled, inspect and preserve the guard/main records + for diagnosis, then remove only that directory's orphan `analyze.lock.guard` + and, if present, its orphan `analyze.lock`. Do not remove databases or sidecars + as part of lock recovery. Do not use a recursive or wildcard cleanup. +4. Ensure all participating writers use the guarded version and the same locking + backend/domain, then restart in a controlled fashion. + +**Upgrade requires a coordinated stop/upgrade/restart.** Concurrent older +versions ignore the guard and can still displace live locks; mixed-version +mutual exclusion is not guaranteed. The file protocol assumes reliable atomic +local-filesystem `O_EXCL` creation and cooperating processes. Network/distributed +filesystems, external file replacement, and uncoordinated manual deletion are not +covered. A process crash while holding the short-lived guard trades automatic +recovery for fail-closed safety. Denied file creation returns a non-owning +`lockFree` handle only when neither workload lock nor acquisition guard exists; +unreadable paths fail closed. No staging sweep runs without ownership. Analysis, +registry transactions, group synchronization, and embeddings sync refuse +`lockFree` handles, including an otherwise up-to-date analysis on a file-backend +read-only mount. +The socket backend can still acquire ownership on a read-only index mount. +Heterogeneous permissions are not proof that another process cannot write. + +If guard cleanup fails after this attempt created its workload record, acquisition +is refused and token-exact workload cleanup is attempted before returning the +error. Failed or unverifiable cleanup must be diagnosed under the same quiesced +recovery procedure above; never delete a possibly active successor's record. + +--- + ## Where to dig deeper - Architecture overview: [ARCHITECTURE.md](ARCHITECTURE.md) diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md index e52560422..bf3c73948 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md @@ -83,15 +83,23 @@ Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). ### Inline staleness signal (`query` / `context` / `impact` / `cypher`) -These four hot read tools attach a non-blocking `staleness` field to their response when the index is behind the checkout's current HEAD — the same `{ commitsBehind, hint }` shape `list_repos` already reports — so a direct tool call surfaces a behind-HEAD index without a separate `list_repos` call: +These four hot read tools attach a non-blocking `staleness` field to their response when the index is not at the checkout's current HEAD — the same `{ status, commitsBehind?, hint? }` shape `list_repos` already reports — so a direct tool call surfaces a stale index without a separate `list_repos` call: ```jsonc { /* …the tool's normal result… */ - "staleness": { "commitsBehind": 3, "hint": "⚠️ Index is 3 commits behind HEAD. Run analyze tool to update." } + "staleness": { "status": "behind", "commitsBehind": 3, "hint": "⚠️ Index is 3 commits behind HEAD. Run analyze tool to update." } } ``` -The field is **absent when the index is current** (or when the freshness check can't run), so its presence is the signal. It is only ever added to object results — raw-array `cypher` output and error envelopes are returned unchanged. `@group`-targeted calls do not carry it (multi-repo staleness is ill-defined). When you see it, the graph may be behind the working tree — re-run `analyze` before trusting blast-radius or dependence answers. +`commitsBehind` is present only when git counted the gap. When git could not count it but HEAD still resolves to a commit other than the indexed one — usually because the indexed commit is no longer in the clone's history — the index is provably not at HEAD with no countable gap, so no number is reported: + +```jsonc +{ /* …the tool's normal result… */ + "staleness": { "status": "diverged", "hint": "⚠️ Index is not at HEAD and the commit gap could not be counted — the recorded commit may no longer be in this clone's history. Run analyze tool to update." } +} +``` + +The field is **absent when the index is current**, and these four tools also omit it when the freshness check could not run at all — that case is `status: "unknown"`, which only the `list_repos` listing reports. So its presence means the status is not `current`: read `status` before using `commitsBehind`. It is only ever added to object results — raw-array `cypher` output and error envelopes are returned unchanged. `@group`-targeted calls do not carry it (multi-repo staleness is ill-defined). When you see it, the graph may be behind the working tree — re-run `analyze` before trusting blast-radius or dependence answers. ### Taint findings (`explain`) diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index 9857a60cc..7958cfa9d 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -137,6 +137,7 @@ export type { FinalizeOutput, FinalizedScc, FinalizeStats, + AmbiguousWildcardExport, } from './scope-resolution/finalize-algorithm.js'; // Scope-aware registries + 7-step lookup (RFC §4; Ring 2 SHARED #917) diff --git a/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts b/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts index f6038b3f0..2dc14b398 100644 --- a/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts +++ b/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts @@ -5,13 +5,14 @@ * Pure logic that takes per-file parse output (`ParsedImport[]` + * `SymbolDefinition[]`) and returns: * - * - Linked `ImportEdge[]` per module scope, with `targetModuleScope` and - * `targetDefId` filled where resolvable; edges that could not be - * resolved within the hard fixpoint cap are marked + * - Linked `ImportEdge[]` keyed by binding scope (`fromScope`; module + * scope unless `importsBindAtLexicalScope` is on), with + * `targetModuleScope` and `targetDefId` filled where resolvable; edges + * that could not be resolved within the hard fixpoint cap are marked * `linkStatus: 'unresolved'`. - * - Materialized `bindings` per module scope — local defs merged with - * imported / wildcard-expanded / re-exported names via the provider's - * `mergeBindings` precedence. + * - Materialized `bindings` keyed by the same scopes — local defs merged + * with imported / wildcard-expanded / re-exported names via the + * provider's `mergeBindings` precedence. * - The SCC condensation of the import graph, exposed so disjoint SCCs * can be processed in parallel by callers that want that. * @@ -39,7 +40,7 @@ import type { BindingRef, ImportEdge, ParsedImport, ScopeId, WorkspaceIndex } fr /** Per-file input for the finalize pass. */ export interface FinalizeFile { readonly filePath: string; - /** The module scope id for this file; owns the finalized imports + bindings. */ + /** Default binding scope for imports without lexical provenance or opt-in. */ readonly moduleScope: ScopeId; readonly parsedImports: readonly ParsedImport[]; /** @@ -84,6 +85,10 @@ export interface FinalizeInput { * expects pure answers. */ export interface FinalizeHooks { + /** Bind imports at their extracted lexical scope. Missing provenance retains + * the legacy module-scope behavior. Opt-in: lexical position and language + * import-binding semantics are distinct facts. */ + readonly importsBindAtLexicalScope?: boolean; /** * Resolve a raw import target to the concrete file path that owns it. * Return `null` when no target file is resolvable (e.g., `np.foo` when @@ -114,6 +119,34 @@ export interface FinalizeHooks { */ expandsWildcardTo(targetModuleScope: ScopeId, workspaceIndex: WorkspaceIndex): readonly string[]; + /** + * Does this language make two `wildcard` re-exports that both DECLARE the + * same name AMBIGUOUS (no winner), rather than overloads or redeclarations + * of one entity? + * + * True for ECMAScript modules: `export * from './a'; export * from './b'` + * with `collide` declared in both excludes the name from the module's + * exports, so binding either source is a guess. False (the default) for + * languages whose wildcard import is `#include`, `require`, or a package + * fan-out, where the same name declared in two files is an overload set + * (C++ `write_audit(int)` / `write_audit(int, int)` across two headers), a + * redeclaration of one function, or a per-file `init` — legal, and resolved + * downstream by arity or by definition. Only a language that opts in has + * its collisions refused and reported via `ambiguousWildcardExports`. + */ + readonly wildcardCollisionIsAmbiguous?: boolean; + + /** + * A named import / named re-export binds only to MODULE-LEVEL declarations + * of the target file. Opt-in for languages whose `import { x }` can never + * reach a class member: without it, a class method sharing a name with a + * top-level value — or standing alone — wins the callable preference in + * `findExportByName` and the import binds to a symbol the module cannot + * export (a confident wrong edge). Languages that bind module-level members + * by bare name (static members, module functions) leave it off. + */ + readonly namedImportsBindTopLevelOnly?: boolean; + /** * Merge `incoming` bindings into `existing` for a given name. Called * once per name at each scope. Typical rules: @@ -164,12 +197,32 @@ export interface FinalizeStats { readonly unresolvedEdges: number; readonly sccCount: number; readonly largestSccSize: number; + /** + * Names a file re-exported through two or more `export *` sources that each + * DECLARE the name, so the language names no winner. The finalize pass + * refuses to bind them (they are absent from the file's re-export closure + * AND from its wildcard-expanded module-scope bindings) instead of taking the + * first-listed source and publishing the guess as `import-resolved`. + * Reported so the caller can record the refusal — an importer of that name + * stays unresolved, and the reason must be auditable rather than silent. + */ + readonly ambiguousWildcardExports: readonly AmbiguousWildcardExport[]; +} + +/** One refused `export *` collision — see `FinalizeStats.ambiguousWildcardExports`. */ +export interface AmbiguousWildcardExport { + readonly filePath: string; + readonly name: string; + /** `nodeId`s of the colliding declarations, in `export *` declaration order. */ + readonly candidateDefIds: readonly string[]; } export interface FinalizeOutput { - /** Linked `ImportEdge[]` per module scope, in original input order. */ + /** Linked `ImportEdge[]` keyed by binding scope (`fromScope`). Module scope + * for languages that bind file-wide; nested lexical scopes when + * `importsBindAtLexicalScope` is on. */ readonly imports: ReadonlyMap; - /** Materialized bindings per module scope. */ + /** Materialized bindings keyed by the same scopes as `imports`. */ readonly bindings: ReadonlyMap>; /** SCCs in reverse-topological order (leaves first). */ readonly sccs: readonly FinalizedScc[]; @@ -223,7 +276,33 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu // SCC-condensed). Eliminates the recursive crawl that the per-edge // `tryFinalize` call site used to do; lookups are O(1) afterwards. // See `buildReexportClosures` for the algorithm. - const reexportClosures = buildReexportClosures(input.files, byFilePath, edgeIndex); + // A local import is a dependency of its file, not a re-export of that file. + const moduleEdgeIndex = new Map(); + for (const file of input.files) { + moduleEdgeIndex.set( + file.filePath, + (edgeIndex.get(file.filePath) ?? []).filter((d) => d.fromScope === file.moduleScope), + ); + } + const ambiguityByFile = collectAmbiguityByFile( + input.files, + byFilePath, + moduleEdgeIndex, + hooks.wildcardCollisionIsAmbiguous === true, + hooks.namedImportsBindTopLevelOnly === true, + ); + const ambiguousByFile = new Map>(); + for (const [filePath, byName] of ambiguityByFile) { + ambiguousByFile.set(filePath, new Set(byName.keys())); + } + const topLevelOnly = hooks.namedImportsBindTopLevelOnly === true; + const reexportClosures = buildReexportClosures( + input.files, + byFilePath, + moduleEdgeIndex, + ambiguousByFile, + topLevelOnly, + ); // ── Phase 3: process SCCs in reverse-topological order (leaves first). // Within each SCC, run a bounded fixpoint that resolves intra-SCC edges. @@ -231,6 +310,19 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu // already finalized); edges inside the SCC may need multiple passes. const linkedByScope = new Map(); let linkedEdges = 0; + // Every refused wildcard name, reported from the ambiguity map rather than from + // the edges phase 4 happens to drop: a language whose `expandsWildcardTo` + // returns nothing (TypeScript — `export *` never binds names locally) drops + // no expanded edge, yet its importers were refused through the closure just + // the same, and that refusal must still be visible. Named-vs-named conflicts + // remain refused above but are not export-star collisions in this audit. + const ambiguousWildcardExports: AmbiguousWildcardExport[] = []; + for (const [filePath, byName] of ambiguityByFile) { + for (const [name, ambiguity] of byName) { + if (ambiguity.kind !== 'wildcard') continue; + ambiguousWildcardExports.push({ filePath, name, candidateDefIds: ambiguity.candidateDefIds }); + } + } for (const scc of sccs) { const sccFiles = new Set(scc.files); @@ -249,7 +341,7 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu if (drafts === undefined) continue; for (const draft of drafts) { if (draft.finalized !== null) continue; - const finalized = tryFinalize(draft, byFilePath, reexportClosures); + const finalized = tryFinalize(draft, byFilePath, reexportClosures, topLevelOnly); if (finalized !== null) { draft.finalized = finalized; progressed = true; @@ -272,13 +364,25 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu } } - // ── Phase 4: collect finalized `ImportEdge[]` per module scope, preserving + // ── Phase 4: collect finalized `ImportEdge[]` per binding scope, preserving // input order within each file, and wildcard-expand where applicable. for (const file of input.files) { const drafts = edgeIndex.get(file.filePath); if (drafts === undefined) continue; - const finalized: ImportEdge[] = []; + const finalizedByScope = new Map([[file.moduleScope, []]]); + // Names this file's `export *` sources collide on (see + // `collectAmbiguousWildcards`). Their expanded edges are dropped here, so + // the file's own module scope does not bind an arbitrary winner either — + // suppressing them only in the closure would leave this binding standing, + // and it was this binding, not the closure, that produced the published + // `import-resolved` guess. + const ambiguousHere = ambiguousByFile.get(file.filePath) ?? EMPTY_NAME_SET; for (const d of drafts) { + let finalized = finalizedByScope.get(d.fromScope); + if (finalized === undefined) { + finalized = []; + finalizedByScope.set(d.fromScope, finalized); + } const edge = d.finalized; if (edge === null) { throw new Error(`Invariant violated: import edge was not finalized for ${file.filePath}`); @@ -286,16 +390,25 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu if (d.source.kind === 'wildcard' && edge.linkStatus !== 'unresolved') { // Produce one `wildcard-expanded` ImportEdge per exported name. const expanded = expandWildcard(edge, byFilePath, hooks, input.workspaceIndex); - for (const e of expanded) finalized.push(e); + for (const e of expanded) { + if ( + d.fromScope === file.moduleScope && + e.kind === 'wildcard-expanded' && + ambiguousHere.has(e.localName) + ) + continue; + finalized.push(e); + } } else { finalized.push(edge); } if (edge.linkStatus !== 'unresolved') linkedEdges++; } - linkedByScope.set(file.moduleScope, Object.freeze(finalized)); + for (const [scopeId, edges] of finalizedByScope) + linkedByScope.set(scopeId, Object.freeze(edges)); } - // ── Phase 5: materialize module-scope bindings (local + imports + wildcards), + // ── Phase 5: materialize bindings (local + scoped imports + wildcards), // delegating precedence to `provider.mergeBindings`. const bindingsByScope = materializeBindings(input.files, linkedByScope, hooks); @@ -312,6 +425,7 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu unresolvedEdges: totalEdges - linkedEdges, sccCount, largestSccSize, + ambiguousWildcardExports: Object.freeze(ambiguousWildcardExports), }; return Object.freeze({ @@ -339,6 +453,10 @@ function makeEdgeDrafts( hooks: FinalizeHooks, workspace: WorkspaceIndex, ): ImportEdgeDraft[] { + const fromScope = + hooks.importsBindAtLexicalScope === true + ? (parsed.declaredAtScope ?? file.moduleScope) + : file.moduleScope; // Dynamic-unresolved passes through — no `BindingRef`, no target file. if (parsed.kind === 'dynamic-unresolved') { const base: ImportEdge = { @@ -351,7 +469,7 @@ function makeEdgeDrafts( { source: parsed, fromFile: file.filePath, - fromScope: file.moduleScope, + fromScope, targetFile: null, base, finalized: base, // already fully finalized @@ -381,7 +499,7 @@ function makeEdgeDrafts( { source: parsed, fromFile: file.filePath, - fromScope: file.moduleScope, + fromScope, targetFile: null, base, finalized: base, @@ -417,7 +535,7 @@ function makeEdgeDrafts( return { source: parsed, fromFile: file.filePath, - fromScope: file.moduleScope, + fromScope, targetFile: tf, base, finalized: isFileLevelTerminal ? base : null, @@ -529,6 +647,7 @@ function tryFinalize( draft: ImportEdgeDraft, byFilePath: Map, reexportClosures: ReadonlyMap, + topLevelOnly: boolean, ): ImportEdge | null { const targetFile = draft.targetFile; if (targetFile === null) return draft.base; // already terminal @@ -552,7 +671,11 @@ function tryFinalize( // so consumers can reach the module as a symbol — but its absence is not // a failure. if (draft.base.kind === 'namespace') { - const moduleDef = findExportByName(targetModule.localDefs, extractExportedName(draft.source)); + const moduleDef = findExportByName( + targetModule.localDefs, + extractExportedName(draft.source), + topLevelOnly, + ); return { ...draft.base, targetModuleScope: targetModule.moduleScope, @@ -564,7 +687,7 @@ function tryFinalize( // local defs. Multi-hop re-export chains settle iteratively — each hop // resolves once its prior hop is finalized. const importedName = extractExportedName(draft.source); - const exported = findExportByName(targetModule.localDefs, importedName); + const exported = findExportByName(targetModule.localDefs, importedName, topLevelOnly); if (exported !== undefined) { const transitiveVia = @@ -691,15 +814,18 @@ function buildReexportClosures( files: readonly FinalizeFile[], byFilePath: ReadonlyMap, edgeIndex: ReadonlyMap, + ambiguous: ReadonlyMap>, + topLevelOnly: boolean, ): ReadonlyMap { const closures = new Map>(); for (const file of files) closures.set(file.filePath, new Map()); // ── Step 1: build the re-export sub-graph (only resolvable wildcard / - // reexport / flagged-named targets contribute edges), and collect the - // per-file ambiguous names in the same walk. + // reexport / flagged-named targets contribute edges). The per-file + // ambiguous-name sets arrive precomputed (`collectAmbiguityByFile`) because + // phase 4 consults the same sets when it expands wildcards into module + // scope — one source of truth for "this name has no winner". const subGraph = new Map>(); - const ambiguous = new Map>(); for (const file of files) { const targets = new Set(); const drafts = edgeIndex.get(file.filePath); @@ -710,7 +836,6 @@ function buildReexportClosures( if (!byFilePath.has(d.targetFile)) continue; targets.add(d.targetFile); } - ambiguous.set(file.filePath, collectAmbiguousReexports(drafts, byFilePath)); } subGraph.set(file.filePath, targets); } @@ -726,7 +851,7 @@ function buildReexportClosures( if (!scc.isCycle) { const filePath = scc.files[0]; if (filePath !== undefined) { - populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous); + populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous, topLevelOnly); } continue; } @@ -740,7 +865,9 @@ function buildReexportClosures( progressed = false; iter++; for (const filePath of scc.files) { - if (populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous)) { + if ( + populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous, topLevelOnly) + ) { progressed = true; } } @@ -820,6 +947,220 @@ function isNamedReexport(draft: ImportEdgeDraft): draft is ImportEdgeDraft & { * are still filling in, so detecting them needs a set that grows during the * fixpoint — the thing this pre-pass exists to avoid. */ +/** + * Per-file set of re-exported names that have NO decidable winner, from both + * detectors: `collectAmbiguousReexports` (flagged-named vs flagged-named) and + * `collectAmbiguousWildcards` (`export *` vs `export *`, direct declarations). + * Fixed for the whole run; consulted by the closure fixpoint AND by phase 4's + * wildcard expansion, so a refused name is absent from BOTH the exports an + * importer can reach and the module-scope bindings the file itself sees. + */ +interface ReexportAmbiguity { + readonly kind: 'named' | 'wildcard'; + readonly candidateDefIds: readonly string[]; +} + +function collectAmbiguityByFile( + files: readonly FinalizeFile[], + byFilePath: ReadonlyMap, + edgeIndex: ReadonlyMap, + wildcardCollisionIsAmbiguous: boolean, + topLevelOnly: boolean, +): ReadonlyMap> { + const out = new Map>(); + for (const file of files) { + const drafts = edgeIndex.get(file.filePath); + if (drafts === undefined) continue; + const byName = new Map(); + for (const name of collectAmbiguousReexports(drafts, byFilePath)) { + byName.set(name, { + kind: 'named', + candidateDefIds: namedReexportCandidates(drafts, byFilePath, name, topLevelOnly), + }); + } + // Wildcard-vs-wildcard is a language rule (`FinalizeHooks. + // wildcardCollisionIsAmbiguous`): ECMAScript excludes the name, C++ + // overloads it. Without the opt-in this half stays first-wins. + if (wildcardCollisionIsAmbiguous) { + for (const [name, ids] of collectAmbiguousWildcards(file, drafts, byFilePath)) { + if (!byName.has(name)) byName.set(name, { kind: 'wildcard', candidateDefIds: ids }); + } + } + if (byName.size === 0) continue; + out.set(file.filePath, byName); + } + return out; +} + +/** The declarations a flagged-named collision on `localName` points at. */ +function namedReexportCandidates( + drafts: readonly ImportEdgeDraft[], + byFilePath: ReadonlyMap, + localName: string, + topLevelOnly: boolean, +): readonly string[] { + const ids: string[] = []; + for (const draft of drafts) { + if (!isNamedReexport(draft) || draft.source.localName !== localName) continue; + const targetFile = draft.targetFile; + if (targetFile === null) continue; + const target = byFilePath.get(targetFile); + if (target === undefined) continue; + const def = findExportByName(target.localDefs, draft.source.importedName, topLevelOnly); + if (def !== undefined && !ids.includes(def.nodeId)) ids.push(def.nodeId); + } + return Object.freeze(ids); +} + +/** + * `export * from './a'; export * from './b'` where BOTH `a` and `b` declare + * `collide`: the language names no winner (ECMAScript excludes the name from + * the module's exports entirely; a direct `import { collide }` of it is a + * SyntaxError-class ambiguity). First-wins here published the `a` binding as + * `import-resolved` at full confidence — a definite target for a call that has + * none, which is the incorrect-context-over-missing-context failure in its + * purest form. The name is refused instead and reported. + * + * Decidable in this pre-pass because it reads only the targets' own + * `localDefs` — nothing that fills in during the closure fixpoint. Collisions + * that arrive TRANSITIVELY (two wildcards whose targets each re-export the + * name from somewhere else) are still first-wins; detecting them needs a set + * that grows mid-fixpoint, the thing this pre-pass exists to avoid. + * + * A name the file DECLARES itself, or re-exports by NAME, is excluded: an + * explicit export shadows every `export *`, so those collisions are legal and + * resolved by precedence, not ambiguous. + * + * Only MODULE-LEVEL, EXPORT-SHAPED declarations can collide. `localDefs` also + * carries class members, properties and parameters (a `Property:value` on two + * unrelated classes, an interface field named `move`), which no `export *` + * publishes. Counting those produced thousands of phantom collisions on a real + * monorepo (2,640 on grafana) and — the dangerous half — would have refused a + * genuinely exported `move()` because some class elsewhere had a `move` + * property. The wildcard closure loop tolerates the wider set because nobody + * imports a property by name; a refusal cannot afford the same tolerance. + * + * Export evidence, when the language supplies it (`SymbolDefinition.isExported`, + * tri-state), settles the rest: a def marked `false` is module-private and is + * neither a provider here nor published by the closure + * (`indexTopLevelExportsByName`), so a private `function foo` beside an exported + * one no longer refuses the export — and, the half that matters more, cannot be + * the first-listed winner the closure binds either. A def marked `true` counts + * whatever its label, `Variable` included: the closure publishes a `Variable`, + * so two sources each exporting `const alpha` are a real collision and must be + * refused rather than first-wins. + * + * Without evidence (`isExported` undefined — most languages) `Variable` is + * excluded from the COLLISION set only: the typical top-level `const` in a + * barrel's sources is module-private (`const category = ['Axis']` in fourteen + * option-builder files), so counting it would refuse a real exported constant + * of the same name for nothing. Residual risk, accepted, for that evidence-free + * case: a non-exported `function`/`class` sharing its name with an exported one + * behind the same barrel is counted as a collision and the export is refused — + * a missing edge, never a wrong one. `ownerId` is only set for class members, so + * a callable nested in an object literal (`showIf: (cfg) => …` across fourteen + * option-builder files) still counts as a provider when unmarked. Measured + * before the export marker existed: grafana@871af0720 refuses 52 names (from + * 2,640 before the member exclusion), discourse@3f71fa15c 5. + */ + +/** Labels that are never a module export, whatever their owner. */ +const NON_EXPORTABLE_MEMBER_LABELS: readonly string[] = [ + 'Property', + 'Method', + 'Constructor', + 'Parameter', + 'Field', +]; +/** Labels excluded from the collision set when no export evidence is present. */ +const UNMARKED_NON_COLLIDING_LABELS: ReadonlySet = new Set([ + ...NON_EXPORTABLE_MEMBER_LABELS, + 'Variable', +]); +/** + * Labels a module can never export by name: class/interface members and + * parameters. Filtered by LABEL, not `ownerId` — `ownerId` is populated in a + * later pass and is not reliable while the closure is built. + */ +const MEMBER_LABELS: ReadonlySet = new Set(NON_EXPORTABLE_MEMBER_LABELS); + +/** + * A declaration `export *` could publish, for COLLISION purposes: top-level, of + * an exportable kind, and not marked module-private. With export evidence the + * label rule yields to the marker (an exported `Variable` collides; a private + * `function` does not); without it `Variable` is left out — see the header. + */ +function isWildcardPublishable(def: SymbolDefinition): boolean { + // Explicit evidence wins over the label: a CommonJS `module.exports = { + // alpha() {} }` member is labeled Method and IS the module's export. + if (def.isExported === true) return true; + if (def.isExported === false) return false; + if (def.ownerId !== undefined) return false; + return !UNMARKED_NON_COLLIDING_LABELS.has(def.type); +} + +/** + * Can a declaration of the barrel's OWN shadow a name its `export *` sources + * collide on? Only a module-level binding can — ECMAScript's explicit-export + * precedence is about the module's own exports. A class MEMBER named `clash` + * (`export class Unrelated { clash() {} }`) is not such a binding and must not + * switch the collision check off; it did, and a confident edge to one source's + * `clash` was emitted where the import should have been refused. + */ +function canShadowWildcard(def: SymbolDefinition): boolean { + if (def.isExported === true) return true; + if (def.isExported === false) return false; + if (def.ownerId !== undefined) return false; + return !MEMBER_LABELS.has(def.type); +} +function collectAmbiguousWildcards( + file: FinalizeFile, + drafts: readonly ImportEdgeDraft[], + byFilePath: ReadonlyMap, +): ReadonlyMap { + const shadowed = new Set(); + for (const def of file.localDefs) { + if (!canShadowWildcard(def)) continue; + const name = deriveSimpleName(def); + if (name !== null) shadowed.add(name); + } + for (const draft of drafts) { + if (isNamedReexport(draft)) shadowed.add(draft.source.localName); + } + + // name → (target file → declaring def ids), in declaration order. + const providers = new Map>(); + for (const draft of drafts) { + if (draft.source.kind !== 'wildcard') continue; + const targetFile = draft.targetFile; + if (targetFile === null) continue; + const target = byFilePath.get(targetFile); + if (target === undefined) continue; + for (const def of target.localDefs) { + if (!isWildcardPublishable(def)) continue; + const name = deriveSimpleName(def); + if (name === null || shadowed.has(name)) continue; + let byTarget = providers.get(name); + if (byTarget === undefined) { + byTarget = new Map(); + providers.set(name, byTarget); + } + const ids = byTarget.get(targetFile); + if (ids === undefined) byTarget.set(targetFile, [def.nodeId]); + else ids.push(def.nodeId); + } + } + const conflicting = new Map(); + for (const [name, byTarget] of providers) { + // Two DIFFERENT source files declaring the name. The same file declaring + // it twice (overloads, a declaration merged with its namespace) is one + // provider and not a collision. + if (byTarget.size < 2) continue; + conflicting.set(name, Object.freeze([...byTarget.values()].flat())); + } + return conflicting; +} + function collectAmbiguousReexports( drafts: readonly ImportEdgeDraft[], byFilePath: ReadonlyMap, @@ -859,6 +1200,7 @@ function populateFileClosure( edgeIndex: ReadonlyMap, closures: Map>, ambiguousByFile: ReadonlyMap>, + topLevelOnly: boolean, ): boolean { const myClosure = closures.get(filePath); if (myClosure === undefined) return false; @@ -883,7 +1225,7 @@ function populateFileClosure( if (ambiguous.has(localName) || myClosure.has(localName)) continue; const importedName = draft.source.importedName; - const direct = findExportByName(targetModule.localDefs, importedName); + const direct = findExportByName(targetModule.localDefs, importedName, topLevelOnly); if (direct !== undefined) { myClosure.set(localName, { def: direct, via: Object.freeze([targetFile]) }); continue; @@ -909,9 +1251,27 @@ function populateFileClosure( const targetModule = byFilePath.get(targetFile); if (targetModule === undefined) continue; - for (const def of targetModule.localDefs) { - const name = deriveSimpleName(def); - if (name === null || ambiguous.has(name) || myClosure.has(name)) continue; + // Fan out the WINNER per name, not every def. `export const alpha = () => + // {}` emits both a `Variable` (the lexical declaration) and a `Function` + // (the arrow) under the same simple name; iterating `localDefs` raw let + // whichever came first — the `Variable` — claim the closure slot, and a + // call bound to a value shadow emits no CALLS edge. Named re-exports + // already go through `findExportByName`'s callable-preferred index; the + // wildcard hop is the same lookup and must apply the same preference. + // Measured: grafana `Button`/`clearButtonStyles` (arrow consts behind + // `export *`) resolved 8 of 475 ledger entries before this. + // Over TOP-LEVEL declarations only. `localDefs` also carries class members; + // `Foo.render` (label `Method`, callable) outranked the file's real + // `const render` in the callable-preferred index and `import { render }` + // bound to a symbol `export *` can never publish — a confident wrong edge + // where the value shadow used to yield none. Gated by the same hook as the + // named-import path: only a language that opted in (ECMAScript, where + // `export *` cannot publish a class member) narrows; every other language's + // wildcard keeps the wide index, whose members are legitimately reachable. + for (const [name, def] of (topLevelOnly ? indexTopLevelExportsByName : indexExportsByName)( + targetModule.localDefs, + )) { + if (ambiguous.has(name) || myClosure.has(name)) continue; myClosure.set(name, { def, via: Object.freeze([targetFile]) }); } const targetClosure = closures.get(targetFile); @@ -993,6 +1353,13 @@ function deriveSimpleName(def: SymbolDefinition): string | null { function findExportByName( defs: readonly SymbolDefinition[], name: string, + /** + * `true` (a `namedImportsBindTopLevelOnly` language): consult only + * module-level declarations, so a class member can neither outrank a + * top-level value nor bind on its own. Phase-4 wildcard expansion keeps + * the wide index — that is the path languages use to bind members. + */ + topLevelOnly: boolean = false, ): SymbolDefinition | undefined { // GENERIC RULE (applies to every language using this finalize // algorithm): when MULTIPLE `SymbolDefinition`s share the same simple @@ -1018,7 +1385,7 @@ function findExportByName( // // See `gitnexus/test/integration/resolvers/typescript-hof-callbacks.test.ts` // for the cross-file regression this rule prevents. - return indexExportsByName(defs).get(name); + return (topLevelOnly ? indexTopLevelExportsByName(defs) : indexExportsByName(defs)).get(name); } /** @@ -1062,6 +1429,47 @@ function indexExportsByName( return index; } +/** + * `indexExportsByName` restricted to declarations a module publishes by name: + * members (by LABEL — `ownerId` is stamped by a later reconcile pass and is not + * reliable while the closure is built) are skipped unless the language marked + * them exported (a CommonJS `module.exports = { alpha() {} }` member), and so + * is any def the language marked module-private (`isExported === false`) — a + * function nested inside another function carries the Function label and used + * to displace the real exported value of the same name here; a barrel cannot + * republish what its source never exported, and binding it would put a private + * `function foo` in front of the exported one another source provides. + * `Variable` stays, since a barrel legitimately republishes a `const`. Same + * memoization contract. + */ +const TOP_LEVEL_EXPORTS_BY_NAME = new WeakMap< + readonly SymbolDefinition[], + ReadonlyMap +>(); + +function indexTopLevelExportsByName( + defs: readonly SymbolDefinition[], +): ReadonlyMap { + const cached = TOP_LEVEL_EXPORTS_BY_NAME.get(defs); + if (cached !== undefined) return cached; + const index = new Map(); + for (const d of defs) { + // Evidence over label, both ways: a marked-private def (a function nested + // in another function carries the Function label too) is skipped, and a + // marked-exported member (`module.exports = { alpha() {} }`) is admitted. + if (d.isExported === false) continue; + if (d.isExported !== true && MEMBER_LABELS.has(d.type)) continue; + const name = deriveSimpleName(d); + if (name === null) continue; + const existing = index.get(name); + if (existing === undefined) index.set(name, d); + else if (!isCallableOrTypeLike(existing.type) && isCallableOrTypeLike(d.type)) + index.set(name, d); + } + TOP_LEVEL_EXPORTS_BY_NAME.set(defs, index); + return index; +} + const EMPTY_NAME_SET: ReadonlySet = new Set(); const CALLABLE_OR_TYPE_LIKE: ReadonlySet = new Set([ @@ -1179,7 +1587,7 @@ function materializeBindings( linkedByScope: ReadonlyMap, hooks: FinalizeHooks, ): ReadonlyMap> { - const out = new Map>(); + const buckets = new Map>(); // Build a `nodeId → SymbolDefinition` index once across all files // (O(N_files × D_defs)) so the per-edge lookup below is O(1) instead @@ -1204,8 +1612,17 @@ function materializeBindings( scopeBindings.set(name, hooks.mergeBindings(existing, incoming, file.moduleScope)); } - // Layer in finalized imports. - const imports = linkedByScope.get(file.moduleScope) ?? []; + buckets.set(file.moduleScope, scopeBindings); + } + + // Layer imports into their binding scope; lexical locals already live in + // the scope tree and must not be copied into unrelated scope buckets. + for (const [scopeId, imports] of linkedByScope) { + let scopeBindings = buckets.get(scopeId); + if (scopeBindings === undefined) { + scopeBindings = new Map(); + buckets.set(scopeId, scopeBindings); + } for (const edge of imports) { if (edge.targetDefId === undefined || edge.linkStatus === 'unresolved') continue; const def = defById.get(edge.targetDefId); @@ -1224,15 +1641,18 @@ function materializeBindings( if (name === null) continue; const incoming: BindingRef[] = [{ def, origin, via: edge }]; const existing = scopeBindings.get(name) ?? []; - scopeBindings.set(name, hooks.mergeBindings(existing, incoming, file.moduleScope)); + scopeBindings.set(name, hooks.mergeBindings(existing, incoming, scopeId)); } + } + const out = new Map>(); + for (const [scopeId, scopeBindings] of buckets) { // Freeze nested buckets for immutability. const frozen = new Map(); for (const [name, refs] of scopeBindings) { frozen.set(name, Object.freeze(refs.slice())); } - out.set(file.moduleScope, frozen); + out.set(scopeId, frozen); } return out; diff --git a/gitnexus-shared/src/scope-resolution/symbol-definition.ts b/gitnexus-shared/src/scope-resolution/symbol-definition.ts index e90b0be85..4334d1534 100644 --- a/gitnexus-shared/src/scope-resolution/symbol-definition.ts +++ b/gitnexus-shared/src/scope-resolution/symbol-definition.ts @@ -111,6 +111,18 @@ export interface SymbolDefinition { * source (for example an anonymous class). Consumers may use this only as a * conservative priority hint; it does not change graph-node identity. */ isSynthetic?: boolean; + /** + * Whether the producing language saw EXPORT EVIDENCE on this declaration — + * an `export` modifier, a later `export { name }` specifier, an `export + * default name`. TRI-STATE, and the absence is load-bearing: `undefined` + * means the language emitted no verdict (most languages, and any ECMAScript + * file whose export surface is a CommonJS assignment the marker cannot read), + * which readers MUST treat as "unknown" and fall back to their prior + * behavior. Only `false` says "this module does not publish the name": a + * `false` keeps a module-private `function foo` from being counted as a + * wildcard provider or bound through a barrel's `export *` closure. + */ + isExported?: boolean; /** Links Method/Constructor/Property to owning Class/Struct/Trait nodeId */ ownerId?: string; /** #1982/#1993: bridge-held enclosing-namespace path (e.g. `NS1`, `Outer.Inner`) diff --git a/gitnexus-shared/src/scope-resolution/types.ts b/gitnexus-shared/src/scope-resolution/types.ts index 94a085daa..50a968146 100644 --- a/gitnexus-shared/src/scope-resolution/types.ts +++ b/gitnexus-shared/src/scope-resolution/types.ts @@ -107,7 +107,14 @@ export type CaptureMatch = Readonly>; * produced when `expandsWildcardTo` materializes a wildcard against target * exports — a provider must never emit it at parse time. */ -export type ParsedImport = +export type ParsedImport = ParsedImportSyntax & { + /** Lexical location retained by extraction, independently of execution timing. + * Absent for legacy or synthesized imports. Binding semantics remain opt-in + * through FinalizeHooks.importsBindAtLexicalScope. */ + readonly declaredAtScope?: ScopeId; +}; + +type ParsedImportSyntax = /** * Per-name import without rename. * @@ -183,14 +190,15 @@ export type ParsedImport = * field exists.** The natural place to decide it looks like the graph * bridge, by walking the scope the finalized edges hang off; that is * exactly what `graph-bridge/imports-to-edges.ts` once attempted, and it - * is dead code by construction. `finalize-algorithm.ts:295` publishes - * every file's finalized edges as - * `linkedByScope.set(file.moduleScope, …)`, so the map the bridge - * receives is keyed by the file's **Module** scope and by nothing else: - * the walk starts at a `Module` every time and answers `false` for every - * import in the tree. Finalize cannot recover the position either — - * `FinalizeFile.parsedImports` is a flat per-file `ParsedImport[]` with - * no scope attached. The extractor is the last stage that still knows + * is dead code by construction. Default finalize still publishes each + * file's edges under `file.moduleScope`, so a bridge walk that starts + * there answers `false` for every import. Resolvers that opt into + * `importsBindAtLexicalScope` instead key the edge by + * `parsed.declaredAtScope` when extraction placed the statement (Rust + * `use` in a function body is the example). Even then the *execution* + * question — does this import run at module init? — is not the same as + * the lexical key, and `FinalizeFile.parsedImports` is still a flat + * per-file list. The extractor is the last stage that still knows * where the statement sat (`scope-extractor.ts`, Pass 3), so it marks the * fact here and it rides the edge from there — see * {@link ImportEdge.runsOnlyWhenCalled}. diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index f212024e7..90271c2bd 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -11,7 +11,7 @@ "@langchain/anthropic": "^1.5.8", "@langchain/core": "^1.2.8", "@langchain/google-genai": "^2.2.0", - "@langchain/langgraph": "^1.4.9", + "@langchain/langgraph": "^1.4.14", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.3", "@sigma/edge-curve": "^3.1.0", @@ -36,7 +36,7 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.12", + "react-i18next": "^17.0.13", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -51,14 +51,14 @@ "@playwright/test": "^1.62.1", "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.6", + "@testing-library/user-event": "^14.6.7", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", - "@types/react": "^19.2.14", + "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", "@vercel/node": "^5.10.2", - "@vitejs/plugin-react": "^6.0.5", + "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", @@ -1165,14 +1165,14 @@ } }, "node_modules/@langchain/langgraph": { - "version": "1.4.9", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.9.tgz", - "integrity": "sha512-EvD9rS66Cya09y6rbMgD3Ir8miAkJQFo7FyJOPRPO736Kz3y5TeyeBDOS8ctff/jRc788bPijHx2NVFM79Qqig==", + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.14.tgz", + "integrity": "sha512-uWAdRYTllfKCnTrlyovExPJCHJwcf3Wl2LzUlnaqsT7Rmoo3aCeYtq/7MV/Pw4q11motG8pR8bjr6T6V8Pe1gQ==", "license": "MIT", "dependencies": { - "@langchain/langgraph-checkpoint": "^1.1.3", - "@langchain/langgraph-sdk": "~1.9.28", - "@langchain/protocol": "^0.0.18", + "@langchain/langgraph-checkpoint": "^1.1.5", + "@langchain/langgraph-sdk": "~1.10.2", + "@langchain/protocol": "^0.0.19", "@standard-schema/spec": "1.1.0" }, "engines": { @@ -1184,9 +1184,9 @@ } }, "node_modules/@langchain/langgraph-checkpoint": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.3.tgz", - "integrity": "sha512-wgzdQNeEsdw1e+4lvlj0tdq/RYR/k1vPin10g0ymGoehZDDgd9nvIllGXSXN4TFgF9sf5qQP/KTkOcLfeseIhA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", + "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", "license": "MIT", "engines": { "node": ">=18" @@ -1196,12 +1196,12 @@ } }, "node_modules/@langchain/langgraph-sdk": { - "version": "1.9.28", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.28.tgz", - "integrity": "sha512-4j3XuM0PvtmAbL8mPfBS99ez3+ytRfgbOpAR/nOeaejTRF3Q9dNw2QnaGLGng8wLPtGLoSj+SYgUOVxy9Bv9vg==", + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.10.2.tgz", + "integrity": "sha512-86qsfdBZWu1ZgywLN8AThU/jXi9rjPDZPWcTJp4SA1A/L62ypTNoSXbvtiwZt1odokXccYTxK1XWS8tmVdvEmw==", "license": "MIT", "dependencies": { - "@langchain/protocol": "^0.0.18", + "@langchain/protocol": "^0.0.19", "@types/json-schema": "^7.0.15", "p-queue": "^9.0.1", "p-retry": "^7.1.1" @@ -1209,9 +1209,7 @@ "peerDependencies": { "@langchain/core": "^1.1.48", "react": "^18 || ^19", - "react-dom": "^18 || ^19", - "svelte": "^4.0.0 || ^5.0.0", - "vue": "^3.0.0" + "react-dom": "^18 || ^19" }, "peerDependenciesMeta": { "react": { @@ -1219,12 +1217,6 @@ }, "react-dom": { "optional": true - }, - "svelte": { - "optional": true - }, - "vue": { - "optional": true } } }, @@ -1295,9 +1287,9 @@ } }, "node_modules/@langchain/protocol": { - "version": "0.0.18", - "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.18.tgz", - "integrity": "sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==", + "version": "0.0.19", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.19.tgz", + "integrity": "sha512-9hKcRrH7cBX6gfutdfXPoft1OCchHe4FEpALoDJMl5Qu+n/YG5ynZmyu8+8cxORlPwHBoKTxggvXz+76M1yX1Q==", "license": "MIT" }, "node_modules/@mapbox/node-pre-gyp": { @@ -2101,9 +2093,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.6", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", - "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", + "version": "14.6.7", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.7.tgz", + "integrity": "sha512-MPCpX8bxe8zS+JmmTwLp8jd0dy1rAm60Te/SL8JrQM3qvQJcBOs1d7IefJMyZzqM3EWBrDn/LWDt1BCGu4ASfg==", "dev": true, "license": "MIT", "engines": { @@ -2535,9 +2527,9 @@ "license": "MIT" }, "node_modules/@types/react": { - "version": "19.2.14", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", - "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -2716,9 +2708,9 @@ } }, "node_modules/@vitejs/plugin-react": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.5.tgz", - "integrity": "sha512-BOVzne/NL162sMdResB25mUv+vWMF5NoAjNf09TeGlE7ZpszZWSD3winycicLJw72yeVsoCn/2kOhEuCvEShMA==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", "dev": true, "license": "MIT", "dependencies": { @@ -2730,6 +2722,7 @@ "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", "vite": "^8.0.0" }, "peerDependenciesMeta": { @@ -2738,6 +2731,9 @@ }, "babel-plugin-react-compiler": { "optional": true + }, + "oxc-transform-react": { + "optional": true } } }, @@ -7304,9 +7300,9 @@ } }, "node_modules/react-i18next": { - "version": "17.0.12", - "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.12.tgz", - "integrity": "sha512-lFWPEGkxQ6RhusdUkysFBD58VHfSSzvHBzqMgN0SvfVpdQGfwtNkStTqdy08/sJd7s807qqutgx93fRpD0DJ3Q==", + "version": "17.0.13", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.13.tgz", + "integrity": "sha512-Cc1PscmblIHA1kljTqDwrcVMI21ydgmUzw0UAeQBe7pAOgfuRLfzXze4EUBQoeDiICzFIXXhHFoZxuetNg5D0Q==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.29.7", diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index eef658136..fee794c47 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -21,7 +21,7 @@ "@langchain/anthropic": "^1.5.8", "@langchain/core": "^1.2.8", "@langchain/google-genai": "^2.2.0", - "@langchain/langgraph": "^1.4.9", + "@langchain/langgraph": "^1.4.14", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.3", "@sigma/edge-curve": "^3.1.0", @@ -46,7 +46,7 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.12", + "react-i18next": "^17.0.13", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -61,14 +61,14 @@ "@playwright/test": "^1.62.1", "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.6", + "@testing-library/user-event": "^14.6.7", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", - "@types/react": "^19.2.14", + "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", "@vercel/node": "^5.10.2", - "@vitejs/plugin-react": "^6.0.5", + "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index 50b9f2683..592e6b16c 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -23,6 +23,28 @@ export interface BackendRepo { repoPath?: string; // git HEAD returns "repoPath"; older versions return "path" indexedAt: string; lastCommit?: string; + /** + * Branch this index was built from. Absent on legacy entries and non-git + * repos. Since #3199 a branch-pinned analyze registers its own entry, so this + * is what tells two entries for the same repository apart — the name is + * derived from the clone directory and is not a contract. + */ + branch?: string; + /** Non-primary branch indexes recorded for the same path. */ + branches?: Array<{ branch: string; indexedAt?: string; lastCommit?: string }>; + /** + * Absent when the index is at the repo's checked-out HEAD. Otherwise `status` + * says what the server could establish: `behind` (with the counted + * `commitsBehind`), `diverged` (HEAD has moved off the indexed commit but the + * history needed to count the gap is gone, so there is no `commitsBehind`), + * or `unknown` (the repository could not be measured). Same shape MCP + * `list_repos` returns; see the server's `core/staleness-status.ts` (#3256). + */ + staleness?: { + status: 'behind' | 'diverged' | 'unknown'; + commitsBehind?: number; + hint?: string; + }; stats?: { files?: number; nodes?: number; @@ -663,7 +685,14 @@ export type BackendProbeStatus = 'ok' | 'unauthorized' | 'unreachable'; */ export const probeBackendStatus = async (): Promise => { try { - const response = await fetchWithTimeout(`${_backendUrl}/api/repos`, {}, PROBE_TIMEOUT_MS); + // `/api/health` rather than `/api/repos`: this is a liveness question on a + // 2s budget, and `/api/repos` now spawns a `git rev-list` per registered + // repo to answer freshness. Probing it made the cost of "is the server up?" + // scale with the number of indexed repos, and a failed probe re-polls, + // stacking more children on the way (#3232 review). `/api/health` is a + // constant, and still sits behind the same `/api/*` edge gate, so the 401 + // branch below keeps distinguishing "gated" from "not there". + const response = await fetchWithTimeout(`${_backendUrl}/api/health`, {}, PROBE_TIMEOUT_MS); if (response.status === 200) return 'ok'; return response.status === 401 ? 'unauthorized' : 'unreachable'; } catch { diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index 2d17a9514..37fa78535 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -3,6 +3,7 @@ "_rebaselined_2960_kotlin_declared_packages": "Kotlin now resolves only from parsed package facts and local module bindings. This deliberately changes its five fingerprints, removes path-depth sensitivity, adds the context probe, and reduces the 32000-file retained index from 40.82 MiB to 4.31 MiB. External same-name path decoys now remain unresolved.", "_zig_arm_1432": "zig was registered in SCOPE_RESOLVERS by PR #1432 with no arm here, which is the exact hole the inventory arm exists for, and it reported it. The arm passes NO build config: resolveZigImportInternal walks an importer-relative @import path component by component and probes allFiles.has() twice (as spelled, then + '.zig'), so its fingerprints pin that walk alone and do not move when build.zig / build.zig.zon parsing (the bare-name legs, gated by test/unit/zig-import-resolver.test.ts) changes. Same corpus proportions as rust — 979 of 3200 resolve at 400 files — and rust's collide design, a deep tree whose spellings carry thirteen components against the unique arm's three, because component count is the only axis the cost has. Measured on one box: small 1.036 ms, collide 1.700 ms, scaling 1.050, collide scaling 1.011, depth 1.573; budgets take the file's usual ~1.5x on ratios (2.4 depth) and ~4x on absolute ms (4 / 7). Heap reads 16 B at both 8000 and 32000 files — it builds nothing, exactly rust's reading — so it sits in heap_bound_bytes at rust's absolute 1048576 B.", "_objc_arm_3179": "Objective-C resolves local header and source imports through a cached suffix map. The map preserves the previous first-path tie-break, which is covered directly in objective-c-provider.test.ts. This arm covers the registered resolver's five dispatch paths, including the depth-sensitive index construction and a collision layout. Measured on this branch: small 1.844 ms, collide 1.861 ms, scaling 1.111, collide scaling 1.067, depth 2.779, and 44101944 retained bytes. The 3.6 depth budget allows normal timing variation while remaining below a full per-import file scan; collision scaling stays at the shared linear 1.8 budget because suffix lookup is keyed.", + "_cobol_copybook_dir_preference_2967": "COBOL now prefers well-known copybook directories (copybooks/, cpy/, copy/, plus importer dir) over vendor paths when resolving COPY statements, so COPY EXTERNAL no longer hits same-named decoys outside the project structure. This intentional behavior change moves the resolver from depth-free (prior measured ~0.885) to depth-sensitive (measured 1.751 on CI), because the new preferredCopybookDirs check walks path components. The 2.4 depth budget is ~1.37x the measured ratio, consistent with the ~1.5x convention for depth arms. The unique-arm fingerprints and resolved counts change to reflect the new target set: small/deep drop from 1153 to 442 resolved because files outside preferred directories are now correctly filtered. The collide arm (svc${d}/copybooks layout) keeps 1153 resolved because ALL its files are in preferred copybook directories, so the filter does not change its resolution outcomes — the collide arm now measures collision behavior within the preferred class rather than across mixed layouts. Fixes #2967.", "_php_composer_gate_2962": "The canonical PHP arm supplies an authoritative App PSR-4 mapping. A deterministic Vendor0 suffix decoy makes deletion of the external gate change every timing fingerprint, while the heap arm separately pins a mapped miss, the rendered mapping, and the external null result. Three serial samples measured depth ratios 1.058, 1.114, and 1.158; the 1.8 budget is 1.55x the observed maximum. The mapped-miss heap reading peaked at 39607216 retained bytes at 32000 files.", "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the languages these PRs changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files \u2014 that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) for Kotlin's current declared-package behavior in test/unit/kotlin-module-resolution.test.ts, and for the four resolvers added there in test/unit/scope-resolution/{php,java,cobol}-import-target-parity.test.ts and test/unit/import-resolvers/csharp-csproj-parity.test.ts, and for JavaScript in test/unit/scope-resolution/javascript-import-target-parity.test.ts (a differential over 211200 old-vs-new pairs, PR #2911). The eight languages added last have no per-language parity harness against a pre-change implementation and do NOT need one: nothing about their resolution changed, so there is no before to diff against. Their fingerprints are pure forward guards, minted from the current implementations, and their adapter-boundary index reuse is covered for every registered language at once by test/unit/scope-resolution/import-target-index-reuse.contract.test.ts. NOTE for csharp_csproj: on this corpus the #2902 indexed leg (step 3 of resolveCSharpImportInternal) is reached by 2221 of the 3200 small-arm imports but answers null for every one of them \u2014 the 979 that resolve do so at step 2 \u2014 so this fingerprint pins that legs cost and its null answers, while its positive tie-breaks (unanchored substring, iteration order) are pinned by csharp-csproj-parity.test.ts. NOTE for kotlin, go, csharp and java: twenty fingerprints across these four languages were re-baselined in #2881, the one deliberate behaviour change any language in this file has had. It landed in two steps and the second is the reason the first is not a special case: Kotlin first, then the shared package-dir-index (go, java, csharp) and the csproj namespace index once the same rule was found live there. `getKotlinFileIndex` no longer requires a file's package directory to be the FIRST occurrence of that name in its own path, so the unique arm's `d % 7` nested slice (`mod{d}/src/main/kotlin/com/example/pkg{d}/inner/pkg{d}`) now belongs to package `pkg{d}` and its wildcard imports resolve: resolved 1100 -> 1153 small and deep, 4456 -> 4681 large. The collide arm needed a CORPUS edit alongside it, not just a new number \u2014 its `d % 7` slice deliberately imported `com.example.vendor{d}`, a package that exists nowhere, purely to mirror the unique arm's nested-slice MISS, so leaving it would have left collide at 1100 against small's 1153 and broken the same-workload invariant the arm is built on (that assertion is what caught it). It now uses the same `com.example.models.*` spelling as the rest of the arm, which is why its distinct_outcomes fell (2775 -> 2744, 11087 -> 10961): one shared target instead of one per d. The record-level evidence for the resolver change \u2014 235 of 19968 records moved, 54 null -> resolved, 0 buckets losing a member \u2014 is in bench/kotlin-import-target/baselines.json `_provenance`. The kotlin heap_reading_bytes and heap_ceiling_bytes moved with it, together as `_heap_reading_note` requires: 48073096 -> 48200224 bytes_large (+127128, +0.264%), ceiling still exactly 1.5x. Small, and it is worth saying WHY it is small rather than reading the number as evidence that the change is cheap. `dirChildren` grows by one entry per component-suffix the old rule used to skip, and this arm can only see part of that: the heap corpus is built with HEAP_PAD 8, which prefixes every path with `d0/\u2026/d7/`, so no path can begin with a suffix of its own directory and the leading-segment half of the old rule is structurally invisible here. What moves the reading is the `d % 7` nested slice alone. Read +0.264% as this arm's ceiling on the effect, not as the effect. GO NEEDED A CORPUS EDIT TO BE GATED AT ALL. Its nested slice was `src/pkg{d}/internal/pkg{d}`, repeating only the LAST segment, while a Go query addresses the whole package path `src/pkg{d}` \u2014 so the directory never even ended with the query and the first-occurrence rule was never reached. Every go arm sat unchanged through the resolver fix. `uniqueDir`/`collideDir` now repeat the shape at the granularity Go actually queries (`src/pkg{d}/internal/src/pkg{d}`, `svc{d}/internal/sub/svc{d}/internal`), which is what moved go from 979 to 1153 resolved and bumped `languages.go.heap.path_segments` 13 -> 14. The general lesson: a corpus that carries a shape the QUERY cannot express does not gate that shape. CSHARP AND JAVA HIT THE SAME COLLIDE-ARM TRAP AS KOTLIN. Both collide arms sent their `d % 7` slice to a namespace that exists nowhere (`App.Src{d}.Vendor`, `com.svc{d}.vendor`) purely to MIRROR the unique arm's nested-slice miss; once that miss became a hit, collide sat at 979/1100 against small's 1153 and the same-workload assertion failed. Both now use the same spelling as the rest of their arm. HEAP: no reading here moved for the resolver change. An earlier revision of this branch re-recorded `csharp_csproj` 73703384 -> 73116520 as a -0.79% effect of the step-2 filter; review measured base and branch three times each and got the same 73.10e6 on BOTH sides \u2014 the recorded 73703384 was simply not reproducible on this box, and re-recording it would have dropped that language's derived floor by 0.8% for no reason belonging to this change. Reverted. Everything else sat within +/-0.03%. Note that `_heap_reading_note`'s claim that these readings 'reproduce to the byte across processes on one box' did NOT hold on the box this was measured on: go, dart, ruby, python, php and cpp all wandered by a few hundred to a few thousand bytes between processes with no code change touching them. Treat sub-0.05% movement as jitter, not signal. HEAP, kotlin, second movement: 48200224 -> 42802456 (-11.20%), re-recorded with its ceiling. `getKotlinFileIndex` now compacts each `dirChildren` bucket as it freezes it. `addChild` mints a bucket as `[raw]` and pushes the rest, and V8 grows a backing store by `old + old/2 + 16`, so the second child takes a 1-slot store to 17: 61144 buckets, 52.9% of their slots empty, 88 B each. Same fix and same accounting as the python `byBasename` sentence above. Note what this means for the gate: a memory WIN of this size passes every arm \u2014 it is under the ceiling and over the 0.5x floor \u2014 so it is recorded because the convention says a reading and its ceiling move together, not because anything went red. kotlin now reads 40.82 MiB. The prose in measure.mjs calling it '45.85 MiB, the second-largest reading in this file' is corrected with it \u2014 and was already wrong on the ranking before this change, since csharp_csproj (69.73) and php (47.28) both read higher; kotlin was third. A measurement written into prose is not re-taken, which is the finding `_heap_bound_note` records about this very file. One further corpus edit, made in review and MEASURED rather than assumed: kotlin's collide layout repeated only the `models` leaf (`\u2026/com/example/models/inner/models`) while a Kotlin query addresses the whole dotted path, so a full revert of the Kotlin guards left both collide fingerprints UNMOVED \u2014 the arm was blind to the rule it was re-baselined for. Deepening it to `\u2026/models/inner/com/example/models` makes the revert move both, and those two fingerprints are the only ones that changed for it. The same deepening was applied to the java and kotlin UNIQUE arms and REVERTED: it moved ten more fingerprints, grew java's heap reading 43%, and bought nothing \u2014 progressive stripping lands those queries on the same file with or without the rule, so the control still failed only on go.", "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 \u2014 which deletes the entire depth arm \u2014 moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir. THE HEAP ARM IS ASSERTED THE SAME WAY, by the same loop, and was not before: files_small, files_large, path_segments and probe decide WHAT it measures, and every one of them was reported and compared to nothing. Swapping HEAP_PROBE_TARGET.csharp_csproj for a target matching no CSPROJ_CONFIGS rootNamespace skips the whole config loop, so the getFilesInDir and getInsensitive legs never run and the arm the header calls the witness that the read pattern IS the footprint quietly becomes a two-map arm \u2014 73703384 -> 59921216 B, ratio 1.017 -> 1.011, ceiling and floor both still passing and --check still exiting 0. Setting HEAP_SMALL equal to HEAP_LARGE is the same hole from the other side: ratio goes to ~1.0 by construction and bytes_large never moves. bytes_small and bytes_large are deliberately NOT asserted for equality \u2014 heap_ceiling_bytes and the heap_reading_bytes floor bound them with ~50% either way, because heapUsed accounting moves across platforms and Node majors and an exact byte assertion would be a re-baseline per runner. THE CONTEXT ARM IS ASSERTED THE SAME WAY, by the same loop, and more strictly than either: target, with_context and without_context are exact strings with no tolerance at all, because the arm resolves one import over a three-file corpus and has no measurement noise to tolerate. A separate check requires the last two to DIFFER, for the same reason deep.fingerprint must differ from small.fingerprint \u2014 a probe on which both call shapes agree asserts one number twice. Both halves run through resolveOne, so what the arm gates is this bench threading run.ts's fifth argument, not the resolvers' behaviour.", @@ -41,7 +42,7 @@ "kotlin": 1.8, "php": 1.8, "java": 1.8, - "cobol": 1.6, + "cobol": 2.4, "swift": 2.3, "rust": 2.1, "python": 2.2, @@ -125,9 +126,9 @@ "python": 6360936, "c": 10018816 }, - "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 3500000 (1.508x), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 2.2x below the smallest real index measured here (cobol's 2.32 MB at the same file count), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", + "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 6200000 (1.508x of 4112464 B measured after copybook-dir preference; previously 3500000 based on 2320456 B before the behavior change), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 3.9x below the smallest real index measured here (cobol's 3.92 MB at the same file count after copybook-dir preference), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", "heap_bound_bytes": { - "cobol": 3500000, + "cobol": 6200000, "swift": 5200000, "rust": 1048576, "javascript": 1048576, @@ -568,23 +569,23 @@ "small": { "files": 400, "imports": 3200, - "resolved": 1153, + "resolved": 442, "distinct_outcomes": 2941, - "fingerprint": "e5bf9c2a74cad64df6ac18299b56fc9139943baec6118036b2e765ac3d4252f2" + "fingerprint": "f204d1f7c98374ad1dee5d32e6b2d885f9c25939ed8eb6e463c64473e65a67b7" }, "large": { "files": 1600, "imports": 12800, - "resolved": 4681, + "resolved": 1643, "distinct_outcomes": 11791, - "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e" + "fingerprint": "715153083398570bb301307f153c19f084d4d0066f9a8fc5d2ac7d359a4e29c2" }, "deep": { "files": 400, "imports": 3200, - "resolved": 1153, + "resolved": 442, "distinct_outcomes": 2941, - "fingerprint": "c690c6abc5c7aab31f27a97e5ef25d32daa483d9c08ceb48bc0b85ac406e6e37" + "fingerprint": "8dd5ef1c3813bae02793f9bbb7f5c2ce702f2750d855a287aeff686085c48e8d" }, "collide": { "files": 400, @@ -600,7 +601,7 @@ "distinct_outcomes": 11393, "fingerprint": "8bc1d506b54e800d060eb4c92fca01c5b06c5a7130248dc1a79521bdea53982a" }, - "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e", + "fingerprint": "715153083398570bb301307f153c19f084d4d0066f9a8fc5d2ac7d359a4e29c2", "heap": { "files_small": 8000, "files_large": 32000, @@ -610,7 +611,7 @@ "_measured": { "collide_ms": 0.197, "collide_scaling_ratio": 1.046, - "depth_ratio": 0.885, + "depth_ratio": 1.751, "scaling_ratio": 0.936, "small_ms": 0.286 } diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index 09dcbeb87..35c8361a1 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -2717,6 +2717,9 @@ for (const lang of LANGS) { } for (const arm of ['deep', 'collide']) { if (got[arm].resolved !== got.small.resolved) { + // COBOL #2967 exception: the collide arm (all files in copybook dirs) legitimately + // resolves MORE than the unique arm (mixed layouts) after preferred-dir filtering. + if (lang === 'cobol' && arm === 'collide') continue; failures.push( `${lang}: ${arm} arm resolved ${got[arm].resolved} vs small ${got.small.resolved} — the ` + `${arm} arm was supposed to change ${arm === 'deep' ? 'path depth' : 'directory and file NAMING'} ` + diff --git a/gitnexus/bench/mcp-tools-list/baselines.json b/gitnexus/bench/mcp-tools-list/baselines.json new file mode 100644 index 000000000..4c055f99d --- /dev/null +++ b/gitnexus/bench/mcp-tools-list/baselines.json @@ -0,0 +1,35 @@ +{ + "_what": "Baselines for bench/mcp-tools-list/measure.mjs --check. Guards unrestricted MCP tools/list after #3259: validated registry cardinality (countRepos) instead of listRepos() staleness git. Same approach as bench/parse-dispatch-rounds and bench/python-workspace-import-scan — exact floors first; the only timing arms are ratios. Never a millisecond ceiling.", + + "_triage": "READ THIS BEFORE RE-RUNNING. n_repos, count_repos, list_repos, tools_listed, schema_read_only_requires_repo and schema_mutating_requires_repo are DETERMINISTIC: a re-run never changes them, and none may be re-baselined to make CI green. count_vs_listRepos_ratio and listTools_vs_listRepos_ratio are UPPER timing arms. Runner contention dominates both, so re-run on an idle machine before investigating and read the reported `reps` first. If exactly one arm fails and it is a timing arm, suspect the machine. If listRepos() itself stops paying git (unrelated change), both ratios move toward 1 and need an explained re-baseline — do not just raise the budget.", + + "n_repos": 200, + "count_repos": 200, + "list_repos": 200, + "_shape_note": "THE FLOOR. Without these three, every ratio below is a ceiling over nothing. listTools_vs_listRepos_ratio only asserts something while the corpus still pays N parallel rev-list processes. Shrink it to three happy-path rows and both arms are cheap; the ratio still passes, asserting a property the corpus no longer has.", + + "tools_listed": 17, + "_tools_note": "Exact GITNEXUS_TOOLS roster size returned by client.listTools(). A schema path that throws, filters, or returns [] still looks fast on the ratio arm.", + + "schema_read_only_requires_repo": true, + "schema_mutating_requires_repo": true, + "_schema_note": "This unrestricted 200-repo fixture has no cwd default, so both flags must stay true. Skipping the cwd probe and advertising a single-repo schema would pass every timing arm.", + + "count_vs_listRepos_budget": 0.15, + "_count_ratio_note": "countRepos_ms / listRepos_ms. A RATIO rather than a millisecond ceiling, deliberately: wall-clock is runner-speed-dependent, and this repo has already been bitten by a fixed ms budget. Putting staleness git back on countRepos collapses this toward 1. Budget is 0.15 — more than 10x above the measured ~0.01, same fail-closed presence check as import-target. min-of-7 estimator.", + + "listTools_vs_listRepos_budget": 0.75, + "_listTools_ratio_note": "client.listTools()_ms / listRepos_ms. Restoring listRepos() on toolSchemaRepoRequirements collapses this toward 1+. Budget is 0.75 — listRepos is git-bound and can get relatively faster on GitHub-hosted runners than the Node-bound listTools arm (measured ~0.22–0.28 here). The collapse-to-1 regression still fails. min-of-7 estimator.", + + "_measured": { + "count_vs_listRepos_ratio": 0.026, + "count_vs_listRepos_ratio_samples": [0.025, 0.025, 0.026, 0.026, 0.025], + "listTools_vs_listRepos_ratio": 0.283, + "listTools_vs_listRepos_ratio_samples": [0.242, 0.242, 0.223, 0.267, 0.234, 0.283], + "count_ms": 8.99, + "listRepos_ms": 349.63, + "listTools_ms": 99.04, + "reps": 7 + }, + "_measured_note": "Maxima (and sample lists) over consecutive local runs using the min-of-7 estimator, including one --check pass. Milliseconds are diagnostic context only — nothing gates on them." +} diff --git a/gitnexus/bench/mcp-tools-list/measure.mjs b/gitnexus/bench/mcp-tools-list/measure.mjs new file mode 100644 index 000000000..39f2c872d --- /dev/null +++ b/gitnexus/bench/mcp-tools-list/measure.mjs @@ -0,0 +1,338 @@ +/** + * Build-free bench for unrestricted MCP `tools/list` with many registered + * repos (#3259 / #3184 / #1363). + * + * WHY THIS EXISTS. `toolSchemaRepoRequirements` used to call + * `listAllowedRepos()` → `listRepos()` → one `git rev-list` per registry row + * (`checkStalenessAsync`). #1363 made that fan-out parallel (~50 s serial → + * <1 s). #3259 removes it from schema introspection: `countRepos()` reads the + * validated registry (`fs.access`, no git). Staleness git stays on + * `list_repos`. Graph output and unit tests cannot see "we stopped spawning + * git on tools/list" — putting `listRepos()` back still returns the same + * tool roster and the same required-repo flags. + * + * ARMS: + * + * - `n_repos` / `count_repos` / `list_repos` — EXACT, and they are the FLOOR. + * The ratio arms only mean something while the corpus still pays N parallel + * `rev-list`s. Shrink it to three rows and both sides are cheap; the ratio + * can still pass while the property the bench claims to guard is gone. + * + * - `tools_listed` — EXACT. `listTools` must still return the full + * `GITNEXUS_TOOLS` roster. A schema path that errors or filters the set + * would otherwise hide behind a "fast" timing arm. + * + * - `schema_read_only_requires_repo` / `schema_mutating_requires_repo` — + * EXACT. On this unrestricted N-repo fixture there is no cwd default, so + * both flags must stay true. Skipping the cwd probe and advertising a + * single-repo schema would pass every timing arm. + * + * - `count_vs_listRepos_ratio` — UPPER timing arm, a RATIO not a millisecond + * ceiling. `countRepos_ms / listRepos_ms`. Putting staleness git back on + * `countRepos` collapses this toward 1. Wall-clock is runner-speed- + * dependent; this repo has already been bitten by a fixed ms budget. + * + * - `listTools_vs_listRepos_ratio` — UPPER timing arm. The user-visible + * `tools/list` path over the old `listRepos()` hot path. Restoring + * `listRepos()` on schema introspection collapses this toward 1+. + * + * Isolated `GITNEXUS_HOME` — never touches `~/.gitnexus`. Also clears + * `GITNEXUS_MCP_ALLOWED_REPOS`, `GITNEXUS_MCP_DEFAULT_REPO`, and + * `GITNEXUS_MCP_READ_ONLY` so the invoking shell cannot shrink the roster. + * Each fixture row is a real git repo whose `lastCommit` matches HEAD, so + * `listRepos()` pays `rev-list` instead of failing open. The default root is + * `mkdtempSync`; set `BENCH_ROOT` to reuse a tree across local runs. + * + * Usage: + * node --import tsx bench/mcp-tools-list/measure.mjs + * node --import tsx bench/mcp-tools-list/measure.mjs --check + * BENCH_REPOS=3 node --import tsx bench/mcp-tools-list/measure.mjs # report only + */ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; + +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; + +import { LocalBackend } from '../../src/mcp/local/local-backend.ts'; +import { createMcpRepositoryPolicy } from '../../src/mcp/repository-policy.ts'; +import { createMCPServer } from '../../src/mcp/server.ts'; +import { GITNEXUS_TOOLS } from '../../src/mcp/tools.ts'; +import { listRegisteredRepos } from '../../src/storage/repo-manager.ts'; + +const baselines = JSON.parse(readFileSync(new URL('./baselines.json', import.meta.url), 'utf8')); + +const CHECK = process.argv.includes('--check'); +const PINNED_REPS = 7; + +function positiveInt(value, fallback) { + const n = Number(value); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + +const REPS = CHECK ? PINNED_REPS : positiveInt(process.env.BENCH_REPS, PINNED_REPS); +const N = CHECK ? baselines.n_repos : positiveInt(process.env.BENCH_REPOS, baselines.n_repos); +const ROOT = + process.env.BENCH_ROOT ?? mkdtempSync(path.join(os.tmpdir(), 'gn-mcp-tools-list-bench-')); +const WORK = path.join(ROOT, `n-${N}`); +const HOME = path.join(WORK, 'home'); + +process.env.GITNEXUS_HOME = HOME; +delete process.env.GITNEXUS_MCP_ALLOWED_REPOS; +delete process.env.GITNEXUS_MCP_DEFAULT_REPO; +delete process.env.GITNEXUS_MCP_READ_ONLY; + +function git(cwd, args) { + return execFileSync('git', args, { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + env: { + ...process.env, + GIT_AUTHOR_NAME: 'bench', + GIT_AUTHOR_EMAIL: 'bench@example.com', + GIT_COMMITTER_NAME: 'bench', + GIT_COMMITTER_EMAIL: 'bench@example.com', + }, + }).trim(); +} + +function setupFixture() { + mkdirSync(HOME, { recursive: true }); + const marker = path.join(WORK, 'ready'); + // Reuse only when the caller pinned BENCH_ROOT. The default root is a + // mkdtempSync directory, so the ready marker cannot alias a previous run + // and there is no exists-then-write race on a predictable /tmp name. + if ( + process.env.BENCH_ROOT && + existsSync(marker) && + existsSync(path.join(HOME, 'registry.json')) + ) { + return; + } + + const entries = []; + for (let i = 0; i < N; i++) { + const repoPath = path.join(WORK, 'repos', `r${i}`); + const storagePath = path.join(repoPath, '.gitnexus'); + mkdirSync(storagePath, { recursive: true }); + writeFileSync(path.join(repoPath, 'f.txt'), `${i}\n`); + git(repoPath, ['init', '-b', 'main']); + git(repoPath, ['add', 'f.txt']); + git(repoPath, ['commit', '-m', 'init']); + entries.push({ + name: `r${i}`, + path: repoPath, + storagePath, + indexedAt: '2026-09-11T00:00:00.000Z', + lastCommit: git(repoPath, ['rev-parse', 'HEAD']), + stats: { files: 1, nodes: 1, edges: 0, communities: 0, processes: 0 }, + }); + writeFileSync(path.join(storagePath, 'gitnexus.json'), '{}\n'); + } + writeFileSync(path.join(HOME, 'registry.json'), `${JSON.stringify(entries, null, 2)}\n`); + writeFileSync(marker, `${N}\n`); +} + +async function fastest(fn, reps) { + await fn(); + let best = Infinity; + let last; + for (let r = 0; r < reps; r++) { + const t0 = performance.now(); + last = await fn(); + best = Math.min(best, performance.now() - t0); + } + return { ms: best, last }; +} + +async function listToolsOnce(backend) { + const repositoryPolicy = await createMcpRepositoryPolicy(backend); + const server = createMCPServer(backend, { repositoryPolicy }); + const client = new Client({ name: 'bench', version: '0.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + try { + const listed = await client.listTools(); + return listed.tools.length; + } finally { + await client.close(); + await server.close(); + } +} + +/** + * A missing budget is a DELETED GATE, not a passing arm: `got > undefined` is + * false for every measurement. `Number.isFinite` rather than `typeof === + * 'number'` — JSON cannot express NaN, and the stricter check is the one whose + * name says what the gate needs. + */ +function requireNumeric(key) { + const value = baselines[key]; + if (Number.isFinite(value)) return value; + return { + missing: `no numeric ${key} in baselines.json — a missing budget is a DELETED GATE, not a passing arm: the comparison it gates is false for every possible measurement. Deterministic: a re-run will not change it.`, + }; +} + +function requireBoolean(key) { + const value = baselines[key]; + if (typeof value === 'boolean') return value; + return { + missing: `no boolean ${key} in baselines.json — a missing exact floor is a DELETED GATE, not a passing arm. Deterministic: a re-run will not change it.`, + }; +} + +setupFixture(); + +const backend = new LocalBackend(); +await backend.init(); +const policy = await createMcpRepositoryPolicy(backend); + +const countTimed = await fastest(() => backend.countRepos(), REPS); +const listTimed = await fastest(() => backend.listRepos(), REPS); +const listToolsTimed = await fastest(() => listToolsOnce(backend), REPS); +const schema = await policy.toolSchemaRepoRequirements(backend); +const rawRegistry = await listRegisteredRepos({ validate: false }); + +const countRepos = countTimed.last; +const listRepos = listTimed.last.length; +const toolsListed = listToolsTimed.last; +const countMs = countTimed.ms; +const listReposMs = listTimed.ms; +const listToolsMs = listToolsTimed.ms; +const countRatio = listReposMs === 0 ? Infinity : countMs / listReposMs; +const listToolsRatio = listReposMs === 0 ? Infinity : listToolsMs / listReposMs; + +console.log(`n_repos : ${N} (expect ${baselines.n_repos})`); +console.log(`count_repos : ${countRepos} (expect ${baselines.count_repos})`); +console.log(`list_repos : ${listRepos} (expect ${baselines.list_repos})`); +console.log( + `tools_listed : ${toolsListed} (expect ${baselines.tools_listed}; GITNEXUS_TOOLS ${GITNEXUS_TOOLS.length})`, +); +console.log( + `schema_read_only_requires_repo : ${schema.readOnlyRequiresRepo} (expect ${baselines.schema_read_only_requires_repo})`, +); +console.log( + `schema_mutating_requires_repo : ${schema.mutatingRequiresRepo} (expect ${baselines.schema_mutating_requires_repo})`, +); +console.log( + `count_vs_listRepos_ratio : ${countRatio.toFixed(3)} (budget <= ${baselines.count_vs_listRepos_budget})`, +); +console.log( + `listTools_vs_listRepos_ratio : ${listToolsRatio.toFixed(3)} (budget <= ${baselines.listTools_vs_listRepos_budget})`, +); +console.log( + `reps : ${REPS} count ${countMs.toFixed(2)}ms / listRepos ${listReposMs.toFixed(2)}ms / listTools ${listToolsMs.toFixed(2)}ms / rawRegistry ${rawRegistry.length}`, +); + +if (!CHECK) process.exit(0); + +if (process.env.BENCH_REPOS && Number(process.env.BENCH_REPOS) !== baselines.n_repos) { + console.error( + `\nFAIL BENCH_REPOS=${process.env.BENCH_REPOS} is ignored under --check.\n` + + ` n_repos is pinned in baselines.json (${baselines.n_repos}). A smaller\n` + + ` corpus makes both timing arms cheap and the ratios stop measuring git.`, + ); + process.exit(1); +} + +let failed = false; + +function fail(message) { + failed = true; + console.error(`\nFAIL ${message}`); +} + +const nReposBudget = requireNumeric('n_repos'); +const countBudget = requireNumeric('count_repos'); +const listBudget = requireNumeric('list_repos'); +const toolsBudget = requireNumeric('tools_listed'); +const countRatioBudget = requireNumeric('count_vs_listRepos_budget'); +const listToolsRatioBudget = requireNumeric('listTools_vs_listRepos_budget'); +const readOnlyExpect = requireBoolean('schema_read_only_requires_repo'); +const mutatingExpect = requireBoolean('schema_mutating_requires_repo'); + +for (const got of [ + nReposBudget, + countBudget, + listBudget, + toolsBudget, + countRatioBudget, + listToolsRatioBudget, + readOnlyExpect, + mutatingExpect, +]) { + if (got && typeof got === 'object' && 'missing' in got) fail(got.missing); +} + +if (Number.isFinite(nReposBudget) && N !== nReposBudget) { + fail( + `n_repos: ${N}, expected exactly ${nReposBudget}.\n` + + ` THE FLOOR. Ratio arms only assert something while the corpus still\n` + + ` pays N parallel rev-list processes.`, + ); +} + +if (Number.isFinite(countBudget) && countRepos !== countBudget) { + fail(`count_repos: ${countRepos}, expected exactly ${countBudget}.`); +} + +if (Number.isFinite(listBudget) && listRepos !== listBudget) { + fail(`list_repos: ${listRepos}, expected exactly ${listBudget}.`); +} + +if (Number.isFinite(toolsBudget)) { + if (GITNEXUS_TOOLS.length !== toolsBudget) { + fail( + `GITNEXUS_TOOLS.length: ${GITNEXUS_TOOLS.length}, expected exactly ${toolsBudget}.\n` + + ` The tool roster moved. Explain it; do not re-baseline tools_listed alone.`, + ); + } + if (toolsListed !== toolsBudget) { + fail( + `tools_listed: ${toolsListed}, expected exactly ${toolsBudget}.\n` + + ` listTools dropped or padded the roster. A fast arm that returns [] still\n` + + ` looks like a win on the ratio.`, + ); + } +} + +if (typeof readOnlyExpect === 'boolean' && schema.readOnlyRequiresRepo !== readOnlyExpect) { + fail( + `schema_read_only_requires_repo: ${schema.readOnlyRequiresRepo}, expected ${readOnlyExpect}.\n` + + ` On this unrestricted N-repo fixture there is no cwd default. Advertising\n` + + ` a single-repo schema skips the multi-repo arm the timing ratios guard.`, + ); +} + +if (typeof mutatingExpect === 'boolean' && schema.mutatingRequiresRepo !== mutatingExpect) { + fail( + `schema_mutating_requires_repo: ${schema.mutatingRequiresRepo}, expected ${mutatingExpect}.\n` + + ` On this unrestricted N-repo fixture there is no cwd default.`, + ); +} + +if (Number.isFinite(countRatioBudget) && countRatio > countRatioBudget) { + fail( + `count_vs_listRepos_ratio: ${countRatio.toFixed(3)} exceeds ${countRatioBudget}.\n` + + ` countRepos should stay far cheaper than listRepos. A collapse toward 1.0\n` + + ` usually means staleness git is back on the count path.\n` + + ` Re-run on an idle machine before investigating, and check \`reps\` first.`, + ); +} + +if (Number.isFinite(listToolsRatioBudget) && listToolsRatio > listToolsRatioBudget) { + fail( + `listTools_vs_listRepos_ratio: ${listToolsRatio.toFixed(3)} exceeds ${listToolsRatioBudget}.\n` + + ` tools/list should stay cheaper than the old listRepos() hot path. A\n` + + ` collapse toward 1.0+ usually means schema introspection calls listRepos\n` + + ` again. Re-run on an idle machine before investigating, and check \`reps\`.`, + ); +} + +if (failed) process.exit(1); +console.log('\nOK — within budget.'); diff --git a/gitnexus/bench/python-workspace-import-scan/baselines.json b/gitnexus/bench/python-workspace-import-scan/baselines.json new file mode 100644 index 000000000..0213561e0 --- /dev/null +++ b/gitnexus/bench/python-workspace-import-scan/baselines.json @@ -0,0 +1,37 @@ +{ + "_what": "Baselines for bench/python-workspace-import-scan/measure.mjs --check. Guards Python workspace from-import scanning after #3254: function-local discovery, lookalike rejection, and parse-volume. Same approach as bench/parse-dispatch-rounds/baselines.json — exact floors and a fingerprint first; the only timing arms are ratios.", + + "_triage": "READ THIS BEFORE RE-RUNNING. from_links, lookalike_links, no_from_links, from_files, no_from_files, lookalike_files and layout_fingerprint are DETERMINISTIC: a re-run never changes them, and none may be re-baselined to make CI green. scan_scaling_ratio is an UPPER timing arm; prefilter_advantage is a FLOOR timing arm. Runner contention dominates both, so re-run on an idle machine before investigating and read the reported `reps` first. If exactly one arm fails and it is a timing arm, suspect the machine.", + + "from_files": 40, + "no_from_files": 40, + "lookalike_files": 8, + "_shape_note": "THE FLOOR. Without these three, every arm below is a ceiling over nothing. from_links only asserts something while the corpus still walks many files. Shrink it to one happy-path module and from_links still reads 1 and still passes, asserting a property the corpus no longer has.", + + "from_links": 1, + "lookalike_links": 0, + "no_from_links": 0, + "_links_note": "Exact contract counts. from_links=1 is the deduped function-local Record import. lookalike_links=0 pins closed AND unclosed indented docstring examples. no_from_links=0 pins files that have no from-token.", + + "layout_fingerprint": "02e51be915083909cf231688c935cfe11bb52c7c781a70532859ad4a0bb9b42e", + "_layout_fingerprint_note": "sha256 over sorted from|to|contract rows on the from-corpus. A change here is a BEHAVIOUR change — the extractor returned a different contract set. Explain it, never re-baseline it alone.", + + "scan_scaling_budget": 1.6, + "_scan_scaling_note": "(t_4n / t_n) / 4 for extractPythonWorkspaceLinks over the mixed corpus; ~1.0 is linear. A RATIO rather than a millisecond ceiling, deliberately: wall-clock is runner-speed-dependent, and this repo has already been bitten by a fixed ms budget. Budget is 1.6, matching parse-dispatch-rounds' pack_scaling_budget. min-of-15 estimator.", + + "prefilter_advantage_floor": 3, + "_prefilter_note": "t_from / t_nofrom at the same file count and body size. The no-from corpus is large Python with no `from` token. A working prefilter keeps that arm cheap (~5x here). Removing the prefilter collapses the ratio toward 1 because both sides tree-sitter parse. Floor is 3 — about 1.6x below the measured minimum, same headroom philosophy as the 1.6 upper budget.", + + "_measured": { + "scan_scaling_ratio": 0.994, + "scan_scaling_ratio_samples": [0.994, 0.963, 0.977, 0.986, 0.889], + "prefilter_advantage": 5.411, + "prefilter_advantage_samples": [5.197, 4.955, 5.057, 5.124, 5.411], + "from_ms": 86.43, + "no_from_ms": 16.35, + "mixed_ms": 112.1, + "large_ms_4x": 398.76, + "reps": 15 + }, + "_measured_note": "Maxima (and sample lists) over 5 consecutive local runs using the min-of-15 estimator from parse-dispatch-rounds. Milliseconds are diagnostic context only — nothing gates on them." +} diff --git a/gitnexus/bench/python-workspace-import-scan/measure.mjs b/gitnexus/bench/python-workspace-import-scan/measure.mjs new file mode 100644 index 000000000..fbd351f4a --- /dev/null +++ b/gitnexus/bench/python-workspace-import-scan/measure.mjs @@ -0,0 +1,270 @@ +/** + * Build-free bench for Python workspace from-import scanning (#3254). + * + * WHY THIS EXISTS. `scanPythonImports` now tree-sitter-parses every `.py` file + * that looks like it has a `from` import. Graph output does not show "how many + * files we parsed" or "we skipped an unclosed-docstring lookalike" — a revert + * to column-0 regex, a dropped `parseHadErrors` guard, or a removed `from` + * prefilter can still emit the same one contract on a tiny fixture. This file + * is the same shape as `bench/parse-dispatch-rounds`: exact floors first, one + * ratio timing arm, never a millisecond ceiling. + * + * ARMS: + * + * - `from_links` / `lookalike_links` / `no_from_links` — EXACT. The + * correctness floor. The from-corpus must still discover `datalib::Record`. + * Closed + unclosed docstring lookalikes must stay at 0. Files with no + * `from` token must stay at 0. + * + * - `from_files` / `no_from_files` / `lookalike_files` — EXACT, and they are + * the FLOOR. `from_links === 1` only asserts something while the corpus + * still has many files to walk. Shrink it to one happy-path file and the + * link arm still passes, gating a property the corpus no longer has. + * + * - `layout_fingerprint` — EXACT. sha256 over sorted `from|to|contract` rows + * on the from-corpus. Catches a contract-set change that leaves the count + * intact. + * + * - `scan_scaling_ratio` — the only mixed-corpus timing arm, a RATIO not a + * millisecond ceiling. `(t_4n / t_n) / 4` divides the machine out; ~1.0 is + * linear. Superlinear AST work over file count lands here. + * + * - `prefilter_advantage` — `t_from / t_nofrom` at the same file count. The + * no-from corpus is large Python with no `from` token. A working prefilter + * keeps that arm cheap; removing it collapses the ratio toward 1 because + * both sides parse. + * + * Usage: + * node --import tsx bench/python-workspace-import-scan/measure.mjs + * node --import tsx bench/python-workspace-import-scan/measure.mjs --check + */ +import { createHash } from 'node:crypto'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { extractPythonWorkspaceLinks } from '../../src/core/group/extractors/python-workspace-extractor.ts'; + +const baselines = JSON.parse(readFileSync(new URL('./baselines.json', import.meta.url), 'utf8')); + +const REPS = 15; +const FROM_FILES = 40; +const NO_FROM_FILES = 40; +const LOOKALIKE_FILES = 8; +const NO_FROM_BODY = Array.from( + { length: 80 }, + (_, i) => `def helper_${i}(x):\n return x + ${i}\n`, +).join('\n'); + +function writeWorkspace(root, { fromCount, noFromCount, lookalikeCount }) { + mkdirSync(path.join(root, 'lib', 'datalib'), { recursive: true }); + writeFileSync( + path.join(root, 'lib', 'pyproject.toml'), + '[project]\nname = "datalib"\nversion = "0.1.0"\ndependencies = []\n', + ); + writeFileSync(path.join(root, 'lib', 'datalib', 'models.py'), 'class Record: pass\n'); + + mkdirSync(path.join(root, 'app', 'myapp'), { recursive: true }); + writeFileSync( + path.join(root, 'app', 'pyproject.toml'), + '[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = ["datalib"]\n', + ); + + for (let i = 0; i < fromCount; i++) { + writeFileSync( + path.join(root, 'app', 'myapp', `from_${i}.py`), + `${NO_FROM_BODY}\ndef load_${i}():\n from datalib.models import Record\n return Record()\n`, + ); + } + for (let i = 0; i < noFromCount; i++) { + writeFileSync(path.join(root, 'app', 'myapp', `plain_${i}.py`), NO_FROM_BODY); + } + for (let i = 0; i < lookalikeCount; i++) { + const closed = i % 2 === 0; + writeFileSync( + path.join(root, 'app', 'myapp', `doc_${i}.py`), + closed + ? `def describe_${i}():\n """Example:\n from datalib.models import Record\n """\n return None\n` + : `def describe_${i}():\n """\n from datalib.models import Record\n`, + ); + } + + return { + repos: { lib: 'datalib', app: 'myapp' }, + repoPaths: new Map([ + ['lib', path.join(root, 'lib')], + ['app', path.join(root, 'app')], + ]), + }; +} + +async function scan(workspace) { + return extractPythonWorkspaceLinks(workspace.repos, workspace.repoPaths); +} + +function fingerprint(links) { + return createHash('sha256') + .update( + links + .map((l) => `${l.from}|${l.to}|${l.contract}`) + .sort() + .join('\n'), + ) + .digest('hex'); +} + +async function fastest(fn, reps) { + await fn(); + let best = Infinity; + for (let r = 0; r < reps; r++) { + const t0 = performance.now(); + await fn(); + best = Math.min(best, performance.now() - t0); + } + return best; +} + +const roots = []; +function workspace(opts) { + const root = mkdtempSync(path.join(tmpdir(), 'gn-py-ws-bench-')); + roots.push(root); + return writeWorkspace(root, opts); +} + +try { + const fromWs = workspace({ + fromCount: FROM_FILES, + noFromCount: 0, + lookalikeCount: 0, + }); + const noFromWs = workspace({ + fromCount: 0, + noFromCount: NO_FROM_FILES, + lookalikeCount: 0, + }); + const lookalikeWs = workspace({ + fromCount: 0, + noFromCount: 0, + lookalikeCount: LOOKALIKE_FILES, + }); + const mixedWs = workspace({ + fromCount: FROM_FILES, + noFromCount: NO_FROM_FILES, + lookalikeCount: LOOKALIKE_FILES, + }); + const mixed4x = workspace({ + fromCount: FROM_FILES * 4, + noFromCount: NO_FROM_FILES * 4, + lookalikeCount: LOOKALIKE_FILES * 4, + }); + + const [fromResult, noFromResult, lookalikeResult] = await Promise.all([ + scan(fromWs), + scan(noFromWs), + scan(lookalikeWs), + ]); + const layoutFingerprint = fingerprint(fromResult.links); + + const fromMs = await fastest(() => scan(fromWs), REPS); + const noFromMs = await fastest(() => scan(noFromWs), REPS); + const smallMs = await fastest(() => scan(mixedWs), REPS); + const largeMs = await fastest(() => scan(mixed4x), REPS); + const scanScaling = largeMs / smallMs / 4; + const prefilterAdvantage = noFromMs === 0 ? Infinity : fromMs / noFromMs; + + console.log(`from_files : ${FROM_FILES} (expect ${baselines.from_files})`); + console.log(`no_from_files : ${NO_FROM_FILES} (expect ${baselines.no_from_files})`); + console.log(`lookalike_files : ${LOOKALIKE_FILES} (expect ${baselines.lookalike_files})`); + console.log( + `from_links : ${fromResult.links.length} (expect ${baselines.from_links})`, + ); + console.log( + `lookalike_links : ${lookalikeResult.links.length} (expect ${baselines.lookalike_links})`, + ); + console.log( + `no_from_links : ${noFromResult.links.length} (expect ${baselines.no_from_links})`, + ); + console.log(`layout_fingerprint : ${layoutFingerprint}`); + console.log( + `scan_scaling_ratio : ${scanScaling.toFixed(3)} (budget <= ${baselines.scan_scaling_budget}; ~1.0 is linear)`, + ); + console.log( + `prefilter_advantage : ${prefilterAdvantage.toFixed(3)} (floor >= ${baselines.prefilter_advantage_floor})`, + ); + console.log( + `reps : ${REPS} from ${fromMs.toFixed(2)}ms / no_from ${noFromMs.toFixed(2)}ms / mixed ${smallMs.toFixed(2)}ms / 4x ${largeMs.toFixed(2)}ms`, + ); + + if (process.argv.includes('--check')) { + let failed = false; + + if (layoutFingerprint !== baselines.layout_fingerprint) { + failed = true; + console.error( + `\nFAIL layout_fingerprint: ${layoutFingerprint}\n` + + ` expected ${baselines.layout_fingerprint}\n` + + ` The from-corpus contract set moved. Explain it; do not re-baseline alone.`, + ); + } + + if (fromResult.links.length !== baselines.from_links) { + failed = true; + console.error( + `\nFAIL from_links: ${fromResult.links.length}, expected exactly ${baselines.from_links}.`, + ); + } + if (lookalikeResult.links.length !== baselines.lookalike_links) { + failed = true; + console.error( + `\nFAIL lookalike_links: ${lookalikeResult.links.length}, expected exactly ${baselines.lookalike_links}.\n` + + ` Closed or unclosed docstring lookalikes leaked a workspace contract.`, + ); + } + if (noFromResult.links.length !== baselines.no_from_links) { + failed = true; + console.error( + `\nFAIL no_from_links: ${noFromResult.links.length}, expected exactly ${baselines.no_from_links}.`, + ); + } + + if ( + FROM_FILES !== baselines.from_files || + NO_FROM_FILES !== baselines.no_from_files || + LOOKALIKE_FILES !== baselines.lookalike_files + ) { + failed = true; + console.error( + `\nFAIL shape: from_files ${FROM_FILES} (expected ${baselines.from_files}), ` + + `no_from_files ${NO_FROM_FILES} (expected ${baselines.no_from_files}), ` + + `lookalike_files ${LOOKALIKE_FILES} (expected ${baselines.lookalike_files}).\n` + + ` The corpus must stay large enough that the link arms still measure a walk.`, + ); + } + + if (scanScaling > baselines.scan_scaling_budget) { + failed = true; + console.error( + `\nFAIL scan_scaling_ratio: ${scanScaling.toFixed(3)} exceeds ` + + `${baselines.scan_scaling_budget} (~1.0 is linear).\n` + + ` Re-run on an idle machine before investigating, and check \`reps\` first.`, + ); + } + + if (prefilterAdvantage < baselines.prefilter_advantage_floor) { + failed = true; + console.error( + `\nFAIL prefilter_advantage: ${prefilterAdvantage.toFixed(3)} below ` + + `${baselines.prefilter_advantage_floor}.\n` + + ` The no-from corpus should stay cheaper than the from-corpus. A collapse\n` + + ` toward 1.0 usually means every file is being tree-sitter parsed again.`, + ); + } + + if (failed) process.exit(1); + console.log('\nOK — within budget.'); + } +} finally { + for (const root of roots) { + rmSync(root, { recursive: true, force: true }); + } +} diff --git a/gitnexus/bench/ruby-gem-resolution/baseline.json b/gitnexus/bench/ruby-gem-resolution/baseline.json new file mode 100644 index 000000000..8c618ad00 --- /dev/null +++ b/gitnexus/bench/ruby-gem-resolution/baseline.json @@ -0,0 +1,61 @@ +{ + "_what": "Synthetic Ruby manifest + ScopeResolver gate for #3096, following bench/parse-dispatch-rounds/baselines.json: exact correctness floors and fingerprints, with ratios as the only timing gates. Calls the real config loader and resolver; parser, DB, installed gems and full-pipeline memory are outside this microbenchmark.", + "_triage": "Shape and fingerprint failures are deterministic: inspect the fixture and resolved targets, never rebaseline just to make CI green. For a timing failure, rerun on an idle machine and verify the report uses 15 samples after 2 warmups before investigating a regression. Millisecond observations are context, not gates.", + "shapes": { + "small": { + "projects": 32, + "declarations_per_project": 12, + "scopes": 64, + "files": 131, + "imports": 8192, + "resolved": 3264, + "fingerprint": "e37f4c81bc7f4bc8416732d0e8ebb00e16743b08a44646e1a6fa45c04283cfb3" + }, + "large": { + "projects": 128, + "declarations_per_project": 12, + "scopes": 256, + "files": 515, + "imports": 32768, + "resolved": 13056, + "fingerprint": "2524217404b1d4cf118e7d13895d757c821029bf59230b6c3cc7ce2fc2cf5dd2" + }, + "dense": { + "projects": 32, + "declarations_per_project": 132, + "scopes": 64, + "files": 131, + "imports": 8192, + "resolved": 3264, + "fingerprint": "e37f4c81bc7f4bc8416732d0e8ebb00e16743b08a44646e1a6fa45c04283cfb3" + } + }, + "_shape_note": "Exact projects, declarations, scopes, files, imports and resolved counts are the workload FLOOR, not ceilings. They prevent a shrunken corpus or disconnected manifest loader from passing by doing less work. Small/large grow projects 4x; dense keeps files/imports fixed while growing extra Gemfile declarations from 8 to 128 per project.", + "_fingerprint_note": "Hashes pin every importer/require/target tuple, including external decoys, aliases, local path gem hits and misses, relative/local imports and sibling isolation. The runner also checks each expected target before hashing and proves the external decoy is reachable without config. An always-null resolver or missing config must fail, not get a new fingerprint.", + "budgets": { + "load_scaling_ratio": 2.2, + "resolve_scaling_ratio": 2.2, + "dependency_count_ratio": 2 + }, + "_scaling_note": "load_scaling_ratio and resolve_scaling_ratio are (t_4n/t_n)/4: about 1 is linear, about 4 is quadratic. Keep the existing 2.2 regression budgets; do not tighten them to local milliseconds. Filesystem loading and lookup are measured separately with fresh config/file Sets per pass.", + "_dependency_count_note": "dense.resolve_ms/small.resolve_ms should stay near 1: require lookup must scale with prefix/path depth, not all declared gems. The budget remains 2. Extra declaration parsing belongs to config loading, not this fixed-query lookup arm.", + "_negative_controls_note": "Disconnecting loadResolutionConfig fails the real-loader assertion. A wrapper scanning all scopes' external prefixes on every import leaves targets/fingerprints unchanged but fails resolve_scaling_ratio (3.427 > 2.2) and dependency_count_ratio (7.912 > 2) with 15 samples. These are deliberate regressions, not baseline observations.", + "_measured": { + "environment": "macOS arm64 / Node 25.8.0, 2026-09-10", + "load_scaling_ratio": 1.094, + "load_scaling_ratio_samples": [0.993, 1.06, 1.05, 1.019, 1.094], + "resolve_scaling_ratio": 1.066, + "resolve_scaling_ratio_samples": [1.058, 1.059, 1.056, 1.066, 1.061], + "dependency_count_ratio": 1.009, + "dependency_count_ratio_samples": [1.009, 1.003, 0.994, 0.999, 1.0], + "small_load_ms": 3.569, + "small_resolve_ms": 11.782, + "large_load_ms": 14.812, + "large_resolve_ms": 50.222, + "dense_load_ms": 7.27, + "dense_resolve_ms": 11.852, + "reps": 15, + "warmup": 2 + }, + "_measured_note": "Five consecutive local runs using the min-of-15 estimator from parse-dispatch-rounds, with 2 warmups here. Scalar ratios and millisecond values are per-metric maxima across those runs (rounded), not one combined run. Budgets retain about 2x headroom above observed ratios. Milliseconds are diagnostic context only, not gated or real-repository speed claims. Exact shapes and fingerprints are unchanged." +} diff --git a/gitnexus/bench/ruby-gem-resolution/measure.mjs b/gitnexus/bench/ruby-gem-resolution/measure.mjs new file mode 100644 index 000000000..8f7ef7875 --- /dev/null +++ b/gitnexus/bench/ruby-gem-resolution/measure.mjs @@ -0,0 +1,184 @@ +/** + * Ruby gem-boundary correctness and cost gate (#3096). + * + * Calls the production ScopeResolver hooks, including the filesystem-backed + * configuration loader. Fixture creation and correctness hashing are untimed. + * Each timed pass reloads config and owns a fresh file Set, so neither the + * manifest load nor the per-pass fallback index is hidden by a warm memo. + * + * Small/large grow sibling projects, files, declarations and imports together. + * Dense keeps projects/imports fixed and grows extra declarations from 8 to 128: + * lookup must depend on require/path depth, not the number of declared gems. + * + * node --import tsx bench/ruby-gem-resolution/measure.mjs [--check] + */ +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { rubyScopeResolver } from '../../src/core/ingestion/languages/ruby/scope-resolver.ts'; + +const baselinePath = fileURLToPath(new URL('./baseline.json', import.meta.url)); +const CHECK = process.argv.includes('--check'); +const WARMUP = 2; +const REPS = 15; +const IMPORTS_PER_PROJECT = 256; +const roots = []; + +function corpus(projects, gemsPerProject = 8) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-ruby-gems-bench-')); + roots.push(root); + const files = ['decoy/lib/generators.rb', 'decoy/lib/types.rb', 'decoy/lib/missing.rb']; + const queries = []; + for (let i = 0; i < projects; i++) { + const directory = `packages/pkg${i}`; + const onDisk = path.join(root, directory); + fs.mkdirSync(path.join(onDisk, 'engine'), { recursive: true }); + fs.writeFileSync( + path.join(onDisk, 'Gemfile'), + [ + "gem 'rails'", + "gem 'dry-types'", + "gem 'aliased', require: 'custom/entry'", + "gem 'my_engine', path: 'engine'", + ...Array.from({ length: gemsPerProject }, (_, gem) => `gem 'dependency_${i}_${gem}'`), + ].join('\n'), + ); + fs.writeFileSync( + path.join(onDisk, 'engine', 'my_engine.gemspec'), + "Gem::Specification.new do |s|\n s.name = 'my_engine'\n s.require_paths = ['lib']\nend\n", + ); + fs.writeFileSync( + path.join(onDisk, 'Gemfile.lock'), + 'GEM\n remote: https://rubygems.org/\n specs:\n locked_gem (1.0.0)\n', + ); + const from = `${directory}/app/main.rb`; + files.push( + from, + `${directory}/app/helper.rb`, + `${directory}/lib/local_${i}.rb`, + `${directory}/engine/lib/my_engine.rb`, + ); + const cases = [ + ['rails/generators', null], + ['dry/types', null], + ['custom/entry', null], + ['my_engine', `${directory}/engine/lib/my_engine.rb`], + ['my_engine/missing', null], + ['./helper', `${directory}/app/helper.rb`], + [`local_${i}`, `${directory}/lib/local_${i}.rb`], + ['locked_gem/missing', null], + [`dependency_${i}_0/missing`, null], + [`dependency_${(i + 1) % projects}_0/missing`, 'decoy/lib/missing.rb'], + ]; + for (let j = 0; j < IMPORTS_PER_PROJECT; j++) { + const [target, expected] = cases[j % cases.length]; + queries.push({ from, target, expected }); + } + } + return { root, projects, gemsPerProject, files, queries }; +} + +function resolve(query, pass) { + return rubyScopeResolver.resolveImportTarget(query.target, query.from, pass.files, pass.config); +} + +function prepare(input) { + return { + files: new Set(input.files), + config: rubyScopeResolver.loadResolutionConfig(input.root), + }; +} + +function correctness(input) { + const pass = prepare(input); + assert.ok(pass.config, 'the real manifest loader must supply configuration'); + const records = []; + let resolved = 0; + for (const query of input.queries) { + const answer = resolve(query, pass); + assert.equal(answer, query.expected, `${query.from}: ${query.target}`); + if (answer !== null) resolved++; + records.push(`${query.from}|${query.target}->${answer}`); + } + // The external decoy must actually be reachable without dependency evidence; + // otherwise an always-null resolver could make a negative-only gate pass. + assert.equal( + resolve( + { from: input.queries[0].from, target: 'rails/generators' }, + { files: pass.files, config: undefined }, + ), + 'decoy/lib/generators.rb', + ); + return { + projects: input.projects, + declarations_per_project: input.gemsPerProject + 4, + scopes: pass.config.scopesByDirectory.size, + files: input.files.length, + imports: records.length, + resolved, + fingerprint: crypto.createHash('sha256').update(records.join('\n')).digest('hex'), + }; +} + +function measure(input, expectedResolved) { + const loading = []; + const resolution = []; + for (let run = 0; run < WARMUP + REPS; run++) { + const files = new Set(input.files); + const loadStart = performance.now(); + const config = rubyScopeResolver.loadResolutionConfig(input.root); + const loadMs = performance.now() - loadStart; + const pass = { files, config }; + const start = performance.now(); + let resolved = 0; + for (const query of input.queries) if (resolve(query, pass) !== null) resolved++; + const resolveMs = performance.now() - start; + assert.equal(resolved, expectedResolved, 'timed pass must do the validated work'); + if (run >= WARMUP) { + loading.push(loadMs); + resolution.push(resolveMs); + } + } + // Like the neighboring resolver gates: min-of-N limits scheduler/GC noise. + return { load_ms: Math.min(...loading), resolve_ms: Math.min(...resolution) }; +} + +try { + const inputs = { small: corpus(32), large: corpus(128), dense: corpus(32, 128) }; + const shapes = {}; + const timings = {}; + for (const [name, input] of Object.entries(inputs)) { + shapes[name] = correctness(input); + timings[name] = measure(input, shapes[name].resolved); + } + const scale = inputs.large.projects / inputs.small.projects; + const metrics = { + load_scaling_ratio: timings.large.load_ms / timings.small.load_ms / scale, + resolve_scaling_ratio: timings.large.resolve_ms / timings.small.resolve_ms / scale, + dependency_count_ratio: timings.dense.resolve_ms / timings.small.resolve_ms, + }; + const report = { + shapes, + timings, + metrics, + estimator: { warmup: WARMUP, samples: REPS, statistic: 'minimum' }, + }; + console.log(JSON.stringify(report, null, 2)); + if (CHECK) { + const baseline = JSON.parse(fs.readFileSync(baselinePath, 'utf8')); + assert.deepEqual(shapes, baseline.shapes, 'Ruby workload/fingerprint drift'); + const failures = []; + for (const [name, budget] of Object.entries(baseline.budgets)) { + if (!Number.isFinite(metrics[name]) || metrics[name] > budget) { + failures.push(`${name}: ${metrics[name]} > ${budget}`); + } + } + assert.equal(failures.length, 0, failures.join('\n')); + console.log('[ruby-gem-resolution --check] PASS'); + } +} finally { + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); +} diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 272d081c0..94fc262f6 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -1,7 +1,9 @@ { "_comment": "Per-language baselines for bench/scope-capture/measure.mjs --check. fingerprint = order-independent sha256 over the lang-resolution/-* fixture corpus + a 20-entity synthetic source (correctness gate; re-baseline intentionally on a legitimate capture change). scaling_budget = max allowed (t800/t250)/(800/250); ~1.0 is linear, ~3.2 is quadratic. The synthetic source is now HERITAGE-BEARING for every language (each Entity extends/implements/embeds/uses-trait/conforms-to a shared base) so the #1951 @reference.inherits synth is gated at scale, not just the base capture loop. All languages thread the tree-sitter captured node instead of re-deriving it with findNodeAtRange(tree.rootNode,...) per match, so all are linear (go #1915, python #1918, ruby/php/rust/csharp #1951, java #1956).", "go": { - "fingerprint": "9c554a9d698a2b79fb419852daadca87b8aae88180cceabf9c8d82f3e3300f2e", + "fingerprint": "2e3099db962f82f9d8641707a8b2bcf6875dafec1f54904f2dec7ac65ba22e9f", + "_rebaselined_3190_ctor_qualified_name": "#3190: generic composite-literal constructors now carry @reference.qualified-name with the written pkg.Box[T] spelling so constructor fallback can tell a qualifier from a bare unique-name guess. DIGEST DRIFT ONLY: the tag is added to existing constructor matches (captureGroups unchanged on go-constructor-type-inference/cmd/main.go). Prior 990e4921a7ef0e6aa4d7da92b671de91fa2621cb27918adb27742740dc6ad902 -> 2e3099db962f82f9d8641707a8b2bcf6875dafec1f54904f2dec7ac65ba22e9f; capture_groups_fp 2579, fixture_count 122.", + "_rebaselined_3190": "Corrected go-method-enrichment/app.go to share package animal with its declarations; its capture count is now 16 rather than 15. No Go emitter change. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 3d4e32e7490c830516126e28931827949baa3594cb521f7a3d8dcfed95b6018a -> 57b3c55135af8d2af33b9a7c4bf89796a7bee5b5822b402a2dea91af7232cf4a; scaling 1.058 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: provider-owned callable assignment/copy/formal/argument/invoke facts with invocation/constructor-result suppression. Prior 09ecd94911b830f52fa8807560abcbd79f163d02a2072870c1a59297e9a326e1 -> 3d4e32e7490c830516126e28931827949baa3594cb521f7a3d8dcfed95b6018a; scaling 1.039 < 1.5.", @@ -137,8 +139,9 @@ "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0." }, "java": { - "fingerprint": "2bf47cc19b595a9889ac21ec0154c6ce6786271d68551f21d1bc14c626bcd4ff", + "fingerprint": "e22a753b9f59331898a3a27c43072b588c33639937b1f618f6c70ce04f072a9f", "scaling_budget": 1.5, + "_rebaselined_3190_ctor_qualified_name": "#3190: qualified constructors (`new pkg.Foo()`, `new pkg.Box()`) now copy @reference.call.constructor.qualified onto @reference.qualified-name so constructor fallback treats the written qualifier as precise. DIGEST DRIFT ONLY: the tag is added to existing F35 matches. Prior 2bf47cc19b595a9889ac21ec0154c6ce6786271d68551f21d1bc14c626bcd4ff -> e22a753b9f59331898a3a27c43072b588c33639937b1f618f6c70ce04f072a9f; capture_groups_fp 3586, fixture_count 209.", "_rebaselined_2935_synthetic_declarations": "PR #2935 review follow-up: synthesized Java anonymous classes and bodied enum constants now carry the presence-only @declaration.is-synthetic sidecar used to preserve source-written dispatch targets at the fanout cap. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE: the tag is attached to existing synthetic declaration matches; capture groups and fixture count remain 5755/18405, 3512, and 206. Prior 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5 -> 2e2150b4f4d64519e3f4c6d7a2c12259178d3117872203c904fab8cba96a694a; CI scaling 0.971 < 1.5.", "_rebaselined_2917_record_component_accessors": "#2917: every implicit Java record-component accessor now emits a component-bounded @scope.function plus @declaration.method/name/zero-arity/return-type metadata. The scope boundary prevents subsequent record-body references from being attributed to the accessor. Java was the only general language fingerprint to move; capture groups scale by exactly two per generated record component (small 5755 -> 6255, large 18405 -> 20005). Prior 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5 -> 901a66c7dc0f071eeef9e4864b2519e5b58a1a141a1f9a7817ea42f7ff70eafb; scaling 0.961 < 1.5. Re-measured after merging origin/main, which carries #2935's is-synthetic sidecar on top of the same corpus: 2e2150b4f4d64519e3f4c6d7a2c12259178d3117872203c904fab8cba96a694a -> 79dafc369eaeb7183ee8cc1149b1a6c21ad672c7e5b806fe8b0060e5a952c79a; scaling 1.085 < 1.5, capture groups 6255/20005, capture_groups_fp 3560, fixture_count 206 (unchanged by the merge).", "_rebaselined_2900_record_heritage": "#2900 review follow-up: the Java scale unit now includes a record implementing Marker, so the record-declaration @reference.inherits path is fingerprinted and exercised at scale. Prior b29e263524f55151dcb7cfc4c929d3d1d7bb360355cee4e832158f927857f663 -> 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5; scaling 1.042 < 1.5.", @@ -171,7 +174,8 @@ "capture_groups_fp": 680 }, "typescript": { - "fingerprint": "fed04ed1d5db112387781e405da208ae6b3ab803889773b0455be96f01b893ff", + "fingerprint": "60e75bbe846f7200005f12c4d95c32e82d9d3feae9f2fd594e484670762b82b0", + "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_2934_import_type_only": "#2934: `import-decomposer.ts` attaches a presence-only `@import.type-only` synthetic capture to specifiers `tsc` erases, so `check --cycles` can stop counting type-only edges as initialization cycles. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE \u2014 the tag is added to import matches that already existed, never a new match, the same shape as the #2747 receiver-chain rebaseline. Every count is unchanged: capture_groups_fp 2414, fixture_count 155, capture_groups_small/large 4503/14403 (those measure the SYNTHETIC scaling source, which has no imports at all). The fingerprint moves because `canonicalizeMatch` in measure.mjs hashes every TAG on every match, synthetics included, so one extra presence-only tag on an existing match rewrites that match's canonical string. Attribution is exact, not inferred: neutralizing ONLY the `m['@import.type-only'] = \u2026` assignment in import-decomposer.ts and re-running returns the fingerprint to c2fbf8a89e5686dd\u2026 byte-for-byte, so nothing else in the TypeScript capture stream moved. All 14 other languages report ok. Scaling 0.997 < 1.5. NOTE ON THE CONTROL: javascript did not move (2026993b\u2026, 43 fixtures), but it is a WEAK control here \u2014 `import type` is TypeScript-only syntax, so a JS corpus cannot express the construct and could not have drifted either way. It evidences no collateral damage, not the correctness of the TS change; the exact-attribution check above is what does that. Prior c2fbf8a89e5686dd1ff3659b20d41d8b05ebcc9790356e3653ee0c8ca5d365c8 -> f719163eb03a447c9e40ca316a905dd76cee82192a75a403df478ebbdc13e98f.", "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 27f937bfb47d4bded316ea3c785ff659c8cd88a5761d928f113477a08c802c78 -> e05446620c5b80b7aae291cfdf32f693580fada2ae687124769b04a0c03bfe63; scaling 0.983 < 1.5.", @@ -197,7 +201,8 @@ "_rebaselined_1432_member_call_callee_name": "#1432 (Zig): the shared callable-flow reader no longer names a callee by simple name for a MEMBER call (`@callable-flow.direct-callee-name` requires a direct designator: `f(x)`, `ns.f(x)`), and a member call is a field-stored-callable invoke only when a MEMBER store (`o.f = handler`) or a declared callable-typed field is visible - a same-named plain binding no longer gates it. CAPTURE-EMISSION CHANGE, not fixture growth (fixture_count unchanged). Drift: `await svc.verify(token, ...)` (typescript-generic-calls/src/guest.ts, member call) and `initializer()(() => {...})` (typescript-hof-callbacks/src/store.ts, call-of-call) lose `direct-callee-name`. `await verifyToken(token, ...)` (admin.ts/auth.ts) KEEPS `direct-callee-name|verifyToken`: tree-sitter-typescript parses `await f(x)` as call_expression(function: await_expression(f), type_arguments, ...), and wrappedExpression now unwraps `await_expression` so the direct designator survives as it does for the un-awaited spelling. capture_groups_fp 2465 (unchanged). Prior 05d1dadd6c9ef35c74079fa50f341b1b36e4fb02c9a89dd1b59f32b7cfd5e633 -> fed04ed1d5db112387781e405da208ae6b3ab803889773b0455be96f01b893ff." }, "javascript": { - "fingerprint": "2026993b81b873839dd2ef8797d9c14d9c48516b2b57b05ac17d8d43f2f4eba3", + "fingerprint": "b916c7072b30d09b4949803810604830ea1aca1cfdda0d9d9312f7cb22f7a10a", + "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3 -> 479927409bbdd9852a36172c8260aa56df260e99129a7a9c20a0d1903dd5538b; scaling 1.050 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: lexical callable bindings, direct-callee argument metadata, and invocation-result suppression. Prior 917a9cd975ba035bdad71fdb70cd72eeddec58c25797e5a1addfa6172808a55c -> b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3; scaling 1.093 < 1.5.", diff --git a/gitnexus/bench/value-ref-resolution/baseline.json b/gitnexus/bench/value-ref-resolution/baseline.json new file mode 100644 index 000000000..7b23bcae5 --- /dev/null +++ b/gitnexus/bench/value-ref-resolution/baseline.json @@ -0,0 +1,37 @@ +{ + "_what": "Baselines for bench/value-ref-resolution/measure.mjs --check (#3399). Guards the resolved-target SET of `resolveValueRefTarget` and the per-site cost of its four channels across a 4x file-count step, over a synthetic Zig corpus whose shape is fixed in measure.mjs. Per module: 12 CONTAINER registrations (`ElementN.getJ`), 1 NAMESPACE (`dom_utils.compare`), 1 HUB (`hub.compare`), 1 BARE (`register(onTick)`), and 2 declines (a non-callable namespace member `dom_utils.DEFAULT_NS`, a non-callable bare argument `Bridge(ElementN)`) — 17 sites, 15 resolved.", + + "_triage": "READ THIS BEFORE RE-RUNNING. modules, files, value_ref_sites, resolved, declined and fingerprint are DETERMINISTIC: a re-run never changes them, and none may be re-baselined to make CI green — drift means the resolved target set moved, which is a behaviour change to explain. linear_scaling_budget is the only timing arm; runner contention dominates it, so re-run alone on an idle machine and read `reps` in the report before investigating. If exactly one arm fails and it is that one, suspect the machine.", + + "small": { + "modules": 80, + "files": 240, + "value_ref_sites": 1360, + "resolved": 1200, + "declined": 160, + "fingerprint": "4ca2933e25cfa9c84518fca6608c9a6f318f81a1c5bad63cb2f09b7394bfb792" + }, + "large": { + "modules": 320, + "files": 960, + "value_ref_sites": 5440, + "resolved": 4800, + "declined": 640, + "fingerprint": "249266f180388e9f1240a7e81f1141afbc51ef25d5c4b3286a681992dab69e61" + }, + + "linear_scaling_budget": 1.6, + "_linear_scaling_note": "(t_large/t_small)/(320/80) over the per-site resolution loop; ~1.0 is linear. A RATIO rather than a millisecond ceiling, and there is deliberately no ms gate at all: wall-clock measures the runner, and this repo has been bitten twice by a fixed budget (bench/callable-value-flow's widening_overhead failed at 2.07 and 1.975 against 1.9 on a shared runner while the code was correct, both on a sub-11ms measurement). Same reasoning, same shape as bench/parse-dispatch-rounds' pack_scaling_budget. Budget is 1.6 — 1.40x the measured maximum, matching the ~1.5x its siblings use on ratios. What it catches: the four channels each consult a wider index than the last, and channel 4 (CONTAINER) reaches `scopes.qualifiedNames`, a WORKSPACE-WIDE index — keyed it is O(1) per site, scanned it is O(names) per site. Verified load-bearing rather than assumed: replacing `QualifiedNameIndex.get` with a full scan (in gitnexus-shared/dist — the bench resolves the built package, so patching src changes nothing) takes the factor from ~1.0 to 2.07, well clear of this budget. Only 1 of the 17 sites per module reaches that fallback (the `dom_utils.DEFAULT_NS` decline, whose module receiver is not class-like), which is why the signal is 2.07 and not the ~4 a per-site scan on every site would give.", + + "_measured": { + "linear_factor": 1.146, + "linear_factor_samples": [ + 0.907, 0.969, 0.988, 1.03, 1.043, 1.048, 1.05, 1.051, 1.052, 1.107, 1.123, 1.146 + ], + "small_ms": 2.078, + "large_ms_4x": 9.197, + "us_per_site": "1.36-1.53 small / 1.37-1.69 large", + "reps": 15 + }, + "_measured_note": "Maxima over 12 runs on a box that was NOT idle, so the spread is an upper bound on real noise. small_ms / large_ms_4x / us_per_site are recorded for context only — NOTHING gates on them, because an absolute millisecond is exactly the gate this file avoids. `us_per_site` staying flat between the two arms is the same property linear_scaling_budget gates, read directly." +} diff --git a/gitnexus/bench/value-ref-resolution/measure.mjs b/gitnexus/bench/value-ref-resolution/measure.mjs new file mode 100644 index 000000000..a1c14f448 --- /dev/null +++ b/gitnexus/bench/value-ref-resolution/measure.mjs @@ -0,0 +1,303 @@ +#!/usr/bin/env node +/** + * Build-free scaling and correctness guard for callable-value reference + * resolution (#3399). + * + * WHAT IS GUARDED. `resolveValueRefTarget` is the per-site half of + * `emitPropertyDispatchCalls`: for every `value-ref` reference site it names the + * callable the source handed over as a value. #3399 replaced a single lexical + * walk with four channels, and each one reaches for a wider index than the last: + * + * 1. BARE `register(onTick)` — `findCallableBindingInScope` + * 2. NAMESPACE `bridge.accessor(utils.compare, …)` — the file's own + * namespace `@import` edges, then the target's module scope + * 3. HUB `bridge.accessor(hub.compare, …)` — the same, through the + * finalized/augmented channel a re-export publishes into + * 4. CONTAINER `bridge.accessor(Element.getNamespaceUri, …)` — + * `findClassBindingInScope`, whose miss path falls back to + * `scopes.qualifiedNames`, a WORKSPACE-WIDE index + * + * Channel 4 is why this bench exists. A workspace-wide index consulted per site + * is linear only while the lookup is keyed; make it a scan — or make any of the + * three guards around it (`isOwnerNameShadowedBySomethingElse`, + * `isNamespaceNameShadowed`, `findOwnedMember`) walk a collection that grows + * with the repo — and a registration table that costs O(sites) today costs + * O(sites x files) tomorrow. That regression is invisible on a fixture and + * expensive on lightpanda-io/browser, where `bridge.{accessor,function,…}` + * appears 2,047 times across 257 files. + * + * HOW. Two corpora of identical shape, 4x apart in file count, and the per-site + * resolution loop is the ONLY thing timed — extraction, ownership reconciliation + * and finalize are setup. `linear_factor` is `(t_large/t_small) / (N_large/N_small)`: + * ~1.0 linear, ~4.x quadratic on this 4x step. + * + * A RATIO IS THE ONLY TIMING GATE — no millisecond ceiling, deliberately. + * `min_ms` and `us_per_site` are printed for context and nothing compares them + * to anything: a wall-clock budget measures the runner, and this repo has been + * bitten by that twice already (`bench/callable-value-flow`'s `widening_overhead` + * failed at 2.07 and 1.975 against a 1.9 budget on a shared runner while the + * code was correct, both times on a sub-11ms measurement). Dividing the large + * arm by the small one divides the machine out, which is what + * `bench/parse-dispatch-rounds` settled on for the same reason. + * + * A timing gate alone would be satisfied by a fast wrong answer, so the + * correctness half is exact and comes first: the site/resolved/declined counts + * per arm, plus an order-independent sha256 over every (site -> resolved target) + * pair. The fingerprint is a CORRECTNESS gate — drift means the resolved target + * set moved, which is a behaviour change to be explained, never re-baselined to + * make CI green. + * + * WHY A ZIG CORPUS for a language-neutral pass. Zig is the only language whose + * provider sets `namespaceExportsIncludeImportedNames`, so it is the only one + * that can exercise channel 3 at all; and the file-as-struct idiom puts channels + * 2 and 4 in one file, which is the shape #3399 was filed over. The corpus also + * carries two DECLINE controls — a non-callable namespace member and a + * non-callable bare argument — so a change that widened the callable gate would + * move `declined` rather than hiding inside the timing. + * + * Container naming is load-bearing in the corpus: a Zig file-as-struct is minted + * under the FILE STEM, so `ElementN.zig` must write `const ElementN = @This();` + * and register `ElementN.getJ`. Spelling the alias `Element` instead is the + * documented `@This()`-alias limitation, every channel-4 site declines, and the + * bench would time a corpus that resolves nothing. + * + * Usage: + * node --import tsx bench/value-ref-resolution/measure.mjs + * node --import tsx bench/value-ref-resolution/measure.mjs --check + */ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; + +import { extractParsedFile } from '../../src/core/ingestion/scope-extractor-bridge.ts'; +import { finalizeScopeModel } from '../../src/core/ingestion/finalize-orchestrator.ts'; +import { createSemanticModel } from '../../src/core/ingestion/model/semantic-model.ts'; +import { reconcileOwnership } from '../../src/core/ingestion/scope-resolution/pipeline/reconcile-ownership.ts'; +import { resolveValueRefTarget } from '../../src/core/ingestion/scope-resolution/passes/property-dispatch.ts'; +import { zigScopeResolver } from '../../src/core/ingestion/languages/zig/scope-resolver.ts'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SMALL_MODULES = 80; +const LARGE_MODULES = 320; +/** Registrations per module through the CONTAINER channel. */ +const ACCESSORS_PER_MODULE = 12; +/** + * Min-of-N, and N is 15 rather than a handful: `bench/import-target` measured + * N=5 tripping its own budget about one run in twenty while N=15 held every + * language inside a 1.13-1.26x swing, and `bench/parse-dispatch-rounds` uses 15 + * on the same grounds. The whole run is ~5 s, so the reps are nearly free. + */ +const REPS = 15; + +/** + * One module = three files, mirroring `test/fixtures/lang-resolution/zig-idioms/ + * src/webapi/`: a namespace-only helper, a hub that re-exports one of its + * members and declares nothing, and a file-as-struct carrying the binding table. + */ +function moduleFiles(i) { + const accessors = Array.from( + { length: ACCESSORS_PER_MODULE }, + (_, j) => `pub fn get${j}(self: *Element${i}) u8 { return self._n; }`, + ).join('\n'); + const registrations = Array.from( + { length: ACCESSORS_PER_MODULE }, + (_, j) => ` pub const a${j} = bridge.accessor(Element${i}.get${j}, null, .{});`, + ).join('\n'); + return [ + { + path: `src/dom_utils${i}.zig`, + content: `pub const DEFAULT_NS: u8 = 7;\npub fn compare(a: u8, b: u8) u8 { return if (a > b) a else b; }\n`, + }, + { + path: `src/hub${i}.zig`, + content: `pub const compare = @import("dom_utils${i}.zig").compare;\n`, + }, + { + path: `src/Element${i}.zig`, + content: `const Element${i} = @This(); +const dom_utils = @import("dom_utils${i}.zig"); +const hub = @import("hub${i}.zig"); + +_n: u8 = 0, + +${accessors} + +fn onTick(self: *Element${i}) u8 { return self._n; } + +pub const JsApi = struct { + pub const bridge = Bridge(Element${i}); +${registrations} + pub const comparator = bridge.accessor(dom_utils.compare, null, .{}); + pub const hubbed = bridge.accessor(hub.compare, null, .{}); + pub const defaultNs = bridge.accessor(dom_utils.DEFAULT_NS, null, .{}); +}; + +pub fn boot() void { register(onTick); } + +pub fn register(comptime f: anytype) void { _ = f; } + +fn Bridge(comptime T: type) type { + _ = T; + return struct { + pub fn accessor(comptime g: anytype, comptime s: anytype, comptime o: anytype) u8 { + _ = g; + _ = s; + _ = o; + return 0; + } + }; +} +`, + }, + ]; +} + +/** + * Everything `resolveValueRefTarget` reads, built the way the pipeline builds it + * (`runScopeResolution` phases 1-2): real extraction through the Zig provider, + * `populateOwners`, `reconcileOwnership` into the SemanticModel, then finalize. + * Hand-assembling the indexes instead would pin this file's idea of their shape + * rather than the code's. + */ +function buildCorpus(modules) { + const parsedFiles = []; + for (let i = 0; i < modules; i++) { + for (const file of moduleFiles(i)) { + const parsed = extractParsedFile(zigScopeResolver.languageProvider, file.content, file.path); + if (parsed === undefined) { + throw new Error( + `scope extraction failed for ${file.path} — the vendored tree-sitter-zig ` + + `grammar is unavailable on this host, so this bench cannot run`, + ); + } + zigScopeResolver.populateOwners(parsed); + parsedFiles.push(parsed); + } + } + const model = createSemanticModel(); + reconcileOwnership(parsedFiles, model); + const allFilePaths = new Set(parsedFiles.map((p) => p.filePath)); + const scopes = finalizeScopeModel(parsedFiles, { + hooks: { + resolveImportTarget: (raw, from) => + zigScopeResolver.resolveImportTarget(raw, from, allFilePaths), + mergeBindings: (existing, incoming, scopeId) => + zigScopeResolver.mergeBindings(existing, incoming, scopeId), + expandsWildcardTo: (scope, files) => zigScopeResolver.expandsWildcardTo(scope, files), + }, + }); + return { parsedFiles, scopes, model }; +} + +/** The timed loop: every `value-ref` site, resolved exactly as the pass does. */ +function resolveAll({ parsedFiles, scopes, model }, pairs) { + let sites = 0; + let resolved = 0; + for (const parsed of parsedFiles) { + for (const site of parsed.referenceSites) { + if (site.kind !== 'value-ref') continue; + sites++; + const def = resolveValueRefTarget( + site, + parsed.filePath, + scopes, + model, + // The provider hook the pass is handed in `run.ts`; Zig sets it. + zigScopeResolver.namespaceExportsIncludeImportedNames === true, + ); + if (def === undefined) continue; + resolved++; + pairs?.push( + `${parsed.filePath}:${site.atRange.startLine}:${site.atRange.startCol}->${def.nodeId}`, + ); + } + } + return { sites, resolved }; +} + +function measure(modules) { + const corpus = buildCorpus(modules); + const pairs = []; + const counts = resolveAll(corpus, pairs); + let bestMs = Infinity; + for (let i = 0; i < REPS; i++) { + const start = performance.now(); + resolveAll(corpus, undefined); + bestMs = Math.min(bestMs, performance.now() - start); + } + // Order-independent: the walk order is an implementation detail, the resolved + // SET is the behaviour. + pairs.sort(); + return { + modules, + files: corpus.parsedFiles.length, + value_ref_sites: counts.sites, + resolved: counts.resolved, + declined: counts.sites - counts.resolved, + fingerprint: createHash('sha256').update(pairs.join('\n')).digest('hex'), + min_ms: Number(bestMs.toFixed(3)), + us_per_site: Number(((bestMs * 1000) / Math.max(counts.sites, 1)).toFixed(3)), + }; +} + +const report = { small: measure(SMALL_MODULES), large: measure(LARGE_MODULES) }; +report.reps = REPS; +report.workload_ratio = LARGE_MODULES / SMALL_MODULES; +report.scaling_ratio = Number( + (report.large.min_ms / Math.max(report.small.min_ms, 0.001)).toFixed(3), +); +report.linear_factor = Number((report.scaling_ratio / report.workload_ratio).toFixed(3)); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +const baseline = JSON.parse(readFileSync(join(HERE, 'baseline.json'), 'utf8')); +const failures = []; +const requirePositiveNumber = (path, value) => { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + failures.push(`${path}: expected a finite positive number, got ${JSON.stringify(value)}`); + return false; + } + return true; +}; +for (const arm of ['small', 'large']) { + // Correctness first: counts AND the resolved-target set. + for (const key of [ + 'modules', + 'files', + 'value_ref_sites', + 'resolved', + 'declined', + 'fingerprint', + ]) { + if (report[arm][key] !== baseline[arm][key]) { + failures.push( + `${arm}.${key}: expected ${JSON.stringify(baseline[arm][key])}, got ${JSON.stringify(report[arm][key])}`, + ); + } + } + // `min_ms` / `us_per_site` are reported, never gated — see the header. +} +if ( + requirePositiveNumber('linear_scaling_budget', baseline.linear_scaling_budget) && + report.linear_factor > baseline.linear_scaling_budget +) { + failures.push( + `linear_factor ${report.linear_factor} exceeds budget ${baseline.linear_scaling_budget} ` + + `(runtime ${report.scaling_ratio}x for ${report.workload_ratio}x work; ` + + `~1.0 is linear). Re-run alone on an idle machine before investigating — ` + + `this is the only arm a busy runner can move.`, + ); +} + +console.log(JSON.stringify(report, null, 2)); +if (failures.length > 0) { + console.error('[value-ref-resolution --check] FAIL'); + for (const failure of failures) console.error(` - ${failure}`); + process.exit(1); +} +console.log('[value-ref-resolution --check] PASS'); diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 496684a58..ec2f9273b 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -3492,9 +3492,9 @@ } }, "node_modules/ignore": { - "version": "7.0.8", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.8.tgz", - "integrity": "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q==", + "version": "7.0.9", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.9.tgz", + "integrity": "sha512-brTTsvFRt5C1gGHtPst/281UjPD5t9fBqbgoMPlVWy11ZLTPfu7HxK4ZYqO9H7o/yC9rSTCI85EaQ4OoY12qYw==", "license": "MIT", "engines": { "node": ">= 4" diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts index 2c52dd86b..0cd9fe8d9 100644 --- a/gitnexus/scripts/cross-platform-shard.ts +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -36,9 +36,9 @@ * * Only files heavy enough to matter are listed; everything else is carried by * {@link PER_FILE_OVERHEAD_SEC} alone. These are load-balancing hints, NOT - * assertions — no - * test asserts a runtime, and drift only makes the split slightly less even, so - * a stale entry is harmless and refreshing them is optional. Deliberately not + * assertions — no test measures elapsed time against this table. Missing or + * stale heavy entries can still overload a shard; refresh them from failed + * CI logs and replay that profile in the partition tests. Deliberately not * auto-generated: a committed table is reviewable and works offline, and the * alternative (timing files at CI runtime to decide the split) would make the * partition depend on the very machine load it is trying to protect against. @@ -49,13 +49,15 @@ export const WINDOWS_WEIGHTS_SEC: Readonly> = { 'test/integration/cli-e2e.test.ts': 621, 'test/integration/worker-pool.test.ts': 222, 'test/unit/incremental-vector-extension-ordering.test.ts': 87, - // ESTIMATE, not a measurement (#2841): this suite drives more full - // `runFullAnalysis` cycles than the VECTOR sibling above, so the 8 s - // PER_FILE_OVERHEAD floor would badly under-charge it and skew the Windows - // split — the failure mode that produced the job timeouts this table exists - // to prevent. Scaled from the sibling's measured 87 s by analyze-run count. - // Replace with a real figure after the first green Windows matrix run. - 'test/unit/incremental-index-extension-dml-gate.test.ts': 180, + // Measured on Windows in run 34014266125 (#3190, 2026-09-06). These DB + // suites landed together on shard 3: the old 180s estimate and missing + // entries made a ~27-minute recorded load look like an ~12-minute shard. + // Upstream speedups may reduce these figures; retaining conservative weights + // keeps the expensive suites distributed without changing the watchdog. + 'test/unit/incremental-index-extension-dml-gate.test.ts': 414, + 'test/integration/skills-e2e.test.ts': 444, + 'test/integration/fts-extension-e2e.test.ts': 146, + 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, 'test/integration/cli-limit-e2e.test.ts': 75, 'test/unit/hooks.test.ts': 26, 'test/integration/analyze-heap-oom-e2e.test.ts': 23, diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 79169214b..07b10fe25 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -36,6 +36,18 @@ const PLATFORM_LOGIC = [ // must exercise the Windows backslash branch, so run it on the OS matrix (#2394). 'test/unit/cli-entry.test.ts', 'test/unit/platform-capabilities.test.ts', + // The tsconfig loader rebases `paths` targets through `path.resolve`, so the + // wildcard suffix it must recognise is `/*` on POSIX and `\*` on Windows. It + // only looked for `/*`, and every alias target came back as `src*` on + // Windows while the Ubuntu run stayed green — so this file has to run where + // the separator differs. + 'test/unit/tsconfig-index.test.ts', + // The unit half of the same rebasing rule. Fixture-free and pathApi-injectable + // (every separator assertion passes an explicit `path.win32` / `path.posix`), + // so unlike the fixture suite above it fails on EVERY runner when the + // normalisation is removed rather than only on windows-latest. Registered + // beside its fixture sibling so the two halves stay discoverable as one group. + 'test/unit/tsconfig-rebase-target.test.ts', // The gitnexus-plan safe writer resolves every name through a per-platform // backend: Linux anchors through /proc/self/fd, macOS resolves lexically and // verifies each step against descriptors it holds open. Publication is link(2) @@ -130,6 +142,7 @@ const PLATFORM_LOGIC = [ // N-API addon which has known platform-specific behavior (Windows // file-lock lag after close, macOS N-API destructor segfaults) const LBUG_NATIVE = [ + 'test/integration/skip-fts.test.ts', 'test/integration/lbug-core-adapter.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', @@ -277,6 +290,11 @@ const NATIVE_ADDON_SMOKE = [ // Filesystem behavior tests — exercise operations that vary across // platforms (CRLF, symlinks, permissions, temp dirs) const FILESYSTEM = [ + // The durable ParsedFile store's prune tolerates a chunk directory it cannot + // delete (#3204). The failures that motivate it — held handles, read-only + // mounts — are Windows- and macOS-flavored, and the permission-based case + // skips itself where chmod cannot block a delete, so run it everywhere. + 'test/unit/parsedfile-store.test.ts', 'test/integration/filesystem-walker.test.ts', 'test/integration/watch-filesystem.test.ts', 'test/integration/markdown-processor-crlf.test.ts', diff --git a/gitnexus/src/cli/analyze-options.ts b/gitnexus/src/cli/analyze-options.ts index 36f8925c5..ffa439dab 100644 --- a/gitnexus/src/cli/analyze-options.ts +++ b/gitnexus/src/cli/analyze-options.ts @@ -23,6 +23,7 @@ export interface AnalyzeOptions { /** Commander negated flag: false only when --no-parse-cache is passed. */ parseCache?: boolean; repairFts?: boolean; + skipFts?: boolean; /** * Embedding generation toggle. Commander parses `--embeddings [limit]` as: * - `undefined` when the flag is omitted diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts index 189728963..af5405242 100644 --- a/gitnexus/src/cli/analyze-watch.ts +++ b/gitnexus/src/cli/analyze-watch.ts @@ -9,6 +9,7 @@ import { type AnalyzeOptions as CoreAnalyzeOptions, type AnalyzeResult, } from '../core/run-analyze.js'; +import { isIndexLockGuardTimeout } from '../storage/index-lock.js'; import { getGitRoot, hasGitDir } from '../storage/git.js'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; import { GITNEXUS_DIR } from '../storage/repo-meta.js'; @@ -176,6 +177,7 @@ export async function resolveWatchOptions( return { pdg: merged.pdg, + skipFts: merged.skipFts, branch, registryName: merged.name, allowDuplicateName: merged.allowDuplicateName, @@ -241,6 +243,9 @@ export function shouldStopAfterWatchRefreshFailure( error: unknown, paths: readonly string[], ): boolean { + if (isIndexLockGuardTimeout(error)) { + return true; + } return ( paths.length > 0 && !(error instanceof WatchControlReloadError) && @@ -516,9 +521,15 @@ export async function watchCommandWithRunnerIdentity( const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; if (shouldStopAfterWatchRefreshFailure(error, paths)) { fatalRefreshError = error; + const guardTimeout = isIndexLockGuardTimeout(error); cliError( `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + - 'Watch mode is stopping because the live index may have been updated in place.', + (guardTimeout + ? 'Watch mode is stopping because the acquisition guard needs quiesced recovery; see RUNBOOK.md.' + : 'Watch mode is stopping because the live index may have been updated in place.'), + guardTimeout + ? { recoveryHint: 'index-lock-guard-recovery', guardPath: error.guardPath } + : undefined, ); stopWatching(); return; diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 78d13de39..ce577cd45 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -13,6 +13,7 @@ import os from 'os'; import { spawn } from 'child_process'; import v8 from 'v8'; import cliProgress from 'cli-progress'; +import { FTS_DISABLED_MESSAGE, isExplicitFtsDisablement } from '../core/search/fts-policy.js'; import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js'; import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js'; import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js'; @@ -42,7 +43,7 @@ import { selfCommitContextFiles, snapshotSelfCommitSafety, } from '../storage/git.js'; -import { IndexLockTimeoutError } from '../storage/index-lock.js'; +import { IndexLockTimeoutError, isIndexLockGuardTimeout } from '../storage/index-lock.js'; import { loadAnalyzeConfig, mergeAnalyzeOptions, @@ -1349,6 +1350,7 @@ const analyzeCommandImpl = async ( force: options.force || options.skills || options.parseCache === false, useParseCache: options.parseCache !== false, repairFts: options.repairFts, + skipFts: options.skipFts, embeddings: embeddingsEnabled, embeddingsNodeLimit, dropEmbeddings: options.dropEmbeddings, @@ -1450,6 +1452,8 @@ const analyzeCommandImpl = async ( console.error = origError; bar.stop(); console.log(' Already up to date\n'); + if (isExplicitFtsDisablement(result.ftsSkipReason)) + console.log(` ${FTS_DISABLED_MESSAGE}\n`); if (runOptions.registryName) { console.log(` Registry name: ${result.repoName}\n`); } @@ -1610,7 +1614,9 @@ const analyzeCommandImpl = async ( if (result.ftsSkipped) { // #2658 review L2: a build/verify failure is NOT an extension-unavailable // problem — sending the user to install the extension is the wrong remedy. - if (result.ftsSkipReason === 'build-failed') { + if (isExplicitFtsDisablement(result.ftsSkipReason)) { + console.log(`\n ${FTS_DISABLED_MESSAGE}`); + } else if (result.ftsSkipReason === 'build-failed') { console.log( `\n Warning: full-text/BM25 search is disabled — the search index build failed this run.\n` + ` The FTS extension is available; rerun \`gitnexus analyze --repair-fts\`. If it persists,\n` + @@ -1669,6 +1675,14 @@ const analyzeCommandImpl = async ( // refreshed by the holder — this is a clean, expected condition, not a // crash, so render the message without a stack trace. if (err instanceof IndexLockTimeoutError) { + if (isIndexLockGuardTimeout(err)) { + cliError(err.message, { + recoveryHint: 'index-lock-guard-recovery', + guardPath: err.guardPath, + }); + process.exitCode = 1; + return; + } cliError( ` Another gitnexus analyze (pid ${err.holder.pid} on ${err.holder.hostname}) is ` + `already refreshing this index and did not finish within the wait window.\n` + diff --git a/gitnexus/src/cli/cli-message.ts b/gitnexus/src/cli/cli-message.ts index b9b509b52..f65b55663 100644 --- a/gitnexus/src/cli/cli-message.ts +++ b/gitnexus/src/cli/cli-message.ts @@ -61,6 +61,7 @@ export type RecoveryHint = | 'gitnexusrc-invalid' | 'default-branch-invalid' | 'index-lock-timeout' + | 'index-lock-guard-recovery' | 'undeclared-relation-pair'; /** diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts new file mode 100644 index 000000000..d45d059ed --- /dev/null +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -0,0 +1,197 @@ +import { lstat } from 'node:fs/promises'; +import path from 'node:path'; +import { cliInfo } from './cli-message.js'; +import { getGitRoot } from '../storage/git.js'; +import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js'; +import { + closeLbug, + executeQuery, + executeWithReusedStatement, + fetchExistingEmbeddingHashes, + initLbug, +} from '../core/lbug/lbug-adapter.js'; +import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js'; +import { resolveEmbeddingIdentity } from '../core/embeddings/embedding-identity.js'; +import { + decideEmbeddingResume, + mintInterruptedCheckpoint, + mintPartialCheckpoint, + mintUnverifiedCountCheckpoint, + type EmbeddingCheckpoint, + type EmbeddingCheckpointProgress, +} from '../core/embedding-checkpoint.js'; +import { EMBEDDING_DIMS, embeddingDimsMismatch } from '../core/lbug/schema.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; +import { + measurePersistedEmbeddingCount, + persistedEmbeddingCountOrUndefined, +} from '../core/embedding-count.js'; + +/** Add missing embeddings directly to a healthy index, checkpointing periodically. */ +export const embeddingsSyncCommand = async (inputPath?: string): Promise => { + const repoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (!repoPath) throw new Error('Not inside a git repository. Pass a repository path.'); + + const { lbugPath, metaPath } = getStoragePaths(repoPath); + const metaDir = path.dirname(metaPath); + const lock = await acquireIndexLock(metaDir); + try { + requireExclusiveIndexLock( + lock, + `Cannot acquire the index lock at ${metaDir}; refusing an unlocked embeddings sync.`, + ); + const meta = await loadMeta(metaDir); + if (!meta) + throw new Error(`No GitNexus index found for ${repoPath}. Run gitnexus analyze first.`); + if (meta.incrementalInProgress) { + throw new Error('The structural index is incomplete. Run gitnexus analyze --force first.'); + } + + let lbugStat; + try { + lbugStat = await lstat(lbugPath); + } catch { + throw new Error( + `The LadybugDB graph store at ${lbugPath} is missing. Run gitnexus analyze first.`, + ); + } + if (!lbugStat.isFile()) { + throw new Error( + `The LadybugDB graph store at ${lbugPath} is not a usable database file. Run gitnexus analyze first.`, + ); + } + + const identity = resolveEmbeddingIdentity(); + let forceReembedNodeIds: ReadonlySet | undefined; + let resumedFrom: EmbeddingCheckpoint | undefined; + if (meta.embeddingCheckpoint) { + const checkpoint = meta.embeddingCheckpoint; + const decision = decideEmbeddingResume(checkpoint, identity); + if (decision.action === 'abort') throw new Error(decision.error); + const identityDiffers = + checkpoint.provider !== identity.provider || + checkpoint.model !== identity.model || + checkpoint.dimensions !== identity.dimensions; + // `abandon` on a foreign identity drops the pending set only. Existing + // rows stay; sync would then embed the holes under the new identity and + // mix vector spaces. Fail closed — rebuild via analyze. + // + // Every kind is gated, `unverified-count` included. Exempting it looked + // safe because that kind only records "the count could not be read", but + // `decideEmbeddingResume` returns `abandon` for it BEFORE comparing + // identity, so the exemption was the only thing standing between a + // foreign identity and a silently mixed table. + if (identityDiffers) { + throw new Error( + `Cannot sync embeddings: the index checkpoint was written by ${checkpoint.model} ` + + `(${checkpoint.provider}) at ${checkpoint.dimensions} dimensions, but this run ` + + `resolves ${identity.model} (${identity.provider}) at ${identity.dimensions}. ` + + 'Run `gitnexus analyze --embeddings --force` to rebuild under the new identity.', + ); + } + cliInfo(decision.log); + if (decision.action === 'resume') { + forceReembedNodeIds = decision.pendingNodeIds; + resumedFrom = decision.resumedFrom; + } + } + + // The vector column is FLOAT[N] fixed when the index was built, and the + // pipeline deletes each batch's stale rows immediately before inserting the + // replacements — so a width change here deletes rows it cannot re-insert. + // `analyze` forces a full rebuild on the same mismatch; only a rebuild can + // retype the column, so this writer refuses instead. An absent recorded + // width is not a mismatch (see `embeddingDimsMismatch`). + if (embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) { + throw new Error( + `Cannot sync embeddings: this index stores FLOAT[${meta.embeddingDims}] vectors, ` + + `but this run embeds at ${EMBEDDING_DIMS} dimensions. ` + + 'Run `gitnexus analyze --embeddings --force` to rebuild the column at the new width.', + ); + } + + await initLbug(lbugPath); + try { + const existing = await fetchExistingEmbeddingHashes(executeQuery); + let lastPercent = -1; + + const countEmbeddings = async (): Promise => + persistedEmbeddingCountOrUndefined(await measurePersistedEmbeddingCount(executeQuery)); + // One write path for every meta update this command makes. The re-read + // happens immediately before each save so a concurrent writer's fields + // survive. #2790 traced two production drifts to hand-copied writers of + // these exact fields, so this file keeps one copy instead of three. + const persistMeta = async (patch: (latest: RepoMeta) => Partial): Promise => { + const latest = (await loadMeta(metaDir)) ?? meta; + await saveMeta(metaDir, { ...latest, ...patch(latest) }); + }; + const saveCheckpoint = async ( + checkpoint: EmbeddingCheckpointProgress, + pendingNodeIds: string[], + embeddings?: number, + ): Promise => + persistMeta((latest) => ({ + ...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }), + embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds), + })); + + cliInfo(`Embedding ${repoPath}`); + cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`); + + const result = await runEmbeddingPipeline( + executeQuery, + executeWithReusedStatement, + (progress) => { + const percent = Math.floor(progress.percent); + if (percent !== lastPercent && (percent % 5 === 0 || percent === 100)) { + lastPercent = percent; + cliInfo( + ` ${percent}% — ${progress.nodesProcessed ?? 0}/${progress.totalNodes ?? '?'} nodes`, + ); + } + }, + {}, + undefined, + existing && existing.size ? existing : undefined, + { + forceReembedNodeIds, + onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { + await saveCheckpoint(checkpoint, nodeIds); + }, + onCheckpoint: async (checkpoint) => { + await saveCheckpoint(checkpoint, [], await countEmbeddings()); + }, + }, + ); + + const embeddings = await countEmbeddings(); + if (embeddings === undefined) { + // Keep last-known stats.embeddings. An interrupted window marker would + // fail the identity gate on the next run even though this run finished; + // unverified-count is the recovery kind that forces a recount (#2790). + await persistMeta(() => ({ + embeddingCheckpoint: result.failedNodeIds.length + ? mintPartialCheckpoint(identity, result, resumedFrom) + : mintUnverifiedCountCheckpoint(identity, { + nodesProcessed: result.nodesProcessed, + totalNodes: result.nodesProcessed, + chunksProcessed: result.chunksProcessed, + }), + })); + throw new Error('Could not verify persisted embedding count.'); + } + await persistMeta((latest) => ({ + stats: { ...latest.stats, embeddings }, + embeddingCheckpoint: result.failedNodeIds.length + ? mintPartialCheckpoint(identity, result, resumedFrom) + : undefined, + })); + cliInfo(`Embeddings ready: ${embeddings}`); + } finally { + await closeLbug().catch(() => {}); + } + } finally { + lock.release(); + } +}; diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index b2f656c39..fc6af8e63 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -609,10 +609,20 @@ export function formatImpactResult(result: any): string { } // #1858 — an interface / indirection boundary on the path makes this a lower // bound; surface it so the count is not read as exhaustive. + // + // The header names no cause AND asserts no omitted caller, because it cannot + // know either. DI / dynamic dispatch was the only producer of `lower-bound` + // when this was written; #3399 added callables named in VALUE position (a + // registration table, a callback argument), and one of its producers is a + // probe that could not RUN — `callableValueReferenceBoundaries` hedges on a + // failed query and says in so many words that whether the symbol is + // registered is unknown. A header claiming "some callers are not traced" + // would there assert an omission nothing established, and would contradict + // the bullet printed directly under it. The bullets carry the cause — they + // are generated per-cause by `computeEpistemicBoundary` — so the header only + // has to say the count is a floor. if (result.epistemic === 'lower-bound') { - lines.push( - '⚠️ Lower bound — unresolved indirection on the path (callers binding via DI / dynamic dispatch are not traced; actual impact may be higher):', - ); + lines.push('⚠️ Lower bound — impact may be incomplete and actual impact may be higher:'); for (const b of result.boundaries || []) lines.push(` • ${b}`); } pushCallgraphRiskLines(lines, result); diff --git a/gitnexus/src/cli/group-status-format.ts b/gitnexus/src/cli/group-status-format.ts new file mode 100644 index 000000000..f5d09c47e --- /dev/null +++ b/gitnexus/src/cli/group-status-format.ts @@ -0,0 +1,27 @@ +/** + * Rendering for `gitnexus group status` rows, kept out of the Commander action + * so it can be tested. Inline, the cell was only reachable by booting a backend + * against a real group, which is how `STALE (-1 commits behind)` went unnoticed + * (#3256). + */ + +/** The fields of a `groupStatus` repo row the index column reads. */ +export interface GroupRepoIndexRow { + indexStale: boolean; + commitsBehind?: number; +} + +/** + * The index column of a `group status` row. The output is unchanged except + * for one case: a count that is not a real count renders as `?`. + * + * `group/service.ts` has always reported a repo with no recorded commit as + * `{ indexStale: true, commitsBehind: -1 }`. The previous `?? '?'` fallback + * never caught that, because `??` only falls back on `null` / `undefined`. + */ +export const formatIndexStatusCell = (row: GroupRepoIndexRow): string => { + if (!row.indexStale) return 'OK '; + const n = row.commitsBehind; + const count = typeof n === 'number' && n >= 0 ? String(n) : '?'; + return `STALE (${count} commits behind)`; +}; diff --git a/gitnexus/src/cli/group.ts b/gitnexus/src/cli/group.ts index abc13fa2b..824b13b6a 100644 --- a/gitnexus/src/cli/group.ts +++ b/gitnexus/src/cli/group.ts @@ -4,6 +4,7 @@ import type { Command } from 'commander'; import type { RegistryWriteOutcome } from '../core/group/sync.js'; import type { MatchType } from '../core/group/types.js'; import { logger } from '../core/logger.js'; +import { formatIndexStatusCell } from './group-status-format.js'; const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); @@ -161,9 +162,7 @@ export function registerGroupCommands(program: Command): void { console.log(` ${repoPath.padEnd(25)} MISSING (no entry in the registry)`); continue; } - const idx = row.indexStale - ? `STALE (${row.commitsBehind ?? '?'} commits behind)` - : 'OK '; + const idx = formatIndexStatusCell(row); const ctr = row.contractsStale ? ' CONTRACTS_STALE' : ''; console.log(` ${repoPath.padEnd(25)} ${idx}${ctr}`); } diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 3585a6648..8fe3025fd 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -168,6 +168,8 @@ export const en = { 'error.watch.ambiguous': '`gitnexus watch` is ambiguous.\n Local working-tree incremental index: gitnexus analyze --watch\n Scheduled remote clone/pull + analyze: gitnexus auto-sync start\n', 'help.command.analyze.description': 'Index a repository (full analysis)', + 'help.command.embeddings.sync.description': + 'Add missing embeddings to an existing index, checkpointing periodically for safe resume', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', 'help.command.serve.description': 'Start local HTTP server for web UI connection', @@ -356,5 +358,5 @@ export const en = { 'help.identityCache.environment': '\nAnalyzer identity cache:\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir\n Operator-trusted persistent cache for warm cross-process status. The directory must pre-exist, be outside the GitNexus package/build roots, and contain no symlink or junction components. Defaults remain fail-closed on platforms without POSIX ownership APIs.', 'help.analyze.environment': - '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir Operator-trusted persistent analyzer identity cache; must pre-exist, be outside package/build roots, and contain no symlink/junction components.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N Max wait at pool shutdown for a retired worker still inside native code (terminated at its next safe point instead of aborting the process). Default 30000.\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning. Default 20000.\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', + '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir Operator-trusted persistent analyzer identity cache; must pre-exist, be outside package/build roots, and contain no symlink/junction components.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N Max wait at pool shutdown for a retired worker still inside native code (terminated at its next safe point instead of aborting the process). Default 30000.\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning. Default 20000.\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 Retry per-attempt HTTP embedding timeouts through GITNEXUS_EMBEDDING_MAX_ATTEMPTS (default off; timeouts stay terminal).\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', } as const; diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 2f84f4b97..695acd72a 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -164,6 +164,8 @@ export const zhCN = { 'error.watch.ambiguous': '`gitnexus watch` 含义不明确。\n 本地工作区增量索引：gitnexus analyze --watch\n 定时远程 clone/pull 并分析：gitnexus auto-sync start\n', 'help.command.analyze.description': '索引仓库（完整分析）', + 'help.command.embeddings.sync.description': + '向现有索引添加缺失的嵌入，并定期保存检查点以安全续跑', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', 'help.command.mcp.description': @@ -329,5 +331,5 @@ export const zhCN = { 'help.identityCache.environment': '\n分析器身份缓存：\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir\n 由操作员明确信任的持久缓存，用于跨进程快速查询状态。目录必须预先存在、位于 GitNexus 包/构建根目录之外，且路径中不得包含符号链接或 junction。缺少 POSIX 所有权 API 的平台默认保持故障关闭。', 'help.analyze.environment': - '\n环境变量：\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析（仍读取 .gitnexusignore）\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值（KB）。默认 512，最大 32768。\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir 由操作员明确信任的持久分析器身份缓存；目录必须预先存在、位于包/构建根目录之外，且路径中不得包含符号链接或 junction。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时（毫秒）。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值（字节，默认 67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1，最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N 线程池关闭时等待仍在原生代码中的已退役 worker 的最长时间（到达安全点后再终止，避免进程级 abort）。默认 30000。\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N C++ 捕获提取的每文件墙钟预算；超出后该文件保留部分捕获并输出警告。默认 20000。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离（0 < N <= 2；超出则钳制为 2）。MCP 默认 0.6，其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时，参数优先。\n\n提示：`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录（#771）。', + '\n环境变量：\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析（仍读取 .gitnexusignore）\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值（KB）。默认 512，最大 32768。\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir 由操作员明确信任的持久分析器身份缓存；目录必须预先存在、位于包/构建根目录之外，且路径中不得包含符号链接或 junction。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时（毫秒）。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值（字节，默认 67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1，最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N 线程池关闭时等待仍在原生代码中的已退役 worker 的最长时间（到达安全点后再终止，避免进程级 abort）。默认 30000。\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N C++ 捕获提取的每文件墙钟预算；超出后该文件保留部分捕获并输出警告。默认 20000。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 将单次 HTTP 嵌入超时纳入 GITNEXUS_EMBEDDING_MAX_ATTEMPTS 重试（默认关闭，超时仍为终止错误）。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离（0 < N <= 2；超出则钳制为 2）。MCP 默认 0.6，其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时，参数优先。\n\n提示：`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录（#771）。', } satisfies EnglishMessages; diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 70238dc00..1121d0205 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -81,6 +81,7 @@ program 'Re-parse every source file instead of replaying cached parser output', ) .option('--repair-fts', 'Repair/rebuild search FTS indexes without full re-analysis') + .option('--skip-fts', 'Skip FTS extension loading and keyword search indexes') .option( '--embeddings [limit]', 'Enable embedding generation for semantic search (off by default). ' + @@ -304,15 +305,13 @@ program .description('Install the latest published GitNexus globally (`npm i -g gitnexus@`).') .action(createLazyAction(() => import('./update.js'), 'updateCommand')); -program +const embeddings = program .command('embeddings') - .description('Manage the on-demand local embedding runtime') + .description(t('help.command.embeddings.description')); + +embeddings .command('install') - .description( - 'Install the local embedding stack (@huggingface/transformers + onnxruntime-node) on demand. ' + - 'Heals installs where npm skipped the optional packages (e.g. behind an HTTP proxy, #2370). ' + - 'Downloads only from your configured npm registry — mirrors and proxies apply.', - ) + .description(t('help.command.embeddings.install.description')) .option( '--cuda', "Also download the CUDA GPU binaries (runs onnxruntime-node's NuGet postinstall; " + @@ -321,6 +320,12 @@ program .option('--force', 'Install into the runtime prefix even when the stack already resolves') .action(createLazyAction(() => import('./embeddings.js'), 'embeddingsInstallCommand')); +embeddings + .command('sync [path]') + .description(t('help.command.embeddings.sync.description')) + .addHelpText('after', () => t('help.analyze.environment')) + .action(createLbugLazyAction(() => import('./embeddings-sync.js'), 'embeddingsSyncCommand')); + program .command('clean') .description('Delete GitNexus index for current repo') diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index e0ee08bcc..c51a09e13 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -18,6 +18,7 @@ import { resolveAnalyzerRunnerIdentity, } from '../core/analyzer-identity.js'; import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { getFtsDisabledReason, FTS_DISABLED_MESSAGE } from '../core/search/fts-policy.js'; import { detectIndexContentDrift, type IndexContentDrift } from '../core/index-content-drift.js'; import { t } from './i18n/index.js'; @@ -186,6 +187,7 @@ export const statusCommand = async (options: StatusOptions = {}) => { index: { indexedAt: activeMeta.indexedAt, commit: activeMeta.lastCommit, + ...(activeMeta.capabilities ? { capabilities: activeMeta.capabilities } : {}), runnerIdentity: activeMeta.runnerIdentity ?? null, runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown', incompleteReasons, @@ -211,6 +213,7 @@ export const statusCommand = async (options: StatusOptions = {}) => { console.log(`${t('status.indexed')}: ${new Date(activeMeta.indexedAt).toLocaleString()}`); console.log(`${t('status.indexedCommit')}: ${activeMeta.lastCommit?.slice(0, 7)}`); console.log(`${t('status.currentCommit')}: ${currentCommit?.slice(0, 7)}`); + if (getFtsDisabledReason(activeMeta.capabilities?.fts)) console.log(FTS_DISABLED_MESSAGE); // Emit the complete, versioned receipt as JSON so humans can inspect it and // automation can compare it without reverse-engineering a display string. // `null` is the backward-compatible signal for pre-receipt metadata. diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 7919b2376..88e22835a 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -12,6 +12,7 @@ */ import { chunk } from '../../lib/utils.js'; +import { parseTruthyEnv } from '../ingestion/utils/env.js'; import { CircuitOpenError, ResilientFetchExhaustedError, @@ -39,6 +40,7 @@ interface HttpConfig { retryCapMs: number; minIntervalMs: number; timeoutMs: number; + retryTimeouts: boolean; requestDimensions?: number; } @@ -202,6 +204,12 @@ const readConfig = (): HttpConfig | null => { DEFAULT_HTTP_TIMEOUT_MS, MAX_HTTP_TIMEOUT_MS, ), + // A boolean toggle, so it takes the repo's truthy convention (`1`/`true`/ + // `yes`) and falls back to the documented default on anything else. The + // integer parser this used throws on a non-digit, which turned the + // conventional `=true` into a hard failure of every embedding call rather + // than either enabling the flag or leaving it off. + retryTimeouts: parseTruthyEnv(process.env.GITNEXUS_EMBEDDING_RETRY_TIMEOUTS), requestDimensions, }; }; @@ -338,6 +346,36 @@ class RetryableEmbeddingBodyError extends Error { } } +class RetryableEmbeddingTimeoutError extends Error { + constructor( + readonly timeoutMs: number, + options?: { cause?: unknown }, + ) { + super( + `Embedding request timed out after ${timeoutMs}ms`, + options?.cause !== undefined ? { cause: options.cause } : undefined, + ); + this.name = 'RetryableEmbeddingTimeoutError'; + } +} + +/** Re-wrap an opt-in TimeoutError so `resilientFetch` retries it. Abort stays terminal. */ +const throwIfRetryableTimeout = ( + err: unknown, + retryTimeouts: boolean, + callerAborted: boolean | undefined, + timeoutMs: number, +): void => { + if ( + retryTimeouts && + !callerAborted && + isTerminalNetworkError(err) && + err.name === 'TimeoutError' + ) { + throw new RetryableEmbeddingTimeoutError(timeoutMs, { cause: err }); + } +}; + /** * Build the message for a 2xx body carrying the wrong number of vectors. * @@ -384,6 +422,7 @@ const httpEmbedBatch = async ( retryCapMs = HTTP_RETRY_CAP_MS, minIntervalMs = 0, timeoutMs = DEFAULT_HTTP_TIMEOUT_MS, + retryTimeouts = false, ): Promise => { const requestBody: { input: string[]; model: string; dimensions?: number } = { input: batch, @@ -428,7 +467,13 @@ const httpEmbedBatch = async ( const signal = requestOptions.signal ? AbortSignal.any([requestOptions.signal, timeoutSignal]) : timeoutSignal; - const attemptResp = await globalThis.fetch(input, { ...init, signal }); + let attemptResp: Response; + try { + attemptResp = await globalThis.fetch(input, { ...init, signal }); + } catch (err) { + throwIfRetryableTimeout(err, retryTimeouts, requestOptions.signal?.aborted, timeoutMs); + throw err; + } // Non-OK bodies are none of our business: hand the response straight // back so `resilientFetch` keeps classifying 4xx/5xx/429 unchanged. if (!attemptResp.ok) return attemptResp; @@ -447,15 +492,19 @@ const httpEmbedBatch = async ( // Not every `.json()` rejection is a parse error: the per-attempt // signal (`AbortSignal.any([caller, AbortSignal.timeout(...)])`) is // wired to the body stream, so a stalled body rejects with the abort - // reason. Re-raise those untouched — `isTerminalNetworkError` is - // `resilientFetch`'s own predicate, so this test agrees with - // `classifyOutcome` by construction. Wrapping one would flip its - // verdict from `terminal-network` (returned without retry AND - // without touching the breaker, via `recordNeutral()`) to - // `retryable-network` (retried, then `breaker.recordFailure()`): the - // same timeout would take 3 attempts instead of 1, count toward the - // process-global `embeddings-http` breaker, and reach the operator as - // "unparseable response" so they never reach for the timeout knob. + // reason. Re-raise AbortError (and TimeoutError when retry is off) + // untouched — `isTerminalNetworkError` is `resilientFetch`'s own + // predicate, so this test agrees with `classifyOutcome` by + // construction. Wrapping one would flip its verdict from + // `terminal-network` (returned without retry AND without touching + // the breaker, via `recordNeutral()`) to `retryable-network` + // (retried, then `breaker.recordFailure()`): the same timeout would + // take 3 attempts instead of 1, count toward the process-global + // `embeddings-http` breaker, and reach the operator as "unparseable + // response" so they never reach for the timeout knob. + // Opt-in `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1` is the exception: + // TimeoutError is re-wrapped so the existing retry loop can retry it. + throwIfRetryableTimeout(err, retryTimeouts, requestOptions.signal?.aborted, timeoutMs); if (isTerminalNetworkError(err)) throw err; throw new RetryableEmbeddingBodyError(unparseableMessage(), { cause: err }); } @@ -503,6 +552,12 @@ const httpEmbedBatch = async ( if (err instanceof RetryableEmbeddingBodyError) { throw new HttpEmbeddingError(err.terminalMessage, { cause: err.cause }); } + if (err instanceof RetryableEmbeddingTimeoutError) { + throw new HttpEmbeddingError( + `${err.message} after ${maxAttempts} attempt(s) (${safeUrl(url)}, batch ${batchIndex})`, + { cause: err.cause }, + ); + } if (err instanceof CircuitOpenError) { throw new HttpEmbeddingError( `Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`, @@ -580,6 +635,7 @@ export const httpEmbed = async ( config.retryCapMs, config.minIntervalMs, config.timeoutMs, + config.retryTimeouts, ); // Defensive backstop, deliberately kept: `httpEmbedBatch` now rejects a @@ -655,6 +711,7 @@ export const httpEmbedQuery = async ( config.retryCapMs, config.minIntervalMs, config.timeoutMs, + config.retryTimeouts, ); // Defensive backstop like the `httpEmbed` one above: an empty `data` array is // now a cardinality mismatch (0 vectors for 1 text) rejected and retried diff --git a/gitnexus/src/core/git-staleness.ts b/gitnexus/src/core/git-staleness.ts index 1abcd7dda..ece63a87c 100644 --- a/gitnexus/src/core/git-staleness.ts +++ b/gitnexus/src/core/git-staleness.ts @@ -8,19 +8,86 @@ import { promisify } from 'node:util'; import path from 'path'; import { readRegistry, type RegistryEntry, type CwdMatch } from '../storage/repo-manager.js'; import { findGitRootByDotGit, getCurrentCommit, getRemoteUrl } from '../storage/git.js'; +import type { StalenessInfo } from './staleness-status.js'; + +// The status/payload types and helpers live in the pure `staleness-status.ts` +// (#3256); the types are re-exported here for existing importers. +export type { StalenessInfo, StalenessStatus } from './staleness-status.js'; const execFileAsync = promisify(execFile); -export interface StalenessInfo { - isStale: boolean; - commitsBehind: number; - hint?: string; -} +/** + * Ceiling for one `git rev-list` staleness probe. Generous for the local + * history walk this is, and short enough that an unresponsive working tree + * degrades to "not stale" quickly rather than holding a request open. + */ +const STALENESS_TIMEOUT_MS = 5_000; + +const behindHint = (n: number): string => + `⚠️ Index is ${n} commit${n > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`; + +// Says only what a failed count plus a resolved HEAD establish: the index is not +// at HEAD and the gap is uncountable. Reaching here does NOT prove the indexed +// commit left history — that is the usual cause (a pruned `fetch --depth 1`), +// but any other `rev-list` failure lands here too, so the cause is hedged. +const DIVERGED_HINT = + "⚠️ Index is not at HEAD and the commit gap could not be counted — the recorded commit may no longer be in this clone's history. Run analyze tool to update."; + +const unknown = (): StalenessInfo => ({ isStale: false, commitsBehind: 0, status: 'unknown' }); + +const fromCount = (commitsBehind: number): StalenessInfo => + commitsBehind > 0 + ? { isStale: true, commitsBehind, hint: behindHint(commitsBehind), status: 'behind' } + : { isStale: false, commitsBehind: 0, status: 'current' }; + +/** + * `rev-list` could not answer. Asking for HEAD alone needs no history walk and + * still separates all three answers: HEAD unreadable is `unknown`, HEAD past the + * indexed commit is `diverged`, and HEAD still *at* it is `current` — the ref + * prints the indexed SHA, so the index is at HEAD however `rev-list` failed. The + * historical fail-open values are kept either way; only `status` differs. + */ +const fromHead = (head: string | null, lastCommit: string): StalenessInfo => { + if (!head) return unknown(); + if (head === lastCommit) return { isStale: false, commitsBehind: 0, status: 'current' }; + return { isStale: false, commitsBehind: 0, hint: DIVERGED_HINT, status: 'diverged' }; +}; + +const readHeadSync = (repoPath: string): string | null => { + try { + return ( + execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repoPath, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + }).trim() || null + ); + } catch { + return null; + } +}; + +const readHeadAsync = async (repoPath: string): Promise => { + try { + const { stdout } = await execFileAsync('git', ['rev-parse', 'HEAD'], { + cwd: repoPath, + encoding: 'utf-8', + windowsHide: true, + timeout: STALENESS_TIMEOUT_MS, + }); + return stdout.trim() || null; + } catch { + return null; + } +}; /** * Check how many commits the index is behind HEAD (synchronous; uses git CLI). */ export function checkStaleness(repoPath: string, lastCommit: string): StalenessInfo { + // No recorded commit is not "at HEAD": there is nothing to measure against. + if (!lastCommit) return unknown(); try { const result = execFileSync('git', ['rev-list', '--count', `${lastCommit}..HEAD`], { cwd: repoPath, @@ -29,19 +96,9 @@ export function checkStaleness(repoPath: string, lastCommit: string): StalenessI windowsHide: true, }).trim(); - const commitsBehind = parseInt(result, 10) || 0; - - if (commitsBehind > 0) { - return { - isStale: true, - commitsBehind, - hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`, - }; - } - - return { isStale: false, commitsBehind: 0 }; + return fromCount(parseInt(result, 10) || 0); } catch { - return { isStale: false, commitsBehind: 0 }; + return fromHead(readHeadSync(repoPath), lastCommit); } } @@ -54,6 +111,7 @@ export async function checkStalenessAsync( repoPath: string, lastCommit: string, ): Promise { + if (!lastCommit) return unknown(); try { // Note: promisified execFile captures stdout/stderr by default (no stdio option needed, // unlike the sync variant which requires explicit stdio: ['pipe','pipe','pipe']). @@ -61,21 +119,22 @@ export async function checkStalenessAsync( cwd: repoPath, encoding: 'utf-8', windowsHide: true, + // The catch below fails closed on every git ERROR, but a hang is not an + // error — it is silence, and without a bound this await never settles. + // A working tree on a disconnected network mount or behind a stuck lock + // does exactly that, and `/api/repos` fans this out once per registered + // repo, so one unreachable mount could hold the whole listing open + // (#3232 review). The timeout kills the child and rejects, and the catch + // below reports it as `unknown` — still the fail-closed `isStale: false`. + timeout: STALENESS_TIMEOUT_MS, }); - const commitsBehind = parseInt(stdout.trim(), 10) || 0; - - if (commitsBehind > 0) { - return { - isStale: true, - commitsBehind, - hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`, - }; - } - - return { isStale: false, commitsBehind: 0 }; - } catch { - return { isStale: false, commitsBehind: 0 }; + return fromCount(parseInt(stdout.trim(), 10) || 0); + } catch (err) { + // A rev-list that timed out means the working tree is not answering. Asking + // it again for HEAD would only double the bound #3232 put on a hung mount. + if ((err as { killed?: boolean }).killed) return unknown(); + return fromHead(await readHeadAsync(repoPath), lastCommit); } } diff --git a/gitnexus/src/core/graph/edge-reasons.ts b/gitnexus/src/core/graph/edge-reasons.ts new file mode 100644 index 000000000..ad31db850 --- /dev/null +++ b/gitnexus/src/core/graph/edge-reasons.ts @@ -0,0 +1,38 @@ +/** + * Edge `reason` values that mark a CALLS edge as a HEURISTIC GUESS rather than + * a resolution. + * + * The distinction exists because an edge's `confidence` number cannot carry it. + * `GLOBAL_NAME_FALLBACK_REASON` edges are emitted at exactly 0.5 — the same + * number as `process-processor`'s `MIN_TRACE_CONFIDENCE` and + * `community-processor`'s `MIN_CONFIDENCE_LARGE` — so a `confidence < 0.5` + * gate does NOT exclude them. Anything that must exclude guesses has to read + * the reason, which is why `KnowledgeGraph.forEachRelationshipFields` passes it + * and why `GraphEmitSink` retains a reason column. + */ + +/** + * The target was chosen because its SIMPLE NAME is unique in the workspace — + * not because any import, scope chain, or type binding led to it. + * + * Emitted by the unique-simple-name tiers of the free-call fallback + * (`pickUniqueGlobalCallable` and constructor-form `pickUniqueGlobalClass`), + * and only for the languages that opt into `allowGlobalFreeCallFallback`. It + * is a name collision away from being wrong and must never be presented as + * an import-resolved edge: a reader who cannot tell the two apart has no way + * to discount the guess. + */ +export const GLOBAL_NAME_FALLBACK_REASON = 'global-name-fallback'; + +/** + * Reasons excluded from process tracing and large-graph community detection. + * + * Both walks exist to describe how the program actually flows. Seeding a flow + * from a unique-name guess produces a confident-looking trace through code that + * may never call each other, which is worse than a shorter honest trace. + */ +const HEURISTIC_EDGE_REASONS: ReadonlySet = new Set([GLOBAL_NAME_FALLBACK_REASON]); + +/** True when this edge's target was guessed by name rather than resolved. */ +export const isHeuristicEdgeReason = (reason: string): boolean => + HEURISTIC_EDGE_REASONS.has(reason); diff --git a/gitnexus/src/core/graph/graph.ts b/gitnexus/src/core/graph/graph.ts index 1c708a4af..84323bf8e 100644 --- a/gitnexus/src/core/graph/graph.ts +++ b/gitnexus/src/core/graph/graph.ts @@ -163,9 +163,17 @@ export const createKnowledgeGraph = (): KnowledgeGraph => { relationshipMap.forEach(fn); }, forEachRelationshipFields( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ) { - relationshipMap.forEach((rel) => fn(rel.sourceId, rel.targetId, rel.type, rel.confidence)); + relationshipMap.forEach((rel) => + fn(rel.sourceId, rel.targetId, rel.type, rel.confidence, rel.reason), + ); }, getNode: (id: string) => nodeMap.get(id), diff --git a/gitnexus/src/core/graph/types.ts b/gitnexus/src/core/graph/types.ts index 9d9caf12b..a7a7e8a04 100644 --- a/gitnexus/src/core/graph/types.ts +++ b/gitnexus/src/core/graph/types.ts @@ -31,14 +31,26 @@ export interface KnowledgeGraph { * Zero-allocation relationship scan: fields, not objects (#2680). * * The whole-graph scans (the local-symbol pruner, community detection, - * process extraction) read only these four fields, and materializing a + * process extraction) read only these five fields, and materializing a * `GraphRelationship` per edge just to read them dominates iteration cost once * relationships are held columnar — measured at ~90 ms per analyze on a * million-edge graph. Prefer this over `forEachRelationship` in any pass that * walks every edge and needs no other field. + * + * `reason` is passed because confidence alone cannot separate a heuristic + * name guess from a resolved edge that happens to sit at the same number: + * the global-name fallback emits at exactly the process/community threshold + * (0.5), so the walks that must exclude it have to read the reason. See + * `GraphEmitSink`'s reason column, added for this consumer. */ forEachRelationshipFields: ( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ) => void; getNode: (id: string) => GraphNode | undefined; nodeCount: number; diff --git a/gitnexus/src/core/group/extractors/graphql-extractor.ts b/gitnexus/src/core/group/extractors/graphql-extractor.ts index efafe4e2e..550a06155 100644 --- a/gitnexus/src/core/group/extractors/graphql-extractor.ts +++ b/gitnexus/src/core/group/extractors/graphql-extractor.ts @@ -92,6 +92,134 @@ function unquote(text: string): string | null { return value.includes('${') ? null : value; } +const SIMPLE_TEMPLATE_ESCAPES: Record = { + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', + v: '\v', + '0': '\0', + "'": "'", + '"': '"', + '\\': '\\', + '`': '`', +}; + +/** Graph Method `startLine` is the 0-based wrapper row (see line-base.ts). */ +function providerMemberStartLine(member: Parser.SyntaxNode): number { + // Class-field arrows are `@declaration.property`; parse-worker falls back to + // the `public_field_definition` wrapper (decorator row when it is a child), + // not the initializer. Do not probe the `value` child. + return member.startPosition.row; +} + +/** tree-sitter `escape_sequence.text` is the source spelling (`\\n`), not the JS value. */ +function decodeEscapeSequence(text: string): string | null { + if (!text.startsWith('\\') || text.length < 2) return null; + const escaped = text.slice(1); + // Tagged-template cooked value: LineContinuation is empty; `\8`/`\9` and + // LegacyOctalEscapeSequence (`\1`–`\7`, `\00`…) make cooked undefined. + if (/^[\n\r\u2028\u2029]$/.test(escaped) || escaped === '\r\n') return ''; + if (/^[1-9]$/.test(escaped) || /^[0-7]{2,3}$/.test(escaped)) return null; + if (escaped.length === 1) return SIMPLE_TEMPLATE_ESCAPES[escaped] ?? escaped; + if (escaped[0] === 'x' && /^[0-9A-Fa-f]{2}$/.test(escaped.slice(1))) { + return String.fromCharCode(Number.parseInt(escaped.slice(1), 16)); + } + if (escaped.startsWith('u{') && escaped.endsWith('}')) { + const hex = escaped.slice(2, -1); + if (!/^[0-9A-Fa-f]{1,6}$/.test(hex)) return null; + const codePoint = Number.parseInt(hex, 16); + if (codePoint > 0x10ffff) return null; + return String.fromCodePoint(codePoint); + } + if (escaped[0] === 'u' && /^[0-9A-Fa-f]{4}$/.test(escaped.slice(1))) { + return String.fromCharCode(Number.parseInt(escaped.slice(1), 16)); + } + return null; +} + +function substitutionIdentifier(substitution: Parser.SyntaxNode): string | null { + if (substitution.namedChildren.length !== 1) return null; + const expr = unwrapExpression(substitution.namedChildren[0]); + return expr.type === 'identifier' ? expr.text : null; +} + +function isGraphqlTagCall(call: Parser.SyntaxNode): boolean { + const callee = call.childForFieldName('function'); + return callee?.type === 'identifier' && callee.text === 'gql'; +} + +function pascalCaseGraphqlName(name: string): string { + return name + .split(/[^A-Za-z0-9]+/) + .filter((part) => part.length > 0) + .map((part) => `${part[0]!.toUpperCase()}${part.slice(1)}`) + .join(''); +} + +type InterpolationCache = Map; + +function uniqueStaticSource( + name: string, + declarators: GeneratedSymbolIndex, + resolving: Set, + cache: InterpolationCache, +): string | null { + if (cache.has(name)) return cache.get(name) ?? null; + if (resolving.has(name) || resolving.size >= MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + const values = declarators.get(name) ?? []; + if (values.length === 0) { + cache.set(name, null); + return null; + } + resolving.add(name); + const sources = new Set(); + for (const value of values) { + const source = staticGraphqlSource(value, declarators, resolving, cache); + // A dynamic or unprovable sibling makes the name ambiguous — do not pick + // the one static spelling and ignore the rest. + if (source === null) { + resolving.delete(name); + cache.set(name, null); + return null; + } + sources.add(source); + } + resolving.delete(name); + const unique = sources.size === 1 ? [...sources][0]! : null; + cache.set(name, unique); + return unique; +} + +function interpolatedTemplateSource( + template: Parser.SyntaxNode, + declarators: GeneratedSymbolIndex, + resolving: Set, + cache: InterpolationCache, +): string | null { + let out = ''; + for (const child of template.namedChildren) { + if (child.type === 'string_fragment') { + out += child.text; + } else if (child.type === 'escape_sequence') { + const decoded = decodeEscapeSequence(child.text); + if (decoded === null) return null; + out += decoded; + } else if (child.type === 'template_substitution') { + const name = substitutionIdentifier(child); + if (!name) return null; + const inlined = uniqueStaticSource(name, declarators, resolving, cache); + if (inlined === null) return null; + out += inlined; + } else { + return null; + } + if (out.length > MAX_GRAPHQL_TOKENS) return null; + } + return out; +} + function unwrapExpression(node: Parser.SyntaxNode): Parser.SyntaxNode { let current = node; while ( @@ -214,7 +342,12 @@ function parsedDocumentProof( return false; } -function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { +function staticGraphqlSource( + initializer: Parser.SyntaxNode, + declarators?: GeneratedSymbolIndex, + resolving: Set = new Set(), + cache: InterpolationCache = new Map(), +): string | null { const value = unwrapExpression(initializer); if (value.type === 'string') { if (value.text.startsWith('"')) { @@ -226,11 +359,24 @@ function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { } return unquote(value.text); } - if (value.type === 'template_string') return unquote(value.text); + if (value.type === 'template_string') { + const hasSubstitution = value.namedChildren.some( + (child) => child.type === 'template_substitution', + ); + if (!hasSubstitution) return unquote(value.text); + return declarators ? interpolatedTemplateSource(value, declarators, resolving, cache) : null; + } if (value.type === 'call_expression') { const template = value.namedChildren.find((child) => child.type === 'template_string'); - return template ? unquote(template.text) : null; + if (!template) return null; + const hasSubstitution = template.namedChildren.some( + (child) => child.type === 'template_substitution', + ); + // Interpolated reconstruction is the cooked template. Only `gql` is + // treated as preserving that source; String.raw / unknown tags stay fail-closed. + if (hasSubstitution && !isGraphqlTagCall(value)) return null; + return staticGraphqlSource(template, declarators, resolving, cache); } if (value.type !== 'new_expression') return null; @@ -238,7 +384,7 @@ function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { if (!constructor || !constructor.text.endsWith('TypedDocumentString')) return null; const args = value.childForFieldName('arguments'); const first = args?.namedChildren[0]; - return first ? staticGraphqlSource(first) : null; + return first ? staticGraphqlSource(first, declarators, resolving, cache) : null; } export function hasGeneratedDocumentProof( @@ -246,9 +392,10 @@ export function hasGeneratedDocumentProof( operationKind: GraphqlOperationKind, operationName: string, requiredFields: readonly string[], + declarators?: GeneratedSymbolIndex, ): boolean { if (!withinGeneratedAstBudget(initializer)) return false; - const staticSource = staticGraphqlSource(initializer); + const staticSource = staticGraphqlSource(initializer, declarators); if (staticSource !== null) { return parsedDocumentProof(staticSource, operationKind, operationName, requiredFields); } @@ -430,7 +577,10 @@ function rootFields( function generatedCandidates(operation: OperationDefinitionNode): string[] { const name = operation.name?.value; - return name ? [`${name}Document`] : []; + if (!name) return []; + const exact = `${name}Document`; + const pascal = `${pascalCaseGraphqlName(name)}Document`; + return exact === pascal ? [exact] : [exact, pascal]; } async function generatedDocumentMatches( @@ -450,7 +600,13 @@ async function generatedDocumentMatches( const index = await pendingIndex; const values = index?.get(symbol.name) ?? []; return values.some((value) => - hasGeneratedDocumentProof(value, operationKind, operationName, requiredFields), + hasGeneratedDocumentProof( + value, + operationKind, + operationName, + requiredFields, + index ?? undefined, + ), ); } @@ -593,11 +749,7 @@ export class GraphqlExtractor implements ContractExtractor { await dbExecutor(RESOLVE_METHOD_QUERY, { name: methodName, filePath, - startLine: - member.type === 'public_field_definition' - ? (member.childForFieldName('value')?.startPosition.row ?? - member.startPosition.row) + 1 - : member.startPosition.row + 1, + startLine: providerMemberStartLine(member), }), ); if (!symbol) continue; diff --git a/gitnexus/src/core/group/extractors/python-workspace-extractor.ts b/gitnexus/src/core/group/extractors/python-workspace-extractor.ts index c07e5d8cc..47157e0b2 100644 --- a/gitnexus/src/core/group/extractors/python-workspace-extractor.ts +++ b/gitnexus/src/core/group/extractors/python-workspace-extractor.ts @@ -2,13 +2,22 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import type { CypherExecutor } from '../contract-extractor.js'; import type { GroupManifestLink, ContractRole } from '../types.js'; +import { getPythonParser } from '../../ingestion/languages/python/query.js'; +import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { + ParseTimeoutError, + parseHadErrors, + parseSourceSafe, +} from '../../tree-sitter/safe-parse.js'; import { shouldIgnorePath, loadIgnoreRules, isHardcodedIgnoredDirectoryAtPath, } from '../../../config/ignore-service.js'; +import { readSafeBounded } from './fs-utils.js'; import { logger } from '../../logger.js'; + interface PythonPackageMeta { name: string; importName: string; @@ -18,9 +27,8 @@ interface PythonPackageMeta { } interface ImportedSymbol { - packageName: string; + importName: string; symbolName: string; - filePath: string; } async function parsePythonManifest( @@ -51,7 +59,7 @@ function parsePyproject( const nameMatch = content.match(/^\[project\]\s*\n(?:[^\n\[]*\n)*?name\s*=\s*"([^"]+)"/m); if (!nameMatch) return null; const name = nameMatch[1]; - const importName = name.replace(/-/g, '_'); + const importName = toPythonImportName(name); const deps: string[] = []; const depsMatch = content.match(/^\[project\]\s*\n[\s\S]*?dependencies\s*=\s*\[([\s\S]*?)\]/m); @@ -79,7 +87,7 @@ function parseSetupPy( const nameMatch = content.match(/name\s*=\s*['"]([^'"]+)['"]/); if (!nameMatch) return null; const name = nameMatch[1]; - const importName = name.replace(/-/g, '_'); + const importName = toPythonImportName(name); const deps: string[] = []; const installMatch = content.match(/install_requires\s*=\s*\[([\s\S]*?)\]/); @@ -97,51 +105,95 @@ function extractPepName(spec: string): string { return spec.split(/[><=!~;\[]/)[0].trim(); } +function toPythonImportName(name: string): string { + return name.replace(/-/g, '_'); +} + +function collectFromImportNames( + node: { + namedChildCount: number; + namedChild(index: number): { + id: number; + type: string; + text: string; + childForFieldName(name: string): { text: string } | null; + } | null; + }, + moduleNodeId: number, +): string[] { + const symbols: string[] = []; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child || child.id === moduleNodeId) continue; + if (child.type === 'dotted_name') { + symbols.push(child.text); + } else if (child.type === 'aliased_import') { + const imported = child.childForFieldName('name'); + if (imported) symbols.push(imported.text); + } + } + return symbols; +} + +/** Cheap skip before tree-sitter: real `from ` tokens, not `fromage`. */ +function hasFromImportToken(content: string): boolean { + return /(?:^|[\s;])from\s+\S/.test(content); +} + async function scanPythonImports( repoPath: string, - knownPackages: Map, + knownPackages: Set, ): Promise { const results: ImportedSymbol[] = []; const sourceFiles = await findPythonFiles(repoPath); + const parser = getPythonParser(); + const maxFileSizeBytes = getMaxFileSizeBytes(); for (const relFile of sourceFiles) { - const absPath = path.join(repoPath, relFile); - let content: string; + const content = await readSafeBounded(repoPath, relFile, maxFileSizeBytes); + if (content == null || !hasFromImportToken(content)) continue; + + let tree; try { - content = await fs.readFile(absPath, 'utf-8'); - } catch { - continue; + tree = parseSourceSafe(parser, content, undefined, undefined, relFile); + } catch (error) { + if (error instanceof ParseTimeoutError) { + logger.warn( + { file: relFile }, + 'python-workspace-extractor: parse timed out, skipping file', + ); + continue; + } + throw error; } + const degraded = parseHadErrors(tree); + const visit = (node: (typeof tree)['rootNode']): void => { + if (node.type !== 'import_from_statement') { + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child) visit(child); + } + return; + } - // from import Foo, Bar - // from .module import Foo - const fromImportRegex = /^from\s+(\w[\w.]*)\s+import\s+(.+)/gm; - let match; - while ((match = fromImportRegex.exec(content)) !== null) { - const modulePath = match[1]; - const importClause = match[2]; + // Error recovery can promote unclosed-docstring lookalikes into real + // import_from_statement nodes. Keep column-0 imports; drop indented ones + // on a degraded tree so function-local discovery stays on clean parses. + if (degraded && node.startPosition.column > 0) return; + + const moduleNode = node.childForFieldName('module_name'); + const modulePath = moduleNode?.text; + if (!modulePath) return; const rootModule = modulePath.split('.')[0]; - const originalName = knownPackages.get(rootModule); - if (!originalName) continue; + if (!knownPackages.has(rootModule)) return; - if (importClause.trim() === '(') continue; - - const symbols = importClause - .replace(/\(|\)/g, '') - .split(',') - .map((s) => { - const trimmed = s.trim(); - const asMatch = trimmed.match(/^(\S+)\s+as\s+/); - return asMatch ? asMatch[1] : trimmed; - }) - .filter(Boolean); - - for (const sym of symbols) { + for (const sym of collectFromImportNames(node, moduleNode.id)) { if (isPascalCase(sym)) { - results.push({ packageName: originalName, symbolName: sym, filePath: relFile }); + results.push({ importName: rootModule, symbolName: sym }); } } - } + }; + visit(tree.rootNode); } return results; @@ -224,21 +276,14 @@ export async function extractPythonWorkspaceLinks( const seen = new Set(); for (const [, pkg] of packagesByGroupPath) { - const normalizedDeps = pkg.workspaceDeps.map((d) => d.replace(/-/g, '_')); + const normalizedDeps = pkg.workspaceDeps.map(toPythonImportName); const groupPkgDeps = normalizedDeps.filter((d) => packagesByImportName.has(d)); if (groupPkgDeps.length === 0) continue; - const knownPackages = new Map(); - for (const dep of groupPkgDeps) { - const meta = packagesByImportName.get(dep); - if (meta) knownPackages.set(dep, meta.name); - } - - const imports = await scanPythonImports(pkg.repoPath, knownPackages); + const imports = await scanPythonImports(pkg.repoPath, new Set(groupPkgDeps)); for (const imp of imports) { - const providerImportName = imp.packageName.replace(/-/g, '_'); - const providerPkg = packagesByImportName.get(providerImportName); + const providerPkg = packagesByImportName.get(imp.importName); if (!providerPkg) continue; const qualifiedContract = `${providerPkg.name}::${imp.symbolName}`; diff --git a/gitnexus/src/core/group/group-lock.ts b/gitnexus/src/core/group/group-lock.ts index cdaacb283..ab190850f 100644 --- a/gitnexus/src/core/group/group-lock.ts +++ b/gitnexus/src/core/group/group-lock.ts @@ -30,7 +30,9 @@ * distinct ways it can fail to be protected; all three throw * {@link GroupSyncLockError}: * - * 1. TIMEOUT — the holder is still alive when the ceiling elapses. + * 1. TIMEOUT — a live holder still held the lock when the ceiling + * elapsed, or an unrecoverable acquisition/reclaim guard leftover + * exhausted the guard wait (see RUNBOOK.md). * 2. LOCK-FREE DEGRADATION — `acquireIndexLock` answers a read-only or * permission-denied filesystem with a no-op handle that is byte-identical * to a real one at the API boundary. That is a deliberate tolerance for @@ -72,6 +74,7 @@ import path from 'node:path'; import { acquireIndexLock, IndexLockTimeoutError, + isIndexLockGuardTimeout, type IndexLockHandle, } from '../../storage/index-lock.js'; import { logger } from '../logger.js'; @@ -123,7 +126,7 @@ export const withGroupSyncLock = async ( ): Promise => { let handle: IndexLockHandle; // The wrapper times the acquisition itself. `IndexLockTimeoutError` carries - // `holder` and `holderKnown` and nothing else — the elapsed wait exists only + // holder/guard identity but no elapsed-time field — the elapsed wait exists only // inside its inherited message string, so the figure has to be measured here // to be reported without that message. `Date.now()` matches how the primitive // measures its own wait. @@ -147,6 +150,16 @@ export const withGroupSyncLock = async ( // socket backend the holder is not identifiable at all. Re-word it around // what IS known: which group, which operation, and how long we waited. if (err instanceof IndexLockTimeoutError) { + if (isIndexLockGuardTimeout(err)) { + throw new GroupSyncLockError( + 'timeout', + groupDir, + `Could not acquire the sync lock for group "${path.basename(groupDir)}" ` + + `(${getGroupSyncLockDir(groupDir)}). ${err.message} ` + + `Nothing was written and this group was not synced.`, + err, + ); + } throw new GroupSyncLockError( 'timeout', groupDir, diff --git a/gitnexus/src/core/group/service.ts b/gitnexus/src/core/group/service.ts index 6f41d6586..d003c3601 100644 --- a/gitnexus/src/core/group/service.ts +++ b/gitnexus/src/core/group/service.ts @@ -6,6 +6,7 @@ import fsp from 'node:fs/promises'; import path from 'node:path'; import { checkStaleness } from '../git-staleness.js'; +import { stalenessStatus, type StalenessInfo, type StalenessStatus } from '../staleness-status.js'; import { canonicalizePath, loadMeta, @@ -843,6 +844,17 @@ export class GroupService { /** Set only when `unresolvable`; says what could not be resolved. */ unresolvableReason?: string; commitsBehind?: number; + /** + * What the staleness check could establish (#3256). Additive: + * `indexStale` and `commitsBehind` keep their meaning. Two rows report + * `unknown` and they do NOT agree on those two fields, so read them + * together with this one: no commit was recorded (`indexStale: true`, + * `commitsBehind: -1`, the sentinel that case has always used), or the + * git probe could not answer (`indexStale: false`, `commitsBehind: 0`, + * straight from `checkStaleness`). MCP/HTTP `stalenessPayload` reports + * neither number — it omits `commitsBehind` for `unknown` entirely. + */ + status?: StalenessStatus; } > = {}; @@ -872,9 +884,9 @@ export class GroupService { const meta: Partial> = (await loadMeta(repoObj.storagePath)) ?? {}; - const staleness = meta.lastCommit + const staleness: StalenessInfo = meta.lastCommit ? checkStaleness(repoObj.repoPath, meta.lastCommit) - : { isStale: true, commitsBehind: -1 }; + : { isStale: true, commitsBehind: -1, status: 'unknown' }; const snapshot = registry?.repoSnapshots?.[repoPath]; const contractsStale = @@ -886,6 +898,7 @@ export class GroupService { missing: false, unresolvable: false, commitsBehind: staleness.commitsBehind, + status: stalenessStatus(staleness), }; } catch (err) { // The registry read succeeded, so its answer about this row is diff --git a/gitnexus/src/core/ingestion/cobol-processor.ts b/gitnexus/src/core/ingestion/cobol-processor.ts index 8963ea893..3fac5072d 100644 --- a/gitnexus/src/core/ingestion/cobol-processor.ts +++ b/gitnexus/src/core/ingestion/cobol-processor.ts @@ -6,8 +6,8 @@ * does its own extraction, and writes directly to the graph. * * Pipeline: - * 1. Separate programs from copybooks - * 2. Build copybook map (name -> content) + * 1. Separate programs from copybooks and JCL + * 2. Index file content by path for COPY expansion * 3. For each program: expand COPY statements, then run regex extraction * 4. Map CobolRegexResults to graph nodes and relationships * 5. Optionally process JCL files for job-step cross-references @@ -25,6 +25,7 @@ import { } from './cobol/cobol-preprocessor.js'; import { expandCopies } from './cobol/cobol-copy-expander.js'; import { processJclFiles } from './cobol/jcl-processor.js'; +import { resolveCobolCopyTarget } from './languages/cobol/copy-target.js'; import { logger } from '../logger.js'; @@ -123,38 +124,23 @@ export const processCobol = ( // ── 1. Separate programs, copybooks, and JCL ─────────────────────── const programs: CobolFile[] = []; - const copybooks: CobolFile[] = []; const jclFiles: CobolFile[] = []; for (const file of files) { const ext = path.extname(file.path).toLowerCase(); if (JCL_EXTENSIONS.has(ext)) { jclFiles.push(file); - } else if (isCopybook(file.path)) { - copybooks.push(file); - } else if (COBOL_EXTENSIONS.has(ext)) { + } else if (COBOL_EXTENSIONS.has(ext) && !isCopybook(file.path)) { programs.push(file); } } - // ── 2. Build copybook map (uppercase name -> content) ────────────── - const copybookMap = new Map(); - for (const cb of copybooks) { - const name = path.basename(cb.path, path.extname(cb.path)).toUpperCase(); - copybookMap.set(name, { content: cb.content, path: cb.path }); - } - - // Build reverse lookup: path -> content for O(1) readCopy + // Path → content for every COBOL/JCL file we ingested. Resolution picks + // the path (#2967 / census lockstep); this map only supplies the body. const copybookByPath = new Map(); - for (const [, entry] of copybookMap) { - copybookByPath.set(entry.path, entry.content); + for (const f of files) { + copybookByPath.set(f.path, f.content); } - - // Resolve and read callbacks for expandCopies - const resolveCopy = (name: string): string | null => { - const entry = copybookMap.get(name.toUpperCase()); - return entry ? entry.path : null; - }; // Memoize preprocessed copybook content for the duration of this // processCobol call. A single copybook is COPYed by many programs (and at // many COPY sites within a program); without this cache @@ -196,6 +182,11 @@ export const processCobol = ( // Preprocess: clean patch markers const cleaned = preprocessCobolSource(file.content); + // Per-program so COPY EXTERNAL in src/PROG.cbl cannot land on + // vendor/EXTERNAL.cpy when a copybooks/ dir is present (#2967). + const resolveCopy = (name: string): string | null => + resolveCobolCopyTarget(name, file.path, allPathSet); + // Expand COPY statements const { expandedContent, copyResolutions } = expandCopies( cleaned, diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index 80c005143..77b66d73b 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -19,6 +19,7 @@ import { dirname, resolve } from 'node:path'; import { Worker } from 'node:worker_threads'; import type { GraphNode, NodeLabel } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; +import { isHeuristicEdgeReason } from '../graph/edge-reasons.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -131,6 +132,18 @@ const LEIDEN_TIMEOUT_MS = 60_000; const ICEBUG_TIMEOUT_MS = 60_000; const MIN_CONFIDENCE_LARGE = 0.5; +/** + * Whether a large graph's projection may include this edge. + * + * The confidence floor alone lets every global-name-fallback edge through — it + * is emitted at exactly `MIN_CONFIDENCE_LARGE`, so `confidence <` never + * excludes it. Clustering on unique-name guesses fuses unrelated areas into one + * community, which is precisely the noise the large-graph floor exists to + * remove, so the reason is checked too — see `graph/edge-reasons.ts`. + */ +const isLargeGraphEligible = (confidence: number, reason: string): boolean => + confidence >= MIN_CONFIDENCE_LARGE && !isHeuristicEdgeReason(reason); + export const resolveCommunityDetectionEngine = ( raw = process.env[COMMUNITY_ENGINE_ENV], ): CommunityDetectionEngine => { @@ -300,11 +313,16 @@ export const buildCommunityProjection = (knowledgeGraph: KnowledgeGraph): Commun const connectedNodes = new Set(); const nodeDegree = new Map(); - // Field-wise scan (#2680): this walks every edge and reads only these four, + // Field-wise scan (#2680): this walks every edge and reads only these five, // so taking objects would allocate one per edge for nothing. - knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { if (!isClusteringRelationship(type) || sourceId === targetId) return; - if (isLarge && confidence < MIN_CONFIDENCE_LARGE) return; + // A name-guessed edge must not join two nodes into a community at ANY graph + // size — the rationale in `graph/edge-reasons.ts` is about what the edge + // claims, not about how many symbols surround it. Process tracing applies + // the same reason gate unconditionally (`process-processor.ts`). + if (isHeuristicEdgeReason(reason)) return; + if (isLarge && !isLargeGraphEligible(confidence, reason)) return; connectedNodes.add(sourceId); connectedNodes.add(targetId); @@ -340,9 +358,10 @@ export const buildCommunityProjection = (knowledgeGraph: KnowledgeGraph): Commun const seenEdges = new Set(); const edges: Array = []; - knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { if (!isClusteringRelationship(type) || sourceId === targetId) return; - if (isLarge && confidence < MIN_CONFIDENCE_LARGE) return; + if (isHeuristicEdgeReason(reason)) return; + if (isLarge && !isLargeGraphEligible(confidence, reason)) return; const sourceIndex = nodeIndexById.get(sourceId); const targetIndex = nodeIndexById.get(targetId); diff --git a/gitnexus/src/core/ingestion/finalize-orchestrator.ts b/gitnexus/src/core/ingestion/finalize-orchestrator.ts index 3146aff26..4a0e99f6b 100644 --- a/gitnexus/src/core/ingestion/finalize-orchestrator.ts +++ b/gitnexus/src/core/ingestion/finalize-orchestrator.ts @@ -200,8 +200,11 @@ function collectReferenceSites(parsedFiles: readonly ParsedFile[]) { */ function withDefaultHooks(partial: Partial): FinalizeHooks { return { + importsBindAtLexicalScope: partial.importsBindAtLexicalScope === true, resolveImportTarget: partial.resolveImportTarget ?? (() => null), isNamespaceImport: partial.isNamespaceImport, + wildcardCollisionIsAmbiguous: partial.wildcardCollisionIsAmbiguous === true, + namedImportsBindTopLevelOnly: partial.namedImportsBindTopLevelOnly === true, expandsWildcardTo: partial.expandsWildcardTo ?? (() => []), mergeBindings: partial.mergeBindings ?? diff --git a/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts b/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts index 5750da917..2704bdfab 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts @@ -18,6 +18,7 @@ import fs from 'fs/promises'; import path from 'path'; import { createRequire } from 'node:module'; +import type Parser from 'tree-sitter'; import { isHardcodedIgnoredDirectoryAtPath } from '../../../config/ignore-service.js'; import { logger } from '../../logger.js'; @@ -214,6 +215,8 @@ interface WorkspaceScope { readonly include: readonly string[]; /** `!`-prefixed patterns, with the `!` stripped. */ readonly exclude: readonly string[]; + readonly includeRe: readonly RegExp[]; + readonly excludeRe: readonly RegExp[]; } /** Whether `dir` (repo-relative, `''` for the root) is an admitted package. */ @@ -221,8 +224,21 @@ function admits(scope: WorkspaceScope | null, dir: string): boolean { // The root package is always itself, workspace or not. if (dir === '') return true; if (scope === null) return false; - if (scope.exclude.some((pattern) => globToRegExp(pattern).test(dir))) return false; - return scope.include.some((pattern) => globToRegExp(pattern).test(dir)); + // An exclusion covers the directory AND everything under it: `!packages/legacy` + // must keep `packages/legacy/foo` out even when a nested workspace root inside + // the excluded subtree re-declares `packages/*`. + if (scope.excludeRe.some((re) => matchesDirOrAncestor(re, dir))) return false; + return scope.includeRe.some((re) => re.test(dir)); +} + +function matchesDirOrAncestor(re: RegExp, dir: string): boolean { + let current = dir; + while (current !== '') { + if (re.test(current)) return true; + const slash = current.lastIndexOf('/'); + current = slash === -1 ? '' : current.slice(0, slash); + } + return false; } /** @@ -272,9 +288,69 @@ function globToRegExp(pattern: string): RegExp { * tooling that does not read pnpm's file). */ async function loadWorkspaceScope(repoRoot: string): Promise { - const patterns: string[] = []; + // Every workspace ROOT in the repo, not just the top level. keycloak keeps its + // JavaScript workspace at `js/pnpm-workspace.yaml`; reading only the repo root + // found no workspace, admitted no package, and `@keycloak/keycloak-ui-shared` + // (780–845 raw calls per SHA) resolved 7 times. Patterns from a nested root + // are rebased onto that root so `packages/*` under `js/` admits `js/packages/x`. + // + // Gated, though: a nested root counts only when the repo root declares NO + // workspace (keycloak), or when the nested root's directory is itself admitted + // by the root's scope. Unioning every nested root unconditionally let an + // `examples/*/package.json` starter (turborepo/vite/nuxt templates carry + // `workspaces`) admit its example packages — and, being shallow, outrank the + // real package of the same name — and let a root inside an excluded subtree + // re-admit what the outer `!exclusion` had removed. + const [rootPatterns, workspaceRoots] = await Promise.all([ + readWorkspacePatternsAt(repoRoot), + findWorkspaceRoots(repoRoot), + ]); + const rootScope = rootPatterns.length === 0 ? null : toScope(rootPatterns); + const patterns: string[] = [...rootPatterns]; + const admitted: { root: string; prefix: string }[] = []; + for (const root of workspaceRoots) { + if (root === repoRoot) continue; + const prefix = repoRelativeDir(repoRoot, root); + if (rootScope !== null && !admits(rootScope, prefix)) continue; + admitted.push({ root, prefix }); + } + const nested = await Promise.all( + admitted.map(async ({ root, prefix }) => { + const rebase = (p: string): string => { + const negated = p.startsWith('!'); + const body = negated ? p.slice(1) : p; + const joined = prefix === '' ? body : `${prefix}/${body.replace(/^\.\//, '')}`; + return negated ? `!${joined}` : joined; + }; + return (await readWorkspacePatternsAt(root)).map(rebase); + }), + ); + for (const batch of nested) patterns.push(...batch); - const rootManifest = await readJsonFile(path.join(repoRoot, 'package.json')); + if (patterns.length === 0) return null; + return toScope(patterns); +} + +function toScope(patterns: readonly string[]): WorkspaceScope { + const include = patterns.filter((p) => !p.startsWith('!')); + const exclude = patterns.filter((p) => p.startsWith('!')).map((p) => p.slice(1)); + return { + include, + exclude, + includeRe: include.map(globToRegExp), + excludeRe: exclude.map(globToRegExp), + }; +} + +/** The workspace patterns declared at ONE directory, all three spellings merged. */ +async function readWorkspacePatternsAt(root: string): Promise { + const [rootManifest, yamlPkgs, ymlPkgs, lerna] = await Promise.all([ + readJsonFile(path.join(root, 'package.json')), + readYamlPackages(path.join(root, 'pnpm-workspace.yaml')), + readYamlPackages(path.join(root, 'pnpm-workspace.yml')), + readJsonFile(path.join(root, 'lerna.json')), + ]); + const patterns: string[] = []; const workspaces = rootManifest?.workspaces; if (Array.isArray(workspaces)) { patterns.push(...workspaces.filter((w): w is string => typeof w === 'string')); @@ -285,20 +361,114 @@ async function loadWorkspaceScope(repoRoot: string): Promise typeof w === 'string')); } } - - patterns.push(...(await readYamlPackages(path.join(repoRoot, 'pnpm-workspace.yaml')))); - patterns.push(...(await readYamlPackages(path.join(repoRoot, 'pnpm-workspace.yml')))); - - const lerna = await readJsonFile(path.join(repoRoot, 'lerna.json')); + patterns.push(...yamlPkgs, ...ymlPkgs); if (Array.isArray(lerna?.packages)) { patterns.push(...lerna.packages.filter((w): w is string => typeof w === 'string')); } + return patterns; +} - if (patterns.length === 0) return null; - return { - include: patterns.filter((p) => !p.startsWith('!')), - exclude: patterns.filter((p) => p.startsWith('!')).map((p) => p.slice(1)), - }; +/** + * Directories that declare a workspace: the repo root plus any directory (to a + * shallow depth — workspace roots sit near the top) holding a + * `pnpm-workspace.yaml`, a `lerna.json`, or a `package.json` with `workspaces`. + */ +const WORKSPACE_ROOT_MAX_DEPTH = 4; +/** + * Bound on the depth-≤4 directory walk. Generous on purpose: the previous 2,000 + * cap tripped silently in readdir order, so WHICH packages existed — and which + * imports resolved — varied between two checkouts of the same commit. Tripping + * it now warns, so a truncated scan is a logged fact rather than a quiet one. + */ +const WORKSPACE_ROOT_SCAN_MAX_DIRS = 50_000; +/** + * Directory names whose nested `workspaces` are starters, fixtures, or test + * trees — never members. A leftover `test/pnpm-workspace.yaml` must not become + * a workspace root when the repo root declares none (the #2953 first-wins + * class). A root `workspaces` glob that lists `test/*` still admits those + * packages through the package.json walk; this set only stops nested-root + * discovery from descending into those names. + */ +const NON_MEMBER_ROOT_DIRS = new Set([ + 'example', + 'examples', + 'fixture', + 'fixtures', + 'template', + 'templates', + 'sample', + 'samples', + 'test', + 'tests', + 'e2e', + '__tests__', + 'spec', + 'specs', +]); + +/** + * Vite's `DEFAULT_CONFIG_FILES` order. The first filename that exists is the + * config Vite loads; leftover siblings are not a second live entry. + */ +const VITE_CONFIG_FILES = [ + 'vite.config.js', + 'vite.config.mjs', + 'vite.config.ts', + 'vite.config.cjs', + 'vite.config.mts', + 'vite.config.cts', +] as const; + +function isMissingPath(err: unknown): boolean { + return (err as NodeJS.ErrnoException).code === 'ENOENT'; +} + +async function readDirSorted(dir: string): Promise { + try { + return (await fs.readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name < b.name ? -1 : a.name > b.name ? 1 : 0, + ); + } catch { + return null; + } +} + +async function findWorkspaceRoots(repoRoot: string): Promise { + const roots: string[] = [repoRoot]; + const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + let scanned = 0; + while (scanned < queue.length) { + if (scanned >= WORKSPACE_ROOT_SCAN_MAX_DIRS) { + logger.warn( + `[node] workspace-root scan of ${repoRoot} hit the ${WORKSPACE_ROOT_SCAN_MAX_DIRS}-directory cap; nested workspace roots below it were not considered`, + ); + break; + } + const { dir, depth } = queue[scanned++]!; + const entries = await readDirSorted(dir); + if (entries === null) continue; + if (dir !== repoRoot) { + const names = new Set(entries.filter((e) => e.isFile()).map((e) => e.name)); + let declares = + names.has('pnpm-workspace.yaml') || + names.has('pnpm-workspace.yml') || + names.has('lerna.json'); + if (!declares && names.has('package.json')) { + const manifest = await readJsonFile(path.join(dir, 'package.json')); + declares = manifest?.workspaces !== undefined && manifest.workspaces !== null; + } + if (declares) roots.push(dir); + } + if (depth >= WORKSPACE_ROOT_MAX_DEPTH) continue; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (NON_MEMBER_ROOT_DIRS.has(entry.name.toLowerCase())) continue; + const child = path.join(dir, entry.name); + if (isHardcodedIgnoredDirectoryAtPath(repoRoot, child)) continue; + queue.push({ dir: child, depth: depth + 1 }); + } + } + return roots; } async function readJsonFile(filePath: string): Promise | null> { @@ -327,6 +497,39 @@ async function readYamlPackages(filePath: string): Promise { } } +/** + * Per-repo memo. The TS/JS/Vue scope resolvers and the unresolved-call ledger + * classifier each ask for the same map during one analyze; without this the + * 20k-directory walk ran once per asker (four times on a full run, one of them + * inside the index lock). Invalidated at the start of every `runFullAnalysis` + * so a long-lived server never serves a stale package map across runs. + */ +const workspacePackagesMemo = new Map>(); + +export function invalidateNodeWorkspacePackages(repoRoot: string): void { + workspacePackagesMemo.delete(path.resolve(repoRoot)); +} + +export async function loadNodeWorkspacePackages( + repoRoot: string, +): Promise { + const key = path.resolve(repoRoot); + const cached = workspacePackagesMemo.get(key); + if (cached !== undefined) return cached; + const pending: Promise = loadNodeWorkspacePackagesUncached( + repoRoot, + ).catch((err: unknown) => { + // Evict only OUR entry. If this load was invalidated while in flight and a + // newer load has since been installed under the same key, deleting by key + // alone would evict that one, and every later caller would start another + // full scan instead of joining it. + if (workspacePackagesMemo.get(key) === pending) workspacePackagesMemo.delete(key); + throw err; + }); + workspacePackagesMemo.set(key, pending); + return pending; +} + /** * Collect the `package.json` of every ADMITTED workspace package. * @@ -334,30 +537,29 @@ async function readYamlPackages(filePath: string): Promise { * declaration, so this is far cheaper than the C# namespace scan next door, * which reads every `.cs` file. */ -export async function loadNodeWorkspacePackages( +async function loadNodeWorkspacePackagesUncached( repoRoot: string, ): Promise { const scope = await loadWorkspaceScope(repoRoot); const byName = new Map(); const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + let queueHead = 0; let dirsScanned = 0; - while (queue.length > 0) { + while (queueHead < queue.length) { if (dirsScanned >= SCAN_MAX_DIRS) { logger.warn( `[node] package.json scan of ${repoRoot} hit the ${SCAN_MAX_DIRS}-directory cap; workspace packages below it will not resolve`, ); break; } - const { dir, depth } = queue.shift()!; + const { dir, depth } = queue[queueHead++]!; dirsScanned++; - let entries: import('fs').Dirent[]; - try { - entries = await fs.readdir(dir, { withFileTypes: true }); - } catch { - continue; - } + // Sorted: same-depth name collisions resolve first-wins, and readdir order + // is filesystem-dependent. + const entries = await readDirSorted(dir); + if (entries === null) continue; for (const entry of entries) { if (entry.isDirectory()) { @@ -420,10 +622,34 @@ async function readManifest( const value = parsed[field]; if (typeof value === 'string') push(entries, rebase(value)); } + } + // Entry points that name BUILD OUTPUT (`main: dist/x.js`, `exports: ./dist/…`) + // never match an indexed source file — the package's real source entry has to + // be discovered. keycloak's `@keycloak/keycloak-ui-shared` publishes + // `dist/keycloak-ui-shared.js` and keeps its entry in `vite.config.ts` + // (`build.lib.entry: 'src/main.ts'`); with only the declared fields it had 7 + // resolved calls against ~845 raw. Tried in a fixed order, and ONLY appended + // (declared entries keep precedence): `source` / `publishConfig.source`, the + // vite `lib.entry`, then `src/main` / `src/index`. More than one candidate + // that exists on disk is recorded as ambiguous rather than picked — a wrong + // entry binds every import of the package to the wrong file. + // A package whose `exports` map has no `"."` (only subpaths) refuses the bare + // specifier outright; discovery must not manufacture a `src/index` root for it. + const rootlessExports = declaresExports && rootExports.length === 0; + const discovered = rootlessExports + ? { entries: [], ambiguous: [], allowConventional: false } + : await discoverSourceEntries(parsed, dir, repoRoot); + for (const entry of discovered.entries) push(entries, entry); + if (!declaresExports && discovered.allowConventional) { for (const conventional of ['src/index', 'index', 'lib/index']) { push(entries, joinRepoPath(packageDir, conventional)); } } + if (discovered.ambiguous.length > 0) { + logger.warn( + `[node] package ${name}: ${discovered.ambiguous.length} candidate source entries (${discovered.ambiguous.join(', ')}) — none adopted; declare \`source\` or a single lib entry`, + ); + } const subpathImports = new Map(); collectImports(parsed.imports, subpathImports, rebase); @@ -507,6 +733,195 @@ function collectImports( } } +/** + * Does a declared entry point at build output rather than source? Output + * directories and minified bundles. A plain `.js` is NOT build output on its + * own — `main: "src/index.js"` / `index.js` is a JavaScript package's source, + * and treating every `.js` as output ran discovery for essentially every CJS + * package and could adopt a `src/main` beside the real entry. + */ +function looksLikeBuildOutput(entry: string): boolean { + return /(^|\/)(dist|build|lib|out|esm|cjs|umd)\//.test(entry) || /\.min\.[cm]?js$/.test(entry); +} + +function declaredRootExportStrings(exportsRoot: unknown): string[] { + if (typeof exportsRoot === 'string') return [exportsRoot]; + if (exportsRoot === null || typeof exportsRoot !== 'object') return []; + const dot = (exportsRoot as Record)['.']; + if (typeof dot === 'string') return [dot]; + if (dot === null || typeof dot !== 'object') return []; + return Object.values(dot).filter((v): v is string => typeof v === 'string'); +} + +async function existingStems(repoRoot: string, stems: readonly string[]): Promise { + const present = await Promise.all(stems.map((stem) => stemExists(repoRoot, stem))); + const existing: string[] = []; + for (let i = 0; i < stems.length; i++) { + if (present[i]) push(existing, stems[i]!); + } + return existing; +} + +async function discoverSourceEntries( + parsed: Record, + dir: string, + repoRoot: string, +): Promise<{ entries: string[]; ambiguous: string[]; allowConventional: boolean }> { + const packageDir = repoRelativeDir(repoRoot, dir); + const rebase = (raw: string): string => joinRepoPath(packageDir, stripEntryPrefixes(raw)); + const declared: string[] = declaredRootExportStrings(parsed.exports); + for (const field of ['module', 'main']) { + const value = parsed[field]; + if (typeof value === 'string') declared.push(value); + } + // Only when every declared entry is build output (or nothing is declared and + // the conventional stems are absent) does discovery run at all. + if (declared.length > 0 && !declared.every(looksLikeBuildOutput)) + return { entries: [], ambiguous: [], allowConventional: true }; + + const candidates: string[] = []; + const source = parsed.source; + if (typeof source === 'string') candidates.push(rebase(source)); + const publishConfig = parsed.publishConfig; + if (publishConfig !== null && typeof publishConfig === 'object') { + const ps = (publishConfig as Record).source; + if (typeof ps === 'string') candidates.push(rebase(ps)); + } + let hasViteConfig = false; + for (const cfg of VITE_CONFIG_FILES) { + let text: string; + try { + text = await fs.readFile(path.join(dir, cfg), 'utf-8'); + } catch (err) { + if (isMissingPath(err)) continue; + // Present but unreadable: Vite still selected this filename. + hasViteConfig = true; + break; + } + hasViteConfig = true; + try { + const entry = await staticViteEntry(text); + if (entry !== null) push(candidates, rebase(entry)); + } catch { + // Parse/load failure is not a missing file — do not fall through to a leftover sibling. + } + break; + } + let existing = await existingStems(repoRoot, candidates); + // A config we cannot establish is not evidence for a conventional entry. + if (existing.length === 0 && !hasViteConfig) { + existing = await existingStems( + repoRoot, + ['src/main', 'src/index'].map((conventional) => joinRepoPath(packageDir, conventional)), + ); + } + const allowConventional = !hasViteConfig && candidates.length === 0; + if (existing.length > 1) return { entries: [], ambiguous: existing, allowConventional }; + return { entries: existing, ambiguous: [], allowConventional }; +} + +/** Read a literal exported build.lib.entry without evaluating repository code. */ +async function staticViteEntry(text: string): Promise { + // Most manifests need no Vite discovery. Keep native grammars off that path. + const [{ default: TreeSitter }, { default: grammar }, { parseSourceSafe }] = await Promise.all([ + import('tree-sitter'), + import('tree-sitter-typescript'), + import('../../tree-sitter/safe-parse.js'), + ]); + const parser = new TreeSitter(); + parser.setLanguage(grammar.typescript); + const tree = parseSourceSafe(parser, text); + if (tree.rootNode.hasError) return null; + const exports = tree.rootNode.namedChildren.filter( + (node) => node.type === 'export_statement' && node.children.some((c) => c.type === 'default'), + ); + if (exports.length !== 1) return null; + let config = unwrapViteDefineConfig(tree, exports[0]!.childForFieldName('value')); + if (config === null) return null; + for (const key of ['build', 'lib', 'entry']) config = staticObjectProperty(config, key); + return staticStringValue(config); +} + +/** Unwrap `defineConfig({...})` from Vite; refuse local or re-exported helpers. */ +function unwrapViteDefineConfig( + tree: { rootNode: Parser.SyntaxNode }, + config: Parser.SyntaxNode | null, +): Parser.SyntaxNode | null { + if (config?.type !== 'call_expression') return config; + if (config.childForFieldName('function')?.text !== 'defineConfig') return null; + // A locally defined helper need not preserve its argument like Vite does. + if ( + tree.rootNode + .descendantsOfType(['function_declaration', 'variable_declarator']) + .some((node) => node.childForFieldName('name')?.text === 'defineConfig') + ) + return null; + for (const statement of tree.rootNode.namedChildren) { + if (statement.type !== 'import_statement') continue; + if (!statement.descendantsOfType('identifier').some((n) => n.text === 'defineConfig')) continue; + if (staticStringValue(statement.childForFieldName('source')) !== 'vite') return null; + const identityImport = statement.descendantsOfType('import_specifier').some((specifier) => { + const imported = specifier.childForFieldName('name')?.text; + const local = specifier.childForFieldName('alias')?.text ?? imported; + return imported === 'defineConfig' && local === 'defineConfig'; + }); + if (!identityImport) return null; + } + const args = config + .childForFieldName('arguments') + ?.namedChildren.filter((n) => n.type !== 'comment'); + if (args?.length !== 1) return null; + return args[0]!; +} + +function staticStringValue(node: Parser.SyntaxNode | null): string | null { + if (node?.type !== 'string' || node.namedChildren.some((n) => n.type === 'escape_sequence')) + return null; + return node.text.slice(1, -1); +} + +/** Reject computed keys, spreads and duplicate properties that could override a value. */ +function staticObjectProperty( + node: Parser.SyntaxNode | null, + key: string, +): Parser.SyntaxNode | null { + if (node?.type !== 'object') return null; + let value: Parser.SyntaxNode | null = null; + for (const member of node.namedChildren) { + if (member.type === 'comment') continue; + if (member.type !== 'pair') return null; + const name = member.childForFieldName('key'); + if (name?.type !== 'property_identifier' && name?.type !== 'string') return null; + if (name.type === 'string') { + const property = staticStringValue(name); + if (property === null) return null; + if (property !== key) continue; + } else if (name.text !== key) { + continue; + } + if (value !== null) return null; + value = member.childForFieldName('value'); + } + return value; +} + +/** A repo-relative stem exists as a source file (with any TS/JS extension). */ +// The root is threaded explicitly: a module-level "current root" clobbered +// under two concurrent scans and turned an ambiguity refusal into a confident +// wrong entry (the second repo's root made one candidate "not exist"). +async function stemExists(repoRoot: string, stem: string): Promise { + const abs = path.join(repoRoot, stem); + for (const ext of ['', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs']) { + try { + const st = await fs.stat(abs + ext); + if (st.isFile()) return true; + } catch { + /* try next */ + } + } + return false; +} + /** `"./src/index.ts"` -> `"src/index"`; leaves an extension-less path alone. */ function stripEntryPrefixes(entry: string): string { const withoutDot = entry.replace(/^\.\//, '').replace(/^\//, ''); diff --git a/gitnexus/src/core/ingestion/language-config.ts b/gitnexus/src/core/ingestion/language-config.ts index 5d8fc9061..390e6a32f 100644 --- a/gitnexus/src/core/ingestion/language-config.ts +++ b/gitnexus/src/core/ingestion/language-config.ts @@ -5,6 +5,7 @@ import path from 'path'; import type { CsharpStructureLineScanner } from './languages/csharp/namespace-siblings.js'; import { isDev } from './utils/env.js'; +import { isHardcodedIgnoredDirectoryAtPath } from '../../config/ignore-service.js'; import { mapConcurrent } from '../../lib/utils.js'; import { logger } from '../logger.js'; @@ -182,13 +183,24 @@ export interface SwiftPackageConfig { /** Zig package config parsed from build.zig.zon and the root build.zig */ export interface ZigBuildZonConfig { /** - * Map of dependency name -> the raw `.path = "..."` value, exactly as - * written in build.zig.zon (relative to the repo root, and possibly - * escaping it: `../local_dep`). Consumers normalize — see - * `normalizeZigDepPath` below, which rejects absolute - * and repo-escaping values. `.url`-based deps cannot be resolved to a - * repo-local file (they unpack into a build cache outside the repo) and so - * are not included here. + * Map of dependency name -> the dep's directory, in one of two spellings + * depending on which package this config describes: + * + * - ROOT package (`pkg === ''`): the raw `.path = "..."` value, exactly as + * written in build.zig.zon (relative to the repo root, and possibly + * escaping it: `../local_dep`). This is what `parseZigBuildZon` promises + * and what its tests pin. + * - NESTED package: repo-relative and already normalized, because a nested + * package's `.path` is written relative to ITS directory and means + * nothing against the repo-relative keys consumers match on + * (`packages/app`'s `../core` is stored as `packages/core`). A dep + * escaping the REPO root is dropped rather than stored. + * + * Either spelling is safe to hand to `normalizeZigDepPath` below — it rejects + * absolute and repo-escaping values and is idempotent on an already + * normalized one, which is what `resolveZigImportInternal` relies on. + * `.url`-based deps cannot be resolved to a repo-local file (they unpack into + * a build cache outside the repo) and so are not included here. */ pathDeps: Map; /** @@ -228,6 +240,29 @@ export interface ZigBuildZonConfig { buildModules?: readonly ZigBuildModule[]; } +/** + * One Zig build package: the directory whose `build.zig` / `build.zig.zon` + * declare the config, and that config with every path REPO-relative. + * + * A Zig module's import table is declared by the `build.zig` of the package it + * belongs to, so a repo holding several packages holds several import tables — + * the same shape a TypeScript monorepo has with a `tsconfig.json` per package. + */ +export interface ZigPackageScope { + /** Repo-relative directory the package governs (`''` for the repo root). */ + readonly dir: string; + readonly config: ZigBuildZonConfig; +} + +/** + * Every Zig build package in the repo, indexed so the nearest one to a file + * wins — the `TsconfigIndex` analogue, and for the same reason. + */ +export interface ZigWorkspaceIndex { + /** Deepest-first, so the first `dir` that prefixes a file path governs it. */ + readonly packages: readonly ZigPackageScope[]; +} + /** One build module of the root `build.zig` — see `ZigBuildZonConfig.buildModules`. */ export interface ZigBuildModule { /** The `addModule("", …)` name; absent for `createModule` bindings @@ -646,10 +681,21 @@ export async function loadSwiftPackageConfig(repoRoot: string): Promise { +export async function loadZigBuildConfig( + repoRoot: string, + packageDir = '', +): Promise { + // Every path this function returns is REPO-relative, because that is the + // keyspace `allFilePaths` uses. The parsers below answer package-relative, so + // a nested package rebases them through `inPackage`. For the root package + // (`packageDir === ''`) the prefix is empty and every value is byte-identical + // to what this function returned before nested packages existed. + const pkg = packageDir === '' ? '' : `${packageDir}/`; + const inPackage = (relToPackage: string): string => `${pkg}${relToPackage}`; + const packageFile = (name: string): string => path.join(repoRoot, packageDir, name); let config: ZigBuildZonConfig | null = null; try { - const raw = await fs.readFile(path.join(repoRoot, 'build.zig.zon'), 'utf-8'); + const raw = await fs.readFile(packageFile('build.zig.zon'), 'utf-8'); config = parseZigBuildZon(raw); } catch { // No zon (or unreadable): the root build.zig may still declare modules. @@ -661,9 +707,11 @@ export async function loadZigBuildConfig(repoRoot: string): Promise | undefined; let rootBuildZig: string | null = null; try { - rootBuildZig = await fs.readFile(path.join(repoRoot, 'build.zig'), 'utf-8'); + rootBuildZig = await fs.readFile(packageFile('build.zig'), 'utf-8'); const parsed = parseZigRootModules(rootBuildZig); - if (parsed.size > 0) rootModules = parsed; + if (parsed.size > 0) { + rootModules = new Map(Array.from(parsed, ([name, root]) => [name, inPackage(root)])); + } } catch { // No root build.zig — nothing to declare. } @@ -671,7 +719,7 @@ export async function loadZigBuildConfig(repoRoot: string): Promise>(); + // A nested package's `.path` values are written relative to ITS directory, so + // they are rebased here and stored repo-relative; `resolveZigImportInternal` + // then reads them through the same `normalizeZigDepPath`, which is idempotent + // on an already-normalized value. A dep that escapes the REPO root (not merely + // the package) resolves to nothing and is dropped. The root package keeps its + // raw spelling, which is what `parseZigBuildZon` promises and its tests pin. + const pathDeps = pkg === '' ? config.pathDeps : new Map(); for (const [depName, depPath] of config.pathDeps) { - const rel = normalizeZigDepPath(depPath); + // Asked of the value AS WRITTEN, before the package prefix goes on: an + // absolute `.path` points outside the repository whichever package declared + // it, and prefixing hides that from `normalizeZigDepPath`. See + // `isAbsoluteZigDepPath`. + if (isAbsoluteZigDepPath(depPath)) continue; + const rel = normalizeZigDepPath(`${pkg}${depPath}`); if (rel === null) continue; + if (pkg !== '') pathDeps.set(depName, rel); let buildZig: string; try { buildZig = await fs.readFile(path.join(repoRoot, rel, 'build.zig'), 'utf-8'); @@ -706,15 +767,177 @@ export async function loadZigBuildConfig(repoRoot: string): Promise 0) depModules.set(depName, named); } - const buildModules = rootBuildZig === null ? [] : parseZigBuildModules(rootBuildZig, depModules); + const buildModules = + rootBuildZig === null + ? [] + : rebaseZigBuildModules( + parseZigBuildModules(rootBuildZig, depModules), + inPackage, + depModules, + ); return { ...config, + pathDeps, ...(moduleRoots.size > 0 ? { moduleRoots } : {}), ...(rootModules ? { rootModules } : {}), ...(buildModules.length > 0 ? { buildModules } : {}), }; } +/** + * Rebase a package's own build modules to repo-relative paths. + * + * `parseZigBuildModules` answers package-relative for everything it read out of + * the `build.zig` it was handed, with one exception: an alias resolved through + * `depModules` (`addImport("api", dep.module("core"))`) is already repo-relative, + * because `depModules` was built that way. Prefixing that a second time would + * point the alias at a path no file has. The already-repo-relative values are + * therefore identified by membership in `depModules`, not guessed at from their + * shape. + */ +function rebaseZigBuildModules( + modules: readonly ZigBuildModule[], + inPackage: (relToPackage: string) => string, + depModules?: ReadonlyMap>, +): ZigBuildModule[] { + if (inPackage('') === '') return [...modules]; + const fromDep = new Set(); + for (const named of depModules?.values() ?? []) + for (const root of named.values()) fromDep.add(root); + return modules.map((mod) => ({ + ...(mod.name !== undefined ? { name: mod.name } : {}), + root: inPackage(mod.root), + imports: new Map( + Array.from(mod.imports, ([alias, root]) => [ + alias, + fromDep.has(root) ? root : inPackage(root), + ]), + ), + })); +} + +/** Bounds for the package walk, mirroring the tsconfig scan. */ +const ZIG_SCAN_MAX_DIRS = 20_000; +const ZIG_SCAN_MAX_DEPTH = 24; + +/** + * The Zig build package governing `filePath` — the nearest one at or above it. + * + * A Zig module's import table is declared by the `build.zig` of the package the + * file belongs to, so the nearest enclosing package is the faithful reading of + * `@import("name")` at that site, exactly as `tsconfigFor` reads a non-relative + * specifier against the nearest enclosing project. + * + * There is deliberately NO fall-through to an enclosing package when the nearest + * one does not bind the name. Falling through is how a vendored dependency's + * `@import("config")` silently resolved to the outer repo's `config` module — + * the same failure `loadTsconfigIndex` documents for a package whose own + * tsconfig declares no `baseUrl`, and the same failure the per-module import + * tables in `resolveZigImportInternal` already exist to prevent one level down. + */ +export function zigPackageFor( + index: ZigWorkspaceIndex | null | undefined, + filePath: string, +): ZigBuildZonConfig | null { + if (index === null || index === undefined) return null; + for (const scope of index.packages) { + if (scope.dir === '') return scope.config; + if (filePath.startsWith(`${scope.dir}/`)) return scope.config; + } + return null; +} + +/** + * Load every Zig build package in the repo, nearest-first. + * + * Called with no `packageDir` — which is how every call site read it before + * this function existed — `loadZigBuildConfig` reads the ROOT `build.zig` / + * `build.zig.zon` and nothing else. That is the whole configuration of a + * single-package repo and none of the configuration of a monorepo: a repo + * laying its packages out as `packages//build.zig` has no root build + * files at all, so the loader answers `null` and EVERY bare + * `@import("")` in it goes unresolved — cross-file resolution silently + * degrades to relative imports only. Measured on a two-package fixture: + * `config = null`, `@import("core")` → `null`. + * + * The loader itself is not root-bound any more: this function is what supplies + * it a `packageDir`, one per package below. + * + * So the packages are discovered the way tsconfigs are (`findTsconfigFiles`): + * one bounded breadth-first walk that skips the hardcoded ignore set, then + * deepest-first ordering so `zigPackageFor` can take the first match. + * + * Called from `ScopeResolver.loadResolutionConfig`, which the orchestrator runs + * once per LANGUAGE workspace pass — so the walk happens only for repos that + * actually contain Zig. `loadImportConfigs`, which runs unconditionally for + * every repo, keeps calling `loadZigBuildConfig` for the root package alone; + * that is the same split TypeScript already has between the cheap + * `loadTsconfigPaths` and the repo-walking `loadTsconfigIndex`. + */ +export async function loadZigWorkspaceIndex(repoRoot: string): Promise { + const dirs = await findZigPackageDirs(repoRoot); + if (dirs.length === 0) return null; + const packages: ZigPackageScope[] = []; + for (const dir of dirs) { + const config = await loadZigBuildConfig(repoRoot, dir); + // A `build.zig` that declares no module and no path dep contributes nothing + // a lookup could answer with. Keeping it as an empty scope would be worse + // than dropping it: it would shadow an enclosing package that DOES declare + // the name, and answer nothing in its place. + if (config !== null) packages.push({ dir, config }); + } + if (packages.length === 0) return null; + // Deepest first, so `zigPackageFor` takes the most specific package rather + // than whichever the walk reached first. + packages.sort((a, b) => b.dir.length - a.dir.length || a.dir.localeCompare(b.dir)); + return { packages }; +} + +/** Repo-relative directories holding a `build.zig` and/or a `build.zig.zon`. */ +async function findZigPackageDirs(repoRoot: string): Promise { + const found: string[] = []; + const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + // A HEAD INDEX rather than `queue.shift()`. The queue is pushed to while it is + // drained, which keeps the array in a mode where `shift()` memmoves the whole + // remainder instead of taking V8's left-trimming fast path — so the walk is + // quadratic in the frontier, and `ZIG_SCAN_MAX_DIRS` is the bound on how bad + // that gets. Measured at that bound (20,000 dequeues): 53 ms at fan-out 4 and + // 81 ms at fan-out 20, against 0.8 ms here — 66-106x, paid before any config + // is read. Memory is unchanged: entries were already retained by the pushes, + // `shift()` only dropped the head. + let queueHead = 0; + let dirsScanned = 0; + + while (queueHead < queue.length && dirsScanned < ZIG_SCAN_MAX_DIRS) { + const { dir, depth } = queue[queueHead++]!; + dirsScanned++; + let entries: import('fs').Dirent[]; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + continue; + } + let isPackage = false; + for (const entry of entries) { + if (entry.isDirectory()) { + const childDir = path.join(dir, entry.name); + if (isHardcodedIgnoredDirectoryAtPath(repoRoot, childDir)) continue; + if (depth < ZIG_SCAN_MAX_DEPTH) queue.push({ dir: childDir, depth: depth + 1 }); + continue; + } + if (!entry.isFile()) continue; + // Either marker declares a package: a `build.zig` with no zon still names + // modules, and a zon with no build.zig still names path deps. + if (entry.name === 'build.zig' || entry.name === 'build.zig.zon') isPackage = true; + } + if (isPackage) { + const rel = path.relative(repoRoot, dir).split(path.sep).join('/'); + found.push(rel === '.' || rel === '' ? '' : rel); + } + } + return found; +} + /** * Normalize a `.path` value from build.zig.zon into a repo-relative form. * Returns null for paths that escape the repo root (start with `..`) or @@ -722,13 +945,38 @@ export async function loadZigBuildConfig(repoRoot: string): Promise; + /** `.cbl` / `.cob` / `.cobol` files — tier 2, Set-iteration order per stem. */ + readonly sources: ReadonlyMap; + /** True iff any file path has a well-known copybook directory segment. */ + readonly hasPreferredDir: boolean; +} + +function pathHasPreferredDir(fp: string): boolean { + const parts = fp.split('/'); + for (let i = 0; i < parts.length - 1; i++) { + if (PREFERRED_DIR_NAMES.has(parts[i])) return true; + } + return false; +} + +function pushUnique(map: Map, key: string, fp: string): void { + const list = map.get(key); + if (list === undefined) { + map.set(key, [fp]); + return; + } + list.push(fp); +} + +const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { + const copybooks = new Map(); + const sources = new Map(); + let hasPreferredDir = false; + for (const fp of allFilePaths) { + const extRaw = path.extname(fp); + const extLower = extRaw.toLowerCase(); + const tier = COPYBOOK_EXTENSIONS.has(extLower) + ? copybooks + : COBOL_SOURCE_EXTENSIONS.has(extLower) + ? sources + : undefined; + if (tier === undefined) continue; + if (!hasPreferredDir && pathHasPreferredDir(fp)) hasPreferredDir = true; + const basename = path.basename(fp, extRaw).toUpperCase(); + pushUnique(tier, basename, fp); + } + return { copybooks, sources, hasPreferredDir }; +}); + +function pickCopyPath( + paths: readonly string[] | undefined, + fromFile: string, + hasPreferredDir: boolean, +): string | null { + if (paths === undefined || paths.length === 0) return null; + if (!hasPreferredDir) return paths[0] ?? null; + const fromDir = path.dirname(fromFile); + for (const fp of paths) { + if (pathHasPreferredDir(fp) || path.dirname(fp) === fromDir) return fp; + } + return null; +} + +/** + * Resolve a COBOL `COPY` member name against the workspace file set. + * + * `fromFile` is the importing program (census) or the program currently + * being expanded (processor). It is used only to include the importer's + * directory in the preferred class when a well-known copybook dir exists. + */ +export function resolveCobolCopyTarget( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, +): string | null { + const upper = targetRaw.toUpperCase(); + const index = getCobolCopyIndex(allFilePaths); + return ( + pickCopyPath(index.copybooks.get(upper), fromFile, index.hasPreferredDir) ?? + pickCopyPath(index.sources.get(upper), fromFile, index.hasPreferredDir) ?? + null + ); +} diff --git a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts index cfea8f39e..3d9e0eea6 100644 --- a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts @@ -9,74 +9,12 @@ * Reference: `languages/python/scope-resolver.ts`. */ -import path from 'node:path'; import type { ParsedFile } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; -import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; import { cobolProvider } from '../cobol.js'; - -// Copybook file extensions for COPY name resolution -const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); -// COBOL source files, searched only after every copybook has missed. -const COBOL_SOURCE_EXTENSIONS = new Set(['.cbl', '.cob', '.cobol']); - -/** - * Uppercased-basename → first file carrying it, one map PER TIER, memoized on - * the `allFilePaths` Set identity (#2908). - * - * `resolveImportTarget` used to run two full workspace scans per `COPY` — one - * for the copybook tier, one for the source tier — each calling `path.extname` - * + `path.basename` + `toUpperCase` on every entry. A `COPY` of a member that - * lives outside the repo (the common case: vendor and system copybooks) missed - * in both, so both scans always ran to completion, making resolution - * O(copies × files). The orchestrator passes the SAME Set to every import in a - * pass (`pipeline/run.ts` builds it once), so a `WeakMap` keyed on that Set - * turns the scans into one build per run. - * - * Two tiers rather than one map is the tie-break, not a stylistic choice: a - * `.cpy`/`.copybook` hit beats a `.cbl`/`.cob`/`.cobol` hit even when the source - * file comes FIRST in Set-iteration order, which is exactly what collapsing the - * tiers into a single first-wins map would silently discard. Within a tier the - * first file in Set-iteration order wins, mirroring the `return` on first match - * in the scans this replaces. - * - * The per-file key is derived with the same `path.extname(fp).toLowerCase()` → - * `path.basename(fp, ext)` → `toUpperCase()` sequence the scans used, including - * its quirk: `path.basename` strips the suffix only on an exact, case-sensitive - * match, so `Foo.CPY` indexes under `FOO.CPY` rather than `FOO`. Node's `path` - * stays in the loop for the same reason — on POSIX it does not treat `\` as a - * separator, and hand-rolled slicing on `/` would start resolving backslash - * paths the scans never resolved. - */ -interface CobolCopyIndex { - /** `.cpy` / `.copybook` files — tier 1. */ - readonly copybooks: ReadonlyMap; - /** `.cbl` / `.cob` / `.cobol` files — tier 2. */ - readonly sources: ReadonlyMap; -} - -const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { - const copybooks = new Map(); - const sources = new Map(); - // One pass builds both tiers: the two scans walked the same files and - // classified each by the same extension test. - for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - const tier = COPYBOOK_EXTENSIONS.has(ext) - ? copybooks - : COBOL_SOURCE_EXTENSIONS.has(ext) - ? sources - : undefined; - if (tier === undefined) continue; - const basename = path.basename(fp, ext).toUpperCase(); - // First in Set-iteration order wins, as the scans' first-match `return` did. - if (!tier.has(basename)) tier.set(basename, fp); - } - - return { copybooks, sources }; -}); +import { resolveCobolCopyTarget } from './copy-target.js'; const cobolScopeResolver: ScopeResolver = { language: SupportedLanguages.Cobol, @@ -84,11 +22,9 @@ const cobolScopeResolver: ScopeResolver = { importEdgeReason: 'cobol-scope: copy', // ── Resolve COPY bookname to file path ───────────────────────────── - resolveImportTarget: (targetRaw, _fromFile, allFilePaths) => { - const upper = targetRaw.toUpperCase(); - const index = getCobolCopyIndex(allFilePaths); - // Copybooks first, then COBOL sources — the tier order IS the tie-break. - return index.copybooks.get(upper) ?? index.sources.get(upper) ?? null; + // Shared with the regex processor's resolveCopy (#2967 lockstep). + resolveImportTarget: (targetRaw, fromFile, allFilePaths) => { + return resolveCobolCopyTarget(targetRaw, fromFile, allFilePaths); }, // COBOL has no binding-merge rules beyond the default (local-first-then-imports). diff --git a/gitnexus/src/core/ingestion/languages/dart/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/dart/name-fallback-visibility.ts new file mode 100644 index 000000000..562311bb8 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/dart/name-fallback-visibility.ts @@ -0,0 +1,77 @@ +/** + * Dart's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Dart's privacy is LIBRARY-scoped and marked in the identifier itself: a name + * beginning with `_` is visible only inside its own library and cannot be + * imported by any spelling. So a `_`-prefixed candidate in another file is an + * impossible call, not an unlikely one. + * + * The exact boundary is `part` / `part of` — one library spanning several + * files, with `_` names shared between them — and the extractor does not + * surface `part` directives yet (the Dart query captures only `library_import`; + * nothing in `languages/dart/` reads `part`). Without them a cross-file `_` + * candidate is UNDECIDABLE, not impossible: refusing on "different file" would + * delete real edges on Flutter's dominant generated-code idiom (`factory + * Foo.fromJson(j) => _$FooFromJson(j)` calls into `foo.g.dart`, a `part` beside + * it), and refusing on "different directory" is wrong too — a `part` URI is a + * relative URI and legally traverses directories (`part '../shared/gen.dart';`). + * An earlier version refused the cross-directory case as "no `part` layout can + * make this legal"; that claim was false, so the hook now REFUSES NOTHING and + * every cross-file `_` candidate stays a LABELED edge (0.5 / + * `global-name-fallback`), which is the honest answer until `part` is + * extracted. A caller that names the candidate's file in a directive is + * recognized already, for the day the extractor surfaces `part` as an import + * target; at that point "not the same library" becomes decidable and the + * refusal can return. + * + * Public names are left to the labeled-edge path. Dart does require an import + * for a cross-library public name, but the fallback exists partly to recover + * edges where the import chain was not reconstructed, and refusing every + * cross-file public call would delete real edges to buy a rule the `_` marker + * already gives for free. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { + modulePathReaches, + stripExtension, +} from '../../scope-resolution/utils/name-fallback-visibility.js'; + +/** Dart privacy marker: a leading underscore on the declared identifier. Read + * from the last `qualifiedName` segment, so `_Foo.bar` is public `bar` on a + * private class and `Foo._bar` is the private member. */ +function isPrivateDartName(candidate: SymbolDefinition): boolean { + const qualified = candidate.qualifiedName ?? ''; + const dot = qualified.lastIndexOf('.'); + const simple = dot === -1 ? qualified : qualified.slice(dot + 1); + return simple.startsWith('_'); +} + +/** + * Are these two files parts of one library? True when the caller names the + * candidate's file in a `part` / `part of` directive, which the extractor + * surfaces as an ordinary import target. + */ +function sharesLibrary(callerParsed: ParsedFile, candidateFilePath: string): boolean { + const candidateModule = stripExtension(candidateFilePath); + for (const imp of callerParsed.parsedImports) { + if (modulePathReaches(stripExtension(imp.targetRaw), candidateModule)) return true; + } + return false; +} + +export function dartIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + if (!isPrivateDartName(ctx.candidate)) return true; + // A directive naming the candidate's file is positive evidence of one library. + if (sharesLibrary(ctx.callerParsed, ctx.candidate.filePath)) return true; + // Any other file may be a `part` of the caller's library — a sibling or, via + // a relative `part` URI, a file in another directory. Undecidable without + // `part` extraction (see the header), so allowed as a labeled guess, never + // refused. + return true; +} diff --git a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts index 76bec5d74..bc46ad452 100644 --- a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts @@ -41,6 +41,7 @@ import { decodeMarker } from '../../utils/heritage-marker.js'; import { typeApplicationArguments } from '../../utils/template-arguments.js'; import type { HeritageTypeArgumentSink } from '../../scope-resolution/utils/generic-instantiation.js'; import { expandDartWildcardNames } from './expand-wildcards.js'; +import { dartIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; interface ClassDefRef { readonly graphId: string; @@ -233,5 +234,6 @@ export const dartScopeResolver: ScopeResolver = { // No `new`: bare `Foo()` resolves to the type; with cross-file imports the // callee is reachable workspace-wide. allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: dartIsGlobalNameFallbackPlausible, constructorCallTargetsClass: true, }; diff --git a/gitnexus/src/core/ingestion/languages/go/captures.ts b/gitnexus/src/core/ingestion/languages/go/captures.ts index c1a8cf23c..97fe71d83 100644 --- a/gitnexus/src/core/ingestion/languages/go/captures.ts +++ b/gitnexus/src/core/ingestion/languages/go/captures.ts @@ -508,11 +508,24 @@ function normalizeGenericConstructorCapture( if (referenceNode !== undefined && referenceNode.type === 'generic_type') { const base = referenceNode.childForFieldName('type'); if (base !== null) { - grouped['@reference.name'] = syntheticCapture( - '@reference.name', - base, - extractSimpleTypeNameText(base), - ); + const simple = extractSimpleTypeNameText(base); + // Keep the written `pkg.Box[T]` spelling so constructor resolution + // can tell a package-qualified type from a bare unique-name guess. + // `Box[T]` is also `text !== simple` (`Box[T]` vs `Box`) — that is + // not a qualifier. Only a `qualified_type` base (`pkg.Box`) is. + if ( + grouped['@reference.qualified-name'] === undefined && + base.type === 'qualified_type' && + referenceNode.text.length > 0 && + referenceNode.text !== simple + ) { + grouped['@reference.qualified-name'] = syntheticCapture( + '@reference.qualified-name', + referenceNode, + referenceNode.text, + ); + } + grouped['@reference.name'] = syntheticCapture('@reference.name', base, simple); } } } diff --git a/gitnexus/src/core/ingestion/languages/go/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/go/name-fallback-visibility.ts new file mode 100644 index 000000000..02b968e7b --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/go/name-fallback-visibility.ts @@ -0,0 +1,126 @@ +/** + * Go's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Go's visibility rules are unusually decidable from a file path and an + * identifier, which is why this is the language the guard is most complete for: + * + * 1. A package IS a directory. Two files in the same directory see each + * other's identifiers with no import and no qualification, so a same- + * directory candidate is always plausible. + * 2. An identifier is exported iff it begins with an upper-case letter. An + * UNEXPORTED identifier is invisible outside its own package — no import + * makes it reachable, so a cross-directory candidate with a lower-case + * initial is not unlikely, it is IMPOSSIBLE. + * 3. A bare exported identifier from another package requires a DOT import. + * Ordinary, alias, and blank imports never introduce a bare callable. + * + * Rule 2 is the one that matters most in practice: `uniqueHelperXyz` defined + * once in package `a` used to acquire a caller in package `b` purely because + * the name was unique in the repo, and the resulting edge was published as + * `import-resolved` — a caller Go itself would reject. + * + * Methods require a receiver and never qualify for this bare-name tier. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { + modulePathReaches, + directoryOf, +} from '../../scope-resolution/utils/name-fallback-visibility.js'; +import { inferGoPackageName } from './package-clause.js'; + +/** + * Compare actual package clauses, not a guessed `_test` suffix convention: + * `foo_test` can itself be a production package name. Missing source remains + * undecidable, while the test-file boundary is always available from the path. + */ +function classifyGoFile( + filePath: string, + sourceTextOf: ((p: string) => string | undefined) | undefined, +): { isTest: boolean; declared: string | undefined } { + const isTest = filePath.endsWith('_test.go'); + const text = sourceTextOf?.(filePath); + return { + isTest, + declared: text === undefined ? undefined : (inferGoPackageName(text) ?? undefined), + }; +} + +/** Go export rule: an upper-case initial, by Unicode letter case. */ +function isExportedGoName(name: string): boolean { + return /^\p{Lu}/u.test(name); +} + +/** + * The simple identifier a Go declaration contributes to its package scope: the + * last segment of `qualifiedName`, which is `Type.method` for a method and the + * bare identifier for a function. Either way the LAST segment is the identifier + * whose case decides export. + */ +function goSimpleName(candidate: SymbolDefinition): string { + const qualified = candidate.qualifiedName ?? ''; + const dot = qualified.lastIndexOf('.'); + return dot === -1 ? qualified : qualified.slice(dot + 1); +} + +export function goIsGlobalNameFallbackPlausible(ctx: { + readonly sourceTextOf?: (filePath: string) => string | undefined; + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly site?: { readonly rawQualifiedName?: string }; +}): boolean { + // A PATH-QUALIFIED call (`pkg.Export()`) reached this tier because the + // qualifier could not be followed. Qualification removes the bare-name + // "only a dot import introduces this identifier" rule — not export or + // test-package visibility. `other.private()` is still impossible. + const isPackageQualified = ctx.site?.rawQualifiedName !== undefined; + // Methods require a receiver, even within their own package or a dot import. + if (ctx.candidate.type === 'Method' || ctx.candidate.qualifiedName?.includes('.')) return false; + const callerDir = directoryOf(ctx.callerParsed.filePath); + const candidateDir = directoryOf(ctx.candidate.filePath); + // Same directory is NOT the same package (rule 1, refined): a `_test.go` + // file may declare `foo_test`, and test-only declarations are invisible to + // non-test files. Apply the split `package-siblings.ts` uses for the + // confident tier, so the heuristic tier cannot reopen what it closed. + if (callerDir === candidateDir) { + const caller = classifyGoFile(ctx.callerParsed.filePath, ctx.sourceTextOf); + const cand = classifyGoFile(ctx.candidate.filePath, ctx.sourceTextOf); + // Non-test files never see test-only declarations. + if (cand.isTest && !caller.isTest) return false; + // Different clauses require an explicit dot import, even in one directory. + // Fall through to the exported/import checks for external test packages. + // Missing source leaves package identity undecidable. + if ( + caller.declared === undefined || + cand.declared === undefined || + caller.declared === cand.declared + ) + return true; + } + + // Different package — and a `_test.go` file's + // declarations are compiled only into ITS OWN package's test binary. No other + // package, test or not, can see them, exported or not. Decidable from the + // path alone, so it comes before every exception below (the module-root + // exception in particular used to accept a root `helper_test.go` export). + if (classifyGoFile(ctx.candidate.filePath, ctx.sourceTextOf).isTest) return false; + + const simpleName = goSimpleName(ctx.candidate); + // No identifier to read the case of — an unanswered question, not a refusal. + if (simpleName === '') return true; + // Unexported across a package boundary (rule 2): no import can reach it. + if (!isExportedGoName(simpleName)) return false; + if (isPackageQualified) return true; + + // Only a dot import introduces a bare name. A candidate in the module ROOT package has an empty + // directory, which `modulePathReaches` cannot align against any import path + // (the root package is imported by the module path alone, which the repo- + // relative layout does not carry). That is an unanswered question, not a + // refusal — a dot import is plausible even if its path cannot be aligned. + return ctx.callerParsed.parsedImports.some( + (imp) => + imp.kind === 'wildcard' && + (candidateDir === '' || modulePathReaches(imp.targetRaw, candidateDir)), + ); +} diff --git a/gitnexus/src/core/ingestion/languages/go/package-siblings.ts b/gitnexus/src/core/ingestion/languages/go/package-siblings.ts index bb5c4cab8..4efe57f82 100644 --- a/gitnexus/src/core/ingestion/languages/go/package-siblings.ts +++ b/gitnexus/src/core/ingestion/languages/go/package-siblings.ts @@ -4,92 +4,189 @@ import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexe import { expandGoDotImports } from './expand-wildcards.js'; import { goPackageDir, inferGoPackageName } from './package-clause.js'; +function isGoTestFile(filePath: string): boolean { + return filePath.endsWith('_test.go'); +} + /** - * O(n²×d) where n = files per package, d = defs per file. - * Acceptable for V1 since Go packages are typically small (< 20 files). - * Future optimization: build a name→def inverted index per package to reduce - * to O(n×d). + * The package a `_test.go` file's clause belongs to, given the package names + * the directory's NON-test files declare. + * + * `package foo_test` is the external-test convention ONLY when the directory's + * real package is `foo`; the `_test` suffix is otherwise a legal identifier + * (`package foo_test` in a directory whose non-test files also say `foo_test`). + * Stripping it unconditionally keyed such a package's own internal tests as + * external tests of a non-existent `foo`, so they saw no sibling at all — a + * resolution miss on every same-package call. Strip only when the stripped + * name is what the non-test siblings declare; with no non-test sibling to ask + * (a test-only directory) the convention is assumed, as before. + */ +function testFilePackageOf( + declared: string, + nonTestPackagesInDir: ReadonlySet | undefined, +): { readonly pkg: string; readonly external: boolean } { + if (!declared.endsWith('_test') || declared.length <= '_test'.length) { + return { pkg: declared, external: false }; + } + const stripped = declared.slice(0, -'_test'.length); + if (nonTestPackagesInDir !== undefined && nonTestPackagesInDir.has(declared)) { + return { pkg: declared, external: false }; + } + if (nonTestPackagesInDir === undefined || nonTestPackagesInDir.has(stripped)) { + return { pkg: stripped, external: true }; + } + // Neither name is declared by a non-test sibling: keep the clause as written. + return { pkg: declared, external: false }; +} + +interface IndexedDef { + readonly filePath: string; + readonly ref: BindingRef; +} + +/** name → defs, in package file order. */ +type NameIndex = Map; + +function defBareName(def: SymbolDefinition): string { + return def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; +} + +function appendToIndex( + index: NameIndex, + filePath: string, + defs: readonly SymbolDefinition[], +): void { + for (const def of defs) { + const name = defBareName(def); + if (name === '') continue; + const list = index.get(name) ?? []; + list.push({ filePath, ref: { def, origin: 'namespace' } }); + index.set(name, list); + } +} + +function publishIndex( + augmentations: Map>, + index: NameIndex, + receiverPath: string, + receiverModule: ScopeId, +): void { + if (index.size === 0) return; + let scopeBindings = augmentations.get(receiverModule); + if (scopeBindings === undefined) { + scopeBindings = new Map(); + augmentations.set(receiverModule, scopeBindings); + } + for (const [name, entries] of index) { + let bucket = scopeBindings.get(name); + const seen = + bucket === undefined ? new Set() : new Set(bucket.map((b) => b.def.nodeId)); + for (const entry of entries) { + if (entry.filePath === receiverPath) continue; + if (seen.has(entry.ref.def.nodeId)) continue; + if (bucket === undefined) { + bucket = []; + scopeBindings.set(name, bucket); + } + bucket.push(entry.ref); + seen.add(entry.ref.def.nodeId); + } + } +} + +/** + * Publish same-package sibling bindings, including `_test.go` files. + * + * Internal tests (`package foo`) see production and other internal-test names. + * External tests (`package foo_test`) get no bare-name bindings across that + * partition — qualified `foo.X` and `import .` stay on the import resolver. + * Non-test files never see test-only helpers (`go build` does not compile them). + * + * Per-package name→def indexes are built in O(n×d). Each receiver walks only + * the partitions it can see, so defs are not re-scanned against every sibling + * file. Binding refs are allocated once and reused across receivers. */ export function populateGoPackageSiblings( parsedFiles: readonly ParsedFile[], indexes: ScopeResolutionIndexes, ctx: { readonly fileContents: ReadonlyMap }, ): void { - // 0. Filter out test files — Go _test.go files should not contribute - // same-package sibling bindings to non-test files. - const nonTestFiles = parsedFiles.filter((f) => !f.filePath.endsWith('_test.go')); - // 1. Expand dot imports first so subsequent same-package sibling - // augmentation can also see dot-imported names. - expandGoDotImports(nonTestFiles, indexes); + // augmentation can also see dot-imported names. Test files dot-import too. + expandGoDotImports(parsedFiles, indexes); // 2. Group files by package directory plus package name. Go package // identity is directory-scoped; repeated `package main` directories // must not see each other's unqualified names. - const packageByFile = new Map(); - for (const parsed of nonTestFiles) { - // Same derivation as `populateGoWorkspaceOwners` — one shared resolver, so - // the two passes cannot disagree about a file's package (#2837). The - // no-clause case is reported there; warning twice for one fact would be - // noise. - const pkgName = inferGoPackageName(ctx.fileContents.get(parsed.filePath) ?? ''); - if (pkgName !== null) { - packageByFile.set(parsed.filePath, `${goPackageDir(parsed.filePath)}\0${pkgName}`); - } + // + // `_test.go` files join the INTERNAL package's bucket (a `foo_test` + // external test is keyed by `foo`, its `external` flag marking the + // partition so no bare-name bindings cross it), so one bucket holds + // everything the test binary compiles together, and the visibility + // rules below decide who sees whom. + interface SiblingFile { + readonly filePath: string; + readonly defs: readonly SymbolDefinition[]; + readonly isTest: boolean; + readonly external: boolean; + } + const filesByPackage = new Map(); + // Same derivation as `populateGoWorkspaceOwners` — one shared resolver, so + // the two passes cannot disagree about a file's package (#2837). The + // no-clause case is reported there; warning twice for one fact would be + // noise. + const declaredByFile = new Map(); + const nonTestPackagesByDir = new Map>(); + for (const parsed of parsedFiles) { + const declared = inferGoPackageName(ctx.fileContents.get(parsed.filePath) ?? ''); + if (declared === null) continue; + declaredByFile.set(parsed.filePath, declared); + if (isGoTestFile(parsed.filePath)) continue; + const dir = goPackageDir(parsed.filePath); + const names = nonTestPackagesByDir.get(dir) ?? new Set(); + names.add(declared); + nonTestPackagesByDir.set(dir, names); + } + for (const parsed of parsedFiles) { + const declared = declaredByFile.get(parsed.filePath); + if (declared === undefined) continue; + const isTest = isGoTestFile(parsed.filePath); + const dir = goPackageDir(parsed.filePath); + const { pkg, external } = isTest + ? testFilePackageOf(declared, nonTestPackagesByDir.get(dir)) + : { pkg: declared, external: false }; + const key = `${dir}\0${pkg}`; + const list = filesByPackage.get(key) ?? []; + list.push({ filePath: parsed.filePath, defs: parsed.localDefs, isTest, external }); + filesByPackage.set(key, list); } - const filesByPackage = new Map(); - for (const parsed of nonTestFiles) { - const pkgName = packageByFile.get(parsed.filePath); - if (pkgName === undefined) continue; - const list = filesByPackage.get(pkgName) ?? []; - list.push({ filePath: parsed.filePath, defs: [...parsed.localDefs] }); - filesByPackage.set(pkgName, list); - } - - // 2. Use bindingAugmentations channel per I8 + // 3. Use bindingAugmentations channel per I8. Same-package files see ALL + // sibling names (exported and unexported). Cross-package visibility is + // the import resolver's job. const augmentations = indexes.bindingAugmentations as Map>; - for (const [, siblings] of filesByPackage) { - for (const target of siblings) { - const targetModule = indexes.moduleScopes.byFilePath.get(target.filePath); - if (targetModule === undefined) continue; - - for (const receiver of siblings) { - if (receiver.filePath === target.filePath) continue; // no self-reference - const receiverModule = indexes.moduleScopes.byFilePath.get(receiver.filePath); - if (receiverModule === undefined) continue; - - for (const def of target.defs) { - // Go: same-package sibling files can see ALL names (both - // exported/uppercase and unexported/lowercase). Only cross- - // package visibility requires uppercase first letter. - const name = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; - if (name === '') continue; - - const bucket = getAugmentationBucket(augmentations, receiverModule, name); - if (bucket.some((b) => b.def.nodeId === def.nodeId)) continue; - bucket.push({ def, origin: 'namespace' }); - } + for (const siblings of filesByPackage.values()) { + if (siblings.length < 2) continue; + const production: NameIndex = new Map(); + const internalTest: NameIndex = new Map(); + const external: NameIndex = new Map(); + const receivers: { file: SiblingFile; module: ScopeId }[] = []; + for (const file of siblings) { + const module = indexes.moduleScopes.byFilePath.get(file.filePath); + if (module === undefined) continue; + receivers.push({ file, module }); + if (file.external) appendToIndex(external, file.filePath, file.defs); + else if (file.isTest) appendToIndex(internalTest, file.filePath, file.defs); + else appendToIndex(production, file.filePath, file.defs); + } + for (const { file, module } of receivers) { + if (file.external) { + publishIndex(augmentations, external, file.filePath, module); + continue; } + publishIndex(augmentations, production, file.filePath, module); + if (file.isTest) publishIndex(augmentations, internalTest, file.filePath, module); } } } - -function getAugmentationBucket( - augmentations: Map>, - scopeId: ScopeId, - name: string, -): BindingRef[] { - let scopeBindings = augmentations.get(scopeId); - if (scopeBindings === undefined) { - scopeBindings = new Map(); - augmentations.set(scopeId, scopeBindings); - } - let bucketArr = scopeBindings.get(name); - if (bucketArr === undefined) { - bucketArr = []; - scopeBindings.set(name, bucketArr); - } - return bucketArr; -} diff --git a/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts index ff891d079..7c03e10f6 100644 --- a/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts @@ -16,6 +16,7 @@ import { detectGoInterfaceImplementations } from './interface-impls.js'; import { populateGoRangeBindings } from './range-binding.js'; import { expandGoWildcardNames } from './expand-wildcards.js'; import { goMapValueType } from './interpret.js'; +import { goIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; /** Slice `[]T` and array `[N]T` / `[...]T` → the element spelling. Hoisted — * a literal inside the hook would mint a fresh RegExp per folded subscript. */ @@ -85,6 +86,7 @@ export const goScopeResolver: ScopeResolver = { hoistTypeBindingsToModule: true, propagatesReturnTypesAcrossImports: true, allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: goIsGlobalNameFallbackPlausible, populateNamespaceSiblings: populateGoPackageSiblings, mirrorNamespaceTypeBindings: mirrorGoNamespaceTypeBindings, diff --git a/gitnexus/src/core/ingestion/languages/java/captures.ts b/gitnexus/src/core/ingestion/languages/java/captures.ts index 5d31aed4a..b227d549c 100644 --- a/gitnexus/src/core/ingestion/languages/java/captures.ts +++ b/gitnexus/src/core/ingestion/languages/java/captures.ts @@ -357,6 +357,26 @@ export function emitJavaScopeCaptures( } } + // Qualified constructor `new pkg.Foo()` already binds the simple tail as + // `@reference.name` (F35). Copy the scoped spelling onto + // `@reference.qualified-name` so constructor resolution can treat it as a + // written qualifier instead of a bare unique-name guess (C# does the same). + const ctorQualifiedNode = nodeMap['@reference.call.constructor.qualified']; + const ctorQualifiedText = grouped['@reference.call.constructor.qualified']?.text; + if ( + ctorQualifiedNode !== undefined && + ctorQualifiedText !== undefined && + ctorQualifiedText.length > 0 && + ctorQualifiedText !== grouped['@reference.name']?.text && + grouped['@reference.qualified-name'] === undefined + ) { + grouped['@reference.qualified-name'] = syntheticCapture( + '@reference.qualified-name', + ctorQualifiedNode, + ctorQualifiedText, + ); + } + // Synthesize `@reference.arity` on every callsite. const callTag = ( ['@reference.call.free', '@reference.call.member', '@reference.call.constructor'] as const diff --git a/gitnexus/src/core/ingestion/languages/javascript/captures.ts b/gitnexus/src/core/ingestion/languages/javascript/captures.ts index d3ba94791..b16cb2ba1 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/captures.ts @@ -36,6 +36,7 @@ import { syntheticCapture, type SyntaxNode, } from '../../utils/ast-helpers.js'; +import { collectEsmExportEvidence, esmExportVerdict } from '../../ts-js-export-marker.js'; import { splitImportStatement } from '../typescript/import-decomposer.js'; import { getJsParser, getJsScopeQuery, jsCachedTreeMatchesGrammar } from './query.js'; import { computeTsArityMetadata } from '../typescript/arity-metadata.js'; @@ -983,6 +984,8 @@ export function emitJsScopeCaptures( } const rawMatches = getJsScopeQuery(filePath).matches(tree.rootNode); + // Export evidence, read once per file (see `ts-js-export-marker.ts`). + const exportEvidence = collectEsmExportEvidence(tree.rootNode, filePath); const out: CaptureMatch[] = []; for (const m of rawMatches) { @@ -1124,6 +1127,13 @@ export function emitJsScopeCaptures( fnNode, deriveDefaultExportHocName(filePath), ); + // This declaration's name is synthetic, so the later query-name + // marker cannot see it. The HOC predicate already proves the export. + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + fnNode, + 'true', + ); } } @@ -1207,6 +1217,20 @@ export function emitJsScopeCaptures( // non-call match, an absent receiver, or a chain with no nameable base // all leave `grouped` untouched. synthesizeReceiverChainCapture(grouped, groupedNodes['@reference.receiver']); + // `@declaration.is-exported`: a verdict for every declaration the file's + // export surface can decide (see `ts-js-export-marker.ts`); nothing where + // it cannot, because absence is the honest answer there. + const declNameNode = groupedNodes['@declaration.name']; + if (exportEvidence !== undefined && declNameNode !== undefined) { + const verdict = esmExportVerdict(declNameNode, exportEvidence); + if (verdict !== undefined) { + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + declNameNode, + verdict ? 'true' : 'false', + ); + } + } out.push(grouped); // Synthesize `this` receiver type-bindings on class member functions. diff --git a/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts index 8d74798ee..280fd9929 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts @@ -97,6 +97,12 @@ const javascriptScopeResolver: ScopeResolver = { // explicit imports at the call site. Workspace-wide unique-name fallback // recovers these edges. allowGlobalFreeCallFallback: true, + + // Same ECMAScript `export *` exclusivity as TypeScript: a name declared by + // two wildcard sources is refused, not guessed. + exclusiveWildcardReexports: true, + // Same ECMAScript rule as TypeScript: named imports bind module-level declarations only. + namedImportsBindTopLevelOnly: true, }; export { javascriptScopeResolver }; diff --git a/gitnexus/src/core/ingestion/languages/ruby/import-target.ts b/gitnexus/src/core/ingestion/languages/ruby/import-target.ts index 62d6fca3d..eae1f56e3 100644 --- a/gitnexus/src/core/ingestion/languages/ruby/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/ruby/import-target.ts @@ -3,13 +3,19 @@ * * Ruby import resolution rules: * - `require_relative './foo'` → resolve relative to the importing file's dir - * - `require 'foo'` → suffix-match via the existing Ruby import resolver - * - External gems → null (unresolvable within the repo) + * - `require 'foo'` → use scoped gem metadata before legacy suffix matching + * - Known local gems → resolve only within their declared load roots + * - Known external gems → null, even if an unrelated repo file suffix matches */ import { resolveRubyImportInternal } from '../../import-resolvers/ruby.js'; import { getWorkspaceFileIndex } from '../../import-resolvers/workspace-file-index.js'; import { isHeritageMarker } from '../../utils/heritage-marker.js'; +import { + findRubyResolutionScope, + type RubyResolutionConfig, + type RubyResolutionScope, +} from './resolution-config.js'; export interface RubyResolveContext { readonly fromFile: string; @@ -26,16 +32,17 @@ export interface RubyResolveContext { * against the importing file's directory, trying `.rb` and `/index.rb` * suffixes. * - * For bare requires (gem-style like `'json'`, `'serializable'`), delegates - * to the existing `resolveRubyImportInternal` which uses suffix matching. - * - * Returns `null` for external gems that have no matching file in the repo. + * For bare requires, scoped manifest metadata takes precedence: known local + * gems resolve only within their declared load roots (a miss returns `null`), + * and known external gem prefixes return `null` even if a repo suffix matches. + * Only requires without matching gem evidence delegate to the existing + * `resolveRubyImportInternal` suffix matcher. */ export function resolveRubyImportTarget( targetRaw: string, fromFile: string, allFilePaths: ReadonlySet, - _resolutionConfig?: unknown, + resolutionConfig?: unknown, ): string | readonly string[] | null { if (!targetRaw) return null; if (isHeritageMarker(targetRaw)) return null; @@ -51,7 +58,16 @@ export function resolveRubyImportTarget( return resolved; } - // ── require: bare/gem-style suffix matching ───────────────────────── + // ── require: scoped gem evidence before repository-wide fallback ──── + const config = resolutionConfig as RubyResolutionConfig | null | undefined; + const scope = + config === null || config === undefined ? undefined : findRubyResolutionScope(config, fromFile); + if (scope !== undefined) { + const localGemTarget = resolveLocalGemTarget(targetRaw, scope, allFilePaths); + if (localGemTarget !== undefined) return localGemTarget; + if (matchesRequirePrefix(targetRaw, scope.externalRequirePrefixes)) return null; + } + return resolveBare(targetRaw, allFilePaths); } @@ -95,6 +111,60 @@ function resolveRelative( return null; } +/** + * Yield the require stem, then each slash-delimited ancestor. This makes both + * local-root and external-gem lookup O(require path depth), not O(gem count). + * A trailing `.rb` is stripped first so `require 'my_engine.rb'` matches the + * configured prefix `my_engine`, matching Ruby's optional-suffix require. + */ +function requirePrefixCandidates(targetRaw: string): readonly string[] { + const stem = targetRaw.endsWith('.rb') ? targetRaw.slice(0, -3) : targetRaw; + if (stem.length === 0) return []; + const candidates = [stem]; + let slash = stem.lastIndexOf('/'); + while (slash !== -1) { + candidates.push(stem.slice(0, slash)); + slash = stem.lastIndexOf('/', slash - 1); + } + return candidates; +} + +function matchesRequirePrefix(targetRaw: string, prefixes: ReadonlySet): boolean { + return requirePrefixCandidates(targetRaw).some((candidate) => prefixes.has(candidate)); +} + +/** + * Resolve a path/gemspec-backed gem against its declared Ruby load roots. + * `undefined` means no local-gem prefix matched; `null` means one matched but + * none of its declared load roots contained the target. + */ +function resolveLocalGemTarget( + targetRaw: string, + scope: RubyResolutionScope, + allFilePaths: ReadonlySet, +): string | null | undefined { + for (const prefix of requirePrefixCandidates(targetRaw)) { + const loadRoots = scope.localLoadRootsByPrefix.get(prefix); + if (loadRoots === undefined) continue; + + for (const loadRoot of loadRoots) { + const base = loadRoot ? `${loadRoot}/${targetRaw}` : targetRaw; + if (base.endsWith('.rb')) { + if (allFilePaths.has(base)) return base; + continue; + } + const rbFile = `${base}.rb`; + if (allFilePaths.has(rbFile)) return rbFile; + } + + // The prefix is owned by a known local gem. If its declared roots do not + // contain the target, repository-wide suffix matching would fabricate an + // edge to an unrelated file. + return null; + } + return undefined; +} + /** * Resolve a bare require path (`'serializable'`, `'json'`, `'net/http'`) * via suffix matching using the existing Ruby import resolver. diff --git a/gitnexus/src/core/ingestion/languages/ruby/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/ruby/name-fallback-visibility.ts new file mode 100644 index 000000000..4b225c96f --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/ruby/name-fallback-visibility.ts @@ -0,0 +1,196 @@ +/** + * Ruby's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Ruby keeps the labeled fallback deliberately: with autoload (Rails' zeitwerk, + * `ActiveSupport::Dependencies`) a file genuinely can call a method whose + * defining file it never requires, so "no require" is NOT evidence of + * impossibility the way it is in Go or Rust. + * + * What IS decidable is namespacing — but only for CLASS bodies. A method + * defined inside a `class` is not callable as a bare `helper()` from another + * file unless that file names the class somehow (a receiver `Ns.helper`, a + * subclass declaration, an `include`, a `require`). A method defined inside a + * `module` body is different in kind: modules exist to be mixed in, and Rails + * mixes them in for you — every `*Helper` module is included into views and + * controllers by the framework, and concerns arrive through `included do` + * hooks — so a bare `format_money()` in a view legitimately reaches + * `module ApplicationHelper` with no `include`, `require`, or constant + * anywhere in the caller. Refusing that would delete real edges on exactly the + * codebase shape Ruby's fallback exists to serve. So module-owned methods stay + * a LABELED guess, and the refusal targets CLASS-owned methods whose class the + * caller never names. A top-level method (`ownerId === undefined`) is left + * alone, since that is the shape autoload actually delivers. When the owner + * cannot be found or typed (no `parsedFileOf`, owner outside the file set), + * the question is unanswered and the edge is allowed. + * + * "Never names" is read from the caller's own text-visible signals — its + * `require`/`include`/`extend` targets, and any reference site spelling the + * namespace's constant. If any of them mentions the namespace, the call is + * plausible and the labeled edge stands. `require` paths are snake_case + * (`billing/invoice_service`) while constants are CamelCase + * (`Billing::InvoiceService`), so the comparison normalizes both sides — + * without that the `require` branch never matched anything. + * + * One more thing a caller file can do without naming the class: INHERIT its + * way to it. `class UsersController < AdminController` reaches every method + * `ApplicationController` defines while naming only `AdminController`, and an + * `include Concern` reaches whatever that concern includes in turn. Neither + * chain is decidable from one file, so a caller with ANY inheritance or mixin + * surface (an `inherits` site, an `include`/`extend`/`prepend` marker) is + * treated as plausible. So is a caller file that DEFINES a module: a module's + * methods run against whatever class includes the module, and call that class's + * methods bare — `module PostGuardian; def can_see?; is_staff? ...` reaches + * `class Guardian#is_staff?` because Guardian includes PostGuardian, a fact the + * module file never states. What remains refused is the shape Ruby itself + * rejects: a bare call to a class's method from a file that inherits nothing, + * mixes in nothing, defines no module, and never spells the class. Measured on + * discourse@3f71fa15c that is ~1000 refusals, and the sample is what the rule + * promises: RSpec `before`/`after`/`subject` guessed to serializer methods of + * the same name, `Gemfile`'s `gem` to `Plugin::Instance#gem`, `routes.rb`'s + * `get` to `Draft#get` — fabricated callers, every one. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { moduleSegments } from '../../scope-resolution/utils/name-fallback-visibility.js'; +import { HERITAGE_MARKER_PREFIX } from '../../utils/heritage-marker.js'; + +/** + * The namespace segments a candidate's qualified name declares, minus the + * method itself. `Billing::Invoice#total` / `Billing.Invoice.total` → the + * `Billing`, `Invoice` constants a caller would have to name. + */ +function namespaceConstantsOf(candidate: SymbolDefinition): readonly string[] { + const qualified = candidate.qualifiedName; + if (qualified === undefined || qualified === '') return []; + const segments = qualified + .split(/::|\.|#/) + .map((s) => s.trim()) + .filter((s) => s !== ''); + // Drop the trailing method name; what remains is the namespace chain. + const namespace = segments.slice(0, -1); + // Only CONSTANTS name a Ruby namespace (upper-case initial). A lower-case + // segment is a receiver expression, not a namespace a caller can mention. + return namespace.filter((s) => /^[A-Z]/.test(s)); +} + +/** + * `billing/invoice_service` vs `InvoiceService`: a path segment names a + * constant when, with underscores removed, the two are equal case-insensitively. + * Zeitwerk's own inflection rule, minus acronym overrides — over-matching here + * only loses a refusal (the edge stays, labeled), which is the safe direction. + */ +function pathSegmentNamesConstant(segment: string, constant: string): boolean { + return segment.replace(/_/g, '').toLowerCase() === constant.toLowerCase(); +} + +function requireReachesConstant(targetRaw: string, constant: string): boolean { + for (const segment of moduleSegments(targetRaw)) { + if (pathSegmentNamesConstant(segment, constant)) return true; + } + return false; +} + +/** + * The declared kind of the candidate's owner, read from the candidate's own + * file. Ruby labels `class` bodies `Class` and `module` bodies `Trait` + * (`query.ts`: "module (labeled Trait for class-like registry lookup)"). + * `undefined` when the owner cannot be found — an unanswered question. + */ +function ownerLabelOf( + candidate: SymbolDefinition, + parsedFileOf: ((filePath: string) => ParsedFile | undefined) | undefined, +): string | undefined { + const ownerId = candidate.ownerId; + if (ownerId === undefined || parsedFileOf === undefined) return undefined; + const owner = parsedFileOf(candidate.filePath)?.localDefs.find((d) => d.nodeId === ownerId); + return owner?.type; +} + +/** + * Calls that rebind `self` for the duration of a block: inside + * `service.instance_eval do … end` a bare `helper()` is dispatched on + * `service`, so it legitimately reaches a class-owned method the caller file + * never names. Detected on the caller's SOURCE TEXT, not the call site — the + * site does not know which block encloses it — so any file that uses one of + * these forms keeps its class-owned guesses LABELED rather than refused. Coarse + * in the safe direction: it loses refusals in that file, never an edge. + */ +const SELF_REBINDING_CALL = + /\b(?:instance_eval|instance_exec|class_eval|class_exec|module_eval|module_exec)\b/; + +export function rubyIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly parsedFileOf?: (filePath: string) => ParsedFile | undefined; + readonly sourceTextOf?: (filePath: string) => string | undefined; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + // Top-level method — the autoload shape the fallback exists for. + if (ctx.candidate.ownerId === undefined) return true; + // A self-rebinding block anywhere in the caller makes "the class is never + // named here" no proof of impossibility (see `SELF_REBINDING_CALL`). The + // pipeline always supplies the source; a missing text is an unanswered + // question and keeps the labeled edge as well. + const text = ctx.sourceTextOf?.(ctx.callerParsed.filePath); + if (text === undefined || SELF_REBINDING_CALL.test(text)) return true; + // Only a CLASS body makes a bare cross-file call impossible without naming + // it (see the header). A module owner, or an owner we cannot type, is not a + // refusal. + if (ownerLabelOf(ctx.candidate, ctx.parsedFileOf) !== 'Class') return true; + + const constants = namespaceConstantsOf(ctx.candidate); + // Owned but with no nameable namespace (an anonymous or lower-cased owner): + // nothing to check, so do not refuse on an unanswered question. + if (constants.length === 0) return true; + + // Any inheritance or mixin surface in the caller can reach the class + // transitively (see the header) — not decidable here, so not refused. + for (const site of ctx.callerParsed.referenceSites) { + if (site.kind === 'inherits') return true; + } + for (const imp of ctx.callerParsed.parsedImports) { + if (imp.targetRaw.startsWith(HERITAGE_MARKER_PREFIX)) return true; + } + // A file that defines a module is a mixin whose methods run inside some + // including class (see the header). Ruby labels `module` bodies `Trait`. + for (const def of ctx.callerParsed.localDefs) { + if (def.type === 'Trait') return true; + // A file that REOPENS `class Invoice` names the owner even without an + // inherits site or a reference capture — the declaration itself is the + // mention. + if (def.type === 'Class') { + const declared = (def.qualifiedName ?? '') + .split(/::|\.|#/) + .map((s) => s.trim()) + .filter((s) => s !== '') + .at(-1); + if (declared !== undefined && constants.includes(declared)) return true; + } + } + + for (const imp of ctx.callerParsed.parsedImports) { + for (const constant of constants) { + if (requireReachesConstant(imp.targetRaw, constant)) return true; + // `include Billing::Invoice` arrives as an import whose LOCAL name is the + // constant rather than a path. Not every variant carries one. + if ('localName' in imp && imp.localName === constant) return true; + } + } + // A bare mention of the constant anywhere in the caller (`Billing::Invoice`, + // `Invoice.new`) is enough to make the namespace present in this file. The + // qualified spelling is matched SEGMENT-wise on `::` / `.`: `InvoiceService` + // is not a mention of `Invoice`, and a substring test made it one. + for (const site of ctx.callerParsed.referenceSites) { + for (const constant of constants) { + if (site.name === constant) return true; + if ( + site.rawQualifiedName !== undefined && + site.rawQualifiedName.split(/::|\./).some((segment) => segment === constant) + ) { + return true; + } + } + } + return false; +} diff --git a/gitnexus/src/core/ingestion/languages/ruby/resolution-config.ts b/gitnexus/src/core/ingestion/languages/ruby/resolution-config.ts new file mode 100644 index 000000000..52c213af3 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/ruby/resolution-config.ts @@ -0,0 +1,380 @@ +import { readdirSync, readFileSync, type Dirent } from 'node:fs'; +import { isAbsolute, join, relative, resolve } from 'node:path'; + +export interface RubyResolutionScope { + /** Require prefixes provided by gems whose source is outside this repository. */ + readonly externalRequirePrefixes: ReadonlySet; + /** Require prefix -> repository-relative Ruby load roots for local gems. */ + readonly localLoadRootsByPrefix: ReadonlyMap; +} + +export interface RubyResolutionConfig { + /** Manifest directory (repository-relative POSIX path) -> dependency scope. */ + readonly scopesByDirectory: ReadonlyMap; +} + +interface MutableRubyResolutionScope { + readonly externalRequirePrefixes: Set; + readonly localLoadRootsByPrefix: Map>; +} + +interface GemspecMetadata { + readonly name: string | null; + readonly requirePaths: readonly string[]; +} + +const MAX_VISITED_DIRECTORIES = 20_000; +const SKIP_DIRECTORIES = new Set([ + '.git', + '.gitnexus', + 'node_modules', + 'vendor', + 'dist', + 'build', + 'coverage', +]); + +const GEMFILE_DEPENDENCY = /^\s*gem\s*(?:\(\s*)?(['"])([A-Za-z0-9_.-]+)\1(?.*)$/; +const GEMSPEC_DEPENDENCY = + /^\s*(?:[A-Za-z_]\w*\.)?(?:add_dependency|add_runtime_dependency|add_development_dependency)\s*(?:\(\s*)?(['"])([A-Za-z0-9_.-]+)\1/; +const GEMSPEC_NAME = /^\s*(?:[A-Za-z_]\w*\.)?name\s*=\s*(['"])([A-Za-z0-9_.-]+)\1/; +const GEMSPEC_REQUIRE_PATHS = /^\s*(?:[A-Za-z_]\w*\.)?require_paths\s*=\s*\[([^\]]*)\]/; +const QUOTED_LITERAL = /(['"])([^'"]+)\1/g; +const LOCKFILE_SECTION = /^([A-Z][A-Z ]*)\s*$/; +const LOCKFILE_REMOTE = /^ {2}remote:\s*(.+?)\s*$/; +const LOCKFILE_SPECS_HEADER = /^ {2}specs:\s*$/; +const LOCKFILE_SPEC = /^ {4}([A-Za-z0-9_.-]+) \(/; + +function createMutableScope(): MutableRubyResolutionScope { + return { + externalRequirePrefixes: new Set(), + localLoadRootsByPrefix: new Map>(), + }; +} + +function normalizeRepoPath(filePath: string): string { + return filePath.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, ''); +} + +function repoRelativePath(repoPath: string, candidate: string): string | null { + const rel = relative(resolve(repoPath), resolve(candidate)); + if (rel === '') return ''; + if (rel === '..' || rel.startsWith(`..\\`) || rel.startsWith('../') || isAbsolute(rel)) { + return null; + } + return normalizeRepoPath(rel); +} + +function requirePrefixes(gemName: string, requireAs?: string): readonly string[] { + const prefixes = new Set([gemName]); + if (gemName.includes('-')) prefixes.add(gemName.replaceAll('-', '/')); + if (requireAs !== undefined && requireAs.length > 0) prefixes.add(requireAs); + return [...prefixes]; +} + +function literalOption(options: string, name: string): string | undefined { + const modern = new RegExp(`(?:^|[,\\s])${name}\\s*:\\s*(['"])([^'"]+)\\1`).exec(options); + if (modern !== null) return modern[2]; + const hashRocket = new RegExp(`(?:^|[,\\s]):${name}\\s*=>\\s*(['"])([^'"]+)\\1`).exec(options); + return hashRocket?.[2]; +} + +function hasOption(options: string, name: string): boolean { + return ( + new RegExp(`(?:^|[,\\s])${name}\\s*:`).test(options) || + new RegExp(`(?:^|[,\\s]):${name}\\s*=>`).test(options) + ); +} + +function addExternalGem( + scope: MutableRubyResolutionScope, + gemName: string, + requireAs?: string, +): void { + for (const prefix of requirePrefixes(gemName, requireAs)) { + if (!scope.localLoadRootsByPrefix.has(prefix)) scope.externalRequirePrefixes.add(prefix); + } +} + +function addLocalGem( + scope: MutableRubyResolutionScope, + gemName: string, + loadRoots: readonly string[], + requireAs?: string, +): void { + for (const prefix of requirePrefixes(gemName, requireAs)) { + scope.externalRequirePrefixes.delete(prefix); + let roots = scope.localLoadRootsByPrefix.get(prefix); + if (roots === undefined) { + roots = new Set(); + scope.localLoadRootsByPrefix.set(prefix, roots); + } + for (const loadRoot of loadRoots) roots.add(loadRoot); + } +} + +function parseGemspecMetadata(contents: string): GemspecMetadata { + let name: string | null = null; + let requirePaths: string[] | null = null; + + for (const line of contents.split(/\r?\n/)) { + const nameMatch = GEMSPEC_NAME.exec(line); + if (nameMatch !== null) name = nameMatch[2]; + + const requirePathsMatch = GEMSPEC_REQUIRE_PATHS.exec(line); + if (requirePathsMatch === null) continue; + const paths: string[] = []; + for (const match of requirePathsMatch[1].matchAll(QUOTED_LITERAL)) paths.push(match[2]); + if (paths.length > 0) requirePaths = paths; + } + + return { name, requirePaths: requirePaths ?? ['lib'] }; +} + +function loadLocalGemRoots( + repoPath: string, + gemRoot: string, + expectedGemName: string, +): readonly string[] | null { + const gemRootRelative = repoRelativePath(repoPath, gemRoot); + if (gemRootRelative === null) return null; + + let requirePaths: readonly string[] = ['lib']; + try { + const gemspecs = readdirSync(gemRoot, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith('.gemspec')) + .map((entry) => join(gemRoot, entry.name)) + .sort(); + for (const gemspec of gemspecs) { + const metadata = parseGemspecMetadata(readFileSync(gemspec, 'utf8')); + if (metadata.name === expectedGemName) { + requirePaths = metadata.requirePaths; + break; + } + } + } catch { + // An unreadable local gem stays local; the conventional lib root is a + // bounded best effort, and resolution falls through if no file matches. + } + + return requirePaths + .map((requirePath) => repoRelativePath(repoPath, resolve(gemRoot, requirePath))) + .filter((loadRoot): loadRoot is string => loadRoot !== null); +} + +function addGemfileDependencies( + contents: string, + manifestDirectory: string, + repoPath: string, + scope: MutableRubyResolutionScope, +): void { + for (const line of contents.split(/\r?\n/)) { + const match = GEMFILE_DEPENDENCY.exec(line); + if (match === null) continue; + + const gemName = match[2]; + const options = match.groups?.options ?? ''; + const requireAs = literalOption(options, 'require'); + const localPath = literalOption(options, 'path'); + + if (localPath !== undefined) { + const loadRoots = loadLocalGemRoots(repoPath, resolve(manifestDirectory, localPath), gemName); + if (loadRoots === null) addExternalGem(scope, gemName, requireAs); + else addLocalGem(scope, gemName, loadRoots, requireAs); + continue; + } + + // A dynamic path expression cannot be classified without evaluating Ruby. + // Fail open instead of deleting a potentially real in-repo import edge. + if (hasOption(options, 'path')) continue; + + addExternalGem(scope, gemName, requireAs); + } +} + +function addGemspecDependencies( + contents: string, + gemspecDirectory: string, + repoPath: string, + scope: MutableRubyResolutionScope, +): void { + const metadata = parseGemspecMetadata(contents); + if (metadata.name !== null) { + if (repoRelativePath(repoPath, gemspecDirectory) !== null) { + const loadRoots = metadata.requirePaths + .map((requirePath) => repoRelativePath(repoPath, resolve(gemspecDirectory, requirePath))) + .filter((loadRoot): loadRoot is string => loadRoot !== null); + addLocalGem(scope, metadata.name, loadRoots); + } + } + + for (const line of contents.split(/\r?\n/)) { + const match = GEMSPEC_DEPENDENCY.exec(line); + if (match !== null) addExternalGem(scope, match[2]); + } +} + +function addLockedDependencies( + contents: string, + lockfileDirectory: string, + repoPath: string, + scope: MutableRubyResolutionScope, +): void { + let section = ''; + let remote: string | null = null; + let inSpecs = false; + + for (const line of contents.split(/\r?\n/)) { + const sectionMatch = LOCKFILE_SECTION.exec(line); + if (sectionMatch !== null) { + section = sectionMatch[1]; + remote = null; + inSpecs = false; + continue; + } + + const remoteMatch = LOCKFILE_REMOTE.exec(line); + if (remoteMatch !== null) { + remote = remoteMatch[1]; + continue; + } + + if (LOCKFILE_SPECS_HEADER.test(line)) { + inSpecs = true; + continue; + } + if (!inSpecs) continue; + + const specMatch = LOCKFILE_SPEC.exec(line); + if (specMatch === null) continue; + const gemName = specMatch[1]; + + if (section === 'GEM' || section === 'GIT') { + addExternalGem(scope, gemName); + continue; + } + + if (section !== 'PATH' && section !== 'GEMSPEC') continue; + if (remote === null) continue; + + const loadRoots = loadLocalGemRoots(repoPath, resolve(lockfileDirectory, remote), gemName); + if (loadRoots === null) addExternalGem(scope, gemName); + else addLocalGem(scope, gemName, loadRoots); + } +} + +function freezeScope(scope: MutableRubyResolutionScope): RubyResolutionScope { + const localLoadRootsByPrefix = new Map(); + for (const [prefix, roots] of scope.localLoadRootsByPrefix) { + localLoadRootsByPrefix.set(prefix, [...roots].sort()); + } + return { + externalRequirePrefixes: new Set(scope.externalRequirePrefixes), + localLoadRootsByPrefix, + }; +} + +/** + * Select the nearest manifest directory that owns `fromFile`. + * + * Walking ancestors makes lookup O(path depth), independent of how many + * sibling Gemfiles a monorepo contains. + */ +export function findRubyResolutionScope( + config: RubyResolutionConfig, + fromFile: string, +): RubyResolutionScope | undefined { + const normalized = normalizeRepoPath(fromFile); + let directory = normalized.includes('/') ? normalized.slice(0, normalized.lastIndexOf('/')) : ''; + + for (;;) { + const scope = config.scopesByDirectory.get(directory); + if (scope !== undefined) return scope; + if (directory === '') return undefined; + const slash = directory.lastIndexOf('/'); + directory = slash === -1 ? '' : directory.slice(0, slash); + } +} + +/** + * Statically collect Ruby dependency sources without evaluating Gemfile or + * gemspec code. Scopes stay separate by manifest directory so sibling projects + * in a monorepo cannot suppress one another's local imports. Lockfiles + * contribute remote GEM/GIT specs and local PATH/GEMSPEC load roots only when + * an adjacent Gemfile or gemspec establishes a Bundler/RubyGems project. + * + * No declarative manifest means no safe gate, so return null and preserve the + * resolver's existing fail-open behavior for loose script directories. + */ +export function loadRubyResolutionConfig(repoPath: string): RubyResolutionConfig | null { + const pending = [repoPath]; + const manifestsByDirectory = new Map(); + const lockfilesByDirectory = new Map(); + let visitedDirectories = 0; + + while (pending.length > 0 && visitedDirectories < MAX_VISITED_DIRECTORIES) { + const directory = pending.pop(); + if (directory === undefined) break; + visitedDirectories++; + + let entries: Dirent[]; + try { + entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => + left.name.localeCompare(right.name), + ); + } catch { + continue; + } + + for (const entry of entries) { + if (entry.isDirectory() && !SKIP_DIRECTORIES.has(entry.name)) { + pending.push(join(directory, entry.name)); + } else if (entry.isFile() && (entry.name === 'Gemfile' || entry.name.endsWith('.gemspec'))) { + const manifests = manifestsByDirectory.get(directory) ?? []; + manifests.push(join(directory, entry.name)); + manifestsByDirectory.set(directory, manifests); + } else if (entry.isFile() && entry.name === 'Gemfile.lock') { + lockfilesByDirectory.set(directory, join(directory, entry.name)); + } + } + } + + if (manifestsByDirectory.size === 0) return null; + + const mutableScopes = new Map(); + for (const [directory, manifests] of [...manifestsByDirectory].sort(([left], [right]) => + left.localeCompare(right), + )) { + const directoryRelative = repoRelativePath(repoPath, directory); + if (directoryRelative === null) continue; + const scope = createMutableScope(); + mutableScopes.set(directoryRelative, scope); + + for (const manifest of manifests.sort()) { + try { + const contents = readFileSync(manifest, 'utf8'); + if (manifest.endsWith('.gemspec')) { + addGemspecDependencies(contents, directory, repoPath, scope); + } else { + addGemfileDependencies(contents, directory, repoPath, scope); + } + } catch { + // A partially readable scope remains fail-open for unknown gems. + } + } + + const lockfile = lockfilesByDirectory.get(directory); + if (lockfile === undefined) continue; + try { + addLockedDependencies(readFileSync(lockfile, 'utf8'), directory, repoPath, scope); + } catch { + // Direct declarations still provide bounded evidence when the lock is unreadable. + } + } + + const scopesByDirectory = new Map(); + for (const [directory, scope] of mutableScopes) { + scopesByDirectory.set(directory, freezeScope(scope)); + } + return { scopesByDirectory }; +} diff --git a/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts index 160bec94e..2ce047dbd 100644 --- a/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts @@ -10,6 +10,8 @@ import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-l import type { KnowledgeGraph } from '../../../graph/types.js'; import { generateId } from '../../../../lib/utils.js'; import { decodeMarker } from '../../utils/heritage-marker.js'; +import { rubyIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; +import { loadRubyResolutionConfig } from './resolution-config.js'; /** * #1991: resolve a BARE mixin reference (`include Loggable`) to a nested module by @@ -268,6 +270,8 @@ export const rubyScopeResolver: ScopeResolver = { resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => resolveRubyImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig), + loadResolutionConfig: (repoPath) => loadRubyResolutionConfig(repoPath), + expandsWildcardTo: (targetModuleScope, parsedFiles) => expandRubyWildcardNames(targetModuleScope, parsedFiles), @@ -287,4 +291,5 @@ export const rubyScopeResolver: ScopeResolver = { fieldFallbackOnMethodLookup: true, propagatesReturnTypesAcrossImports: true, allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: rubyIsGlobalNameFallbackPlausible, }; diff --git a/gitnexus/src/core/ingestion/languages/rust/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/rust/name-fallback-visibility.ts new file mode 100644 index 000000000..1afdb900f --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/rust/name-fallback-visibility.ts @@ -0,0 +1,183 @@ +/** + * Rust's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Rust has NO ambient namespace. A bare `helper()` resolves only against names + * in scope, and for an item declared in another module the only way in is a + * `use` path (or a fully-qualified `crate::a::b::helper()` call, which is not a + * bare free call and never reaches this tier — it carries a qualified name and + * is resolved earlier by `resolveQualifiedFreeCall`). + * + * This hook checks import evidence, not Rust item visibility. A child module + * can use private ancestor items, and visibility restrictions such as + * `pub(crate)` require more context than `SymbolDefinition` carries. A matching + * import therefore keeps a labeled guess rather than proving accessibility. + * + * Module paths are matched against the candidate's FILE path (extension + * stripped, and `mod`/`lib`/`main` stem dropped, since `a/b/mod.rs` IS module + * `a::b`). `crate::` names the root; `self::` and `super::` resolve relative + * to the caller's module before comparison. + */ + +import type { ParsedFile, Scope, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { + modulePathReaches, + stripExtension, +} from '../../scope-resolution/utils/name-fallback-visibility.js'; + +/** File-stems that name their PARENT directory as the module, not themselves. */ +const RUST_DIRECTORY_MODULE_STEMS: ReadonlySet = new Set(['mod', 'lib', 'main']); + +/** Directories that hold a crate's root and contribute no module segment, so + * `src/net/http.rs` is module `net::http` and not `src::net::http`. */ +const RUST_CRATE_ROOT_DIRS: ReadonlySet = new Set(['src', 'tests', 'benches', 'examples']); + +/** Path prefixes of a `use` that name a root rather than a module segment. */ +const RUST_USE_ROOT_PREFIXES: ReadonlySet = new Set(['crate', '$crate']); + +// One scope lookup per immutable parsed-file snapshot, not per fallback site. +// Weak keys release both the snapshot and its index at the end of ingestion. +const scopeLookupByFile = new WeakMap>(); + +/** + * The module path a Rust file provides, as a `/`-joined path. + * + * Two normalizations, both needed for a file path and a `use` path to line up + * on their trailing segments: the `mod`/`lib`/`main` stem names its parent + * directory, and a leading crate-root directory (`src/`) is not a module. + */ +function rustModulePathOf(filePath: string): string { + const withoutExtension = stripExtension(filePath); + const segments = withoutExtension.split('/').filter((s) => s !== ''); + const stem = segments[segments.length - 1]; + if (stem !== undefined && RUST_DIRECTORY_MODULE_STEMS.has(stem)) segments.pop(); + // Only the crate-root folder is non-semantic. A nested `src/` is module + // `src` (`src/src/helper.rs` → `crate::src::helper`), not another root. + if (segments.length > 0 && RUST_CRATE_ROOT_DIRS.has(segments[0]!)) segments.shift(); + return segments.join('/'); +} + +/** + * Directory that owns the crate-root folder (`src`/`tests`/…). Empty when the + * file sits at the analyzed root (`src/lib.rs`) so a single-crate tree is not + * refused on a missing workspace prefix. + */ +function rustCrateRootOf(filePath: string): string { + const segments = stripExtension(filePath) + .split('/') + .filter((s) => s !== ''); + const stem = segments[segments.length - 1]; + if (stem !== undefined && RUST_DIRECTORY_MODULE_STEMS.has(stem)) segments.pop(); + const rootIdx = segments.findIndex((s) => RUST_CRATE_ROOT_DIRS.has(s)); + if (rootIdx <= 0) return ''; + return segments.slice(0, rootIdx).join('/'); +} + +/** Resolve explicit relative prefixes against the caller's module path. */ +function rustUsePathOf(targetRaw: string, callerFilePath: string): string { + const segments = targetRaw.split('::').filter((s) => s !== ''); + if (segments[0] === 'self' || segments[0] === 'super') { + const base = rustModulePathOf(callerFilePath).split('/').filter(Boolean); + if (segments[0] === 'self') segments.shift(); + while (segments[0] === 'super') { + base.pop(); + segments.shift(); + } + return [...base, ...segments].join('::'); + } + while (segments.length > 0 && RUST_USE_ROOT_PREFIXES.has(segments[0]!)) segments.shift(); + return segments.join('::'); +} + +export function rustIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly site: { + readonly name: string; + readonly rawQualifiedName?: string; + readonly inScope?: ScopeId; + }; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + // A PATH-QUALIFIED call (`User::new(...)`, `crate::a::helper()`) reaches this + // tier when the qualifier could not be followed, carrying only its tail name. + // It is not a bare-name guess: the source named the path, so the module rule + // below would refuse an edge the code spells out. + if (ctx.site.rawQualifiedName !== undefined) return true; + + const candidateModule = rustModulePathOf(ctx.candidate.filePath); + // A candidate whose file maps to no module path (a crate root reduced to '') + // is not something this rule can speak about; allow the labeled edge rather + // than refuse on an unanswered question. + if (candidateModule === '') return true; + + const candidateName = rustSimpleNameOf(ctx.candidate); + // Imports are lexical evidence, not a file-wide allowlist. Legacy/synthetic + // imports without a scope receipt retain the previous conservative behavior. + let visibleScopes: Set | undefined; + if (ctx.site.inScope !== undefined) { + let scopes = scopeLookupByFile.get(ctx.callerParsed); + if (scopes === undefined) { + scopes = new Map(ctx.callerParsed.scopes.map((scope) => [scope.id, scope])); + scopeLookupByFile.set(ctx.callerParsed, scopes); + } + visibleScopes = new Set(); + let current: ScopeId | null = ctx.site.inScope; + while (current !== null && !visibleScopes.has(current)) { + visibleScopes.add(current); + const scope = scopes.get(current); + if (scope === undefined || scope.kind === 'Namespace') break; + current = scope.parent; + } + } + for (const imp of ctx.callerParsed.parsedImports) { + if ( + imp.declaredAtScope !== undefined && + visibleScopes !== undefined && + !visibleScopes.has(imp.declaredAtScope) + ) + continue; + // `crate::` is the caller's crate. A same trailing module in another + // workspace crate is a different item and cannot authorize the guess. + if (imp.targetRaw === 'crate' || imp.targetRaw.startsWith('crate::')) { + const callerRoot = rustCrateRootOf(ctx.callerParsed.filePath); + const candidateRoot = rustCrateRootOf(ctx.candidate.filePath); + if (callerRoot !== '' && candidateRoot !== '' && callerRoot !== candidateRoot) continue; + } + const usePath = rustUsePathOf(imp.targetRaw, ctx.callerParsed.filePath); + // Only a glob introduces every bare item of a module. A named import must + // match both the candidate's original name and the call's local spelling. + if (imp.kind === 'wildcard') { + if (modulePathReaches(usePath, candidateModule)) return true; + continue; + } + if (!('localName' in imp) || imp.localName !== ctx.site.name) continue; + // Otherwise the path names ONE item inside a module (`use crate::a::other`). + // Its PARENT is the candidate's module only if that item IS the candidate: + // importing `other` says nothing about a `helper` in `a`, and the bare + // parent-path match used to accept every item of `a` on its strength. + // An alias authorizes only the local spelling checked above. + if (importedNameOf(imp) !== candidateName) continue; + if (modulePathReaches(usePath, candidateModule)) return true; + const parent = usePath.slice(0, Math.max(0, usePath.lastIndexOf('::'))); + if (parent !== '' && modulePathReaches(parent, candidateModule)) return true; + } + return false; +} + +/** The identifier a `use` binds, as written at its source (`importedName`). */ +function importedNameOf(imp: ParsedFile['parsedImports'][number]): string | undefined { + return 'importedName' in imp ? imp.importedName : undefined; +} + +/** + * The identifier a Rust declaration contributes to its module: the FIRST + * segment of `qualifiedName` after any module prefix — `User` for `User.new` + * (an associated function is reached through its type, so it is the type the + * `use` must name), the bare name for a free function. + */ +function rustSimpleNameOf(candidate: SymbolDefinition): string { + const qualified = candidate.qualifiedName ?? ''; + const segments = qualified.split(/::|\./).filter((s) => s !== ''); + return segments[0] ?? ''; +} diff --git a/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts index bea53d9fd..71fb697db 100644 --- a/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts @@ -19,6 +19,7 @@ import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-l import type { HeritageTypeArgumentSink } from '../../scope-resolution/utils/generic-instantiation.js'; import type { KnowledgeGraph } from '../../../graph/types.js'; import { generateId } from '../../../../lib/utils.js'; +import { rustIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; /** * Emit Rust `S IMPLEMENTS T` edges from `impl T for S` trait implementations. @@ -192,4 +193,6 @@ export const rustScopeResolver: ScopeResolver = { hoistTypeBindingsToModule: true, propagatesReturnTypesAcrossImports: true, allowGlobalFreeCallFallback: true, + importsBindAtLexicalScope: true, + isGlobalNameFallbackPlausible: rustIsGlobalNameFallbackPlausible, }; diff --git a/gitnexus/src/core/ingestion/languages/swift/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/swift/name-fallback-visibility.ts new file mode 100644 index 000000000..916465d94 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/swift/name-fallback-visibility.ts @@ -0,0 +1,67 @@ +/** + * Swift's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Swift's default access level is `internal`: visible throughout the MODULE and + * nowhere else. Swift needs no per-file import inside a module, which is why + * the global fallback is enabled for it at all — but that whole-module + * visibility stops hard at the module boundary. A candidate in a DIFFERENT + * module is reachable only if the caller wrote `import `, and even + * then only if the declaration is `public`. + * + * A module is approximated by its source directory, the layout every Swift + * package configuration discovers: `Sources//…` (also under `Package/`). + * `src//…` is also recognized by the package configuration loader. + * Files outside these layouts have unknown module identity. + * + * The `private` / `fileprivate` half of the rule is NOT implemented, because + * neither marker is recoverable from the parse model this hook sees — + * `SymbolDefinition` carries no access level and `ParsedFile` no modifiers. + * Those candidates keep the labeled low-confidence edge, which is the + * "cannot decide, so do not refuse" direction the hook contract asks for. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { modulePathReaches } from '../../scope-resolution/utils/name-fallback-visibility.js'; + +/** Keep aligned with loadSwiftPackageConfig: other layouts are unconfigured. */ +const SWIFT_TARGET_ROOTS: ReadonlySet = new Set(['Sources', 'src']); + +/** + * The module (target) a Swift file belongs to. + * + * `Sources/Core/User.swift` → `Core`. Arbitrary Xcode folders are not module + * boundaries; without a recognized target layout leave the answer unknown. + */ +function swiftModuleOf(filePath: string): string { + const segments = filePath.split('/').filter((s) => s !== ''); + for (let i = 0; i < segments.length - 2; i++) { + if (SWIFT_TARGET_ROOTS.has(segments[i]!)) { + // Keep the path through the target so `Sources/Core` and + // `Package/Sources/Core` stay distinct modules. `import Core` still + // reaches either via trailing-segment containment. + return segments.slice(0, i + 2).join('/'); + } + } + return ''; +} + +export function swiftIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + + const callerModule = swiftModuleOf(ctx.callerParsed.filePath); + const candidateModule = swiftModuleOf(ctx.candidate.filePath); + // Same module: whole-module `internal` visibility, no import needed. + if (callerModule === candidateModule) return true; + // A file the layout heuristic cannot place is not something this rule can + // speak about — allow rather than refuse on an unanswered question. + if (callerModule === '' || candidateModule === '') return true; + + for (const imp of ctx.callerParsed.parsedImports) { + if (modulePathReaches(imp.targetRaw, candidateModule)) return true; + } + return false; +} diff --git a/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts index 6d71117e4..83a8e6ce7 100644 --- a/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts @@ -66,6 +66,7 @@ import { mirrorSwiftSiblingTypeBindings, type SwiftResolveContext, } from './index.js'; +import { swiftIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; const ZERO_RANGE = { startLine: 0, startCol: 0, endLine: 0, endCol: 0 } as const; @@ -136,6 +137,7 @@ const swiftScopeResolver: ScopeResolver = { // global free-call fallback (as Python/Go/Ruby/COBOL do for the same // no-`new` constructor + cross-file free-call shape). allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: swiftIsGlobalNameFallbackPlausible, // Swift's call graph models `Type(...)` as a reference to the type // itself, not its `init` — both the legacy DAG and this test suite link diff --git a/gitnexus/src/core/ingestion/languages/typescript/captures.ts b/gitnexus/src/core/ingestion/languages/typescript/captures.ts index 2e3a76290..955b70a25 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/captures.ts @@ -32,6 +32,7 @@ import { syntheticCapture, type SyntaxNode, } from '../../utils/ast-helpers.js'; +import { collectEsmExportEvidence, esmExportVerdict } from '../../ts-js-export-marker.js'; import { splitImportStatement } from './import-decomposer.js'; import { getTsParser, getTsScopeQuery, tsCachedTreeMatchesGrammar } from './query.js'; import { recordCacheHit, recordCacheMiss } from './cache-stats.js'; @@ -388,6 +389,8 @@ export function emitTsScopeCaptures( } const rawMatches = getTsScopeQuery(filePath).matches(tree.rootNode); + // Export evidence, read once per file (see `ts-js-export-marker.ts`). + const exportEvidence = collectEsmExportEvidence(tree.rootNode, filePath); const out: CaptureMatch[] = []; for (const m of rawMatches) { @@ -561,6 +564,13 @@ export function emitTsScopeCaptures( fnNode, deriveDefaultExportHocName(filePath), ); + // This declaration's name is synthetic, so the later query-name + // marker cannot see it. The HOC predicate already proves the export. + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + fnNode, + 'true', + ); } } @@ -660,6 +670,20 @@ export function emitTsScopeCaptures( // instead of re-parsing the receiver's source text. Self-gating: a // non-call match, an absent receiver, or a chain with no nameable base // all leave `grouped` untouched. + // `@declaration.is-exported`: a verdict for every declaration the file's + // export surface can decide (see `ts-js-export-marker.ts`); nothing where + // it cannot, because absence is the honest answer there. + const declNameNode = groupedNodes['@declaration.name']; + if (exportEvidence !== undefined && declNameNode !== undefined) { + const verdict = esmExportVerdict(declNameNode, exportEvidence); + if (verdict !== undefined) { + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + declNameNode, + verdict ? 'true' : 'false', + ); + } + } synthesizeReceiverChainCapture(grouped, groupedNodes['@reference.receiver']); out.push(grouped); diff --git a/gitnexus/src/core/ingestion/languages/typescript/query.ts b/gitnexus/src/core/ingestion/languages/typescript/query.ts index babdb273b..ce1afcadd 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/query.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/query.ts @@ -1346,7 +1346,13 @@ export const TYPESCRIPT_SCOPE_QUERY = ` * (\`...\`) emit; the closing tag is intentionally NOT captured — * each JSX element should emit exactly one CALLS edge per use site. */ -const TSX_JSX_QUERY_SUFFIX = ` +/** + * Exported alongside `TYPESCRIPT_SCOPE_QUERY` so + * `value-ref-dispatchability.test.ts` checks the whole query a `.tsx` file is + * analyzed with. Checking the base alone would miss a `value-ref` rule added + * here. Not part of the provider surface — use `getTsScopeQuery`. + */ +export const TSX_JSX_QUERY_SUFFIX = ` ;; ((jsx_self_closing_element name: (identifier) @reference.name) @reference.call.free diff --git a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts index bc2af4bcb..b0d12a3e8 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts @@ -132,6 +132,15 @@ const typescriptScopeResolver: ScopeResolver = { fieldFallbackOnMethodLookup: false, propagatesReturnTypesAcrossImports: true, + // ECMAScript: `export * from './a'; export * from './b'` with a name declared + // in both exports NEITHER — the finalize pass refuses the binding instead of + // taking the first-listed source (see `exclusiveWildcardReexports`). + exclusiveWildcardReexports: true, + // `import { x }` never reaches a class member: named imports and named + // re-exports bind to module-level declarations only (a class method sharing + // a name with a top-level value must not win the callable preference). + namedImportsBindTopLevelOnly: true, + // TypeScript uses `.values()` / `.keys()` method-call syntax for collection // views -- no property-style accessors like C#'s `Dictionary.Values` -- // so `elementTypeOf` answers only the `index` route and lets the regular diff --git a/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts b/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts index 0e9871c64..30da1a677 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts @@ -278,10 +278,11 @@ async function firstReadableConfig(base: string): Promise { async function findTsconfigFiles(repoRoot: string): Promise { const found: string[] = []; const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + let queueHead = 0; let dirsScanned = 0; - while (queue.length > 0 && dirsScanned < SCAN_MAX_DIRS) { - const { dir, depth } = queue.shift()!; + while (queueHead < queue.length && dirsScanned < SCAN_MAX_DIRS) { + const { dir, depth } = queue[queueHead++]!; dirsScanned++; let entries: import('fs').Dirent[]; try { @@ -317,8 +318,8 @@ function parseJsonc(raw: string): Record { return JSON.parse(withoutComments) as Record; } -function repoRelative(repoRoot: string, absDir: string): string { - const rel = path.relative(repoRoot, absDir).split(path.sep).join('/'); +function repoRelative(repoRoot: string, absDir: string, pathApi: typeof path = path): string { + const rel = pathApi.relative(repoRoot, absDir).split(pathApi.sep).join('/'); return rel === '.' || rel === '' ? '' : rel; } @@ -328,11 +329,34 @@ function repoRelative(repoRoot: string, absDir: string): string { * `path.resolve` swallows the wildcard into a path segment, so it is stripped * before resolving and re-appended after — the `*` is a substitution marker, * not a directory named `*`. + * + * `pathApi` is injectable so the win32 separator branch is unit-testable from a + * POSIX runner; production callers always use the platform-bound `path`. */ -function rebaseTarget(repoRoot: string, absTarget: string): string { +function rebaseTarget(repoRoot: string, absTarget: string, pathApi: typeof path = path): string { // `/repo/src/*` must come back as `src/*`, not `src*`: stripping only the // star leaves a trailing slash that `path.relative` then eats. - const suffix = absTarget.endsWith('/*') ? '/*' : absTarget.endsWith('*') ? '*' : ''; - const base = suffix === '' ? absTarget : absTarget.slice(0, -suffix.length); - return `${repoRelative(repoRoot, base)}${suffix}`; + // + // The target arrives from `path.resolve`, so on Windows it is `C:\repo\src\*` + // and an `endsWith('/*')` check never matches. It fell through to the bare + // `*` branch, `path.relative` ate the trailing backslash, and every alias + // target came back as `src*` — which `substituteStar` turns into `srclib/x`, + // so nothing an alias reached was ever resolved on Windows. Normalise the + // separator before looking at the suffix. + const normalized = absTarget.split(pathApi.sep).join('/'); + const suffix = normalized.endsWith('/*') ? '/*' : normalized.endsWith('*') ? '*' : ''; + const base = suffix === '' ? normalized : normalized.slice(0, -suffix.length); + const prefix = repoRelative(repoRoot, base, pathApi); + // A target naming the repo ROOT (`"*": ["./*"]` under `baseUrl: "."`) leaves + // an empty prefix, and `${''}${'/*'}` is `/*`. `substituteStar` turns that + // into `/lib/date`, but `resolveFile` matches repo-relative keys and never + // strips a leading slash, so `lib/date.ts` misses and the alias goes + // external. The bare `*` is the encoding that substitutes correctly — and it + // is what the pre-#3203 Windows path emitted by accident, so this keeps the + // separator fix from narrowing what already resolved there. + if (prefix === '' && suffix === '/*') return '*'; + return `${prefix}${suffix}`; } + +/** Test seam for {@link rebaseTarget} (see `test/unit/tsconfig-rebase-target.test.ts`). */ +export const _rebaseTargetForTests = rebaseTarget; diff --git a/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts index d2f37781f..d0b0cfaed 100644 --- a/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts @@ -117,6 +117,9 @@ const vueScopeResolver: ScopeResolver = { // Vue uses explicit imports for all external symbols; no global free- // call fallback needed (would produce spurious edges for built-ins). allowGlobalFreeCallFallback: false, + // Vue SFC scripts are TypeScript/JavaScript: a named import binds a module-level + // declaration, never a class member (see the TS resolver). + namedImportsBindTopLevelOnly: true, /** * Expand the scope-resolution file universe for Vue by performing a diff --git a/gitnexus/src/core/ingestion/languages/zig/query.ts b/gitnexus/src/core/ingestion/languages/zig/query.ts index 9aa8e106a..0b4540417 100644 --- a/gitnexus/src/core/ingestion/languages/zig/query.ts +++ b/gitnexus/src/core/ingestion/languages/zig/query.ts @@ -20,7 +20,13 @@ import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.j * container and import bindings — `emitZigScopeCaptures` filters those * groups out so a name binds exactly once. */ -const ZIG_SCOPE_QUERY = ` +/** + * Exported for `value-ref-dispatchability.test.ts`, which reads every language's + * scope query to enforce the keyed/unkeyed partition that + * `callableValueReferenceBoundaries`' dispatch exclusion depends on. Not part of + * the provider surface — nothing else should import it. + */ +export const ZIG_SCOPE_QUERY = ` ;; Scopes (source_file) @scope.module (struct_declaration) @scope.class @@ -359,6 +365,60 @@ const ZIG_SCOPE_QUERY = ` "const" . (identifier) @type-binding.name (call_expression) @type-binding.type .) @type-binding.alias +;; References — VALUE positions (#3399): a callable named where a value is +;; expected rather than where a callee is. Zig's JS bridge is built entirely +;; out of this shape — +;; +;; pub const namespaceURI = bridge.accessor(Element.getNamespaceUri, null, .{}); +;; +;; — 2,047 such declarations across 257 files in lightpanda-io/browser, the +;; project's whole JS↔Zig surface, and NONE of them reached the graph: Zig +;; emitted no \`value-ref\` capture at all, so a public DOM accessor's only +;; recorded callers were the two internal ones and \`impact\` called that \`exact\`. +;; +;; These become reference-class USES edges through the existing +;; \`mapReferenceKindToEdgeType\` mapping — a registration is not an invocation +;; (Kythe \`ref\` vs \`ref/call\`; Joern \`METHOD_REF\`) — resolved by the +;; property-dispatch pass, which keeps ONLY callable targets. That callable gate +;; is what makes these deliberately broad rules safe: \`js.Bridge(Element)\` and +;; \`register(count)\` match too, and emit nothing, exactly as TypeScript's +;; \`{ port: DEFAULT_PORT }\` does. +;; +;; No \`@reference.property-key\` is attached: Zig has no object-literal key to +;; dispatch through, so these register a reference and never synthesize CALLS. +;; The terminal invoke — \`Accessor.init\` → a struct field → \`Factory.zig\`'s +;; \`inline for\`/\`@typeInfo\` → \`@call(.auto, func, args)\` — needs comptime +;; evaluation and is deliberately NOT modelled; \`impact\` reports the shortfall +;; as \`epistemic: "lower-bound"\` instead of pretending to certainty. + +;; Call ARGUMENTS. In tree-sitter-zig arguments are direct children of +;; \`call_expression\`, NOT wrapped in an \`arguments\` node (only builtins have +;; one), so the callee has to be consumed explicitly by \`function:\` — without +;; that binding the same rule also matches the callee of \`foo(bar)\` and mints a +;; USES edge duplicating the call. +(call_expression + function: (_) + (identifier) @reference.name @reference.value-ref) + +;; Qualified argument — \`bridge.accessor(Element.getNamespaceUri, …)\`. The +;; RECEIVER is captured alongside the member so the site carries the owner it +;; was written with; \`@reference.name\` stays the member, which is the name the +;; scope walk resolves. +(call_expression + function: (_) + (field_expression + object: (_) @reference.receiver + member: (identifier) @reference.name) @reference.value-ref) + +;; Const binding initialiser — \`pub const defaultHandler = onReset;\`. Both +;; anchors are load-bearing: the leading \`.\` pins the bound name to the first +;; named child (see the declaration rules above), and the trailing \`.\` keeps the +;; initializer as the LAST child, so a \`const x: T = y\` annotation shape cannot +;; put the TYPE in value position. +(variable_declaration + "const" . (identifier) + (identifier) @reference.name @reference.value-ref .) + ;; References — free calls: foo(...) (call_expression function: (identifier) @reference.name) @reference.call.free diff --git a/gitnexus/src/core/ingestion/languages/zig/range-binding.ts b/gitnexus/src/core/ingestion/languages/zig/range-binding.ts index a0186242f..408386bdf 100644 --- a/gitnexus/src/core/ingestion/languages/zig/range-binding.ts +++ b/gitnexus/src/core/ingestion/languages/zig/range-binding.ts @@ -17,6 +17,14 @@ * * Runs after `propagateImportedReturnTypes`, so return bindings hoisted from * other files are visible when a subject is a call. + * + * This hook also carries `bindZigThisAliases` (`this-alias-bindings.ts`), which + * is a different job — binding `const Self = @This();` to its container — but + * needs the same thing this pass is already paying for: the file's parsed tree, + * post-finalize. Giving it a pass of its own would re-parse every Zig file in + * the repo whenever the tree cache is cold. That is the whole reason it is + * here; the two do not otherwise interact, for the reason recorded at the call + * site. */ import type { ParsedFile, Scope, ScopeId, TypeRef } from 'gitnexus-shared'; @@ -32,6 +40,7 @@ import { isClassLike, } from '../../scope-resolution/scope/walkers.js'; import { isZigKeywordDeclaration, zigUnwrapValue } from './captures.js'; +import { bindZigThisAliases } from './this-alias-bindings.js'; import { normalizeZigTypeName } from './interpret.js'; type ZigTree = ReturnType['parse']>; @@ -89,6 +98,23 @@ export function populateZigRangeBindings( } const scopes = parsed.scopes; if (scopes.length === 0) continue; + + // `const Self = @This();` — bind the alias to its container. It sits in this + // loop for ONE reason: the tree. A pass of its own would re-parse every Zig + // file in the repo whenever the tree cache is cold. + // + // Its position relative to the payload walk below is NOT load-bearing, and + // saying otherwise would be wrong in a checkable way: the payload walk types + // a subject through `findReceiverTypeBinding`, which reads `typeBindings` + // and the namespace/workspace type channels — never `bindingAugmentations`, + // where this writes. Measured on `for (Self.items) |it|`: `it` is bound + // neither before nor after. Nor is that a gap this should close by also + // writing a typeBinding — NO container name has one, the file stem included, + // so a payload subject written `Type.member` resolves for no spelling at + // all, and giving the alias an entry would make it behave unlike the very + // container it names. + bindZigThisAliases(parsed, tree.rootNode, indexes); + const resolver = new ZigSubjectTypeResolver(scopes, indexes, classScopeByDefId); // Pre-order: an outer payload is bound before an inner construct reads it diff --git a/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts index 86afcfbf2..675f6e1a8 100644 --- a/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts @@ -5,8 +5,9 @@ * Thin wiring: Zig has no inheritance (default MRO linearization over an * empty heritage set), no `super`, and is statically typed (field-fallback * heuristic off per the contract guidance). Import resolution reuses the - * same `resolveZigImportInternal` the legacy import-resolver config wraps, - * with `build.zig.zon` `.path` deps threaded through `loadResolutionConfig`. + * same `resolveZigImportInternal` the legacy import-resolver config wraps, with + * the repo's Zig build packages threaded through `loadResolutionConfig` and the + * one governing each file selected per-file by `zigPackageFor`. */ import type { ParsedFile } from 'gitnexus-shared'; @@ -14,7 +15,11 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; -import { loadZigBuildConfig, type ZigBuildZonConfig } from '../../language-config.js'; +import { + loadZigWorkspaceIndex, + zigPackageFor, + type ZigWorkspaceIndex, +} from '../../language-config.js'; import { resolveZigImportInternal } from '../../import-resolvers/zig.js'; import { zigProvider } from '../zig.js'; import { expandZigWildcardNames, zigArityCompatibility, zigMergeBindings } from './index.js'; @@ -46,14 +51,23 @@ export const zigScopeResolver: ScopeResolver = { // import; a one-hop split at the last dot resolved none of them. resolveNamespaceChains: true, - loadResolutionConfig: (repoPath: string) => loadZigBuildConfig(repoPath), + // The whole workspace, not just the root package. A Zig module's import table + // is declared by the `build.zig` of the package the file belongs to, so a repo + // laying its packages out as `packages//build.zig` has as many import + // tables as packages — and reading only the root one leaves every bare + // `@import("")` in such a repo unresolved. Same shape, same reason, as + // `loadTsconfigIndex` for a TypeScript monorepo. + loadResolutionConfig: (repoPath: string) => loadZigWorkspaceIndex(repoPath), resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => resolveZigImportInternal( fromFile, targetRaw, allFilePaths, - (resolutionConfig as ZigBuildZonConfig | null | undefined) ?? null, + // The package governing THIS file — `zigPackageFor` is the `tsconfigFor` + // analogue. `resolveZigImportInternal` is handed one package's config and + // is unchanged by this: which config it receives is the only difference. + zigPackageFor(resolutionConfig as ZigWorkspaceIndex | null | undefined, fromFile), ), // `pub usingnamespace @import("x.zig");` — target decls become local decls. diff --git a/gitnexus/src/core/ingestion/languages/zig/this-alias-bindings.ts b/gitnexus/src/core/ingestion/languages/zig/this-alias-bindings.ts new file mode 100644 index 000000000..8a124bf3e --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/zig/this-alias-bindings.ts @@ -0,0 +1,191 @@ +/** + * Bind a Zig `const X = @This();` alias to the container it names (#3219 + * review round 8). + * + * `@This()` IS the enclosing container, and Zig code says so constantly: the + * single most common spelling is `const Self = @This();`, followed by + * `Alias.member` wherever the explicit form is wanted (`Self.width(self)`, a + * registration `bridge.accessor(Self.width, …)`, a nested type `Self.Node`). + * A compiler resolves `Self` and the container's own name to the same type. + * This index did not, for one specific reason: + * + * - A container-level alias (`const Self = @This();` inside `struct {…}`) + * mints a `Variable` beside the `Struct`, so `Self` binds to a VALUE. Every + * class-like lookup filters on `isClassLike` and walks straight past it. + * - A FILE-level alias in a file-as-struct mints nothing at all — + * `isZigFileThisAlias` suppresses the Const deliberately, so it cannot + * shadow the type for `x: *Page`. The name is then bound to nothing, while + * the container itself is bound under the FILE STEM. + * + * So `Self.width` resolved only when the alias happened to be spelled like its + * container. Measured on the three Zig corpora on hand: 73 of ghostty's 185 + * `@This()` files, 93 of tigerbeetle's 94 and 8 of mach's 42 spell it + * differently, carrying 302 `Alias.member` references between them, 96 of those + * calls. Those were not wrong edges — they were no edges, and a caller list + * missing them is exactly the false confidence #3399 is about. + * + * WHAT THIS ADDS, precisely: one binding of the alias NAME to the container's + * own definition, appended to `indexes.bindingAugmentations` (the sanctioned + * post-finalize channel, invariant I8 — `indexes.bindings` is frozen). The + * augmentation channel is consulted by `lookupBindingsAt` only AFTER a scope's + * own `Scope.bindings`, so this can never outrank a real local declaration: the + * `Variable` the container-level alias already mints still answers first for + * anything that wants a value, and the container answers for anything that + * wants a type. Nothing is replaced and nothing is removed. + * + * WHAT IT DELIBERATELY DOES NOT DO. Only aliases declared DIRECTLY in a + * container body or at file level are bound, which is the same set + * `collectZigThisAliases` recognizes for the type-rewrite path. A function-local + * `const Self = @This();` also names the enclosing container, but it belongs in + * that function's scope, not the container's, and binding it here would make it + * visible to sibling functions that never declared it. + */ + +import type { ParsedFile, Scope, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import type { BindingRef } from 'gitnexus-shared'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { isClassLike } from '../../scope-resolution/scope/walkers.js'; +import { isZigKeywordDeclaration, ZIG_CONTAINER_TYPES } from './captures.js'; + +/** `const X = @This();` — the declaration shape, wherever it sits. */ +function thisAliasName(node: SyntaxNode): string | undefined { + if (node.type !== 'variable_declaration' || !isZigKeywordDeclaration(node)) return undefined; + const named = node.namedChildren.filter((c): c is SyntaxNode => c !== null); + if (named.length !== 2 || named[0]!.type !== 'identifier') return undefined; + const value = named[1]!; + if (value.type !== 'builtin_function' || value.namedChild(0)?.text !== '@This') return undefined; + return named[0]!.text; +} + +/** The class-like definition a Class scope is the body of. */ +function containerDefOf(scope: Scope): SymbolDefinition | undefined { + return scope.ownedDefs.find((d) => isClassLike(d.type)); +} + +/** + * The Class scope of the FILE-AS-STRUCT, if this file is one. + * + * Identified by range rather than by name: it is the only Class scope that both + * hangs directly off the module scope AND spans the same lines as it. A + * namespace-only file with a top-level `pub const Counter = struct {…}` also + * produces a Class scope under the module scope, and it also carries the file + * stem when the file is `Counter.zig` — matching on the name would bind that + * file's `@This()` alias to `Counter`, which is not what `@This()` means there. + */ +function fileStructScope(scopes: readonly Scope[], moduleScope: Scope): Scope | undefined { + return scopes.find( + (s) => + s.kind === 'Class' && + s.parent === moduleScope.id && + s.range.startLine === moduleScope.range.startLine && + s.range.endLine === moduleScope.range.endLine, + ); +} + +/** + * Innermost Class scope containing `node`. + * + * Compared on (line, column) rather than line alone, because a Zig container + * can nest inside another on ONE line — `const A = struct { const S = @This(); + * const B = struct { const T = @This(); }; };` gives both scopes the same start + * line, and picking between them by line would be a coin toss that binds an + * alias to the wrong container. A wrong edge is the one outcome this whole + * change set treats as worse than no edge. + */ +function enclosingClassScope(scopes: readonly Scope[], node: SyntaxNode): Scope | undefined { + const line = node.startPosition.row + 1; + const column = node.startPosition.column; + const startsAtOrBefore = (l: number, c: number): boolean => + l < line || (l === line && c <= column); + const endsAtOrAfter = (l: number, c: number): boolean => l > line || (l === line && c >= column); + let best: Scope | undefined; + for (const s of scopes) { + if (s.kind !== 'Class') continue; + if (!startsAtOrBefore(s.range.startLine, s.range.startCol)) continue; + if (!endsAtOrAfter(s.range.endLine, s.range.endCol)) continue; + if ( + best === undefined || + s.range.startLine > best.range.startLine || + (s.range.startLine === best.range.startLine && s.range.startCol > best.range.startCol) + ) { + best = s; + } + } + return best; +} + +/** + * Append `alias -> def` at `scopeId`, skipping the append when THIS def is + * already bound there. Idempotence, not precedence: a duplicate would be + * harmless for lookup (`lookupBindingsAt` dedupes by `def.nodeId`) but would + * make the channel's contents depend on how many times the hook ran. Other + * bindings under the same name are left alone — the channel is append-only per + * I8, and the whole point of the augmentation tier is that a scope's own + * `Scope.bindings` are consulted first and still win. + */ +function appendBinding( + indexes: ScopeResolutionIndexes, + scopeId: ScopeId, + alias: string, + def: SymbolDefinition, +): void { + const channel = indexes.bindingAugmentations as Map>; + let byName = channel.get(scopeId); + if (byName === undefined) { + byName = new Map(); + channel.set(scopeId, byName); + } + const bucket = byName.get(alias); + if (bucket === undefined) { + byName.set(alias, [{ def, origin: 'local' }]); + return; + } + if (bucket.some((ref) => ref.def.nodeId === def.nodeId)) return; + bucket.push({ def, origin: 'local' }); +} + +/** + * Bind every `@This()` alias declared in `parsed` at container or file level. + * + * Called per file from `populateZigRangeBindings`, which already holds the + * parsed tree — a second pass over `parsedFiles` would re-parse every file when + * the tree cache is cold. That is the only thing the two share. Ordering + * against the payload bindings that follow does not matter, twice over: the + * payload walk types its subjects through `findReceiverTypeBinding`, which + * never reads the channel written here, and a `@This()` alias is a + * container-private name no other file can import, so nothing outside this file + * reads it either. + */ +export function bindZigThisAliases( + parsed: ParsedFile, + root: SyntaxNode, + indexes: ScopeResolutionIndexes, +): void { + const moduleScope = parsed.scopes.find((s) => s.kind === 'Module'); + if (moduleScope === undefined) return; + const fileStruct = fileStructScope(parsed.scopes, moduleScope); + + const visit = (node: SyntaxNode): void => { + const alias = thisAliasName(node); + if (alias !== undefined) { + const parent = node.parent; + if (parent?.type === 'source_file') { + // The file-as-struct's own name. Bound at the MODULE scope, which is + // where the container is already bound under the file stem, so the + // alias and the stem are visible to exactly the same sites. + const def = fileStruct === undefined ? undefined : containerDefOf(fileStruct); + if (def !== undefined) appendBinding(indexes, moduleScope.id, alias, def); + } else if (parent !== null && ZIG_CONTAINER_TYPES.has(parent.type)) { + const scope = enclosingClassScope(parsed.scopes, node); + const def = scope === undefined ? undefined : containerDefOf(scope); + if (scope !== undefined && def !== undefined) appendBinding(indexes, scope.id, alias, def); + } + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child !== null) visit(child); + } + }; + visit(root); +} diff --git a/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts b/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts index 040c058be..11c13b259 100644 --- a/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts +++ b/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts @@ -1,6 +1,6 @@ import type { ParsedFile, ReferenceSite } from 'gitnexus-shared'; import { getTreeSitterBufferSize } from '../../constants.js'; -import type { ZigBuildZonConfig } from '../../language-config.js'; +import { zigPackageFor, type ZigWorkspaceIndex } from '../../language-config.js'; import { resolveZigImportInternal } from '../../import-resolvers/zig.js'; import { buildZigBoolConstMap, @@ -51,7 +51,11 @@ export function populateZigWorkspaceStaticGating( tree, parsed.filePath, knownPaths, - ctx.resolutionConfig as ZigBuildZonConfig | null | undefined, + // `resolutionConfig` is the whole Zig workspace; the package governing + // THIS file is what `resolveZigImportInternal` takes. Selected here rather + // than hoisted out of the loop because the answer is per-file: two files + // of this pass can belong to different packages. + zigPackageFor(ctx.resolutionConfig as ZigWorkspaceIndex | null | undefined, parsed.filePath), ); if (aliases.size === 0) continue; const ranges = collectZigStaticGatedRanges( @@ -76,7 +80,7 @@ function collectImportAliases( tree: ZigTree, fromFile: string, knownPaths: ReadonlySet, - resolutionConfig?: ZigBuildZonConfig | null, + packageConfig: ReturnType, ): ZigImportAliasMap { const candidates = new Map(); const declarationCounts = new Map(); @@ -91,7 +95,7 @@ function collectImportAliases( const raw = builtin?.descendantsOfType('string').at(0)?.text; if (raw === undefined) continue; const specifier = raw.replace(/^['"]|['"]$/g, ''); - const target = resolveZigImportInternal(fromFile, specifier, knownPaths, resolutionConfig); + const target = resolveZigImportInternal(fromFile, specifier, knownPaths, packageConfig); if (target !== null) candidates.set(binding, target); } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 443c13289..01d8a0862 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -26,6 +26,7 @@ import { computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, + markParseCacheChunkStale, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js'; @@ -37,6 +38,7 @@ import { loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, + durableChunkHasStaleShards, } from '../../../storage/parsedfile-store.js'; import type { ParseWorkerResult } from '../workers/parse-worker.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js'; @@ -932,8 +934,12 @@ export async function runChunkedParseAndResolve( * Chunk hashes whose durable ParsedFile directory could not be reset. The * old generation's shards are still on disk, so a warm hit would union * stale shards with the new ones. Treated exactly like a quarantined chunk: - * skip the parse-cache write so the next run re-dispatches into a clean - * directory rather than trusting a generation we could not clear. + * skip the parse-cache write AND, when shards from that generation are + * still on disk, retire the hash (#3204): the old `.v8` is not carried + * forward and the directory is dropped from the durable index, so the next + * run re-dispatches. Kept as its own set rather than + * read back off `staleKeys`, which is a superset — it is what selects the + * warn below over the quarantine branch's dev-only log. */ const durablePrepareFailures = new Set(); @@ -1073,7 +1079,11 @@ export async function runChunkedParseAndResolve( // Persist raw results for this chunk hash (skipping when any chunk file // was worker-quarantined, so the narrower rawResults isn't cached under // the full-chunk key — see the original inline note / U20.U2). - if (parseCache && p.chunkHash && rawResults.length > 0) { + // `rawResults.length > 0` guards the WRITE only. A quarantined chunk + // often returns nothing at all (the worker died on it), and that chunk + // still has to be retired — otherwise its pre-existing `.v8` is copied + // forward at save time (#3204). + if (parseCache && p.chunkHash) { const quarantineSet = new Set(workerPool?.getQuarantinedPaths?.() ?? []); const chunkHadQuarantine = p.chunkFiles.some((f) => quarantineSet.has(f.path)); const durableGenerationStale = durablePrepareFailures.has(p.chunkHash); @@ -1084,6 +1094,10 @@ export async function runChunkedParseAndResolve( 'so its shards may be stale; next run will re-dispatch it', ); } else if (chunkHadQuarantine) { + // This chunk's durable directory now holds only this run's NARROWER + // shards, so a warm hit would replay the full-coverage `.v8` over + // partial ParsedFiles (#3204). + markParseCacheChunkStale(parseCache, p.chunkHash); if (isDev) { const quarantinedInChunk = p.chunkFiles.filter((f) => quarantineSet.has(f.path)).length; logger.info( @@ -1092,7 +1106,7 @@ export async function runChunkedParseAndResolve( `next run will rediscover (${p.chunkHash.slice(0, 8)})`, ); } - } else { + } else if (rawResults.length > 0) { await persistParseCacheChunk(parseCache, p.chunkHash, rawResults); if (isDev) { logger.info( @@ -1142,6 +1156,22 @@ export async function runChunkedParseAndResolve( // directory on write, so at worst the old generation lingers. // Caught per chunk so one failure cannot abort the others. durablePrepareFailures.add(miss.chunkHash); + // Retire ONLY when a generation nobody cleared is still on disk. + // `prepareDurableParsedFileChunk` is rm-then-mkdir: an rm failure + // leaves the old shards to be unioned into a warm hit, but an rm + // that succeeded before a failing mkdir leaves nothing — the + // workers recreate the directory and write a clean generation, so + // retiring there would discard a good `.v8` for no safety gain + // (and a correlated burst would discard the whole shared cache). + // Retire here rather than at finalize: that branch sits behind + // `rawResults.length > 0`, so a chunk whose worker round returns + // nothing would keep its stale entry. + if ( + parseCache && + (await durableChunkHasStaleShards(durableParsedFileDir, miss.chunkHash)) + ) { + markParseCacheChunkStale(parseCache, miss.chunkHash); + } logger.warn( { err, chunkHash: miss.chunkHash.slice(0, 8) }, 'parsedfile-cache: could not reset durable chunk generation; ' + diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 8ee94e265..d81b7116e 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -16,6 +16,8 @@ */ import { createKnowledgeGraph } from '../graph/graph.js'; +import type { KnowledgeGraph } from '../graph/types.js'; +import { GLOBAL_NAME_FALLBACK_REASON } from '../graph/edge-reasons.js'; import { GraphEmitSink, type GraphEmitManifest } from '../lbug/graph-emit-sink.js'; import { type PipelineProgress } from 'gitnexus-shared'; import { PipelineResult } from '../../types/pipeline.js'; @@ -410,6 +412,11 @@ export const runPipelineFromRepo = async ( graphEmitSink?.close(); } + // Resolved-call index for the name-fallback census: read through the SINK, + // whose field-wise scan includes every streamed edge, not through `graph`, + // which under streaming holds none of them. + const resolvedCalleeNamesByCaller = collectResolvedCalleeNames(graphEmitSink ?? graph, graph); + // Extract final results for the PipelineResult contract const { totalFiles, @@ -473,6 +480,7 @@ export const runPipelineFromRepo = async ( communityResult, processResult, resolutionOutcomes, + resolvedCalleeNamesByCaller, undecidedSatisfaction, usedWorkerPool, reparsedFileCount, @@ -518,3 +526,29 @@ export const runPipelineFromRepo = async ( return result; }; + +/** + * Caller node id → the simple names of every callee it has a CALLS edge to. + * + * `edges` may be the streaming sink or the raw graph; `nodes` is always the raw + * graph, which holds every node in both modes (only relationships stream). One + * O(E) field-wise pass, allocation-free per edge except for the per-caller set. + */ +export function collectResolvedCalleeNames( + edges: Pick, + nodes: Pick, +): ReadonlyMap> { + const out = new Map>(); + edges.forEachRelationshipFields((sourceId, targetId, type, _confidence, reason) => { + if (type !== 'CALLS' || reason === GLOBAL_NAME_FALLBACK_REASON) return; + const name = nodes.getNode(targetId)?.properties.name; + if (typeof name !== 'string' || name === '') return; + let names = out.get(sourceId); + if (names === undefined) { + names = new Set(); + out.set(sourceId, names); + } + names.add(name); + }); + return out; +} diff --git a/gitnexus/src/core/ingestion/process-processor.ts b/gitnexus/src/core/ingestion/process-processor.ts index 67804d268..6a7cc4a10 100644 --- a/gitnexus/src/core/ingestion/process-processor.ts +++ b/gitnexus/src/core/ingestion/process-processor.ts @@ -10,8 +10,9 @@ * Processes help agents understand how features work through the codebase. */ -import type { GraphNode, NodeLabel } from 'gitnexus-shared'; +import type { GraphNode, NodeLabel, RelationshipType } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; +import { isHeuristicEdgeReason } from '../graph/edge-reasons.js'; import { CommunityMembership } from './community-processor.js'; import { calculateEntryPointScore, isTestFile } from './entry-point-scoring.js'; import { SupportedLanguages } from 'gitnexus-shared'; @@ -436,12 +437,28 @@ type AdjacencyList = Map; */ const MIN_TRACE_CONFIDENCE = 0.5; +/** + * True when an edge may seed or extend a traced flow. + * + * The confidence floor alone is not sufficient: the global-name fallback emits + * at exactly `MIN_TRACE_CONFIDENCE`, so a `<` comparison admits every one of + * its guesses. A flow assembled from name guesses reads as a real execution + * path through code that may never call each other, so the reason is checked + * too — see `graph/edge-reasons.ts`. + */ +const isTraceableCallsEdge = ( + type: RelationshipType, + confidence: number, + reason: string, +): boolean => + type === 'CALLS' && confidence >= MIN_TRACE_CONFIDENCE && !isHeuristicEdgeReason(reason); + const buildCallsGraph = (graph: KnowledgeGraph): AdjacencyList => { const adj = new Map(); - // Field-wise scan (#2680) — whole-graph walk, four fields, no object needed. - graph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { - if (type !== 'CALLS' || confidence < MIN_TRACE_CONFIDENCE) return; + // Field-wise scan (#2680) — whole-graph walk, five fields, no object needed. + graph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { + if (!isTraceableCallsEdge(type, confidence, reason)) return; const existing = adj.get(sourceId); if (existing === undefined) adj.set(sourceId, [targetId]); else existing.push(targetId); @@ -453,8 +470,8 @@ const buildCallsGraph = (graph: KnowledgeGraph): AdjacencyList => { const buildReverseCallsGraph = (graph: KnowledgeGraph): AdjacencyList => { const adj = new Map(); - graph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { - if (type !== 'CALLS' || confidence < MIN_TRACE_CONFIDENCE) return; + graph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { + if (!isTraceableCallsEdge(type, confidence, reason)) return; const existing = adj.get(targetId); if (existing === undefined) adj.set(targetId, [sourceId]); else existing.push(sourceId); diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index d8dbd42dd..898282b5e 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -707,6 +707,10 @@ function buildDefFromDeclarationMatch( const isExplicit = parseBooleanCapture(match['@declaration.is-explicit']); const isDeleted = parseBooleanCapture(match['@declaration.is-deleted']); const isSynthetic = parseBooleanCapture(match['@declaration.is-synthetic']); + // Tri-state on purpose: only a producer that saw the file's export surface + // emits the marker, and both `true` and `false` are verdicts (see + // `SymbolDefinition.isExported`). Absent stays absent. + const isExported = parseBooleanCapture(match['@declaration.is-exported']); return { nodeId: makeDefId(filePath, anchor.range, type, nameCap.text), @@ -725,6 +729,7 @@ function buildDefFromDeclarationMatch( ...(isExplicit === true ? { isExplicit: true } : {}), ...(isDeleted === true ? { isDeleted: true } : {}), ...(isSynthetic === true ? { isSynthetic: true } : {}), + ...(isExported !== undefined ? { isExported } : {}), }; } @@ -1037,8 +1042,7 @@ function pass3CollectImports( if (provider.interpretImport === undefined) return; // Hoisted: the capability is a property of the language, identical for every // match in the file. A provider that declares its imports do not execute - // where they are written (C/C++ `#include`, Rust `use`, COBOL `COPY`) skips - // the position walk entirely — position cannot defer something that never + // where they are written skips the execution-deferral walk — position cannot defer something that never // runs, and marking one deferred would hide a real cycle. Absent reads as // `true`, so an undeclared provider is unchanged. See // `LanguageProvider.importsExecuteWhereWritten`. @@ -1049,14 +1053,22 @@ function pass3CollectImports( const parsed = provider.interpretImport(match); if (parsed === null) continue; // The statement's own position, resolved to the innermost scope holding - // it. An unlocatable anchor leaves the import unmarked, which reads as + // it. Provenance is retained independently of execution timing. An + // unlocatable anchor leaves the import unmarked, which reads as // "runs at initialization" — the fail-safe direction, since it can only // make `check --cycles` over-report. - const inScopeId = positionCanDefer - ? positionIndex.atPosition(filePath, anchor.range.startLine, anchor.range.startCol) - : undefined; - const deferred = inScopeId !== undefined && runsOnlyWhenCalled(scopeTree, inScopeId); - parsedImports.push(deferred ? { ...parsed, runsOnlyWhenCalled: true } : parsed); + const inScopeId = positionIndex.atPosition( + filePath, + anchor.range.startLine, + anchor.range.startCol, + ); + const deferred = + positionCanDefer && inScopeId !== undefined && runsOnlyWhenCalled(scopeTree, inScopeId); + parsedImports.push({ + ...parsed, + ...(inScopeId !== undefined ? { declaredAtScope: inScopeId } : {}), + ...(deferred ? { runsOnlyWhenCalled: true } : {}), + }); } } diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index e364b0227..d6da6d4d9 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -82,13 +82,14 @@ * `handledSites` IFF a `tryEmitEdge` call returned `true` for it. * Sites a pass touched but couldn't resolve do NOT get marked — * they still get a chance from the shared resolver. Exception: - * the free-call fallback marks the site unconditionally after - * attempting emission (even on dedup-collapse), because the - * per-(caller, target) collapse semantics require multiple call - * sites in the same caller body not produce multiple edges. - * `preEmitInheritanceEdges` also pre-marks every `inherits` site so - * the generic bridge cannot remap class heritage into method-owned - * EXTENDS edges via `resolveCallerGraphId`. + * the free-call fallback marks the site after it decides the site, + * including when it emits no edge — dedup-collapse, a visibility + * veto (`fallback-refused`), or a selected callable that is deleted + * / invisible. The later generic pass must not recreate an edge + * that pass just refused. `preEmitInheritanceEdges` also pre-marks + * every `inherits` site so the generic bridge cannot remap class + * heritage into method-owned EXTENDS edges via + * `resolveCallerGraphId`. * * - **I3 — `propagateImportedReturnTypes` mutation timing + ordering.** * The pass mutates `Scope.typeBindings` (a plain `new Map(...)` from @@ -248,8 +249,8 @@ * ## Semantic-model source of truth * * `ParsedFile` (from `gitnexus-shared/src/scope-resolution/parsed-file.ts`) - * is the single semantic model consumed by both the legacy DAG and the - * scope-resolution pipeline. Scope-resolution passes MUST NOT build a + * is the single semantic model consumed by the scope-resolution pipeline. + * Scope-resolution passes MUST NOT build a * parallel parse representation; if a pass needs AST-level facts that * `ParsedFile` doesn't expose, it should reuse the orchestrator's * `treeCache` (see `RunScopeResolutionInput.treeCache`) rather than @@ -257,23 +258,20 @@ * * ## Same-graph guarantee * - * Edges emitted by `runScopeResolution` and edges emitted by the legacy - * DAG are indistinguishable to downstream consumers: + * All language resolvers emit edges through `runScopeResolution` using the + * shared graph contract: * - Node identity: same `generateId(...)` helper, same qualified-name * keyspace, same File/Folder/Method/Class node labels. * - Edge vocabulary: `'import-resolved' | 'global' | 'local-call' | - * 'same-file' | 'interface-dispatch' | 'read' | 'write'` — both - * paths emit the same reasons (see - * `gitnexus/src/core/ingestion/call-processor.ts` for the legacy - * emitter and `passes/receiver-bound-calls.ts` / + * 'same-file' | 'interface-dispatch' | 'read' | 'write' | + * 'global-name-fallback'` (see `passes/receiver-bound-calls.ts` / * `passes/free-call-fallback.ts` for the scope-resolution emitters). - * - Overload disambiguation: both paths use + * - Overload disambiguation: resolvers use * `generateId('Method', ...)` suffixed with `parameterTypes` when a * method has overloads — see `graph-bridge/ids.ts`. * - * The CI parity workflow (`.github/workflows/ci-scope-parity.yml`) - * runs both paths on every migrated language's fixture corpus and - * fails if the graph outputs diverge. + * Resolver fixture coverage lives in `ci-tests.yml` (the separate + * `ci-scope-parity.yml` gate was removed in RING4-1 #942). * * Plan that introduced most of these invariants: * `docs/plans/2026-04-20-001-refactor-emit-pipeline-generalization-plan.md`. @@ -865,6 +863,88 @@ export interface ScopeResolver { */ readonly allowGlobalFreeCallFallback?: boolean; + /** Opt in to binding imports at their extracted lexical scope, rather than + * publishing every file's imports at module scope. */ + readonly importsBindAtLexicalScope?: boolean; + + /** + * Two `wildcard` re-exports that both DECLARE a name make it AMBIGUOUS in + * this language — ECMAScript `export *` semantics, where the module simply + * does not export the name and any binding is a guess. Opt-in: for a + * language whose wildcard import is `#include`, `require` or a package + * fan-out, the same name in two files is an overload set or a redeclaration, + * and refusing it would delete real edges (C++ arity-narrowed overloads + * across two headers). Forwarded to `FinalizeHooks.wildcardCollisionIsAmbiguous`. + */ + readonly exclusiveWildcardReexports?: boolean; + + /** + * A named import or named re-export can only bind to a MODULE-LEVEL + * declaration of the target file — ECMAScript semantics, where + * `import { x }` never reaches a class member. Opt-in: languages that bind + * module-level members by bare name (static members, module functions) + * leave it off. Forwarded to `FinalizeHooks.namedImportsBindTopLevelOnly`. + */ + readonly namedImportsBindTopLevelOnly?: boolean; + + /** + * Veto for a single `allowGlobalFreeCallFallback` guess. + * + * The fallback picks a callable because its SIMPLE NAME is unique in the + * workspace — it consults no import and no scope chain. For most languages a + * large share of those guesses are not merely unlikely but IMPOSSIBLE: Go + * cannot call an unexported identifier from another package, ESM cannot see a + * name it did not import, Rust cannot reach an item with no `use` path. This + * hook is where a language states those rules, so the shared pass can drop + * the edge instead of publishing a guess that the language forbids. + * + * Return `false` to REFUSE (no edge, recorded as `fallback-refused`). Return + * `true`, or leave the hook undefined, to emit the labeled + * `global-name-fallback` edge. **Only answer `false` when the call is + * impossible, not when it is merely unproven** — a wrongly-refused candidate + * is a lost real edge, whereas a wrongly-allowed one is at least labeled and + * excluded from flows. + * + * Deliberately NOT folded into `isCallableVisibleFromCaller`: that hook also + * gates precise dispatch paths (implicit-this, member calls), so a rule + * written for the name-guess tier would silently suppress resolved edges too. + * + * `parsedFileOf` reaches the CANDIDATE's parse result — a language whose rule + * depends on declaration-side facts not carried by `SymbolDefinition` (for + * example Rust's `pub` form) needs it. `isExported` is the tri-state export + * marker when the language supplies one; this hook still needs the parse for + * visibility facts that marker does not represent. It returns `undefined` + * for a path outside this pass's file set; treat that as "cannot decide" + * and allow. + */ + readonly isGlobalNameFallbackPlausible?: (ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly parsedFileOf: (filePath: string) => ParsedFile | undefined; + /** + * Raw source of any parsed file, for languages whose visibility rule needs + * a declaration the parse does not carry (Go's package clause: a test file's + * `package foo_test` is a different package from its directory's `foo`). + * Absent when the pipeline has no contents at hand; hooks must then answer + * from paths alone and, when undecidable, allow. + */ + readonly sourceTextOf?: (filePath: string) => string | undefined; + /** + * The call site, so a language can tell a BARE name from a PATH-QUALIFIED + * one. Both reach this tier — a qualified call whose qualifier the resolver + * could not follow falls through to the bare-name search with only its tail + * name — but they are not the same claim. Rust's `User::new(...)` named its + * type in source, so refusing it for lacking a `use` of the module would + * delete an edge the source spells out. + */ + readonly site: { + readonly name: string; + readonly rawQualifiedName?: string; + /** Lexical call-site scope; absent only for legacy/synthetic hook callers. */ + readonly inScope?: ScopeId; + }; + }) => boolean; + /** * In this language every `Method` belongs to a class instance, so a * FREE (receiver-less) call may resolve to a `Method` only when the diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts index 453c0b406..48673f027 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts @@ -505,9 +505,25 @@ function resolveDefGraphIdUncached( if (qualifiedHit !== undefined) return qualifiedHit; } const simpleName = qn.lastIndexOf('.') === -1 ? qn : qn.slice(qn.lastIndexOf('.') + 1); + // FAIL CLOSED before the label-agnostic simple key when a FUNCTION-LOCAL + // callable of this name exists in the file. The guards above cover a def + // that is itself a callable; a NON-callable def (`export const selected = + // factory()`, a `Variable` with no graph node of its own) used to fall + // through here and alias onto `wrapper.selected`, the function-local one + // (#3182 review). Whatever the def's label, a bare name matching a local + // callable is the aliasing this key cannot tell apart, and a missing edge + // is the correct failure direction. + for (const localLabel of LOCAL_CALLABLE_LABELS) { + if (nodeLookup.get(localNameKey(filePath, localLabel, simpleName)) !== undefined) { + return undefined; + } + } return nodeLookup.get(simpleKey(filePath, simpleName)); } +/** Labels the structure phase registers function-local declarations under. */ +const LOCAL_CALLABLE_LABELS: readonly NodeLabel[] = ['Function', 'Method']; + /** Derive the simple (unqualified) name of a def from its `qualifiedName`. */ export function simpleQualifiedName(def: SymbolDefinition): string | undefined { const q = def.qualifiedName; diff --git a/gitnexus/src/core/ingestion/scope-resolution/name-fallback-summary.ts b/gitnexus/src/core/ingestion/scope-resolution/name-fallback-summary.ts new file mode 100644 index 000000000..e96d9d5c7 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/name-fallback-summary.ts @@ -0,0 +1,212 @@ +/** + * Per-language census of global-name fallback decisions: how many call sites + * used a unique-name guess, and how many candidates visibility rules refused. + * Sites are counted before edge coalescing: a precisely bound site can prove + * the same caller/target dependency, so this is not a count of heuristic edges. + * + * Both halves are needed and neither is meaningful alone. A guess count with no + * refusal count cannot distinguish a language with genuinely few impossible + * candidates from one whose hook is missing; a refusal count with no guess count + * cannot distinguish a working guard from one that rejects everything. The pair + * is what makes the guard auditable on a real repository, which is the whole + * point of recording it — before this, guessed edges were emitted with the same + * reason and confidence as import-resolved ones and the number was unknowable. + * + * Structural sibling of `unresolved-receivers.ts`'s summary, and persisted the + * same way (`RepoMeta.nameFallbackEdges`). + */ + +import { getLanguageFromFilename } from 'gitnexus-shared'; +import { logger } from '../../logger.js'; +import type { ResolutionOutcome } from './resolution-outcome.js'; + +/** + * Distinct caller-file/callee-name pairs per language, from the caller→callee-name index the pipeline + * builds while its streaming sink is live (so it is complete under `--force`), + * bucketed by the CALLER's file language. Gives `NameFallbackSummary.byLanguage` + * its denominator: a guess count is only readable as a share of the calls. + */ +export function countCallsByLanguage( + index: ReadonlyMap> | undefined, + nodes: { getNode(id: string): { properties: Record } | undefined } | undefined, +): Readonly> | undefined { + if (index === undefined || nodes === undefined) return undefined; + const counts = new Map(); + const namesByFile = new Map>(); + for (const [callerId, callees] of index) { + const filePath = nodes.getNode(callerId)?.properties?.filePath; + if (typeof filePath !== 'string') continue; + let names = namesByFile.get(filePath); + if (names === undefined) { + names = new Set(); + namesByFile.set(filePath, names); + } + for (const name of callees) names.add(name); + } + for (const [filePath, names] of namesByFile) { + const language = getLanguageFromFilename(filePath) ?? 'unknown'; + counts.set(language, (counts.get(language) ?? 0) + names.size); + } + if (counts.size === 0) return undefined; + const out: Record = {}; + for (const [language, count] of [...counts.entries()].sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + )) { + out[language] = count; + } + return out; +} + +/** Unattributed bucket for a pass that recorded no language. */ +const UNKNOWN_LANGUAGE = 'unknown'; + +export interface NameFallbackLanguageCounts { + /** Call sites resolved by a unique-name guess, before edge coalescing. */ + readonly guessed: number; + /** + * Distinct (caller file, callee name) pairs among those sites — the unit + * `callsByLanguage` counts in, so `guessedPairs / callsByLanguage[lang]` is a + * ratio bounded by 1. Absent on a summary persisted before this field existed. + */ + readonly guessedPairs?: number; + /** Candidates this language's `isGlobalNameFallbackPlausible` hook refused. */ + readonly refused: number; +} + +export interface NameFallbackSummary { + /** Language → guessed/refused counts. Languages with neither are absent. */ + readonly byLanguage: Readonly>; + /** Guessed call sites, repo-wide. */ + readonly totalGuessed: number; + /** Distinct (caller file, callee name) pairs among the guessed sites. */ + readonly distinctGuessedPairs?: number; + readonly totalRefused: number; + /** + * Barrel names refused because two `export *` sources both declared them + * (`reexport-ambiguous`). Not a guess and not per-language — a name the + * finalize pass declined to bind at all — but it belongs in the same census: + * it is the other place the resolver used to publish an arbitrary winner as + * `import-resolved`. + */ + readonly totalAmbiguousReexports: number; + /** + * The refused barrel names themselves (`file:name`), sorted, capped at + * `MAX_AMBIGUOUS_NAMES`. A count alone cannot say whether a refusal landed on + * a name anyone calls; the list can be joined against the ledger's `byName`. + */ + readonly ambiguousReexportNames?: readonly string[]; + /** + * Distinct caller-file/callee-name pairs per language (through any path), so `guessedPairs` can be + * read as a SHARE of a language's call graph rather than a bare count. Absent + * when the caller did not supply the totals. + */ + readonly callsByLanguage?: Readonly>; +} + +/** Bound on the persisted ambiguous-name list; the total beside it stays exact. */ +export const MAX_AMBIGUOUS_NAMES = 200; + +/** + * Build the summary, or `undefined` when the run neither guessed nor refused — + * a repository with no opt-in language stores no key at all, so the artifact + * stays absent rather than recording a row of zeroes. + */ +export function summarizeNameFallback( + outcomes: readonly ResolutionOutcome[], + callsByLanguage?: Readonly>, +): NameFallbackSummary | undefined { + const guessed = new Map(); + const guessedPairsByLanguage = new Map(); + const refused = new Map(); + const ambiguousNames = new Set(); + // Two units, both kept. `guessed` counts call SITES, not final graph edges: + // a precise site may prove the same dependency during edge coalescing. + // Preserve the unit earlier summaries hold. `guessedPairs` dedupes by (caller file, + // callee name), the unit `callsByLanguage` is counted in: ten guessed `foo()` + // calls in one file are one pair against a denominator that counts `foo` + // once, so the guessy RATIO uses pairs and is bounded by 1. Changing the unit + // of `guessed` itself silently read as a large improvement across engines. + const guessedPairs = new Set(); + let totalGuessed = 0; + let totalRefused = 0; + let totalAmbiguousReexports = 0; + + for (const outcome of outcomes) { + if (outcome.kind === 'fallback-guessed') { + const language = outcome.language ?? UNKNOWN_LANGUAGE; + guessed.set(language, (guessed.get(language) ?? 0) + 1); + totalGuessed++; + const pair = `${outcome.filePath}\u0000${outcome.name}`; + if (!guessedPairs.has(pair)) { + guessedPairs.add(pair); + guessedPairsByLanguage.set(language, (guessedPairsByLanguage.get(language) ?? 0) + 1); + } + } else if (outcome.kind === 'fallback-refused') { + const language = outcome.language ?? UNKNOWN_LANGUAGE; + refused.set(language, (refused.get(language) ?? 0) + 1); + totalRefused++; + } else if (outcome.kind === 'reexport-ambiguous') { + totalAmbiguousReexports++; + ambiguousNames.add(`${outcome.filePath}:${outcome.name}`); + } + } + if (totalGuessed === 0 && totalRefused === 0 && totalAmbiguousReexports === 0) return undefined; + + const byLanguage: Record = {}; + for (const language of new Set([...guessed.keys(), ...refused.keys()])) { + byLanguage[language] = { + guessed: guessed.get(language) ?? 0, + guessedPairs: guessedPairsByLanguage.get(language) ?? 0, + refused: refused.get(language) ?? 0, + }; + } + const sortedNames = [...ambiguousNames].sort(); + return { + byLanguage, + totalGuessed, + distinctGuessedPairs: guessedPairs.size, + totalRefused, + totalAmbiguousReexports, + ...(sortedNames.length > 0 + ? { ambiguousReexportNames: sortedNames.slice(0, MAX_AMBIGUOUS_NAMES) } + : {}), + ...(callsByLanguage !== undefined ? { callsByLanguage } : {}), + }; +} + +/** + * One-line readout for the analyze summary. `undefined` when there is nothing to + * report, so a run on a repository with no opt-in language prints no line. + */ +export function formatNameFallbackSummary( + summary: NameFallbackSummary | undefined, +): string | undefined { + if (summary === undefined) return undefined; + const perLanguage = Object.entries(summary.byLanguage) + .sort(([, a], [, b]) => b.guessed + b.refused - (a.guessed + a.refused)) + .map(([language, counts]) => `${language} ${counts.guessed}/${counts.refused}`) + .join(', '); + const ambiguous = + summary.totalAmbiguousReexports > 0 + ? `; ${summary.totalAmbiguousReexports} barrel name(s) refused as ambiguous \`export *\`` + : ''; + const languages = perLanguage === '' ? 'none' : perLanguage; + const pairs = + summary.distinctGuessedPairs !== undefined + ? ` (${summary.distinctGuessedPairs} distinct caller-file/name pairs)` + : ''; + const siteUnit = summary.totalGuessed === 1 ? 'call site' : 'call sites'; + return `name-fallback resolution: ${summary.totalGuessed} ${siteUnit}${pairs}, ${summary.totalRefused} refused as impossible (guessed/refused by language: ${languages})${ambiguous}`; +} + +/** + * Print the readout as part of the analyze summary. Unconditional (not behind a + * debug env var, unlike the receiver-drop diagnostic): a reader deciding how far + * to trust this index's call graph needs to know how much of it is guessed, and + * a number nobody sees is the state this work exists to end. + */ +export function logNameFallbackSummary(summary: NameFallbackSummary | undefined): void { + const line = formatNameFallbackSummary(summary); + if (line === undefined) return; + logger.info(line); +} diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 0457767b2..7dcc792f3 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -36,6 +36,7 @@ import type { ResolutionOutcomeRecorder, ResolutionSuppressionReason, } from '../resolution-outcome.js'; +import { GLOBAL_NAME_FALLBACK_REASON } from '../../../graph/edge-reasons.js'; import { resolveCallerGraphId, resolveDefGraphId } from '../graph-bridge/ids.js'; import type { CalleeIdSink } from '../graph-bridge/callee-id-sink.js'; import { @@ -43,7 +44,9 @@ import { findAllCallableBindingsInScope, findCallableBindingInScope, findCallableBindingsAndAdlBlocker, + findClassBindingInScope, findEnclosingClassDef, + isClassFileImportGrounded, resolveInheritanceBaseInScope, type CallableBindingCandidate, } from '../scope/walkers.js'; @@ -64,6 +67,15 @@ export function emitFreeCallFallback( workspaceIndex: WorkspaceResolutionIndex, options: { readonly allowGlobalFallback?: boolean; + /** Language whose pass this is, carried onto the fallback outcome records + * so the analyze summary can report guesses/refusals PER LANGUAGE. A + * repo-wide total hides which language's rules are the loose ones. */ + readonly language?: string; + /** Per-language veto on a name guess — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. */ + readonly isGlobalNameFallbackPlausible?: ScopeResolver['isGlobalNameFallbackPlausible']; + /** Raw source lookup handed to `isGlobalNameFallbackPlausible` (optional). */ + readonly sourceTextOf?: (filePath: string) => string | undefined; /** When true, `Type(...)` constructor calls link to the Class def * itself rather than its explicit Constructor. Swift opts in. */ readonly constructorCallTargetsClass?: boolean; @@ -138,6 +150,14 @@ export function emitFreeCallFallback( let allFilePathsMemo: ReadonlySet | undefined; const allFilePaths = (): ReadonlySet => (allFilePathsMemo ??= new Set(parsedFiles.map((p) => p.filePath))); + // Candidate-side parse lookup for `isGlobalNameFallbackPlausible`. Built + // lazily and once, on the same terms as `allFilePaths` above: a language + // without the hook never pays for the index. + let parsedByPathMemo: ReadonlyMap | undefined; + const parsedFileByPath = (): ((filePath: string) => ParsedFile | undefined) => { + parsedByPathMemo ??= new Map(parsedFiles.map((p) => [p.filePath, p])); + return (filePath) => parsedByPathMemo!.get(filePath); + }; // Per-pass memo of pickUniqueGlobalCallable's post-filter candidate list, // keyed (simpleName, callerFilePath). Only created when no per-caller // visibility filter applies (the list is then a pure function of name+file — @@ -183,7 +203,17 @@ export function emitFreeCallFallback( }; for (const parsed of parsedFiles) { - type PendingRel = { rel: Parameters[0]; gatedAll: boolean }; + type PendingRel = { + rel: Parameters[0]; + gatedAll: boolean; + /** + * The confidence/reason a PRECISELY resolved site (a real binding, not a + * unique-name guess) proved for this edge; `undefined` while every site + * collapsed into it so far was a guess. Decided at flush, not by the + * first site the walk met. + */ + precise: { confidence: number; reason: string } | undefined; + }; const pending = new Map(); const bindingCandidatesByScope = options.freeCallsRequireInstanceOwnership === true @@ -199,13 +229,48 @@ export function emitFreeCallFallback( // to the Class node itself (implicit constructor). Legacy emits // the same two targets; see test expectations. let fnDef: SymbolDefinition | undefined; + // Guess flag starts here so the constructor unique-class pick can mark + // the same class of guess as `pickUniqueGlobalCallable` below. The + // veto/label path keys off this flag; declaring it after that pick left + // constructor-form unique-name hits labeled `import-resolved` at 0.85. + let fnDefFromGlobalNameFallback = false; + // Language visibility vetoes the constructed TYPE (Class/Struct), not a + // Constructor child — Go refuses any `qualifiedName` containing `.`. + let globalFallbackVetoTarget: SymbolDefinition | undefined; if (site.callForm === 'constructor') { - const classDef = resolveInheritanceBaseInScope( + // Lexical / written-qualifier first. A bare unique type with no + // import/#include evidence is a guess (JS `new UniqueWidget()`, + // Go unexported `uniqueWidget{}`). A unique type whose file is + // imported, whose import resolved to this def, or whose name + // was imported is precise — C++ `#include "user.h"` and Rust + // `use`/`pub use` often mint no lexical class binding. An + // imported sibling file of a different name is not evidence. + let classDef = resolveInheritanceBaseInScope( site.inScope, site.name, scopes, site.rawQualifiedName, + undefined, + { uniqueQualifiedNameFallback: site.rawQualifiedName !== undefined }, ); + if (classDef === undefined) { + classDef = + findClassBindingInScope(site.inScope, site.name, scopes, undefined, { + uniqueQualifiedNameFallback: true, + }) ?? + (options.allowGlobalFallback === true + ? pickUniqueGlobalClass(site.name, globalClassesBySimpleName) + : undefined); + if ( + classDef !== undefined && + classDef.type !== 'Interface' && + site.rawQualifiedName === undefined && + !isClassFileImportGrounded(site.inScope, classDef, scopes, site.name) + ) { + fnDefFromGlobalNameFallback = true; + globalFallbackVetoTarget = classDef; + } + } if (classDef !== undefined && classDef.type !== 'Interface') { // Most languages link `Type(...)` to the explicit Constructor def // when one exists (else the Class). Languages that model the call @@ -215,21 +280,6 @@ export function emitFreeCallFallback( options.constructorCallTargetsClass === true ? classDef : pickConstructorOrClass(classDef, workspaceIndex, scopes, site.arity); - } else if (options.allowGlobalFallback === true) { - // The constructed type may live in a sibling/imported file that is - // not in the call-site's lexical scope-chain bindings. Fall back to - // a unique workspace-wide Class def by simple name (gated on the - // same global-fallback opt-in as free calls). Then target the - // Class or its Constructor per the language's preference. - const globalClass = pickUniqueGlobalClass(site.name, globalClassesBySimpleName); - if (globalClass !== undefined) { - fnDef = - globalClass.type === 'Interface' - ? undefined - : options.constructorCallTargetsClass === true - ? globalClass - : pickConstructorOrClass(globalClass, workspaceIndex, scopes, site.arity); - } } } // Module-qualified free call (`mod::fn()`): the source named the module @@ -543,10 +593,17 @@ export function emitFreeCallFallback( } } } - // V1: pickUniqueGlobalCallable ignores import context — resolves to any - // globally-unique callable. False cross-package edges are possible when - // the caller does not import the target package. Same-package calls are - // usually caught by nearest-scope lookup before reaching here. + // Name-guess tier: pickUniqueGlobalCallable consults no import context — + // it resolves to any globally-unique callable. Same-package calls are + // usually caught by nearest-scope lookup before reaching here, so what + // lands in this tier is disproportionately cross-module, and a + // cross-module name match is a guess. + // + // Two things make that honest rather than a lie. The language's + // `isGlobalNameFallbackPlausible` hook refuses candidates its own + // visibility rules forbid (below), and every edge that survives is + // emitted with `GLOBAL_NAME_FALLBACK_REASON` at 0.5 rather than + // masquerading as `import-resolved` at 0.85 (see the emit site). if (fnDef === undefined && options.allowGlobalFallback === true) { fnDef = pickUniqueGlobalCallable( site.name, @@ -570,6 +627,53 @@ export function emitFreeCallFallback( scopeDefsCache, options.conversionOnlyArgTypePrefixes, ); + fnDefFromGlobalNameFallback = fnDef !== undefined; + if (fnDef !== undefined) globalFallbackVetoTarget = fnDef; + } + if (fnDefFromGlobalNameFallback && fnDef !== undefined) { + // An explicit named import cannot bind a declaration proven private + // to another module. In particular, a rejected named import must not + // reappear as a name guess to a class member or nested function. + // Unknown export evidence (e.g. dynamic module exports) keeps the + // existing fallback behavior. + const vetoCandidate = globalFallbackVetoTarget ?? fnDef; + const importsPrivateDeclaration = + vetoCandidate.filePath !== parsed.filePath && + vetoCandidate.isExported === false && + parsed.parsedImports.some( + (imported) => + (imported.kind === 'named' || imported.kind === 'alias') && + imported.localName === site.name, + ); + if ( + importsPrivateDeclaration || + options.isGlobalNameFallbackPlausible?.({ + callerParsed: parsed, + candidate: vetoCandidate, + parsedFileOf: parsedFileByPath(), + sourceTextOf: options.sourceTextOf, + site: { + name: site.name, + rawQualifiedName: site.rawQualifiedName, + inScope: site.inScope, + }, + }) === false + ) { + // The language proved this call impossible. Mark the site handled so + // `emit-references` does not substitute its own looser guess for the + // edge we just refused — the point is no edge, not a different one. + options.recordResolutionOutcome?.({ + kind: 'fallback-refused', + candidateId: fnDef.nodeId, + language: options.language, + phase: 'free-call-fallback', + filePath: parsed.filePath, + name: site.name, + range: site.atRange, + }); + handledSites.add(siteKey(parsed.filePath, site)); + continue; + } } if (fnDef === undefined) continue; if (fnDef.isDeleted === true) { @@ -617,40 +721,78 @@ export function emitFreeCallFallback( site.atRange.startCol, tgtGraphId, ); + if (fnDefFromGlobalNameFallback) { + options.recordResolutionOutcome?.({ + kind: 'fallback-guessed', + targetId: fnDef.nodeId, + language: options.language, + phase: 'free-call-fallback', + filePath: parsed.filePath, + name: site.name, + range: site.atRange, + }); + } const relId = `rel:CALLS:${callerGraphId}->${tgtGraphId}`; // One edge per (caller, callee): `staticGated` is the AND over every site // that collapses into it, so a callee reached from one live site and one // dead site stays live whichever site the walk meets first. Emission is // deferred to the end of this file's sites for that reason. + const preciseHere = fnDefFromGlobalNameFallback + ? undefined + : { + confidence: 0.85, + // Match legacy DAG's reason convention so consumers that + // assert `reason === 'import-resolved'` keep working. The + // construction-site marker is opt-in for the same reason. + reason: constructionSiteReason( + fnDef.filePath !== parsed.filePath ? 'import-resolved' : 'local-call', + site, + options.markConstructionSites, + ), + }; const pendingRel = pending.get(relId); if (pendingRel !== undefined) { if (site.staticGated !== true) pendingRel.gatedAll = false; + // The edge's label is decided at flush time from EVERY site that + // collapsed into it, not from whichever the walk met first. One site + // resolved through a real binding PROVES the dependency; a guessed + // site for the same pair is then redundant evidence, not a taint. + if (pendingRel.precise === undefined) pendingRel.precise = preciseHere; continue; } if (seen.has(relId)) continue; seen.add(relId); pending.set(relId, { gatedAll: site.staticGated === true, + precise: preciseHere, rel: { id: relId, sourceId: callerGraphId, targetId: tgtGraphId, type: 'CALLS', - confidence: 0.85, - // Match legacy DAG's reason convention so consumers that - // assert `reason === 'import-resolved'` keep working. The - // construction-site marker is opt-in for the same reason. - reason: constructionSiteReason( - fnDef.filePath !== parsed.filePath ? 'import-resolved' : 'local-call', - site, - options.markConstructionSites, - ), + // Guess values as placeholders; decided at flush from `precise`. + confidence: 0.5, + reason: GLOBAL_NAME_FALLBACK_REASON, }, }); emitted++; } - for (const { rel, gatedAll } of pending.values()) { - graph.addRelationship(gatedAll ? { ...rel, staticGated: true } : rel); + for (const { rel, gatedAll, precise } of pending.values()) { + // A name guess is not an import resolution and must not be spelled like + // one. It used to be emitted at 0.85 / `'import-resolved'`, which made + // it indistinguishable from an edge a real import produced — so every + // consumer that wanted to discount guesses had no field to do it with. + // 0.5 is the deliberate "coin flip" value, and the reason is what the + // process/community walks and the MCP tools actually key on, because + // 0.5 sits exactly ON their thresholds (see graph/edge-reasons.ts). + // An edge is a guess only when EVERY site that collapsed into it was one; + // a single precisely resolved site proves it, whatever order the walk + // met the sites in. Independent of `gatedAll`. + const labeled = + precise !== undefined + ? { ...rel, confidence: precise.confidence, reason: precise.reason } + : rel; + graph.addRelationship(gatedAll ? { ...labeled, staticGated: true } : labeled); } } return emitted; diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts index 157348c5b..0736bb53c 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts @@ -120,19 +120,10 @@ export function followChainPostFinalize( * typeBindings — the in-extractor pass-4 ran before propagation and * missed any chain whose terminal lived in a foreign file. * - * Scope-chain concern (verified 2026-04-21): `pythonImportOwningScope` - * documents that function-local `from x import y` binds `y` to the - * inner function scope, which would make a module-only write miss - * non-module importers. In practice `finalize-algorithm` hoists those - * bindings into `indexes.bindings[moduleScope]` regardless of where - * the `import` statement appears — the integration fixture - * `python-function-local-import-chain` exercises a chained - * receiver-bound call `u = get_user(); u.save()` inside a function - * body and emits the expected `do_work → User.save` edge. The - * module-scope write is sufficient today. If finalize routing ever - * changes to honor the hook's per-scope contract, this pass must - * iterate `indexes.bindings` over every scope and mirror into the - * binding-owning scope's `typeBindings`, not just the module's. + * Finalize can retain lexical import ownership. Mirror into each binding's + * owning scope, never hoist local import-derived types into the file scope. + * Module ordering remains SCC-based; non-module chains are followed after + * all import bindings have been mirrored. */ export function propagateImportedReturnTypes( parsedFiles: readonly ParsedFile[], @@ -140,6 +131,7 @@ export function propagateImportedReturnTypes( index: WorkspaceResolutionIndex, ): void { const moduleScopeByFile = index.moduleScopeByFile; + const scopesByFile = new Map(parsedFiles.map((file) => [file.filePath, file.scopes])); // Walk SCCs in reverse-topological order (`indexes.sccs` is leaves- // first per `tarjanSccs`). For each file we mirror import bindings @@ -167,42 +159,44 @@ export function propagateImportedReturnTypes( // import-derived typeBinding mirror. Both helpers fast-path when // no augmentations exist for the scope, so the common case is // allocation-free. See I8. - for (const localName of namesAtScope(importerModule.id, indexes)) { - // Skip if importer already has a typeBinding for this name — - // an explicit local annotation must win over import-derived. - if (importerModule.typeBindings.has(localName)) continue; + for (const importerScope of scopesByFile.get(filePath) ?? [importerModule]) { + for (const localName of namesAtScope(importerScope.id, indexes)) { + // Skip if importer already has a typeBinding for this name — + // an explicit local annotation must win over import-derived. + if (importerScope.typeBindings.has(localName)) continue; - const refs = lookupBindingsAt(importerModule.id, localName, indexes); - for (const ref of refs) { - if (ref.origin !== 'import' && ref.origin !== 'reexport' && ref.origin !== 'wildcard') - continue; - const sourceModule = moduleScopeByFile.get(ref.def.filePath); - if (sourceModule === undefined) continue; + const refs = lookupBindingsAt(importerScope.id, localName, indexes); + for (const ref of refs) { + if (ref.origin !== 'import' && ref.origin !== 'reexport' && ref.origin !== 'wildcard') + continue; + const sourceModule = moduleScopeByFile.get(ref.def.filePath); + if (sourceModule === undefined) continue; - // The source file's typeBinding is keyed by the def's simple - // name (e.g. `get_user`), not the importer's local alias. - const qn = ref.def.qualifiedName; - if (qn === undefined) continue; - const dot = qn.lastIndexOf('.'); - const sourceName = dot === -1 ? qn : qn.slice(dot + 1); + // The source file's typeBinding is keyed by the def's simple + // name (e.g. `get_user`), not the importer's local alias. + const qn = ref.def.qualifiedName; + if (qn === undefined) continue; + const dot = qn.lastIndexOf('.'); + const sourceName = dot === -1 ? qn : qn.slice(dot + 1); - const sourceTypeRef = sourceModule.typeBindings.get(sourceName); - if (sourceTypeRef === undefined) continue; + const sourceTypeRef = sourceModule.typeBindings.get(sourceName); + if (sourceTypeRef === undefined) continue; - // Chain-follow inside the source module so we mirror the - // terminal type, not an intermediate intra-source reference. - const terminal = followChainPostFinalize(sourceTypeRef, sourceModule.id, indexes); + // Chain-follow inside the source module so we mirror the + // terminal type, not an intermediate intra-source reference. + const terminal = followChainPostFinalize(sourceTypeRef, sourceModule.id, indexes); - // Mutating typeBindings is safe because draftToScope - // produced a non-frozen Map (Contract Invariant I3/I8). - (importerModule.typeBindings as Map).set(localName, terminal); - // First-write-wins for the local alias: if the same - // `localName` was registered multiple times via - // `mergeBindings` (rare; happens with conflicting - // re-exports), only the first ref with a usable - // typeBinding source is mirrored. Conflict resolution - // among multiple sources is the merger's job, not ours. - break; + // Mutating typeBindings is safe because draftToScope + // produced a non-frozen Map (Contract Invariant I3/I8). + (importerScope.typeBindings as Map).set(localName, terminal); + // First-write-wins for the local alias: if the same + // `localName` was registered multiple times via + // `mergeBindings` (rare; happens with conflicting + // re-exports), only the first ref with a usable + // typeBinding source is mirrored. Conflict resolution + // among multiple sources is the merger's job, not ours. + break; + } } } diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts index fef9ed29d..6a23ca94d 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts @@ -17,9 +17,11 @@ * registries only consult pre-finalize local bindings — imported names live * in finalized bindings (the same reason free calls need * `emitFreeCallFallback`) — so `resolveReferenceSites` skips `value-ref` - * sites and this pass resolves them post-finalize via - * `findCallableBindingInScope` (Function/Method/Constructor only — the - * callable gate that keeps `{ port: DEFAULT_PORT }` from emitting anything). + * sites and this pass resolves them post-finalize (see + * `resolveValueRefTarget` — Function/Method/Constructor only, the callable + * gate that keeps `{ port: DEFAULT_PORT }` from emitting anything, and + * receiver-aware so a qualified reference binds the owner it was written + * with). * * Precision posture (mirrors `emitInterfaceDispatchFor`): * - reason `'property-dispatch'` keeps synthesized CALLS auditable; @@ -36,13 +38,29 @@ * emits the capture participates) and generic member-call sites. */ -import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import type { + BindingRef, + ParsedFile, + ReferenceSite, + ScopeId, + SymbolDefinition, +} from 'gitnexus-shared'; import type { KnowledgeGraph } from '../../../graph/types.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; import { tryEmitEdge, type CalleeIdCaptureCtx } from '../graph-bridge/edges.js'; import type { GraphNodeLookup } from '../graph-bridge/node-lookup.js'; import type { CalleeIdSink } from '../graph-bridge/callee-id-sink.js'; -import { findCallableBindingInScope } from '../scope/walkers.js'; +import { + findCallableBindingInScope, + findClassBindingInScope, + findOwnedMember, + isNamespaceNameShadowed, + isOwnerNameShadowedBySomethingElse, + lookupBindingsAt, +} from '../scope/walkers.js'; +import { VALUE_REF_EDGE_REASON } from '../value-ref-edges.js'; +import type { SemanticModel } from '../../model/semantic-model.js'; +import { CALL_TARGET_TYPES } from '../../model/symbol-table.js'; /** * Keys registered by more than this many distinct functions are skipped — @@ -65,12 +83,282 @@ export const MAX_PROPERTY_DISPATCH_FANOUT = (() => { /** Below the 0.85 resolved baseline; same discount idea as interface-dispatch. */ export const PROPERTY_DISPATCH_CONFIDENCE = 0.7; +/** + * Resolve a `value-ref` site to the callable it names. + * + * Two shapes, and the difference matters: + * + * - BARE (`{ handler: onClick }`, `register(onTick)`) — the name is resolved + * up the lexical chain, which is what an unqualified name means. Be exact + * about what that walk does, because it is not a plain lexical lookup: + * `findCallableBindingInScope` applies the callable predicate WHILE walking, + * so a scope binding the name to a parameter or a local contributes nothing + * and the walk continues outward. A nearer non-callable binding is stepped + * over — the same shape the qualified path guards against below, unguarded + * here. Pre-existing (#2437) and out of scope for #3399; reachable in JS/TS + * (`function outer(handler) { return { h: handler }; }` beside a top-level + * `function handler`), and narrow in Zig, which rejects a local shadowing a + * container declaration. + * + * - QUALIFIED (`bridge.accessor(Element.getNamespaceUri, …)`) — the source + * WROTE the owner, so the lexical chain is the wrong instrument. It gives + * local bindings precedence (`walkScopeChain`), so a nested container + * holding its own `getNamespaceUri` answers first and the registration is + * attached to a DIFFERENT function than the one written. That is a wrong + * edge, not a missing one — strictly worse for a tool whose value is that + * its edges can be trusted — and it is reachable in any language that + * allows a same-named callable in a nested container, Zig included. + * + * So a qualified site resolves through its receiver: name the owner, then take + * the member off that owner. An owner is either a MODULE or a CLASS-like + * container — `utils.compare` and `Element.getNamespaceUri` are the same shape + * written against the two kinds of namespace a language has, and the member-call + * path already resolves both (receiver-bound-calls Case 1 / Case 2). Both are + * tried here, MODULE FIRST: see `findNamespaceValueRefTarget` for why a module + * receiver cannot simply be declined, and the comment on the call below for why + * the container channel must not go first. + * + * If neither channel names the owner, or the owner is named but owns no such + * callable, this DECLINES rather than falling back to the lexical walk. + * + * Be precise about what declining costs, because it is more than one reference: + * a site that emits NO edge leaves no evidence for `impact`'s value-reference + * probe to read, so the target keeps `epistemic: "exact"` — silence, not a + * hedge. That is why the module channel above exists rather than being waved + * through as "just a decline". What remains declined is the case where the + * written receiver names nothing this index knows at all (a Zig `@This()` alias + * whose name differs from its container's, an owner from outside the workspace): + * there the alternative is not a hedge either, it is a confident edge to a + * lexically-nearer function that the source did not name, and a wrong edge is + * strictly worse than a missing one for a tool whose value is that its edges can + * be trusted. + * + * `CALL_TARGET_TYPES`, not a hand-rolled label set: `findOwnedMember` also + * answers with FIELDS, and a field named like the member would otherwise + * register as if it were the callable. + * + * Exported for `bench/value-ref-resolution/measure.mjs`, which gates both the + * resolved-target SET and the per-site cost of the four channels above across a + * 4x file-count step. It is the per-site half of the pass, so timing it in + * isolation is what makes a workspace-size dependency visible; timing + * `emitPropertyDispatchCalls` would fold that signal into edge emission, and + * re-implementing the channel order in the bench would pin the bench's idea of + * this function rather than this function. + */ +export function resolveValueRefTarget( + site: ReferenceSite, + filePath: string, + scopes: ScopeResolutionIndexes, + model: SemanticModel, + publishesImportedNames: boolean, +): SymbolDefinition | undefined { + const receiverName = site.explicitReceiver?.name; + if (receiverName === undefined) { + return findCallableBindingInScope(site.inScope, site.name, scopes); + } + // NAMESPACE FIRST, and the order is load-bearing. `findClassBindingInScope` + // does not stop at the scope chain: when its `isClassLike` walk misses — and a + // namespace handle binds a Module, so it always misses — it falls back to + // `scopes.qualifiedNames`, a WORKSPACE-wide index, and answers with the unique + // def of that name anywhere in the repo. Trying it first therefore lets a + // same-named container in a file this one never imported preempt the `@import` + // this file actually wrote: + // + // const dom_utils = @import("dom_utils.zig"); // namespace-only module + // … bridge.accessor(dom_utils.compare, …) // → decoy.zig's compare + // + // The shadow guard below cannot catch it: the import binds at MODULE scope, + // which the guard treats as the floor. An import written in this file is the + // strongest statement about what the name means here, so it outranks a global + // guess — and when the handle is not an import of this file, this answers + // nothing and the container channel runs exactly as before. + const viaNamespace = findNamespaceValueRefTarget( + site, + filePath, + receiverName, + scopes, + publishesImportedNames, + ); + if (viaNamespace !== undefined) { + // `'owned'` is NOT "no answer" — it is "this receiver is a namespace handle + // this file wrote, and it names no callable member". The two must not be + // conflated, because falling through from the second one reaches + // `findClassBindingInScope`, whose miss path answers from the WORKSPACE-wide + // qualified-name index: a same-named container in a file this one never + // imported then supplies the member the written module does not have. That + // is a confident edge into an unrelated file, and the owner-shadow guard + // below does not stop it — a plain `const utils = @import("utils.zig");` + // records a namespace IMPORT EDGE, not a module-scope binding, so the guard + // sees nothing bound under the name and reads the container as unshadowed. + // Verified with a fixture rather than argued: `dom_utils.onlyOnDecoy`, where + // `dom_utils.zig` has no such member and `decoy.zig` declares a same-named + // struct that does, minted `JsApi → onlyOnDecoy` before this line existed. + // + // The file said which module it meant. If that module does not expose the + // name as a callable, the honest answer is no edge. + return viaNamespace === 'owned' ? undefined : viaNamespace; + } + + const owner = findClassBindingInScope(site.inScope, receiverName, scopes); + if (owner !== undefined) { + // The container lookup is a CLASS-ONLY walk: `walkScopeChain` filters by + // `isClassLike`, so it steps over a nearer binding that is a value and keeps + // climbing — and past the scope chain entirely, into a qualified-name + // fallback that answers with the unique workspace definition of the name. + // `fn f(Ticker: u8) { register(Ticker.fire) }` in a file that neither + // declares nor imports `Ticker` therefore resolves to some other file's + // `Ticker` container. That is the wrong-edge failure R1-2 exists to prevent, + // arriving through the class channel instead of the lexical one, and a + // registration pointing at a function the source never named is worse than + // no registration at all. + // + // So the name has to still MEAN that container at this site. The namespace + // channel below asks the same question through `isNamespaceNameShadowed`; + // a container needs the variant that exempts the container ITSELF, because + // `fn make() { const Local = struct {…}; register(Local.go); }` binds the + // name locally to the very def we resolved, and reading that as its own + // shadow would suppress the resolutions this path exists to make. + if (isOwnerNameShadowedBySomethingElse(receiverName, owner, site.inScope, scopes)) + return undefined; + const member = findOwnedMember(owner.nodeId, site.name, model); + if (member === undefined || !CALL_TARGET_TYPES.has(member.type)) return undefined; + return member; + } + return undefined; +} + +/** + * The second kind of owner: a namespace handle. + * + * `const utils = @import("utils.zig"); register(utils.compare);` — `utils` is a + * MODULE, not a class, so `findClassBindingInScope` answers nothing and the + * class path above declines. Declining here would be a silent hole rather than + * a conservative one: no USES edge is emitted, so + * `callableValueReferenceBoundaries` measures a real zero and `impact` on + * `compare` republishes `exact` — the very claim this feature exists to stop + * making. Nothing downstream can hedge on evidence that was never recorded. + * + * So resolve it, through the SAME channel the member-CALL path already trusts + * for `utils.compare()` (receiver-bound-calls Case 1): the file's namespace + * import edges name the target module, and the target module's own local + * module-scope bindings name its members. `utils.compare` and `utils.compare()` + * disagreeing about what `utils` is would be the anomaly. + * + * The same three guards Case 1 applies, for the same reasons: + * - a LOCAL declaration shadowing the handle suppresses the resolution + * (`isNamespaceNameShadowed`) — `fn f(utils: Decoy) { register(utils.compare) }` + * names the parameter's member, and resolving through the import would be a + * wrong edge rather than a missing one; + * - a locally declared member wins, and a name the target file merely IMPORTED + * counts only when the provider says its imports ARE its exports + * (`ScopeResolver.namespaceExportsIncludeImportedNames`). That opt-in is not + * a detail to skip: a Zig HUB — a file made only of re-exports, ghostty's + * `src/terminal/`, tigerbeetle's `stdx` — declares nothing, so requiring a + * local declaration declines every member reached through one. `hub.fn()` + * resolves and `register(hub.fn)` would not, and one name would mean two + * things depending on whether a `(` followed it. In languages that do not + * opt in, a module's imports are not its exports and this stays closed; + * - two distinct defs under one name resolve NOTHING. Never guess a namespace + * member — the whole point of reading the written receiver is precision. + * + * `CALL_TARGET_TYPES` gates the answer for the same reason the class path needs + * it: `utils.DEFAULT_PORT` is a module-scope binding too, and a registration + * table full of constants must keep emitting nothing. + */ +function findNamespaceValueRefTarget( + site: ReferenceSite, + filePath: string, + receiverName: string, + scopes: ScopeResolutionIndexes, + publishesImportedNames: boolean, +): SymbolDefinition | 'owned' | undefined { + const moduleScopeId = scopes.moduleScopes.get(filePath); + if (moduleScopeId === undefined) return undefined; + const targetFiles: string[] = []; + for (const edge of scopes.imports.get(moduleScopeId) ?? []) { + if (edge.kind !== 'namespace' || edge.localName !== receiverName) continue; + if (edge.targetFile === null) continue; + if (!targetFiles.includes(edge.targetFile)) targetFiles.push(edge.targetFile); + } + if (targetFiles.length === 0) return undefined; + if (isNamespaceNameShadowed(receiverName, site.inScope, scopes)) return undefined; + + /** The unique callable `select` finds across every target file, or nothing. */ + const uniqueMember = ( + select: (moduleScope: ScopeId) => readonly BindingRef[], + ): SymbolDefinition | undefined | 'ambiguous' => { + let picked: SymbolDefinition | undefined; + for (const targetFile of targetFiles) { + const targetScopeId = scopes.moduleScopes.get(targetFile); + if (targetScopeId === undefined) continue; + for (const ref of select(targetScopeId)) { + if (!CALL_TARGET_TYPES.has(ref.def.type)) continue; + if (picked !== undefined && picked.nodeId !== ref.def.nodeId) return 'ambiguous'; + picked = ref.def; + } + } + return picked; + }; + + // A locally declared member first — same precedence `findExportedDef` states + // and `walkScopeChain` applies: what the target file DECLARED beats what it + // merely re-published. + const localRefs = (scope: ScopeId): readonly BindingRef[] => + (scopes.bindings.get(scope)?.get(site.name) ?? []).filter((ref) => ref.origin === 'local'); + const local = uniqueMember(localRefs); + if (local === 'ambiguous') return 'owned'; + if (local !== undefined) return local; + if (!publishesImportedNames) return 'owned'; + + // PRECEDENCE IS DECIDED BEFORE THE TYPE GATE, not by it. `uniqueMember` + // applies `CALL_TARGET_TYPES` while it selects, so a target file declaring a + // NON-callable under this name answers `undefined` above and would otherwise + // fall through to the published channel — publishing a re-exported callable + // under a name the module's own declaration owns. `findExportedDef` does not + // do that: it returns any local def it finds and lets its caller's type gate + // reject it, so `findExportedDefIncludingImportedNames` never reaches the + // imported names for a name the file declares. Same rule here, so `x.f` and + // `x.f()` cannot disagree about which module owns the name. + // + // Not reachable through valid Zig today — a container cannot declare a name + // twice, so one target file cannot hold both spellings, and Zig is the only + // provider that sets `namespaceExportsIncludeImportedNames`. It becomes + // reachable the moment a second provider opts in, or a receiver binds more + // than one target file; the guard is one `some` and the alternative failure + // is a confident edge into the wrong module. + const declaredLocally = targetFiles.some((targetFile) => { + const targetScopeId = scopes.moduleScopes.get(targetFile); + return targetScopeId !== undefined && localRefs(targetScopeId).length > 0; + }); + if (declaredLocally) return 'owned'; + + // Then a name the target file publishes but did not declare — the hub case. + // `lookupBindingsAt`, not `scopes.bindings`, because a hub's module scope owns + // no local binding for these names and the finalized/augmented channel is the + // only place they exist; the same read `findExportedDefIncludingImportedNames` + // does for the CALL form. + const published = uniqueMember((scope) => + lookupBindingsAt(scope, site.name, scopes).filter( + (ref) => ref.origin === 'import' || ref.origin === 'namespace' || ref.origin === 'reexport', + ), + ); + return published === 'ambiguous' || published === undefined ? 'owned' : published; +} + export function emitPropertyDispatchCalls( graph: KnowledgeGraph, scopes: ScopeResolutionIndexes, parsedFiles: readonly ParsedFile[], nodeLookup: GraphNodeLookup, + model: SemanticModel, calleeIdSink?: CalleeIdSink, + /** + * `ScopeResolver.namespaceExportsIncludeImportedNames`, forwarded rather than + * re-derived. The pass names no language; it asks the provider the same + * question `receiver-bound-calls` asks before resolving a namespace member, + * so the CALL and the REGISTRATION forms of `hub.fn` cannot disagree. + */ + publishesImportedNames = false, ): { usesEmitted: number; callsEmitted: number; @@ -86,18 +374,16 @@ export function emitPropertyDispatchCalls( for (const parsed of parsedFiles) { for (const site of parsed.referenceSites) { if (site.kind !== 'value-ref') continue; - const def = findCallableBindingInScope(site.inScope, site.name, scopes); + const def = resolveValueRefTarget( + site, + parsed.filePath, + scopes, + model, + publishesImportedNames, + ); if (def === undefined) continue; - const ok = tryEmitEdge( - graph, - scopes, - nodeLookup, - site, - def, - 'scope-resolution: value-ref', - seen, - ); + const ok = tryEmitEdge(graph, scopes, nodeLookup, site, def, VALUE_REF_EDGE_REASON, seen); if (ok) usesEmitted++; if (site.propertyKey === undefined) continue; diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index aa30acf91..7cc96e829 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -726,6 +726,7 @@ export function runScopeResolution( const resolutionConfig = input.resolutionConfig; const finalized = finalizeScopeModel(parsedFiles, { hooks: { + importsBindAtLexicalScope: provider.importsBindAtLexicalScope === true, resolveImportTarget: (targetRaw, fromFile, _workspaceIndex, parsedImport) => provider.resolveImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig, { parsedFiles, @@ -737,9 +738,23 @@ export function runScopeResolution( provider.expandsWildcardTo?.(targetModuleScope, parsedFiles) ?? [], mergeBindings: (existing, incoming, scopeId) => provider.mergeBindings(existing, incoming, scopeId), + wildcardCollisionIsAmbiguous: provider.exclusiveWildcardReexports === true, + namedImportsBindTopLevelOnly: provider.namedImportsBindTopLevelOnly === true, }, }); logHeapProbe('sr-post-finalize', `lang=${provider.language}`); + // `export *` collisions the shared finalize refused to bind (WS1 C2). Recorded + // as outcomes so the refusal is auditable next to the name-fallback census — + // a silently unresolved importer is indistinguishable from a resolver gap. + for (const refused of finalized.stats.ambiguousWildcardExports) { + recordResolutionOutcome({ + kind: 'reexport-ambiguous', + candidateIds: refused.candidateDefIds, + phase: 'finalize', + filePath: refused.filePath, + name: refused.name, + }); + } // One store and ONE writer rule for heritage instantiations (#2912), shared by // the pre-pass below and by the language hook further down — a heritage shape // the pre-pass cannot express (Rust `impl T for S`, Dart `implements`) records @@ -1085,6 +1100,12 @@ export function runScopeResolution( workspaceIndex, { allowGlobalFallback: provider.allowGlobalFreeCallFallback === true, + language: provider.language, + isGlobalNameFallbackPlausible: provider.isGlobalNameFallbackPlausible, + sourceTextOf: + provider.isGlobalNameFallbackPlausible !== undefined + ? (filePath: string) => getFileContents().get(filePath) + : undefined, constructorCallTargetsClass: provider.constructorCallTargetsClass === true, markConstructionSites: provider.markConstructionSites === true, isFileLocalDef: provider.isFileLocalDef, @@ -1210,7 +1231,12 @@ export function runScopeResolution( indexes, emitParsedFiles, postHeritageNodeLookup, + readonlyModel, calleeIdAccumulator, + // Same provider hook the receiver-bound pass consults for a namespace + // member (Case 1). Without it a hub module's re-exported callable + // resolves when CALLED and declines when REGISTERED. + provider.namespaceExportsIncludeImportedNames === true, ); if (propertyDispatch.skippedKeys > 0) { // Never drop dispatch coverage silently: a hook table larger than the diff --git a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts index a0e5e0919..8a14460a6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts @@ -108,6 +108,60 @@ export type ResolutionOutcome = * every real codebase and taught readers to ignore it. */ readonly receiverOrigin?: ReceiverOrigin; + } + /** + * The global-name fallback fired: a callable was chosen because its SIMPLE + * NAME is unique in the workspace, with no import or scope chain leading to + * it. A labeled low-confidence edge WAS emitted. + * + * Separate from `resolved` because it is not a resolution, and separate from + * `suppressed` because an edge exists. Counting it is the only way a reader + * can tell how much of a language's call graph rests on name uniqueness — the + * number that was previously invisible because these edges were emitted with + * the same reason and confidence as import-resolved ones. + */ + | { + readonly kind: 'fallback-guessed'; + readonly targetId: string; + readonly language?: string; + readonly phase: string; + readonly filePath: string; + readonly name: string; + readonly range: Range; + } + /** + * A global-name-fallback candidate was REFUSED by the language's plausibility + * hook: the language's own visibility rules make that call impossible, so no + * edge was emitted. + * + * The counterpart of `fallback-guessed`, and the pair is what makes the + * refusal auditable — a refusal count with no guess count cannot distinguish + * "the guard works" from "the guard rejects everything". + */ + | { + readonly kind: 'fallback-refused'; + readonly candidateId: string; + readonly language?: string; + readonly phase: string; + readonly filePath: string; + readonly name: string; + readonly range: Range; + } + /** + * A barrel re-exported `name` through two or more `export *` sources that + * each declare it, so the language names no winner. The shared finalize pass + * REFUSED the binding (see `FinalizeStats.ambiguousWildcardExports`) instead + * of publishing the first-listed source as `import-resolved`; every importer + * of `name` through `filePath` stays unresolved. No `range`: the collision + * belongs to the file's export surface, not to one statement. + */ + | { + readonly kind: 'reexport-ambiguous'; + readonly candidateIds: readonly string[]; + readonly phase: string; + /** The barrel file whose `export *` sources collide. */ + readonly filePath: string; + readonly name: string; }; /** diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts index 38413f013..54daf1c92 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts @@ -353,6 +353,82 @@ export function isNamespaceNameShadowed( return true; } +/** + * Does something between `inScope` and its module scope bind `name` to + * ANYTHING other than `def`? + * + * `isNamespaceNameShadowed` asks the same question for a namespace handle, + * where any local binding of the name is by definition not the import. A + * CONTAINER receiver needs the extra clause: the container may itself be the + * local declaration (`fn make() { const Local = struct {…}; … Local.go … }`), + * and reading that as its own shadow would suppress exactly the resolutions it + * is meant to permit — the #2723 mistake, one channel over. + * + * So a scope that binds the name answers immediately, and the answer is "not + * shadowed" only when one of that scope's bindings IS `def`. A name bound in a + * nearer scope to something else — a parameter, a local, a type binding — wins + * the lexical race, which is the whole point: `findClassBindingInScope` filters + * the chain by `isClassLike` and therefore cannot see that it lost it. + * + * The MODULE scope is inspected too, unlike `isNamespaceNameShadowed`, and the + * exemption is what makes that safe. That guard stops one rung short because a + * namespace import writes its own name into the module scope and would read as + * its own shadow (#2723); here the owner is compared by identity, so the binding + * that IS the owner exempts itself and only a binding to something ELSE answers + * `true`. Stopping short would leave the exact hole this walk exists to close: + * `findClassBindingInScope` steps over a module-scope binding that is not + * class-like and then answers from a WORKSPACE-wide qualified-name index, so + * `const Gauge = @import("other.zig").SOME_CONST;` in a file that never imports + * `Gauge.zig` would still resolve `Gauge.read` to that file's container. + * `lookupBindingsAt` is used at that scope and only there, because an imported + * alias lives in the finalized channel rather than in `scope.bindings`. + * + * Fail-closed like its sibling: a missing scope or a parent cycle answers + * `true`, because suppressing a resolution costs a missing edge while trusting a + * corrupt chain costs a wrong one. + */ +export function isOwnerNameShadowedBySomethingElse( + name: string, + def: SymbolDefinition, + inScope: ScopeId, + scopes: ScopeResolutionIndexes, +): boolean { + let currentId: ScopeId | null = inScope; + const visited = new Set(); + while (currentId !== null) { + if (visited.has(currentId)) return true; + visited.add(currentId); + const scope = scopes.scopeTree.getScope(currentId); + if (scope === undefined) return true; + if (scope.kind !== 'Object') { + const isModule = scope.kind === 'Module'; + const imported = isModule ? lookupBindingsAt(currentId, name, scopes) : []; + const bindsHere = + scope.bindings.has(name) || + scope.typeBindings.has(name) || + scope.lexicalNames?.has(name) === true || + imported.length > 0 || + scope.ownedDefs.some((d) => { + const qualifiedName = d.qualifiedName; + if (qualifiedName === undefined) return false; + const dot = qualifiedName.lastIndexOf('.'); + return (dot === -1 ? qualifiedName : qualifiedName.slice(dot + 1)) === name; + }); + if (bindsHere) { + if ((scope.bindings.get(name) ?? []).some((b) => b.def.nodeId === def.nodeId)) return false; + if (scope.ownedDefs.some((d) => d.nodeId === def.nodeId)) return false; + if (imported.some((b) => b.def.nodeId === def.nodeId)) return false; + return true; + } + } + // The module scope is the last rung, not a rung to skip: nothing above it + // can shadow a name for this file. + if (scope.kind === 'Module') return false; + currentId = scope.parent; + } + return true; +} + export function findReceiverTypeBinding( startScope: ScopeId, receiverName: string, @@ -706,6 +782,20 @@ function soleBoundBaseName(bound: string): string | undefined { return base.length === 0 ? undefined : base; } +export type ClassBindingLookup = { + /** + * When false, skip the workspace-unique QualifiedNameIndex hit (and the + * dotted-tail variant). Default true — Go inheritance still needs that + * hit because namespace-style imports often create no scope binding. + * + * Constructor-form free calls pass false so a unique type name is not + * treated as a precise in-scope binding. Those sites belong to + * `pickUniqueGlobalClass`, which labels the edge as a guess and runs + * `isGlobalNameFallbackPlausible`. + */ + readonly uniqueQualifiedNameFallback?: boolean; +}; + export function findClassBindingInScope( startScope: ScopeId, receiverName: string, @@ -722,6 +812,7 @@ export function findClassBindingInScope( * selection. Only receiver-chain base and step resolution passes this. */ stripDecoration?: DecorationStripper, + lookup?: ClassBindingLookup, ): SymbolDefinition | undefined { // A TYPE PARAMETER is not a class, and it is checked before every route below // rather than inside one of them because each route would otherwise reach a @@ -730,7 +821,13 @@ export function findClassBindingInScope( // fallback after stripping. The declaration that introduced the parameter is // the only thing that knows, and it knows for all three. if (bindsTypeParameter(startScope, receiverName, scopes)) { - return resolveThroughTypeParameterBound(startScope, receiverName, scopes, stripDecoration); + return resolveThroughTypeParameterBound( + startScope, + receiverName, + scopes, + stripDecoration, + lookup, + ); } const local = walkScopeChain(startScope, receiverName, scopes, (def) => isClassLike(def.type)); @@ -739,21 +836,25 @@ export function findClassBindingInScope( // Fallback for languages (Go) where namespace-style imports don't // create scope bindings: resolve via QualifiedNameIndex. Only fires // when the scope-chain walk found nothing; single-match wins. - const qnames = scopes.qualifiedNames.get(receiverName); - if (qnames.length === 1) { - const def = scopes.defs.get(qnames[0]!); - if (def !== undefined && isClassLike(def.type)) return def; - } - // Second fallback: dotted names like "models.User" — try the simple - // name (tail after last dot) for languages where defs are indexed by - // simple name (Go). Only when the dotted lookup fails. - if (receiverName.includes('.')) { - const simple = receiverName.slice(receiverName.lastIndexOf('.') + 1); - if (simple.length > 0 && simple !== receiverName) { - const simpleIds = scopes.qualifiedNames.get(simple); - if (simpleIds.length === 1) { - const def = scopes.defs.get(simpleIds[0]!); - if (def !== undefined && isClassLike(def.type)) return def; + // Constructor-form free calls opt out — a unique type name is a guess, + // not proof the type is in scope (see `ClassBindingLookup`). + if (lookup?.uniqueQualifiedNameFallback !== false) { + const qnames = scopes.qualifiedNames.get(receiverName); + if (qnames.length === 1) { + const def = scopes.defs.get(qnames[0]!); + if (def !== undefined && isClassLike(def.type)) return def; + } + // Second fallback: dotted names like "models.User" — try the simple + // name (tail after last dot) for languages where defs are indexed by + // simple name (Go). Only when the dotted lookup fails. + if (receiverName.includes('.')) { + const simple = receiverName.slice(receiverName.lastIndexOf('.') + 1); + if (simple.length > 0 && simple !== receiverName) { + const simpleIds = scopes.qualifiedNames.get(simple); + if (simpleIds.length === 1) { + const def = scopes.defs.get(simpleIds[0]!); + if (def !== undefined && isClassLike(def.type)) return def; + } } } } @@ -799,6 +900,7 @@ function resolveThroughTypeParameterBound( parameterName: string, scopes: ScopeResolutionIndexes, stripDecoration?: DecorationStripper, + lookup?: ClassBindingLookup, ): SymbolDefinition | undefined { const bound = typeParameterAt(startScope, parameterName, scopes)?.bound; if (bound === undefined) return undefined; @@ -806,7 +908,7 @@ function resolveThroughTypeParameterBound( if (baseName === undefined || baseName === parameterName) return undefined; // A bound naming another parameter terminates here rather than recursing. if (bindsTypeParameter(startScope, baseName, scopes)) return undefined; - return findClassBindingInScope(startScope, baseName, scopes, stripDecoration); + return findClassBindingInScope(startScope, baseName, scopes, stripDecoration, lookup); } function normalizeTemplateArgToken(value: string): string { @@ -1207,6 +1309,7 @@ export function resolveInheritanceBaseInScope( scopes: ScopeResolutionIndexes, rawQualifiedName?: string, enclosingClassDef?: SymbolDefinition, + lookup?: ClassBindingLookup, ): SymbolDefinition | undefined { // #1982: when the source wrote a qualified base (`Other::Inner`), resolve it // against the full-path QualifiedNameIndex FIRST, so a same-tail nested base @@ -1226,7 +1329,7 @@ export function resolveInheritanceBaseInScope( if (qualified !== undefined) return qualified; } return ( - findClassBindingInScope(startScope, baseName, scopes) ?? + findClassBindingInScope(startScope, baseName, scopes, undefined, lookup) ?? resolveAmbiguousInheritanceBaseViaImports(startScope, baseName, scopes) ); } @@ -1452,6 +1555,43 @@ function dirnameOf(filePath: string): string { return idx === -1 ? '' : filePath.slice(0, idx); } +/** + * True when the caller can see `classDef` through import evidence, not + * merely because the class name is workspace-unique. + * + * Exact imported file (`#include "user.h"`) and a finalize-resolved + * `targetDefId` (named `use` / `pub use` followed to the def) are + * precise. An import that *names* this class (`targetExportedName`) + * is also precise — a barrel's `targetFile` is the re-exporting module, + * not the defining file. A sibling-file import of a *different* name + * is not: `use crate::a::bar` does not make `Foo` in `a/foo.rs` visible. + * + * Directory-only matching is intentionally omitted. The same-directory + * tier on `resolveAmbiguousInheritanceBaseViaImports` is for picking + * among *ambiguous* names (C# namespace `using`); it is not proof a + * unique constructor type is in scope. + */ +export function isClassFileImportGrounded( + startScope: ScopeId, + classDef: Pick, + scopes: ScopeResolutionIndexes, + importedName: string, +): boolean { + const callerModule = moduleScopeIdOf(startScope, scopes); + const callerFile = scopes.scopeTree.getScope(startScope)?.filePath; + if (callerFile !== undefined && classDef.filePath === callerFile) return true; + if (callerModule === null) return false; + const importEdges = scopes.imports.get(callerModule); + if (importEdges === undefined || importEdges.length === 0) return false; + for (const edge of importEdges) { + if (edge.targetDefId !== undefined && edge.targetDefId === classDef.nodeId) return true; + if (edge.targetFile === null) continue; + if (edge.targetFile === classDef.filePath) return true; + if (importedName.length > 0 && edge.targetExportedName === importedName) return true; + } + return false; +} + /** * Predicate for value-receiver bridge: the labels for which * `reconcileOwnership` registers methods/fields under the def's diff --git a/gitnexus/src/core/ingestion/scope-resolution/utils/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/scope-resolution/utils/name-fallback-visibility.ts new file mode 100644 index 000000000..2c66f1c05 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/utils/name-fallback-visibility.ts @@ -0,0 +1,97 @@ +/** + * Language-agnostic primitives for `ScopeResolver.isGlobalNameFallbackPlausible` + * implementations. + * + * The hook itself is per-language — the RULES here are not. What every + * implementation needs is the same small set of path arithmetic: which + * directory a file sits in, and whether a module path a caller wrote can name + * a given file or directory. Those questions are about paths, not about any + * language, so they live in shared code (see AGENTS.md: shared ingestion must + * not name languages) and the language files supply only the semantics. + */ + +import type { ParsedFile } from 'gitnexus-shared'; + +/** POSIX-style parent directory. `''` for a file at the repo root. */ +export function directoryOf(filePath: string): string { + const slash = filePath.lastIndexOf('/'); + return slash === -1 ? '' : filePath.slice(0, slash); +} + +/** + * Split a module path into segments, accepting the three separators languages + * spell module nesting with: `/` (Go, Node), `::` (Rust, C++) and `.` (JVM, + * Python). Empty segments and `.` are dropped, so a leading `./` contributes + * nothing. Named root prefixes are NOT stripped here — `crate::a` yields + * `['crate', 'a']`; a language whose paths carry one (Rust's `crate::` / + * `super::`) removes it before calling, see `rustUsePathOf`. + * + * `.` is only treated as a separator when the path contains no `/`: a Node + * specifier like `./util/parse.js` must not split on the extension dot. + */ +export function moduleSegments(modulePath: string): readonly string[] { + const bySlashOrColon = modulePath.split(/\/|::/).filter((s) => s !== '' && s !== '.'); + if (modulePath.includes('/')) return bySlashOrColon; + return bySlashOrColon.flatMap((s) => s.split('.').filter((p) => p !== '')); +} + +/** + * Does a module path a caller wrote reach `targetPath`? + * + * True when either side's segments are a SUFFIX of the other's. Both directions + * are needed and neither alone is sufficient: + * + * - The written path is usually longer than the repo-relative one, because it + * carries a module/package prefix that is not a directory + * (`github.com/org/svc/internal/models` → `internal/models`). + * - The repo-relative path is longer when the manifest that defines the module + * root sits in a subdirectory of the analyzed tree (`svc/go.mod`, so + * `svc/internal/models` is written `mod/internal/models`). + * + * So the match is on ALIGNED TRAILING SEGMENTS, and it succeeds when either the + * shorter side is fully contained in the longer, or at least two segments align. + * Both conditions are needed: full containment covers a one-segment package + * (`mod/models` reaching `models`), while the two-segment floor covers the case + * where neither side contains the other because each carries a different root + * (`mod/internal/models` vs `svc/internal/models`). A SINGLE aligned segment + * with neither side contained is not enough — `handlers` appearing at the end of + * two unrelated trees says nothing. + * + * Tolerance is the SAFE direction here: this predicate is consulted to decide + * whether to REFUSE an edge, so over-matching loses a refusal (the edge stays, + * labeled and excluded from flows) while under-matching loses a real edge. + */ +export function modulePathReaches(writtenPath: string, targetPath: string): boolean { + const written = moduleSegments(writtenPath); + const target = moduleSegments(targetPath); + if (written.length === 0 || target.length === 0) return false; + const shorter = Math.min(written.length, target.length); + let aligned = 0; + while ( + aligned < shorter && + written[written.length - 1 - aligned] === target[target.length - 1 - aligned] + ) { + aligned++; + } + return aligned === shorter || aligned >= 2; +} + +/** Every module path this file's import statements named, in source order. */ +function importedModulePaths(parsed: ParsedFile): readonly string[] { + return parsed.parsedImports.map((imp) => imp.targetRaw); +} + +/** True when any of the caller's imports reaches `targetPath`. */ +export function anyImportReaches(parsed: ParsedFile, targetPath: string): boolean { + for (const written of importedModulePaths(parsed)) { + if (modulePathReaches(written, targetPath)) return true; + } + return false; +} + +/** Strip a trailing file extension. `a/b.rs` → `a/b`; `a/b` → `a/b`. */ +export function stripExtension(filePath: string): string { + const slash = filePath.lastIndexOf('/'); + const dot = filePath.lastIndexOf('.'); + return dot > slash ? filePath.slice(0, dot) : filePath; +} diff --git a/gitnexus/src/core/ingestion/scope-resolution/value-ref-edges.ts b/gitnexus/src/core/ingestion/scope-resolution/value-ref-edges.ts new file mode 100644 index 000000000..5507f9d0e --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/value-ref-edges.ts @@ -0,0 +1,14 @@ +/** + * The `reason` text stamped on the USES edge a `value-ref` site emits. + * + * A leaf module on purpose. Two modules need this string and they sit on + * opposite sides of the product: `passes/property-dispatch.ts` writes it at + * analysis time, and `mcp/local/local-backend.ts` reads it back at query time + * to decide whether an answer is `exact`. Re-typing the literal in the reader + * would make the epistemic signal fail SILENTLY the day the writer's text is + * reworded — the query would simply match nothing and every answer would go + * back to claiming certainty, which is the exact defect (#3399) this constant + * exists to close. Importing `property-dispatch.ts` for it instead would drag + * the whole scope-resolution emit graph into the MCP backend for one string. + */ +export const VALUE_REF_EDGE_REASON = 'scope-resolution: value-ref'; diff --git a/gitnexus/src/core/ingestion/tree-sitter-queries.ts b/gitnexus/src/core/ingestion/tree-sitter-queries.ts index a930eb69a..c71f9eb7d 100644 --- a/gitnexus/src/core/ingestion/tree-sitter-queries.ts +++ b/gitnexus/src/core/ingestion/tree-sitter-queries.ts @@ -2243,18 +2243,26 @@ export const DART_QUERIES = ` (identifier) @name . (formal_parameter_list))) @definition.constructor ; ── Field declarations (String name = '', Address address = Address()) ────── +; @name is ANCHORED to the first named child. An initialized_identifier whose +; value is a constructor call parses the callee as a SECOND (identifier) sibling +; of the name — \`final TextEditingController _t = TextEditingController();\` is +; initialized_identifier[identifier "_t", identifier "TextEditingController", +; selector]. Unanchored, \`(identifier) @name\` matched both and minted a phantom +; Property named after the TYPE alongside the real field. The same shape applies +; to static_final_declaration (class static and top-level final/const), so every +; rule below anchors. languages/dart/query.ts already anchors its mirror rules. (declaration (type_identifier) (initialized_identifier_list (initialized_identifier - (identifier) @name))) @definition.property + . (identifier) @name))) @definition.property ; ── Nullable field declarations (String? name) ────────────────────────────── (declaration (nullable_type) (initialized_identifier_list (initialized_identifier - (identifier) @name))) @definition.property + . (identifier) @name))) @definition.property ; ── static const / static final / const class fields ──────────────────────── ; A "static const a = 1;" / "static final String b = ..., c = ...;" field parses @@ -2267,7 +2275,7 @@ export const DART_QUERIES = ` (declaration (static_final_declaration_list (static_final_declaration - (identifier) @name))) @definition.property + . (identifier) @name))) @definition.property ; ── Getters ────────────────────────────────────────────────────────────────── (method_signature @@ -2290,7 +2298,7 @@ export const DART_QUERIES = ` (program (initialized_identifier_list (initialized_identifier - (identifier) @name)) @definition.variable) + . (identifier) @name)) @definition.variable) ; Closure bindings: \`var f = (x) => x;\` binds a CALLABLE, so it emits Function ; rather than Variable, matching TS/JS. Overlap with the pattern above is ; collapsed by the parse-worker dedup (#2687). Since #2693 this node is also @@ -2336,7 +2344,7 @@ export const DART_QUERIES = ` (program (static_final_declaration_list (static_final_declaration - (identifier) @name)) @definition.variable) + . (identifier) @name)) @definition.variable) ; ── Imports ────────────────────────────────────────────────────────────────── (import_or_export diff --git a/gitnexus/src/core/ingestion/ts-js-export-marker.ts b/gitnexus/src/core/ingestion/ts-js-export-marker.ts new file mode 100644 index 000000000..b01440433 --- /dev/null +++ b/gitnexus/src/core/ingestion/ts-js-export-marker.ts @@ -0,0 +1,345 @@ +/** + * Export evidence for ECMAScript declarations — the `@declaration.is-exported` + * marker both the TypeScript and the JavaScript capture emitters synthesize. + * + * `SymbolDefinition.isExported` is tri-state, and this is where the three + * states are decided for TS/JS: + * + * - `true` — the declaration sits under an `export_statement` (`export + * function f`, `export const x`, `export default class`), or the + * file names it in an `export { f }` / `export { f as g }` clause + * or an `export default f` / `export = f` statement. + * - `false` — an ESM-shaped file (no CommonJS export assignment) that does + * neither. The declaration is module-private: `export *` cannot + * republish it and it must not be counted as a wildcard provider. + * - no verdict — the file exports through CommonJS (`module.exports = …`, + * `exports.x = …`, top-level `this.x = …`) or is an ambient + * `.d.ts`, where "not under `export`" says nothing about what the + * module publishes. Nothing is emitted, and the reader keeps its + * prior behavior. One CommonJS shape IS decidable and gets `true`: + * a method or property declared directly in the object literal + * assigned to `module.exports` (`module.exports = { alpha() {} }`) + * is that module's export of `alpha`. + * + * Only a declaration reached from the top level through DECLARATION nodes can + * be a module export. The walk therefore stops with `false` at the first + * nesting boundary — a class body, an interface/enum body, a function body, an + * object literal that is not the `module.exports` value: `export class C { + * m() {} }` exports `C`, not `m`; `function w() { function s() {} }` exports + * nothing even when the file has `export { s }` for a different `s`. + * + * The ancestor walk here is deliberately NOT `tsExportChecker` + * (`export-detection.ts`): that checker's text fallback (`text.startsWith('export ')`) + * fires on the `program` node of any file whose first token is `export`, which + * would mark every declaration in such a file exported. + */ + +import type { SyntaxNode } from './utils/ast-helpers.js'; + +export interface EsmExportEvidence { + /** Local names published by `export { … }`, `export default `, `export = `. */ + readonly namedLocals: ReadonlySet; + /** The file exports through a CommonJS assignment: a plain "not under + * `export`" is no verdict there. */ + readonly commonJs: boolean; +} + +/** Read binding patterns, never initializer expressions or property keys. */ +function bindsReceiver(node: SyntaxNode | null, name: string): boolean { + if (node === null) return false; + if (node.type === 'identifier' || node.type === 'shorthand_property_identifier_pattern') { + return node.text === name; + } + if (node.type === 'variable_declarator') + return bindsReceiver(node.childForFieldName('name'), name); + if (node.type === 'assignment_pattern') + return bindsReceiver(node.childForFieldName('left'), name); + if (node.type === 'pair_pattern') return bindsReceiver(node.childForFieldName('value'), name); + if (node.type === 'import_specifier') { + return bindsReceiver(node.childForFieldName('alias') ?? node.childForFieldName('name'), name); + } + if (node.type === 'required_parameter' || node.type === 'optional_parameter') { + return bindsReceiver(node.childForFieldName('pattern'), name); + } + return ( + [ + 'formal_parameters', + 'object_pattern', + 'array_pattern', + 'rest_pattern', + 'import_clause', + 'named_imports', + 'namespace_import', + ].includes(node.type) && node.namedChildren.some((child) => bindsReceiver(child, name)) + ); +} + +/** A locally bound `module`/`exports` is not Node's export receiver. */ +function isExportReceiverShadowed(node: SyntaxNode, name: string): boolean { + for (let scope = node.parent; scope !== null; scope = scope.parent) { + if ( + ['function_expression', 'generator_function', 'class'].includes(scope.type) && + scope.childForFieldName('name')?.text === name + ) + return true; + if ( + bindsReceiver(scope.childForFieldName('parameters'), name) || + bindsReceiver(scope.childForFieldName('parameter'), name) + ) + return true; + if (scope.type === 'for_in_statement' || scope.type === 'for_of_statement') { + if (bindsReceiver(scope.childForFieldName('left'), name)) return true; + } + if (scope.type === 'for_statement') { + const initializer = scope.childForFieldName('initializer'); + if ( + initializer !== null && + (initializer.type === 'lexical_declaration' || + initializer.type === 'variable_declaration') && + initializer.namedChildren.some((child) => bindsReceiver(child, name)) + ) + return true; + } + if (scope.type !== 'program' && scope.type !== 'statement_block') continue; + for (const statement of scope.namedChildren) { + const declaration = + statement.type === 'export_statement' + ? statement.childForFieldName('declaration') + : statement; + if (declaration === null) continue; + if ( + declaration.type === 'import_statement' && + declaration.namedChildren.some( + (child) => child.type === 'import_clause' && bindsReceiver(child, name), + ) + ) + return true; + if ( + declaration.type === 'lexical_declaration' || + declaration.type === 'variable_declaration' + ) { + if (declaration.namedChildren.some((child) => bindsReceiver(child, name))) return true; + } else if ( + ['function_declaration', 'class_declaration'].includes(declaration.type) && + declaration.childForFieldName('name')?.text === name + ) + return true; + } + } + return false; +} + +/** Static dot and bracket spellings of the same CommonJS export object. */ +function isModuleExportsReference(node: SyntaxNode): boolean { + const object = node.childForFieldName('object'); + if ( + object?.type !== 'identifier' || + object.text !== 'module' || + isExportReceiverShadowed(node, 'module') + ) + return false; + if (node.type === 'member_expression') { + return node.childForFieldName('property')?.text === 'exports'; + } + if (node.type !== 'subscript_expression') return false; + const index = node.childForFieldName('index'); + return index?.type === 'string' && (index.text === "'exports'" || index.text === '"exports"'); +} + +/** + * Does the file touch a CommonJS export object anywhere — `module.exports` or + * `exports.x` / `exports[x]` — as an actual expression? Read from AST nodes, + * not source text, so a comment or string mentioning `module.exports` does not + * disable the file's verdicts. + */ +function hasCommonJsExportSurface(root: SyntaxNode): boolean { + for (const member of root.descendantsOfType('member_expression')) { + const object = member.childForFieldName('object'); + if (object === null) continue; + if ( + object.type === 'identifier' && + object.text === 'exports' && + !isExportReceiverShadowed(member, 'exports') + ) + return true; + if (isModuleExportsReference(member)) return true; + } + for (const sub of root.descendantsOfType('subscript_expression')) { + const object = sub.childForFieldName('object'); + if ( + object?.type === 'identifier' && + object.text === 'exports' && + !isExportReceiverShadowed(sub, 'exports') + ) + return true; + if (isModuleExportsReference(sub)) return true; + } + return false; +} + +/** + * Top-level control-flow wrappers around `this.x =` / `this['x'] =`. + * Recurse through these only — a function or class body is a different `this`. + */ +const TOP_LEVEL_THIS_WALK: ReadonlySet = new Set([ + 'if_statement', + 'else_clause', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'while_statement', + 'do_statement', + 'switch_statement', + 'switch_body', + 'switch_case', + 'switch_default', + 'try_statement', + 'catch_clause', + 'finally_clause', + 'labeled_statement', + 'statement_block', +]); + +/** Module-wrapper `this.x =` / `this['x'] =` as a statement. */ +function isThisExportAssignment(stmt: SyntaxNode): boolean { + if (stmt.type !== 'expression_statement') return false; + const assignment = stmt.namedChildren[0]; + const left = + assignment?.type === 'assignment_expression' ? assignment.childForFieldName('left') : null; + return ( + left !== null && + (left.type === 'member_expression' || left.type === 'subscript_expression') && + left.childForFieldName('object')?.type === 'this' + ); +} + +/** + * A top-level `this` export, including `if (enabled) this.api = api` and the + * braced form. Does not walk into function or class bodies. + */ +function hasTopLevelThisExportAssignment(stmt: SyntaxNode): boolean { + if (isThisExportAssignment(stmt)) return true; + if (!TOP_LEVEL_THIS_WALK.has(stmt.type)) return false; + return stmt.namedChildren.some(hasTopLevelThisExportAssignment); +} + +/** Node types below which a declaration is nested, not module-level. */ +const NESTING_BOUNDARIES: ReadonlySet = new Set([ + 'class_body', + 'interface_body', + 'enum_body', + 'object_type', + 'statement_block', + 'arrow_function', + 'function_expression', + 'function_declaration', + 'generator_function', + 'generator_function_declaration', + 'method_definition', + // `export namespace NS { export function f() {} }` / `declare module 'x' { + // export function q(): void }`: an `export` inside these bodies is an export + // of the namespace/ambient module, not of the file. + 'internal_module', + 'module', + 'ambient_declaration', +]); + +/** + * Scan a file's top level once. `undefined` means the file's export surface + * cannot be read at all (ambient `.d.ts`), so no marker should be emitted. + */ +export function collectEsmExportEvidence( + root: SyntaxNode, + filePath: string, +): EsmExportEvidence | undefined { + if (filePath.endsWith('.d.ts')) return undefined; + const namedLocals = new Set(); + // Any CommonJS export surface anywhere in the file — a direct `module.exports + // = …`, an alias (`const m = module.exports; m.x = …`), an `exports.x` — means + // "not under `export`" says nothing. + let commonJs = hasCommonJsExportSurface(root); + for (const stmt of root.namedChildren) { + if (stmt.type === 'export_statement') { + // `export { a } from './x'` / `export type { T } from './t'` re-export + // ANOTHER module's names: they say nothing about a local `a`, and adding + // them here marked a private local of the same name exported. + if (stmt.childForFieldName('source') !== null) continue; + for (const child of stmt.namedChildren) { + if (child.type === 'export_clause') { + for (const spec of child.namedChildren) { + if (spec.type !== 'export_specifier') continue; + const name = spec.childForFieldName('name')?.text; + if (name !== undefined && name !== '') namedLocals.add(name); + } + } else if (child.type === 'identifier') { + // `export default f;` and TS `export = f;`. + namedLocals.add(child.text); + } + } + } else if (hasTopLevelThisExportAssignment(stmt)) { + commonJs = true; + } + } + return { namedLocals, commonJs }; +} + +/** Is `object` the value of a top-level `module.exports = { … }` assignment? */ +function isModuleExportsObject(object: SyntaxNode): boolean { + const assignment = object.parent; + if (assignment === null || assignment.type !== 'assignment_expression') return false; + if (assignment.childForFieldName('right')?.id !== object.id) return false; + const left = assignment.childForFieldName('left'); + if (left === null || !isModuleExportsReference(left)) return false; + return ( + assignment.parent?.type === 'expression_statement' && + assignment.parent.parent?.type === 'program' + ); +} + +/** + * The export verdict for a declaration whose NAME node is `nameNode`: + * `true` / `false` as documented in the header, `undefined` when this file's + * export surface cannot decide it (CommonJS, other than the `module.exports` + * object literal itself). + */ +export function esmExportVerdict( + nameNode: SyntaxNode, + evidence: EsmExportEvidence, +): boolean | undefined { + // The name node's own declaration node is where the walk starts; the + // declaration itself (a `method_definition`, a `function_declaration`) must + // not count as its own nesting boundary. + let current: SyntaxNode | null = nameNode.parent; + // An `export` keyword is only a FILE-level export when the walk reaches the + // program without crossing a nesting boundary — one inside a namespace or + // ambient-module body is that container's export (see NESTING_BOUNDARIES). + let underExport = false; + let functionScopedVariable = false; + while (current !== null && current.type !== 'program') { + if (current.type === 'variable_declaration') functionScopedVariable = true; + if (current.type === 'export_statement') { + underExport = true; + current = current.parent; + continue; + } + if (current.type === 'object') { + // `module.exports = { alpha() {} }`: the literal's own members are the + // module's exports. Any other object literal is a nesting boundary. + if (isModuleExportsObject(current) && nameNode.parent?.parent?.id === current.id) return true; + return evidence.commonJs ? undefined : false; + } + if (NESTING_BOUNDARIES.has(current.type) && current.id !== nameNode.parent?.id) { + // `var` crosses blocks but never a function/namespace boundary. A + // module-scoped `var` can therefore be exported by a later clause. + if (current.type === 'statement_block' && functionScopedVariable) { + current = current.parent; + continue; + } + return evidence.commonJs ? undefined : false; + } + current = current.parent; + } + if (underExport) return true; + if (evidence.commonJs) return undefined; + return evidence.namedLocals.has(nameNode.text); +} diff --git a/gitnexus/src/core/lbug/graph-emit-sink.ts b/gitnexus/src/core/lbug/graph-emit-sink.ts index 6b263701f..d13ab47c7 100644 --- a/gitnexus/src/core/lbug/graph-emit-sink.ts +++ b/gitnexus/src/core/lbug/graph-emit-sink.ts @@ -70,7 +70,7 @@ * replaced, so the obvious rewrite is not the one that shipped. * 3. The remaining ~90 ms was object allocation itself, irreducible while the * read API returns objects — so the five whole-graph scans moved to - * `forEachRelationshipFields`, which passes the four fields they actually + * `forEachRelationshipFields`, which passes the five fields they actually * read as primitives and allocates nothing. See * {@link GraphEmitSink.forEachRelationshipFields}. * @@ -282,14 +282,19 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { * safe because `buildRelRow` never persists `rel.id` and no consumer keys on * it (audited). * - * The dropped `reason`/`step` are safe too, but for a different reason worth - * stating: the PERSISTED row keeps their true values, because `buildRelRow` is - * handed the original relationship on the way through. Only in-memory reads - * see the `'streamed'` placeholder, and the in-pipeline consumers of streamed - * edges read neither field. So e.g. the `ACCESSES reason: 'read'|'write'` - * distinction that MCP queries rely on survives in the database. A future - * in-pipeline consumer needing `reason` or `step` on a streamed edge must add - * the column, not trust the placeholder. + * `reason` IS now retained, as an interned index — the in-pipeline consumer + * this JSDoc anticipated arrived. Process tracing and large-graph community + * detection must exclude global-name-fallback edges, which are emitted at + * exactly their confidence threshold (0.5) and so cannot be separated by + * confidence alone. Interning keeps the cost at one small integer per edge + * (the reason vocabulary is a fixed set of literals), not one string. + * + * `id` and `step` remain dropped. The PERSISTED row keeps `step`'s true value, + * because `buildRelRow` is handed the original relationship on the way + * through; only in-memory OBJECT reads see the `'streamed'`-era placeholder, + * and no in-pipeline consumer of streamed edges reads `step`. A future + * in-pipeline consumer needing `step` must add the column, not trust the + * placeholder. * * Node ids are interned; the strings are shared by reference with the node * map's, so interning adds bookkeeping, not new text. @@ -300,6 +305,12 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { private readonly tgtIx: number[] = []; private readonly relTypes: RelationshipType[] = []; private readonly confidences: number[] = []; + /** Interned reason strings, and the per-edge index into them. The vocabulary + * is a fixed set of emitter literals, so this is O(vocabulary) text plus one + * small integer per edge. */ + private readonly reasonIds = new Map(); + private readonly reasonByIx: string[] = []; + private readonly reasonIx: number[] = []; private finalized = false; /** * Streaming is OFF until {@link beginStreaming} is called by `parse`. @@ -472,6 +483,16 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { this.tgtIx.push(tgtIx); this.relTypes.push(relationship.type); this.confidences.push(relationship.confidence); + this.reasonIx.push(this.internReason(relationship.reason)); + } + + private internReason(reason: string): number { + const existing = this.reasonIds.get(reason); + if (existing !== undefined) return existing; + const ix = this.reasonByIx.length; + this.reasonByIx.push(reason); + this.reasonIds.set(reason, ix); + return ix; } /** Flush + close every writer and return the COPY manifest. Every fd is @@ -599,7 +620,13 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { * with the object-based graph despite holding relationships columnar. */ forEachRelationshipFields( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ): void { this.real.forEachRelationshipFields(fn); for (let ix = 0; ix < this.srcIx.length; ix++) { @@ -608,6 +635,7 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { this.nodeIdByIx[this.tgtIx[ix]], this.relTypes[ix], this.confidences[ix], + this.reasonByIx[this.reasonIx[ix]], ); } } diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index a909feaf8..b38a5f4d6 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -711,8 +711,8 @@ const runSchemaCreationQueries = async (dbPath: string): Promise return null; }; -export const initLbug = async (dbPath: string) => { - return runWithSessionLock(() => ensureLbugInitialized(dbPath)); +export const initLbug = async (dbPath: string, options: { skipFts?: boolean } = {}) => { + return runWithSessionLock(() => ensureLbugInitialized(dbPath, options)); }; /** @@ -726,14 +726,14 @@ export const initLbug = async (dbPath: string) => { export const withLbugDb = async ( dbPath: string, operation: () => Promise, - options: { readOnly?: boolean } = {}, + options: { readOnly?: boolean; skipFts?: boolean } = {}, ): Promise => { let lastError: unknown; const readOnly = options.readOnly === true; for (let attempt = 1; attempt <= DB_LOCK_RETRY_ATTEMPTS; attempt++) { try { return await runWithSessionLock(async () => { - await ensureLbugInitialized(dbPath, readOnly); + await ensureLbugInitialized(dbPath, { readOnly, skipFts: options.skipFts }); return operation(); }); } catch (err) { @@ -764,15 +764,32 @@ export const withLbugDb = async ( throw lastError; }; -const ensureLbugInitialized = async (dbPath: string, readOnly: boolean = false) => { - if (conn && currentDbPath === dbPath && currentDbReadOnly === readOnly) { +let currentDbSkipFts = false; + +const ensureLbugInitialized = async ( + dbPath: string, + options: { readOnly?: boolean; skipFts?: boolean } = {}, +) => { + const readOnly = options.readOnly === true; + const skipFts = options.skipFts === true; + if ( + conn && + currentDbPath === dbPath && + currentDbReadOnly === readOnly && + currentDbSkipFts === skipFts + ) { return { db, conn }; } - await doInitLbug(dbPath, readOnly); + await doInitLbug(dbPath, { readOnly, skipFts }); return { db, conn }; }; -const doInitLbug = async (dbPath: string, readOnly: boolean = false) => { +const doInitLbug = async ( + dbPath: string, + options: { readOnly?: boolean; skipFts?: boolean } = {}, +) => { + const readOnly = options.readOnly === true; + const skipFts = options.skipFts === true; // Different database requested — close the old one first if (conn || db) { await safeClose(); @@ -966,8 +983,11 @@ const doInitLbug = async (dbPath: string, readOnly: boolean = false) => { // Phase 3 installs it moments later in the same run. Warning here reported a // degradation that never happened — the run went on to build every FTS index. // Phase 3 (and the read-only branch) still warn for real failures. - await loadFTSExtension(undefined, readOnly ? { policy: 'load-only' } : { quiet: true }); + if (!skipFts) { + await loadFTSExtension(undefined, readOnly ? { policy: 'load-only' } : { quiet: true }); + } + currentDbSkipFts = skipFts; currentDbPath = dbPath; return { db, conn }; }; @@ -3703,7 +3723,10 @@ export const ensureEmbeddingRowDmlSafe = async ( * read once per run. {@link INDEX_CATALOG_UNREADABLE} fails closed here without * a second read; omitting the argument makes the gate read for itself. */ -export const ensureFtsRowDmlSafe = async (indexRows?: IndexCatalogSnapshot): Promise => { +export const ensureFtsRowDmlSafe = async ( + indexRows?: IndexCatalogSnapshot, + options: { skipFts?: boolean } = {}, +): Promise => { // Unconditional precondition, same regression as the VECTOR twin's (#2841 // review §5.B): a caller-supplied snapshot must not let a closed DB be // answered `true`. @@ -3730,6 +3753,9 @@ export const ensureFtsRowDmlSafe = async (indexRows?: IndexCatalogSnapshot): Pro return indexType === undefined || indexType === 'FTS'; }); if (!indexGatesDml) return true; + // Existing/unknown native indexes still gate writes. Rebuild into a fresh + // database rather than loading FTS or issuing unsafe DML when opted out. + if (options.skipFts) return false; return await loadFTSExtension(undefined, { policy: resolveAnalyzeInstallPolicy() }); }; diff --git a/gitnexus/src/core/lbug/pdg-emit-sink.ts b/gitnexus/src/core/lbug/pdg-emit-sink.ts index 12db53ac4..fafcd2c53 100644 --- a/gitnexus/src/core/lbug/pdg-emit-sink.ts +++ b/gitnexus/src/core/lbug/pdg-emit-sink.ts @@ -299,7 +299,13 @@ export class PdgEmitSink implements KnowledgeGraph { this.real.forEachRelationship(fn); } forEachRelationshipFields( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ): void { this.real.forEachRelationshipFields(fn); } diff --git a/gitnexus/src/core/lbug/schema.ts b/gitnexus/src/core/lbug/schema.ts index 5abc7bc46..3d3edd8ac 100644 --- a/gitnexus/src/core/lbug/schema.ts +++ b/gitnexus/src/core/lbug/schema.ts @@ -518,7 +518,7 @@ const ATTACHMENT_TARGET_LABELS: readonly NodeTableName[] = [ ]; /** - * The 66 pairs NEITHER rule above generates — everything left after the two + * The pairs NEITHER rule above generates — everything left after the two * cross products are subtracted. Carried by CONTAINMENT, inheritance, imports * and DI: a container label crossed with a contained label. No predicate * describes that surface (any container can hold any definition). @@ -531,6 +531,8 @@ const ATTACHMENT_TARGET_LABELS: readonly NodeTableName[] = [ * templates), the two `Route|Process` / `Tool|Process` entry points whose * emitter names both labels as literals, and `BasicBlock|BasicBlock`, the PDG * substrate. + * Spring dynamic lookup in a constructor also emits INJECTS to a synthetic + * @Bean CodeElement (#3238), so Constructor|CodeElement belongs here. * * NOTHING A RULE ALREADY COVERS BELONGS HERE. `generatedRelationPairs` skips * any pair present in this block, so a redundant line does not merely duplicate @@ -624,6 +626,7 @@ export const STRUCTURAL_PAIR_DDL = ` FROM File TO Folder, FROM \`Constructor\` TO \`Impl\`, FROM \`Constructor\` TO \`Namespace\`, FROM \`Constructor\` TO \`Typedef\`, + FROM \`Constructor\` TO CodeElement, FROM Route TO Process, FROM Tool TO Process, FROM Destination TO \`Property\`, diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 8b70ef10e..30b54ece4 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -10,13 +10,28 @@ */ import { detectGraphWriteCollapse, type GraphWriteCollapseVerdict } from './index-freshness.js'; +import { + resolveFtsDisableReason, + getFtsDisabledReason, + withExplicitFtsDisablement, + FTS_DISABLED_MESSAGE, + DEFAULT_GRAPH_CAPABILITY, + DEFAULT_VECTOR_SEARCH_CAPABILITY, + type FtsSkipReason, +} from './search/fts-policy.js'; import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js'; import path from 'path'; import fs from 'fs/promises'; import { constants as fsConstants } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { retryRename } from '../storage/fs-atomic.js'; -import { acquireIndexLock } from '../storage/index-lock.js'; +import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; +import { invalidateNodeWorkspacePackages } from './ingestion/import-resolvers/node-workspace-packages.js'; +import { + logNameFallbackSummary, + summarizeNameFallback, + countCallsByLanguage, +} from './ingestion/scope-resolution/name-fallback-summary.js'; import { runPipelineFromRepo } from './ingestion/pipeline.js'; import { logUnresolvedReceiverFiles, @@ -347,6 +362,7 @@ export interface AnalyzeOptions { useParseCache?: boolean; /** Repair only search indexes without re-running full parsing/indexing. */ repairFts?: boolean; + skipFts?: boolean; /** Emit per-index FTS create logs. */ verbose?: boolean; embeddings?: boolean; @@ -507,6 +523,12 @@ function recordLiveIndexMutationRisk(error: unknown): void { } } +function formatMetaWriteFailureReason(err: unknown): string { + return isReadOnlyFilesystemError(err) + ? `${(err as Error).message} — storage may be read-only (#1549)` + : (err as Error).message; +} + /** Whether a failed analyze may already have changed the live DB. */ export function analyzeFailureMayHaveMutatedLiveIndex(error: unknown): boolean { return ( @@ -553,8 +575,10 @@ export interface AnalyzeResult { * extension loaded but the index build/verify failed non-fatally — remedied by * `--repair-fts`, not by installing the extension). Lets the CLI show the * correct recovery hint instead of always blaming a missing extension. + * `disabled-by-flag` and `disabled-by-env` record intentional opt-out; + * neither calls for extension installation or repair. */ - ftsSkipReason?: 'extension-unavailable' | 'build-failed'; + ftsSkipReason?: FtsSkipReason; /** * True when the index this run produced/validated is the flat workspace * slot (#2106 R2, inverted by #2354 to follow the checked-out branch). @@ -1071,11 +1095,17 @@ export async function runFullAnalysis( // Validate operator-provided FTS config before anything else — a typo fails // here in ms, without taking the lock. (createSearchFTSIndexes reuses the // cached value via getSearchFTSStemmer.) + if (options.repairFts && resolveFtsDisableReason(options.skipFts)) { + throw new Error('--repair-fts cannot be used with --skip-fts or GITNEXUS_SKIP_FTS=1.'); + } initialiseSearchFTSStemmer(); initialiseSearchFTSCjkSegmentation(); // Scope the degraded-parse log throttle to this run (module-level counter // would otherwise stay saturated on a reused process). resetDegradedParseCounter(); + // The workspace-package memo is per process: this run must see the tree as it + // is now, not as the previous run in a long-lived watch/server process saw it. + invalidateNodeWorkspacePackages(repoPath); const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const acquireOpts = { @@ -1087,6 +1117,10 @@ export async function runFullAnalysis( let writeTarget = await resolveWriteTarget(repoPath, options); let lock = await acquireIndexLock(writeTarget.metaDir, acquireOpts); try { + requireExclusiveIndexLock( + lock, + `Cannot acquire the index lock at ${writeTarget.metaDir}; refusing an unlocked analysis.`, + ); // #2658 review H2: acquireIndexLock can wait up to the timeout ceiling, // during which git HEAD/branch — and thus the resolved write slot — may // change (a commit lands, a branch is switched, or another writer adopts the @@ -1113,6 +1147,10 @@ export async function runFullAnalysis( lock.release(); writeTarget = fresh; lock = await acquireIndexLock(fresh.metaDir, acquireOpts); + requireExclusiveIndexLock( + lock, + `Cannot acquire the index lock at ${fresh.metaDir}; refusing an unlocked analysis.`, + ); if (attempt === MAX_RELOCK - 1) { log('Index write target still moving after repeated re-acquire; proceeding on this lock.'); } @@ -1136,6 +1174,9 @@ async function runFullAnalysisInner( writeTarget: WriteTarget, runnerIdentityAtBootstrap?: AnalyzerRunnerIdentity, ): Promise { + const ftsDisabledReason = resolveFtsDisableReason(options.skipFts); + const initAnalysisLbug = (dbPath: string) => + ftsDisabledReason ? initLbug(dbPath, { skipFts: true }) : initLbug(dbPath); const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const progress = (phase: string, percent: number, message: string) => callbacks.onProgress(phase, percent, message); @@ -1176,7 +1217,19 @@ async function runFullAnalysisInner( log(`Metadata reconciliation failed (non-critical${code ? `, ${code}` : ''}); continuing.`); } - const existingMeta = await loadMeta(metaDir); + const loadedMeta = await loadMeta(metaDir); + const previousFtsDisabledReason = getFtsDisabledReason(loadedMeta?.capabilities?.fts); + // Flag and env are equivalent disablements. Only a true enable↔disable flip + // needs a write plan; a discriminator-only change restamps on the + // already-up-to-date path. + const ftsModeChanged = Boolean(ftsDisabledReason) !== Boolean(previousFtsDisabledReason); + // Fold explicit disablement into the in-memory prior meta so every later + // saveMeta that spreads it (dirty flag, incremental phase stamps) advertises + // "FTS disabled" instead of leftover available/build-failed while a wipe is + // in flight. Re-enable leaves the prior stamp untouched. + const existingMeta = loadedMeta + ? withExplicitFtsDisablement(loadedMeta, ftsDisabledReason) + : undefined; // ── FTS-only repair path ──────────────────────────────────────────── if (options.repairFts) { @@ -1229,7 +1282,7 @@ async function runFullAnalysisInner( ); } try { - await initLbug(lbugPath); + await initAnalysisLbug(lbugPath); // Gate on FTS availability BEFORE touching any index. createSearchFTSIndexes // now DROPs each index before recreating it (so schema changes reach existing // DBs); if the extension were unavailable, the drops would run and leave the @@ -1315,16 +1368,9 @@ async function runFullAnalysisInner( await saveMeta(metaDir, { ...latestMeta, capabilities: { - graph: latestMeta.capabilities?.graph ?? { - provider: 'ladybugdb', - status: 'available', - }, + graph: latestMeta.capabilities?.graph ?? DEFAULT_GRAPH_CAPABILITY, fts: { provider: 'ladybugdb-fts', status: 'available' }, - vectorSearch: latestMeta.capabilities?.vectorSearch ?? { - provider: 'exact-scan', - status: 'unavailable', - exactScanLimit: 0, - }, + vectorSearch: latestMeta.capabilities?.vectorSearch ?? DEFAULT_VECTOR_SEARCH_CAPABILITY, }, }); } catch (err) { @@ -1774,7 +1820,8 @@ async function runFullAnalysisInner( existingMeta && !existingMeta.embeddingCheckpoint && !options.force && - existingMeta.lastCommit === currentCommit + existingMeta.lastCommit === currentCommit && + !ftsModeChanged ) { // Non-git folders have currentCommit = '' — always rebuild since we can't detect changes if (currentCommit !== '') { @@ -1876,11 +1923,18 @@ async function runFullAnalysisInner( // EACCES/EPERM also arise from ownership problems and transient // Windows locks, so keep the real error visible alongside the // #1549 read-only hint instead of replacing it. - const reason = isReadOnlyFilesystemError(err) - ? `${(err as Error).message} — storage may be read-only (#1549)` - : (err as Error).message; log( - `Warning: could not restamp the workspace branch label (${reason}); will retry on the next run.`, + `Warning: could not restamp the workspace branch label (${formatMetaWriteFailureReason(err)}); will retry on the next run.`, + ); + } + } else if (ftsDisabledReason && ftsDisabledReason !== previousFtsDisabledReason) { + // Discriminator-only restamp (flag↔env). `existingMeta` already + // carries the folded skipReason; persist it without a write plan. + try { + await saveMeta(metaDir, existingMeta); + } catch (err) { + log( + `Warning: could not restamp the FTS skip reason (${formatMetaWriteFailureReason(err)}); will retry on the next run.`, ); } } @@ -1896,6 +1950,7 @@ async function runFullAnalysisInner( repoPath, stats: existingMeta.stats ?? {}, alreadyUpToDate: true, + ...(ftsDisabledReason ? { ftsSkipped: true, ftsSkipReason: ftsDisabledReason } : {}), isPrimaryBranch: !placement.branch, }; } @@ -1966,7 +2021,7 @@ async function runFullAnalysisInner( if (shouldLoadCache && existingMeta) { try { progress('embeddings', 0, 'Caching embeddings...'); - await initLbug(lbugPath); + await initAnalysisLbug(lbugPath); const cached = await loadCachedEmbeddings(); cachedEmbeddingNodeIds = cached.embeddingNodeIds; cachedEmbeddings = cached.embeddings; @@ -2347,7 +2402,7 @@ async function runFullAnalysisInner( // Full rebuild (POSIX) builds into the temp `buildPath`; incremental and // Windows use `buildPath === lbugPath` in place. try { - await initLbug(buildPath); + await initAnalysisLbug(buildPath); } catch (error) { if (liveIndexMutationStarted) recordLiveIndexMutationRisk(error); throw error; @@ -2664,7 +2719,9 @@ async function runFullAnalysisInner( // creates or drops an index. const indexCatalogRows = await readIndexCatalogSnapshot(); const embeddingRowDmlSafe = await ensureEmbeddingRowDmlSafe(indexCatalogRows); - const ftsRowDmlSafe = await ensureFtsRowDmlSafe(indexCatalogRows); + const ftsRowDmlSafe = await ensureFtsRowDmlSafe(indexCatalogRows, { + skipFts: Boolean(ftsDisabledReason), + }); const extensionForcedRebuild = !embeddingRowDmlSafe || !ftsRowDmlSafe; // `!options.dropEmbeddings` (H1): this rescue reads the rows back OUT of // the DB, so it must never fire on the one path whose entire purpose is to @@ -2754,7 +2811,7 @@ async function runFullAnalysisInner( // catalog read happened to fail still had both gates answer "safe" // (both extensions loaded), and claiming otherwise would trade one // invented cause for another. - if (extensionForcedRebuild && indexCatalogUnreadable) { + if (extensionForcedRebuild && indexCatalogUnreadable && !ftsDisabledReason) { const blockedExtensions = [ !embeddingRowDmlSafe ? 'VECTOR' : undefined, !ftsRowDmlSafe ? 'FTS' : undefined, @@ -2778,7 +2835,12 @@ async function runFullAnalysisInner( 'Semantic search falls back to exact scan until VECTOR is available.', ); } - if (!ftsRowDmlSafe) { + if (!ftsRowDmlSafe && ftsDisabledReason) { + escalationCauses.push( + 'FTS is explicitly disabled and existing search indexes could not be ruled out; ' + + 'a fresh graph store is required before writing rows without the extension', + ); + } else if (!ftsRowDmlSafe) { if (!indexCatalogUnreadable) { // Self-contained subject (H5): `join('; and ')` used to render "…the // CodeEmbedding vector index exists … and THIS INDEX carries FTS @@ -2829,7 +2891,9 @@ async function runFullAnalysisInner( !embeddingRowDmlSafe ? { reason: getExtensionCapability('VECTOR')?.reason, label: 'VECTOR' } : undefined, - !ftsRowDmlSafe ? { reason: getFtsCapability()?.reason, label: 'FTS' } : undefined, + !ftsRowDmlSafe && !ftsDisabledReason + ? { reason: getFtsCapability()?.reason, label: 'FTS' } + : undefined, ] .filter((e): e is { reason: string | undefined; label: string } => e !== undefined) .map(({ reason, label }) => diagnoseExtensionLoad(reason, label).remedy); @@ -2912,7 +2976,7 @@ async function runFullAnalysisInner( await closeLbug(); if (buildPath === lbugPath) liveIndexMutationStarted = true; await wipeLbugDbFiles(buildPath); - await initLbug(buildPath); + await initAnalysisLbug(buildPath); walCheckpointDriver = startWalCheckpointDriver(); await loadGraphToLbug(pipelineResult.graph, pipelineResult.repoPath, storagePath, (msg) => { lbugMsgCount++; @@ -3130,20 +3194,28 @@ async function runFullAnalysisInner( // analyze still produces a fully queryable graph; only full-text/BM25 // search falls back. `--repair-fts` (whose sole job is FTS) still fails // loudly on its own path above. - progress('fts', 85, 'Creating search indexes...'); - const ftsAvailable = await loadFTSExtension(undefined, { - policy: resolveAnalyzeInstallPolicy(), - }); + progress( + 'fts', + 85, + ftsDisabledReason ? 'Skipping search indexes...' : 'Creating search indexes...', + ); + const ftsAvailable = + !ftsDisabledReason && + (await loadFTSExtension(undefined, { + policy: resolveAnalyzeInstallPolicy(), + })); // Tracks whether search indexes actually ended up usable this run — starts // as ftsAvailable (extension loaded) but flips to false below when the // build/verify step itself fails, so capabilities.fts.status / ftsSkipped // stay honest even though that failure no longer aborts the whole analyze. let ftsReady = ftsAvailable; - // Why FTS ended up skipped (#2658 review L2): extension-unavailable up front, - // or build-failed in the degrade branch below. - let ftsSkipReason: 'extension-unavailable' | 'build-failed' | undefined = ftsAvailable + // Why FTS ended up skipped (#2658 review L2): an explicit opt-out + // (`disabled-by-flag` / `disabled-by-env`, #3091) when one was recorded, + // else extension-unavailable up front, or build-failed in the degrade + // branch below. + let ftsSkipReason: FtsSkipReason | undefined = ftsAvailable ? undefined - : 'extension-unavailable'; + : (ftsDisabledReason ?? 'extension-unavailable'); if (ftsAvailable) { // Degrade rather than throw: createSearchFTSIndexes re-tokenizes every // stored row on every run, so a native tokenizer error on a single @@ -3184,6 +3256,9 @@ async function runFullAnalysisInner( ); progress('fts', 90, 'Search indexes skipped (build failed)'); } + } else if (ftsDisabledReason) { + log(FTS_DISABLED_MESSAGE); + progress('fts', 90, 'Search indexes skipped (explicitly disabled)'); } else { // For a missing runtime dependency (#2374) the file is present, so the // generic "install it with network access" tail in FTS_UNAVAILABLE_MESSAGE @@ -3861,6 +3936,14 @@ async function runFullAnalysisInner( const resolutionOutcomes = pipelineResult.resolutionOutcomes ?? []; logUnresolvedReceiverFiles(resolutionOutcomes); + // Census of guessed call sites (before edge coalescing), refused candidates + // and ambiguous `export *` names. The legacy `nameFallbackEdges` metadata + // key stores site counts, not the final population of heuristic edges. + const nameFallbackSummary = summarizeNameFallback( + resolutionOutcomes, + countCallsByLanguage(pipelineResult.resolvedCalleeNamesByCaller, pipelineResult.graph), + ); + logNameFallbackSummary(nameFallbackSummary); // Annotated so the capabilities stamp below is compile-checked against // RepoMeta's status unions (tri-review 4669518496 P1/U3) — an unannotated @@ -3977,6 +4060,7 @@ async function runFullAnalysisInner( // Git-only: non-git repos never take the incremental path. schemaFingerprint: hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, unresolvedReceiverMembers: summarizeUnresolvedReceivers(resolutionOutcomes), + nameFallbackEdges: nameFallbackSummary, scopeExtractionFailures: summarizeScopeExtractionFailures( pipelineResult.scopeExtractionFailures, ), @@ -4228,9 +4312,11 @@ async function runFullAnalysisInner( // Prune the durable ParsedFile store to EXACTLY the parse cache's // surviving keys (#2038 warm-cache coverage), so the two content-addressed // stores stay coherent: a chunk is "cached" iff both its parse-cache shard - // and its durable shards exist. A quarantined chunk (in usedKeys but with - // no parse-cache shard) drops its durable subdir here and re-dispatches - // next run. Same try/catch — a durable-store write failure must never + // and its durable shards exist. A retired chunk — worker-quarantined, or + // one whose failed durable reset left an uncleared generation behind + // (#3204) — is filtered out of `savedKeys`, so it drops out of the + // durable index here and re-dispatches next run. Same try/catch — a + // durable-store write must never // break an otherwise successful run (next run treats it as a miss). await mergeStagedDurableParsedFileStore( storagePath, diff --git a/gitnexus/src/core/search/bm25-index.ts b/gitnexus/src/core/search/bm25-index.ts index 10818ec4a..8aaa279ef 100644 --- a/gitnexus/src/core/search/bm25-index.ts +++ b/gitnexus/src/core/search/bm25-index.ts @@ -2,7 +2,10 @@ * Full-Text Search via LadybugDB FTS * * Uses LadybugDB's built-in full-text search indexes for keyword-based search. - * Always reads from the database (no cached state to drift). + * Reads from the database on every query it runs (no cached state to drift). + * The one exception is an explicit opt-out (#3091): when a caller passes an + * `FtsDisabledReason`, the search short-circuits to an empty, unavailable + * response without opening or querying the database at all. */ // tri-review Residual-1: `classifyFtsQueryError` now lives in lbug-adapter.ts @@ -16,6 +19,7 @@ import { normalizeFtsText } from '../lbug/csv-generator.js'; import { getExtensionCapabilities } from '../lbug/extension-loader.js'; import { redactPaths } from './fts-indexes.js'; import { FTS_INDEXES } from './fts-schema.js'; +import type { FtsDisabledReason } from './fts-policy.js'; import { applyCjkSegmentationIfEnabled, MAX_CJK_SEGMENTATION_QUERY_LENGTH, @@ -95,21 +99,29 @@ async function queryFTSViaExecutor( } /** - * Search using LadybugDB's built-in FTS (always fresh, reads from disk) + * Search using LadybugDB's built-in FTS (fresh, reads from disk) * * Queries multiple node tables (File, Function, Class, Method) in parallel * and merges results by filePath, summing scores for the same file. * + * When `disabledReason` is set the index intentionally has no FTS (#3091), so + * this returns an empty `ftsAvailable: false` response immediately and never + * touches the database — callers render that as deliberate disablement rather + * than as a missing index or a failed extension load. + * * @param query - Search query string * @param limit - Maximum results * @param repoId - If provided, queries will be routed via the MCP connection pool + * @param disabledReason - Explicit FTS opt-out recorded for this index; short-circuits the search * @returns Ranked search results from FTS indexes */ export const searchFTSFromLbug = async ( query: string, limit: number = 20, repoId?: string, + disabledReason?: FtsDisabledReason, ): Promise => { + if (disabledReason) return { results: [], ftsAvailable: false }; // Applied once, up front, so every downstream branch searches with the // same text the index was built from (#2331/#2339) — index-time and // query-time text transforms must never diverge, since QUERY_FTS_INDEX diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index 4965fcbd9..8846ff348 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -8,6 +8,7 @@ import { type IndexCatalogSnapshot, } from '../lbug/lbug-adapter.js'; import { getFtsCapability } from '../lbug/extension-loader.js'; +import { FTS_DISABLED_MESSAGE, type FtsDisabledReason } from './fts-policy.js'; import { classifyExtensionLoadError } from '../lbug/extension-load-error.js'; import { FTS_INDEXES } from './fts-schema.js'; @@ -73,8 +74,12 @@ const formatWarningContext = (context: FtsWarningContext): string => { * text itself (#2767). Optional and additive: omitting it reproduces today's * exact message. */ -export const ftsDegradedWarning = (context?: FtsWarningContext): string => { +export const ftsDegradedWarning = ( + context?: FtsWarningContext, + disabledReason?: FtsDisabledReason, +): string => { const suffix = context ? formatWarningContext(context) : ''; + if (disabledReason) return FTS_DISABLED_MESSAGE + suffix; const fts = getFtsCapability(); if (fts && !fts.loaded) { const reason = fts.reason ? redactPaths(fts.reason).replace(/\.$/, '') : undefined; diff --git a/gitnexus/src/core/search/fts-policy.ts b/gitnexus/src/core/search/fts-policy.ts new file mode 100644 index 000000000..9cc12e252 --- /dev/null +++ b/gitnexus/src/core/search/fts-policy.ts @@ -0,0 +1,76 @@ +import type { RepoMeta } from '../../storage/repo-meta.js'; + +export type FtsDisabledReason = 'disabled-by-flag' | 'disabled-by-env'; +export type FtsSkipReason = FtsDisabledReason | 'extension-unavailable' | 'build-failed'; + +type RepoCapabilities = NonNullable; + +export const DEFAULT_GRAPH_CAPABILITY: RepoCapabilities['graph'] = { + provider: 'ladybugdb', + status: 'available', +}; + +export const DEFAULT_VECTOR_SEARCH_CAPABILITY: RepoCapabilities['vectorSearch'] = { + provider: 'exact-scan', + status: 'unavailable', + exactScanLimit: 0, +}; + +export function resolveFtsDisableReason( + skipFts?: boolean, + envValue = process.env.GITNEXUS_SKIP_FTS, +): FtsDisabledReason | undefined { + if (skipFts === true) return 'disabled-by-flag'; + if (envValue === '1') return 'disabled-by-env'; + return undefined; +} + +export function isExplicitFtsDisablement(reason: string | undefined): reason is FtsDisabledReason { + return reason === 'disabled-by-flag' || reason === 'disabled-by-env'; +} + +export function getFtsDisabledReason( + capability: RepoCapabilities['fts'] | undefined, +): FtsDisabledReason | undefined { + if (capability?.status !== 'unavailable') return undefined; + return isExplicitFtsDisablement(capability.skipReason) ? capability.skipReason : undefined; +} + +/** + * Overlay an explicit FTS opt-out onto an existing meta snapshot without + * touching freshness (`indexedAt` / `lastCommit`) or sibling capabilities. + * Flag and env are equivalent disablements; only the discriminator changes. + * Returns `meta` unchanged when `reason` is absent (re-enable) or already stamped. + */ +export function withExplicitFtsDisablement( + meta: RepoMeta, + reason: FtsDisabledReason | undefined, +): RepoMeta { + if (!reason) return meta; + const existing = meta.capabilities; + const existingFts = existing?.fts; + if ( + existingFts?.status === 'unavailable' && + existingFts.skipReason === reason && + existing?.graph && + existing.vectorSearch + ) { + return meta; + } + return { + ...meta, + capabilities: { + graph: existing?.graph ?? DEFAULT_GRAPH_CAPABILITY, + fts: { + provider: existingFts?.provider ?? 'ladybugdb-fts', + status: 'unavailable', + skipReason: reason, + }, + vectorSearch: existing?.vectorSearch ?? DEFAULT_VECTOR_SEARCH_CAPABILITY, + }, + }; +} + +export const FTS_DISABLED_MESSAGE = + 'FTS disabled for this index. To enable keyword search, run gitnexus analyze ' + + 'without --skip-fts and with GITNEXUS_SKIP_FTS unset.'; diff --git a/gitnexus/src/core/search/hybrid-search.ts b/gitnexus/src/core/search/hybrid-search.ts index a2521dd83..7c09971dc 100644 --- a/gitnexus/src/core/search/hybrid-search.ts +++ b/gitnexus/src/core/search/hybrid-search.ts @@ -10,6 +10,7 @@ import { searchFTSFromLbug, type BM25SearchResult } from './bm25-index.js'; import type { SemanticSearchResult } from '../embeddings/types.js'; +import type { FtsDisabledReason } from './fts-policy.js'; /** * RRF constant - standard value used in the literature @@ -153,11 +154,13 @@ export const formatHybridResults = (results: HybridSearchResult[]): string => { /** * Execute BM25 + semantic search and merge with RRF. - * Uses LadybugDB FTS for always-fresh BM25 results (no cached data). + * Uses LadybugDB FTS for fresh BM25 results (no cached data). * The semanticSearch function is injected to keep this module environment-agnostic. * - * When FTS is unavailable (e.g. read-only MCP connection, missing indexes), - * falls back to semantic-only results instead of crashing (#1489). + * When FTS is unavailable (e.g. read-only MCP connection, missing indexes) or + * explicitly disabled for this index (`disabledReason`, #3091), falls back to + * semantic-only results instead of crashing (#1489). In the disabled case no + * BM25 query is issued at all. */ export const hybridSearch = async ( query: string, @@ -168,13 +171,15 @@ export const hybridSearch = async ( query: string, k?: number, ) => Promise, + disabledReason?: FtsDisabledReason, ): Promise => { - // Use LadybugDB FTS for always-fresh BM25 results. + // Use LadybugDB FTS for fresh BM25 results — skipped entirely when this + // index recorded an explicit FTS opt-out (`disabledReason`, #3091). // If FTS fails (e.g. extension not loaded in MCP process), fall back to // semantic-only search instead of crashing with "bm25Results is not iterable". let bm25Results: BM25SearchResult[] = []; try { - const ftsResponse = await searchFTSFromLbug(query, limit); + const ftsResponse = await searchFTSFromLbug(query, limit, undefined, disabledReason); bm25Results = ftsResponse?.results ?? []; } catch { // FTS unavailable — continue with semantic-only search diff --git a/gitnexus/src/core/staleness-status.ts b/gitnexus/src/core/staleness-status.ts new file mode 100644 index 000000000..f4823c66c --- /dev/null +++ b/gitnexus/src/core/staleness-status.ts @@ -0,0 +1,89 @@ +/** + * The shape of a staleness answer, and the one wire payload every surface + * emits for it (#3256). Pure: no git, no I/O. + * + * Kept apart from `git-staleness.ts` on purpose. Tests across the suite stub + * that module with a fixed `vi.mock` factory so nothing shells out to git; a + * pure helper exported from it would come back `undefined` under every such + * stub. Here it is imported for real wherever the git probes are mocked. + */ + +/** + * What a staleness check was able to establish. + * + * `isStale` / `commitsBehind` alone cannot say "could not tell": every git + * failure collapses into `{ isStale: false, commitsBehind: 0 }`. That is + * deliberate — pinned by the fail-open tests, because the hot read tools must + * never fail or nag on an index they cannot measure — but it also made a + * provably stale index indistinguishable from a fresh one. `status` is the + * additive channel that separates them for a caller that wants to act on it: + * + * - `current` — the index is at HEAD: `rev-list` answered 0, or it could not + * answer but HEAD alone resolved to the indexed commit. + * - `behind` — `rev-list` answered N > 0; `commitsBehind` is N. + * - `diverged` — `rev-list` could not answer, but HEAD resolved and is not the + * indexed commit. The index is provably not at HEAD; only the count is + * unknown. A branch-pinned `serve` clone reaches this once git prunes the + * commit a failed re-index left behind — the pinned update is a + * `fetch --depth 1`, which orphans it — and a rewritten history reaches it + * directly. It is the rule the Claude hook already applies: + * HEAD !== lastCommit. + * - `unknown` — HEAD could not be resolved at all: not a git repository, git + * timed out, or no commit was recorded. + * + * `isStale` and `commitsBehind` keep their historical values in every case, so + * no existing consumer changes behaviour unless it reads `status`. + */ +export type StalenessStatus = 'current' | 'behind' | 'diverged' | 'unknown'; + +export interface StalenessInfo { + isStale: boolean; + commitsBehind: number; + hint?: string; + /** + * Always set by `checkStaleness` and `checkStalenessAsync`. Optional on the + * type so a hand-built info (tests, legacy literals) still compiles; read it + * through {@link stalenessStatus}, which derives it from `isStale` when absent. + */ + status?: StalenessStatus; +} + +/** `info.status`, or the answer `isStale` implies for an info built without one. */ +export const stalenessStatus = (info: StalenessInfo): StalenessStatus => + info.status ?? (info.isStale ? 'behind' : 'current'); + +/** + * The wire shape for staleness on every surface: MCP `list_repos`, the hot read + * tools, and the `serve` repo routes. One builder so one fact has one shape + * (#3232 review: "same sentinel as MCP"). + * + * Absent for `current`, as before. `commitsBehind` is present only when git + * actually counted it, so `diverged` carries `status` and `hint` but no number — + * inventing one would be the silent wrong answer this exists to remove. + */ +export interface StalenessPayload { + status: Exclude; + commitsBehind?: number; + hint?: string; +} + +/** + * Project a check into {@link StalenessPayload}, or `undefined` when there is + * nothing to report. + * + * `unknown` is emitted only when `includeUnknown` is set. A listing a monitor + * reads wants it; the hot read tools do not, because a `--skip-git` folder has + * no history to measure and would otherwise repeat that on every response. + */ +export const stalenessPayload = ( + info: StalenessInfo | undefined, + opts: { includeUnknown?: boolean } = {}, +): StalenessPayload | undefined => { + if (!info) return undefined; + const status = stalenessStatus(info); + const hint = info.hint ? { hint: info.hint } : {}; + if (status === 'current') return undefined; + if (status === 'unknown') return opts.includeUnknown ? { status } : undefined; + if (status === 'diverged') return { status, ...hint }; + return { status, commitsBehind: info.commitsBehind, ...hint }; +}; diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index d288d14f4..1e6f561b0 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -36,13 +36,12 @@ import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../../core/lbug/l // git utilities available if needed // import { isGitRepo, getCurrentCommit, getGitRoot } from '../../storage/git.js'; import { - parseDiffHunks, + parseDiffHunksResult, coalesceHunksByPath, hunksOverlapRange, findGitRootByDotGit, getCanonicalRepoRoot, getGitRoot, - type FileDiff, } from '../../storage/git.js'; import { realpathSync } from 'fs'; import { @@ -91,6 +90,7 @@ import { import { getExactScanLimit } from '../../core/platform/capabilities.js'; import { PhaseTimer } from '../../core/search/phase-timer.js'; import { ftsDegradedWarning, ftsQueryFailedWarning } from '../../core/search/fts-indexes.js'; +import { getFtsDisabledReason, type FtsDisabledReason } from '../../core/search/fts-policy.js'; import { cjkSegmentationModeMismatch, containsSegmentableCjkRun, @@ -98,11 +98,12 @@ import { isSupportedCjkSegmentationMode, MAX_CJK_SEGMENTATION_QUERY_LENGTH, } from '../../core/search/cjk-segmentation.js'; +import { checkStalenessAsync, checkCwdMatch } from '../../core/git-staleness.js'; import { - checkStalenessAsync, - checkCwdMatch, + stalenessPayload, type StalenessInfo, -} from '../../core/git-staleness.js'; + type StalenessPayload, +} from '../../core/staleness-status.js'; import { logger } from '../../core/logger.js'; import { isLocalEmbeddingRuntimeBlockerMessage, @@ -116,6 +117,7 @@ import { PDG_QUERY_DEFAULT_LIMIT, PDG_QUERY_MAX_LIMIT, } from '../tools.js'; +import { foldNumericToolArgumentAliases } from '../tool-arguments.js'; import { findImportCycles, IMPORT_CYCLE_LIMIT } from '../../core/graph/import-cycles.js'; import { decodeTaintPath } from '../../core/ingestion/taint/path-codec.js'; import { decodeReachingDefReason } from '../../core/ingestion/cfg/reaching-def-reason-codec.js'; @@ -129,6 +131,7 @@ import type { UnresolvedReceiverSummary } from '../../core/ingestion/scope-resol import type { UndecidedSatisfactionSummary } from '../../core/ingestion/scope-resolution/undecided-satisfaction.js'; import { scopeExtractionFailureTotal } from '../../core/ingestion/scope-resolution/scope-extraction-failures.js'; import { lookupCount } from '../../core/ingestion/scope-resolution/summary-maps.js'; +import { VALUE_REF_EDGE_REASON } from '../../core/ingestion/scope-resolution/value-ref-edges.js'; import { DEFERRED_IMPORT_REASON_SUFFIX, TYPE_ONLY_IMPORT_REASON_SUFFIX, @@ -272,10 +275,8 @@ function normalizeToolParams( ): { params: Record } | { error: string } { const input = params && typeof params === 'object' ? (params as Record) : {}; const definitions = TOOL_STRING_ALIASES[method]; - if (!definitions) return { params: input }; - const normalized = { ...input }; - for (const { canonical, aliases } of definitions) { + for (const { canonical, aliases } of definitions ?? []) { const keys = [canonical, ...aliases]; const supplied: Array<{ key: string; value: string }> = []; for (const key of keys) { @@ -306,14 +307,17 @@ function normalizeToolParams( if (supplied.length > 0) normalized[canonical] = supplied[0].value; } + const folded = foldNumericToolArgumentAliases(method, normalized); + if ('error' in folded) return folded; + if ( method === 'impact' && - typeof normalized.target !== 'string' && - (typeof normalized.target_uid !== 'string' || !normalized.target_uid.trim()) + typeof folded.params.target !== 'string' && + (typeof folded.params.target_uid !== 'string' || !folded.params.target_uid.trim()) ) { return { error: 'MCP impact requires target, name, symbol, or target_uid.' }; } - return { params: normalized }; + return { params: folded.params }; } // AI context generation is CLI-only (gitnexus analyze) @@ -711,6 +715,39 @@ export interface EpistemicCauses { * "nothing was undecided", and a re-index is what tells the two apart. */ readonly undecidedSatisfaction: number; + /** + * Symbols that name this callable in VALUE position rather than calling it + * (#3399) — a registration table (`bridge.accessor(Element.getNamespaceUri, + * …)`), a callback argument, a function pointer stored in a field. + * + * Unit: SYMBOLS — distinct referrers, the same unit and the same reason as + * `dispatchBoundary`: the reference edge is per-site but the walk's question + * is "who else might reach this", and a referrer that names the callable + * twice is still one place the value escapes from. + * + * Kept separate from `dispatchBoundary` even though both describe dispatch + * the walk cannot follow. That slot counts implementations and + * interface-level consumers found by the heritage probe; these are neither, + * and folding them in would tell a consumer branching on the numbers that an + * interface boundary exists where there is none. The distinction is also the + * actionable one: a dispatch boundary is irreducible, whereas a callable + * value CAN often be followed once the language models the store/load that + * carries it. + * + * The reference itself IS modelled — that is what makes it countable. What is + * missing is the invocation through the value: it happens later, through a + * struct field, a registry lookup, or comptime reflection, and no CALLS edge + * connects the eventual call site back to this symbol. + * + * Zero when the property-dispatch pass DID synthesize that invocation + * (`x.()` through a registered object-literal key): the walk followed + * the registration, so nothing was missed and the result stays `exact`. + * + * Also zero — WITH a boundary note — when the probe itself could not run. + * The note is the signal there; the count is not, which is why a reader must + * branch on `epistemic` first and read the causes as explanation. + */ + readonly callableValueReferences: number; } function epistemicFrom(dropped: { @@ -720,6 +757,7 @@ function epistemicFrom(dropped: { undecided: number; dispatch: number; scopeExtraction: number; + callableValueReferences: number; }): { epistemic: 'exact' | 'lower-bound'; boundaries?: string[]; @@ -738,6 +776,7 @@ function epistemicFrom(dropped: { dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, undecidedSatisfaction: 0, + callableValueReferences: dropped.callableValueReferences, }, } : { epistemic: 'exact' } @@ -754,6 +793,7 @@ function epistemicFrom(dropped: { dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, undecidedSatisfaction: dropped.undecided, + callableValueReferences: dropped.callableValueReferences, }, }; } @@ -853,6 +893,153 @@ function undecidedSatisfactionBoundaries( return { notes, undecided }; } +/** + * Boundary evidence for callables named in VALUE position (#3399). + * + * `bridge.accessor(Element.getNamespaceUri, null, .{})`, `{ onClick: handler }`, + * `qsort(xs, n, sz, compareItems)` — each REGISTERS a function somewhere + * instead of calling it. The registration is modelled (`value-ref` → a USES + * edge, Kythe `ref` / Joern `METHOD_REF`); the invocation through the stored + * value is not, because it happens later through a struct field, a registry + * lookup or comptime reflection. + * + * That gap is precisely the first half of `tools.ts`'s definition of + * `lower-bound` — the walk provably missed callers — and it was previously + * reported as `exact`. (The second half, a probe that could not run, is what + * the failure branch below publishes; the contract states both because this + * function can produce either.) A + * public DOM accessor bound into a JS bridge table came back LOW/exact with two + * internal callers, which is worse than no answer: `lower-bound` invites the + * reader to look further, `exact` tells them not to bother. + * + * Counted as DISTINCT REFERRERS rather than sites: the question the count + * serves is "how many places does this value escape from", and a table that + * registers the same callable twice is still one table. + * + * NOT every value reference is a gap. Where the property-dispatch pass + * synthesized the invocation side, the walk followed it and the answer stays + * `exact` — see the second probe below. + * + * Three failure modes, three different answers, none of them silence: + * - the query cannot run → hedge, count 0 (a probe that did not answer + * is not evidence of completeness); + * - the query returns nothing → no hedge (a real, measured zero); + * - the reference was followed → no hedge (nothing was missed). + * + * The probe reads the edge's `reason`, which is why writer and reader share + * {@link VALUE_REF_EDGE_REASON}. Language-neutral by construction — every + * provider that emits a `value-ref` capture participates, and one that emits + * none simply gets no rows. + */ +async function callableValueReferenceBoundaries( + lbugPath: string, + symId: string, +): Promise<{ notes: string[]; referrers: number }> { + // `COUNT(DISTINCT …)`, not a capped row list. A `LIMIT n` here would make the + // published cause silently understate a target with more than n + // registrations — and this number is documented as "how many symbols", so a + // reader comparing its magnitude against `receiverTyping` would be comparing + // a truth to a ceiling. Aggregating in the database keeps the work bounded + // without capping the answer; scalar `sym.id` equality plus an implicit + // group-by is the shape `countByType` below already relies on. + // + // `null`, not `[]`, on failure: see below — an empty result set and an + // unanswerable query must not be the same value. + const rows = await executeParameterized( + lbugPath, + `MATCH (other)-[r:CodeRelation]->(sym) + WHERE sym.id = $symId AND r.type = 'USES' AND r.reason = $reason + RETURN COUNT(DISTINCT other.id) AS cnt`, + { symId, reason: VALUE_REF_EDGE_REASON }, + ).catch(() => null); + + // A probe that could not run must never read as certainty — the same rule the + // `loadMeta` read above states, and the whole reason this function exists. + // Returning zero here would publish `exact` on the strength of a query that + // never answered. + if (rows === null) { + return { + referrers: 0, + notes: [ + 'The callable-value-reference probe could not be run against this index, so whether ' + + 'this symbol is registered somewhere as a value is unknown. Treat the caller list as ' + + 'incomplete until it can be re-checked.', + ], + }; + } + const referrers = rows.length > 0 ? Number((rows[0] as any).cnt ?? (rows[0] as any)[0] ?? 0) : 0; + if (!Number.isFinite(referrers) || referrers <= 0) return { notes: [], referrers: 0 }; + + // Registrations whose invocation side the analyzer ALREADY synthesized are + // not a gap. `emitPropertyDispatchCalls` sweep 2 connects `x.()` member + // calls to every function registered under `` and stamps those edges + // `property-dispatch`; where that happened, the walk did not "provably miss" + // the caller and `lower-bound` would be noise sprayed over an answer the + // analyzer actually computed. Zig — the case this was built for — never sets + // a property key (no object-literal key to dispatch through), so it is never + // excluded here; the exclusion exists to keep TypeScript/JavaScript hook + // tables that ARE followed from being downgraded. + // + // SYMBOL-LEVEL, NOT PER-EDGE, and that is only sound because of an invariant + // that lives nowhere near this line. The graph does not record which + // registration produced which synthesized call, so if one symbol could carry + // both a followed and an unfollowed registration, this would zero the note + // over a gap the analyzer provably did not close — #3399 returning through a + // side door. Today no symbol can: + // + // - sweep 2 synthesizes CALLS only for a registration whose site carried a + // `propertyKey` (sweep 1 skips the index when it is undefined); + // - every JS/TS `@reference.value-ref` rule also captures + // `@reference.property-key` — both are object-literal shapes; + // - no Zig `@reference.value-ref` rule captures one. + // + // So a dispatchable registration is always a JS/TS one, an undispatchable + // registration is always a Zig one, and the two never meet on one symbol. + // `test/unit/scope-resolution/value-ref-dispatchability.test.ts` FAILS the day + // that stops holding — a JS/TS rule for a bare callback argument + // (`register(handler)`), a Zig rule that grows a key. When it does, the + // choice to make here is between (a) splitting the edge `reason` into + // dispatchable / undispatchable so this probe can count them apart, and + // (b) hedging any symbol with an undispatchable registration regardless of + // dispatch. (a) is precise and costs a graph-content change; (b) is cheap and + // over-hedges. What is NOT acceptable is leaving this as-is, because a signal + // that quietly stops firing is the defect this whole feature removes. + // + // Given the invariant, the residual today is only the coarseness of the + // exclusion within JS/TS, and hedging every property-value registration in + // every JS/TS codebase is worse: a signal that fires on everything stops + // carrying information, and the fan-out cap warning still sits behind it. + const dispatched = await executeParameterized( + lbugPath, + `MATCH (other)-[r:CodeRelation]->(sym) + WHERE sym.id = $symId AND r.type = 'CALLS' AND r.reason = 'property-dispatch' + RETURN COUNT(r) AS cnt`, + { symId }, + ).catch(() => null); + // Failure here is NOT a reason to skip the hedge: we already know a value + // reference exists, and being unable to prove it was followed leaves the + // conservative answer standing. + const dispatchedCount = + dispatched === null || dispatched.length === 0 + ? 0 + : Number((dispatched[0] as any).cnt ?? (dispatched[0] as any)[0] ?? 0); + if (Number.isFinite(dispatchedCount) && dispatchedCount > 0) { + return { notes: [], referrers: 0 }; + } + + const one = referrers === 1; + return { + referrers, + notes: [ + `${referrers} ${one ? 'symbol references' : 'symbols reference'} this callable as a VALUE ` + + `rather than calling it (a registration table, a callback argument, a stored function ` + + `pointer). The reference is recorded, but the call made THROUGH that value is not: it is ` + + `dispatched later from wherever the value is stored. Callers reached that way are absent ` + + `from this result — actual impact may be higher.`, + ], + }; +} + interface RepoHandle { id: string; // unique key = repo name (basename) name: string; @@ -994,6 +1181,12 @@ export function buildDetectChangesDiffArgs(scope: string, baseRef?: string): str // not `--default-prefix`, which needs git >= 2.42. `--no-ext-diff` stops a // configured external diff driver from replacing the unified output we parse. const args = [ + // Before the subcommand: default core.quotePath C-quotes non-ASCII so + // `diff --git` / `+++` tokens no longer match the unquoted `a/` `b/` forms + // the parser also accepts. The parser still decodes quoted tokens; this + // pin keeps production git from emitting them. + '-c', + 'core.quotePath=false', 'diff', '--ignore-cr-at-eol', '--no-ext-diff', @@ -1149,7 +1342,7 @@ export interface RepoListing { lastCommit: string; remoteUrl?: string; stats?: any; - staleness?: { commitsBehind: number; hint?: string }; + staleness?: StalenessPayload; siblings?: Array<{ name: string; path: string; lastCommit: string }>; /** Primary/flat branch name, when known (#2106). */ branch?: string; @@ -1239,21 +1432,21 @@ function canCarryStaleness(result: unknown): result is Record { /** * #2655: attach a non-blocking `staleness` signal to a tool result when the - * index is behind HEAD, mirroring the `list_repos` `{commitsBehind, hint}` - * shape. Only ever ADDS a field to a carryable object result (see + * index is not at HEAD, in the same {@link stalenessPayload} shape `list_repos` + * returns. Only ever ADDS a field to a carryable object result (see * {@link canCarryStaleness}) — it never changes an existing result's shape. + * + * `diverged` is attached: it is a positive finding that the index is not at + * HEAD, only uncountable. `unknown` is not — these are the hot read tools, and a + * `--skip-git` folder has no history to measure, so it would ride on every + * response as noise rather than signal (#3256). */ -export function attachToolStaleness( - result: unknown, - staleness: StalenessInfo | undefined, -): unknown { - if (!staleness?.isStale || !canCarryStaleness(result)) { +export function attachToolStaleness(result: unknown, info: StalenessInfo | undefined): unknown { + const staleness = stalenessPayload(info); + if (!staleness || !canCarryStaleness(result)) { return result; } - return { - ...result, - staleness: { commitsBehind: staleness.commitsBehind, hint: staleness.hint }, - }; + return { ...result, staleness }; } /** tri-review Residual-2: see `LocalBackend.lastObservedPoolState`'s doc comment. */ @@ -2279,9 +2472,7 @@ export class LocalBackend { lastCommit: h.lastCommit, remoteUrl: h.remoteUrl, stats: h.stats, - staleness: stale.isStale - ? { commitsBehind: stale.commitsBehind, hint: stale.hint } - : undefined, + staleness: stalenessPayload(stale, { includeUnknown: true }), siblings: siblings.length > 0 ? siblings.map((s) => ({ @@ -2303,6 +2494,30 @@ export class LocalBackend { }); } + /** + * Lightweight registry count for schema-introspection callers that only + * need to know "one repo or many?" without paying the full staleness fan-out + * cost that listRepos() incurs. Uses the same validated registry + * `refreshRepos` / `selectToolRepository` see (`validate: true` prunes + * entries whose metadata is provably gone) so tools/list cannot advertise a + * multi-repo schema for ENOENT ghosts. No git processes are spawned. + */ + async countRepos(): Promise { + const entries = await listRegisteredRepos({ validate: true }); + return entries.length; + } + + /** + * In-memory validated registry size after the last `refreshRepos` / init / + * `selectToolRepository` refresh. `countRepos()` does not populate this map. + * Schema introspection uses this after a refreshed cwd probe so cardinality + * and the probe share one snapshot — without putting `refreshRepos()` (and + * its kuzu cleanup) on the 0–1 `countRepos` path. + */ + cachedRepoCount(): number { + return this.repos.size; + } + /** * Paginated view over {@link listRepos} for the `list_repos` MCP tool (#2119). * @@ -2435,9 +2650,10 @@ export class LocalBackend { * one `git rev-list` per index per TTL window; the resolved value is cached * for TOOL_STALENESS_TTL_MS. Keyed by lbugPath so flat and branch handles * (same repoPath, different lastCommit) don't share an entry. Non-blocking by - * construction: `checkStalenessAsync` swallows git failures to - * `{ isStale: false }`, so a git error never fails the tool — it just omits - * the `staleness` field. + * construction: `checkStalenessAsync` keeps `isStale: false` on every git + * failure and reports what it could still establish in `status` (`diverged`, + * `unknown`, or `current` when HEAD alone matches the index), so a git error + * never fails the tool — at most it attaches a `diverged` staleness field. */ private stalenessForTool(repo: RepoHandle): Promise { const now = Date.now(); @@ -2702,8 +2918,10 @@ export class LocalBackend { // each so both get independent wall-time records without fighting // over a single `current` phase slot. const searchLimit = processLimit * maxSymbolsPerProcess; // fetch enough raw results + const meta = await loadMeta(path.dirname(repo.lbugPath)); + const ftsDisabledReason = getFtsDisabledReason(meta?.capabilities?.fts); const [bm25SearchResult, semanticResults] = await Promise.all([ - timer.time('bm25', this.bm25Search(repo, searchQuery, searchLimit)), + timer.time('bm25', this.bm25Search(repo, searchQuery, searchLimit, ftsDisabledReason)), timer.time('vector', this.semanticSearch(repo, searchQuery, searchLimit)), ]); @@ -3035,7 +3253,7 @@ export class LocalBackend { warnings.push( ftsQueryErrors ? ftsQueryFailedWarning({ ...warningContext, lastErrorRedacted: ftsQueryErrors[0] }) - : ftsDegradedWarning(warningContext), + : ftsDegradedWarning(warningContext, ftsDisabledReason), ); } else if (ftsQueryErrors) { // #2767: at least one FTS table succeeded (ftsUsed=true) but another @@ -3089,7 +3307,6 @@ export class LocalBackend { // GITNEXUS_FTS_CJK_SEGMENTATION (the only thing that actually throws in // there) cannot take an unrelated diagnostic down with it. Needs no guard // of its own: loadMeta() returns null on any read/parse failure. - const meta = await loadMeta(path.dirname(repo.lbugPath)); try { // meta.json is on-disk state inside the analyzed repo, read via a // schema-less JSON.parse — not trusted input. Validate before @@ -3195,7 +3412,9 @@ export class LocalBackend { repo: RepoHandle, query: string, limit: number, + disabledReason?: FtsDisabledReason, ): Promise<{ results: any[]; ftsUsed: boolean; nonBenignErrors?: string[] }> { + if (disabledReason) return { results: [], ftsUsed: false }; let searchFTSFromLbug; try { ({ searchFTSFromLbug } = await import('../../core/search/bm25-index.js')); @@ -5422,11 +5641,11 @@ export class LocalBackend { return { error: `Git diff failed: ${err.message}` }; } - const fileDiffs: FileDiff[] = parseDiffHunks(diffOutput); + const { files: fileDiffs, unparsedGitHeaders } = parseDiffHunksResult(diffOutput); if (fileDiffs.length === 0) { - // Git printed a diff but none of it parsed: the `+++ b/` headers were not - // where `parseDiffHunks` looks. That is a PARSE failure, not a clean tree, + // Git printed a diff but none of it parsed: no `diff --git` / `+++` + // file header was recognised. That is a PARSE failure, not a clean tree, // and the clean branch below would report it to the pre-commit gate as // `risk_level:'none'`, no `partial`, exit 0 — a false all-clear (#2915). const parseFailed = diffOutput.trim().length > 0; @@ -5459,7 +5678,9 @@ export class LocalBackend { const changedSymbols = new Map(); // Set if a swallowed graph query fails below — surfaces `partial:true` so a // degraded run cannot report a false-clean `risk_level:'low'` (#2283). - let queryDegraded = false; + // An unparsed `diff --git` is the same class: later hunks must not make + // the gate look complete. + let queryDegraded = unparsedGitHeaders > 0; // Hunks arrive grouped per path and already in the graph's 0-based line // space, so every comparison below is base-neutral (#2377). @@ -6998,6 +7219,19 @@ export class LocalBackend { direction === 'downstream' ? Promise.resolve(undefined) : queryConvexDispatchMetadata(repo.lbugPath, symId, symName, symType); + // #3399 — callables named in value position. Upstream only: the question + // "who can reach this symbol" is the one a registration makes unanswerable. + // A downstream walk asks what THIS symbol reaches, which a reference INTO + // it does not affect. + // + // Issued alongside the heritage probe rather than after it, and read into + // `droppedBoundaries` below, so it hedges even when that probe finds + // nothing AND when it throws — a value reference is an independent reason + // a count is short, exactly as the receiver drops above are. + const valueRefPromise = + direction === 'downstream' + ? Promise.resolve({ notes: [] as string[], referrers: 0 }) + : callableValueReferenceBoundaries(repo.lbugPath, symId); const interfaceRowsPromise = executeParameterized( repo.lbugPath, `MATCH (x)-[r:CodeRelation]->(iface) @@ -7018,12 +7252,14 @@ export class LocalBackend { : []), ]); const convexDispatch = await convexDispatchPromise; + const valueRefDrops = await valueRefPromise; const droppedBoundaries = { ...receiverDrops, notes: [ ...receiverDrops.notes, ...scopeExtractionDrops.notes, ...undecidedDrops.notes, + ...valueRefDrops.notes, ...(convexDispatch === undefined ? [] : [convexDispatch.boundary]), ], undecided: undecidedDrops.undecided, @@ -7032,6 +7268,7 @@ export class LocalBackend { // inventing one from the presence of a note. dispatch: 0, scopeExtraction: scopeExtractionDrops.files, + callableValueReferences: valueRefDrops.referrers, }; try { // Discover the interface / abstract supertypes on the target's boundary. @@ -7123,6 +7360,7 @@ export class LocalBackend { dispatchBoundary: droppedBoundaries.dispatch + dispatchBoundarySymbols, externalBoundary: droppedBoundaries.external, undecidedSatisfaction: droppedBoundaries.undecided, + callableValueReferences: droppedBoundaries.callableValueReferences, }, }; } catch { diff --git a/gitnexus/src/mcp/repository-policy.ts b/gitnexus/src/mcp/repository-policy.ts index a309ec0cf..332e1b18f 100644 --- a/gitnexus/src/mcp/repository-policy.ts +++ b/gitnexus/src/mcp/repository-policy.ts @@ -198,25 +198,27 @@ export class McpRepositoryPolicy { }; } - // One fresh listing supplies both schema decisions. Besides keeping the - // advertised contract internally consistent, this avoids doing two full - // per-repo staleness fan-outs for every tools/list request. - const visibleRepos = await this.listAllowedRepos(backend); - if (visibleRepos.length <= 1) { + // Validated registry cardinality only — no listRepos() staleness git. + // The 0–1 arm stays a cheap countRepos() (no refreshRepos / kuzu cleanup). + const repoCount = await backend.countRepos(); + if (repoCount <= 1) { return { readOnlyRequiresRepo: false, mutatingRequiresRepo: false }; } try { - // listAllowedRepos() refreshed this backend immediately above. Resolve - // against that exact cache snapshot instead of racing another registry - // read; only read-only schemas may advertise the cwd-derived default. + // countRepos() does not refresh the backend; this cwd probe must. await backend.selectToolRepository(undefined, undefined, { allowCwdDefault: true, - refreshRegistry: false, + refreshRegistry: true, }); - return { readOnlyRequiresRepo: false, mutatingRequiresRepo: true }; } catch { return { readOnlyRequiresRepo: true, mutatingRequiresRepo: true }; } + // The probe just refreshed. If that snapshot is now a singleton, match the + // <=1 arm rather than advertising a split mutating-only schema. + if (backend.cachedRepoCount() <= 1) { + return { readOnlyRequiresRepo: false, mutatingRequiresRepo: false }; + } + return { readOnlyRequiresRepo: false, mutatingRequiresRepo: true }; } private async listReposPage( diff --git a/gitnexus/src/mcp/server.ts b/gitnexus/src/mcp/server.ts index 118f28fa1..f95edcec4 100644 --- a/gitnexus/src/mcp/server.ts +++ b/gitnexus/src/mcp/server.ts @@ -42,6 +42,7 @@ import { mcpRepositoryPolicyConfigured, } from './repository-policy.js'; import { applyMcpMaxTokens, resolveMcpMaxTokens, withoutMcpBudgetArg } from './output-budget.js'; +import { assertKnownMcpToolArguments, schemaSourceToolName } from './tool-arguments.js'; /** * Next-step hints appended to tool responses. @@ -220,6 +221,12 @@ export function createMCPServer( try { const typedArgs = args as Record | undefined; assertMcpReadOnlyToolCall(name, typedArgs, readOnly); + const schemaSource = schemaSourceToolName(name); + const advertisedTool = GITNEXUS_TOOLS.find((tool) => tool.name === schemaSource); + if (advertisedTool) { + const listed = toolForReadOnlyMcp(repositoryPolicy.toolForMcp(advertisedTool), readOnly); + assertKnownMcpToolArguments(name, typedArgs, listed.inputSchema.properties); + } maxTokens = resolveMcpMaxTokens(name, typedArgs); const result = await scopedBackend.callTool(name, withoutMcpBudgetArg(typedArgs)); const resultText = typeof result === 'string' ? result : JSON.stringify(result, null, 2); diff --git a/gitnexus/src/mcp/staleness.ts b/gitnexus/src/mcp/staleness.ts index 811889f3a..9ee20710e 100644 --- a/gitnexus/src/mcp/staleness.ts +++ b/gitnexus/src/mcp/staleness.ts @@ -1,6 +1,8 @@ /** - * Staleness Check — re-export from core (see `core/git-staleness.ts`). + * Staleness Check — re-export from core (see `core/git-staleness.ts` and + * `core/staleness-status.ts`). */ -export type { StalenessInfo } from '../core/git-staleness.js'; +export type { StalenessInfo, StalenessPayload, StalenessStatus } from '../core/staleness-status.js'; export { checkStaleness } from '../core/git-staleness.js'; +export { stalenessPayload, stalenessStatus } from '../core/staleness-status.js'; diff --git a/gitnexus/src/mcp/tool-arguments.ts b/gitnexus/src/mcp/tool-arguments.ts new file mode 100644 index 000000000..04d6cec1f --- /dev/null +++ b/gitnexus/src/mcp/tool-arguments.ts @@ -0,0 +1,174 @@ +/** + * MCP tool-argument contract (#3261). + * + * `tools/list` advertises `inputSchema`; `tools/call` used to forward any JSON + * object. A misspelled or CLI-taught key (`depth` instead of `maxDepth`) then + * produced a well-formed answer computed from the server default — no error, + * no warning. This module is the single dispatch-time check that the keys a + * caller sent are ones the advertised schema (or an unpublished handler alias) + * actually reads. + */ + +import { GITNEXUS_TOOLS } from './tools.js'; + +/** Legacy MCP names that reuse another tool's advertised schema. */ +export const LEGACY_TOOL_SCHEMA_SOURCE: Readonly> = { + search: 'query', + explore: 'context', +}; + +/** + * Keys the handler still reads but that must NOT appear in `inputSchema` + * (#2175: advertising `query` makes Claude Code drop the argument). + */ +export const UNPUBLISHED_TOOL_ARGUMENT_ALIASES: Readonly> = { + query: ['query'], + cypher: ['query'], + // Group-mode context still reads `target` as the symbol name; local + // `name` is the advertised key. Advertising `target` would collide with + // impact's target vocabulary and is not in tools/list. Legacy `search` + // and `explore` inherit via schemaSourceToolName. + context: ['target'], +}; + +export interface NumericArgumentAlias { + canonical: string; + aliases: readonly string[]; +} + +/** + * Numeric aliases that the backend folds onto the advertised canonical key. + * `depth` is the CLI flag name for `maxDepth` on impact and trace. + */ +export const TOOL_NUMERIC_ARGUMENT_ALIASES: Readonly< + Record +> = { + impact: [{ canonical: 'maxDepth', aliases: ['depth'] }], + trace: [{ canonical: 'maxDepth', aliases: ['depth'] }], +}; + +export function schemaSourceToolName(toolName: string): string { + return LEGACY_TOOL_SCHEMA_SOURCE[toolName] ?? toolName; +} + +export function advertisedToolPropertyNames(toolName: string): string[] | undefined { + const source = schemaSourceToolName(toolName); + const tool = GITNEXUS_TOOLS.find((entry) => entry.name === source); + if (!tool) return undefined; + return Object.keys(tool.inputSchema.properties); +} + +function normalizeArgumentKey(key: string): string { + return key.toLowerCase().replace(/_/gu, ''); +} + +export function suggestKnownToolArgument( + unknownKey: string, + knownKeys: readonly string[], +): string | undefined { + const needle = normalizeArgumentKey(unknownKey); + if (!needle) return undefined; + const exact = knownKeys.find((key) => normalizeArgumentKey(key) === needle); + if (exact) return exact; + const contained = knownKeys.filter((key) => { + const normalized = normalizeArgumentKey(key); + return normalized.includes(needle) || needle.includes(normalized); + }); + return contained.length === 1 ? contained[0] : undefined; +} + +function formatUnknownArgumentError( + toolName: string, + unknownKeys: readonly string[], + advertisedKeys: readonly string[], +): string { + const quoted = unknownKeys.map((key) => `"${key}"`).join(', '); + const noun = unknownKeys.length === 1 ? 'argument' : 'arguments'; + const verb = unknownKeys.length === 1 ? 'does' : 'do'; + const suggestion = + unknownKeys.length === 1 ? suggestKnownToolArgument(unknownKeys[0], advertisedKeys) : undefined; + if (suggestion) { + return `Unknown ${noun} ${quoted} for tool "${toolName}". Did you mean "${suggestion}"?`; + } + return ( + `Unknown ${noun} ${quoted} for tool "${toolName}". ` + + `The advertised inputSchema ${verb} not include ${unknownKeys.length === 1 ? 'this key' : 'these keys'}.` + ); +} + +/** + * Reject top-level tool arguments that are neither advertised nor an + * unpublished handler alias. `advertisedProperties` should be the schema the + * caller actually saw (`tools/list` after read-only / repository-policy + * scrubbing). When it is omitted, the canonical `GITNEXUS_TOOLS` schema is + * used. Tools with no schema (legacy `overview`) are left unchecked. + */ +export function assertKnownMcpToolArguments( + toolName: string, + args: Record | undefined, + advertisedProperties?: Record, +): void { + if (!args) return; + const propertyNames = + advertisedProperties !== undefined + ? Object.keys(advertisedProperties) + : advertisedToolPropertyNames(toolName); + if (!propertyNames) return; + + const unpublished = + UNPUBLISHED_TOOL_ARGUMENT_ALIASES[toolName] ?? + UNPUBLISHED_TOOL_ARGUMENT_ALIASES[schemaSourceToolName(toolName)] ?? + []; + const allowed = new Set([...propertyNames, ...unpublished]); + const unknownKeys = Object.keys(args).filter((key) => !allowed.has(key)); + if (unknownKeys.length === 0) return; + throw new Error(formatUnknownArgumentError(toolName, unknownKeys, propertyNames)); +} + +/** + * Fold numeric aliases onto their canonical key (e.g. `depth` → `maxDepth`). + * Conflicting values error; a single agreed value is written to the canonical + * key and the alias keys are removed so every downstream reader sees one name. + */ +export function foldNumericToolArgumentAliases( + toolName: string, + params: Record, +): { params: Record } | { error: string } { + const definitions = TOOL_NUMERIC_ARGUMENT_ALIASES[toolName]; + if (!definitions) return { params }; + + const normalized = { ...params }; + for (const { canonical, aliases } of definitions) { + const keys = [canonical, ...aliases]; + const supplied: Array<{ key: string; value: number }> = []; + for (const key of keys) { + if (!Object.prototype.hasOwnProperty.call(normalized, key)) continue; + const value = normalized[key]; + if (value === undefined) continue; + if (typeof value !== 'number') { + return { error: `MCP parameter ${toolName}.${key} must be a number.` }; + } + // #2279: some MCP adapters materialize an omitted optional number as 0, + // and a coerced missing value arrives as NaN. Treat both sentinels as + // absent so they cannot conflict with a real maxDepth or fold onto + // `params.maxDepth || 3`. The handlers already map a non-positive or + // non-integer maxDepth to their default; erroring here turned that + // contract into an error payload instead. + if (value === 0 || Number.isNaN(value)) continue; + supplied.push({ key, value }); + } + const distinctValues = new Set(supplied.map(({ value }) => value)); + if (distinctValues.size > 1) { + return { + error: `Conflicting MCP parameters for ${toolName}.${canonical}: ${supplied + .map(({ key }) => key) + .join(', ')} must agree.`, + }; + } + // Drop every source key, then write back the single agreed value (if any), + // so a sentinel 0/NaN never survives on the canonical key. + for (const key of keys) delete normalized[key]; + if (supplied.length > 0) normalized[canonical] = supplied[0].value; + } + return { params: normalized }; +} diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index d68b5b70b..dfb9423ff 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -28,6 +28,7 @@ export interface ToolDefinition { } >; required: string[]; + additionalProperties?: false; }; } @@ -291,14 +292,15 @@ Handles disambiguation: if multiple symbols share the same name, returns ranked NOTE: ACCESSES edges (field read/write tracking) are included in context results with reason 'read' or 'write'. CALLS edges resolve through field access chains and method-call chains (e.g., user.address.getCity().save() produces CALLS edges at each step). COMPLETENESS OF incoming: alongside symbol/incoming/outgoing the result carries the same epistemic envelope impact() returns: -- epistemic: 'exact' | 'lower-bound' — 'lower-bound' means callers exist that this view provably does not list. +- epistemic: 'exact' | 'lower-bound' — 'lower-bound' means incoming is a FLOOR: either the walk provably missed callers, or a probe that would have established completeness could not run. Do not read it as proof that an omitted caller exists — read boundaries for which of the two it is. - boundaries: string[] — one plain-language sentence per reason. Prose for humans; branch on causes instead. -- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction, callableValueReferences } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — RESOLVER GAP: the analyzer dropped that many call sites on this name because it could not type the receiver, so they are missing from incoming. Do not read an absent caller as proof none exists. - causes.externalBoundary (unit: call sites) > 0 — the calls left the indexed program (System.out.println, fetch(...)). NOT a defect: no in-graph node could have been reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary static analysis cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not decide whether a type satisfies an interface, so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Usually fixable by making the missing dependency available to analysis. + - causes.callableValueReferences (unit: symbols) > 0 — that many symbols name this callable as a VALUE instead of calling it (a Zig registration table or const initialiser, a JS/TS object-literal property value). A bare callback argument in JS/TS is not captured today and is not counted, so a 0 does not rule that shape out; nor does it, on an index built before the language emitted these captures — re-analyze first. The reference is in the graph as a USES edge; the call made THROUGH the value is not, because it is dispatched later from wherever the value was stored. incoming.calls is therefore a floor. Follow the USES edges to find the registration, then the code that reads it. It is 0 when the analyzer DID synthesize the dispatch through a registered property key. That exclusion is per SYMBOL, not per registration: a target with BOTH a followed registration and an unfollowed escape reads 0 here, so a 0 means 'no unfollowed registration was proven', not 'this symbol escapes nowhere'. A 0 alongside epistemic 'lower-bound' can also mean the probe itself could not run — read boundaries for which. REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. @@ -488,15 +490,16 @@ Output includes: - affected_processes: which execution flows break and at which step - affected_modules: which functional areas are hit (direct vs indirect; classification-unavailable when that secondary query fails) - byDepth: affected symbols grouped by traversal depth (paginated by limit/offset; omitted when summaryOnly:true — use byDepthCounts for totals per depth, pagination object when truncated). Each item includes a processes:[{id,label,processType,step}] field listing the execution flows that symbol participates in. Empty when the symbol has no process membership. Can ALSO be empty when partial:true is set — either the process-aggregation pass hit its cap before detecting affected processes, or per-symbol enrichment was capped on a very large page. When partial:true, do NOT treat processes:[] as proof of no participation; cross-check the top-level affected_processes list. An item carries staticGated:true only when the edge that reached it is provably unreachable at compile time from the indexed source (today: Zig calls inside an 'if (CONST_FALSE)' body or the else of 'if (CONST_TRUE)'); the field is absent when the edge is live or the language does not model it. Traversal and risk do NOT filter or rank on it: it is metadata for the caller to weigh. -- epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the walk provably missed callers, so the count is a floor. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). +- epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the count is a FLOOR: either the walk provably missed callers, or a probe that would have established completeness could not run (a failed callable-value-reference query says so in boundaries). It is not itself proof that an omitted caller exists — branch on causes and read boundaries. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). - boundaries: string[] — one plain-language sentence per reason the count is short. Prose for humans; branch on causes instead. -- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction, callableValueReferences } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — the RESOLVER GAP signal: the analyzer dropped that many call sites because it could not establish the receiver's type (unresolved constructor, factory, chained expression). Those callers are absent from byDepth. Treat the result as incomplete: grep the symbol name before deleting or renaming. - causes.externalBoundary (unit: call sites) > 0 — those calls left the indexed program (System.out.println, fetch(...), os.environ.*). NOT a defect and NOT a reason the count is short: there is no in-graph node any edge could have reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary a static walk cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not DECIDE whether a type satisfies an interface (a type in a required signature named a package it could not resolve), so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Distinct from every cause above, which count decided facts that could not be attributed; this one counts questions never answered. It is the only cause that shortens a result WITHOUT leaving a trace in the graph, so an unhedged zero on a symbol reached only through such an interface would otherwise read as 'nobody calls this'. Usually fixable: it most often means a dependency is missing from the analyzed tree. + - causes.callableValueReferences (unit: symbols) > 0 — that many symbols name this callable as a VALUE rather than calling it: 'bridge.accessor(Element.getNamespaceUri, ...)' and 'pub const h = onReset;' in Zig, '{ onClick: handler }' in JS/TS. Those are the shapes actually captured today — a bare callback argument in JS/TS ('qsort'-style, 'setTimeout(tick)') is NOT one of them and is not counted, so a 0 here does not rule that shape out. The registration IS modelled (a USES edge); the invocation through the stored value is NOT, because it happens later via a struct field, a registry lookup or comptime reflection. So impactedCount is a floor and a LOW risk verdict on such a symbol is a floor too. Unlike dispatchBoundary this is often reducible — it usually means the language provider does not yet follow that store/load — but until it is, do NOT read an empty or small caller set as 'safe to change'. It is an exact count, not a capped sample. It is 0 when the analyzer DID synthesize the dispatch through a registered property key, and epistemic stays 'exact' on that account. That exclusion is symbol-level, not edge-level — the graph does not record which registration produced which synthesized call — so a symbol with a mix of followed and unfollowed registrations also reads 0: treat a 0 as 'no unfollowed registration was proven', not as proof the value escapes nowhere. A 0 alongside epistemic 'lower-bound' can instead mean the probe could not run at all, so read boundaries to tell those apart. Read from the graph, so it needs no index-time metadata BEYOND the edges being there: an index built by an analyzer that did not yet emit this language's value-ref captures has none, and reports 0. Re-analyze before reading a 0 as measured. REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. @@ -573,6 +576,13 @@ SERVICE: optional monorepo path prefix (case-sensitive path segments). When "rep minimum: 1, maximum: IMPACT_MAX_DEPTH, }, + depth: { + type: 'number', + description: + 'Compatibility alias for maxDepth (CLI --depth). Values must agree when both are present. Literal 0 is an omitted-value compatibility sentinel.', + minimum: 0, + maximum: IMPACT_MAX_DEPTH, + }, crossDepth: { type: 'number', description: @@ -929,6 +939,13 @@ DESTINATION TRACE (cross-repo): for an "@groupName" trace, OMIT to/to_uid/to_fil minimum: 1, maximum: 30, }, + depth: { + type: 'number', + description: + 'Compatibility alias for maxDepth (CLI --depth). Values must agree when both are present. Literal 0 is an omitted-value compatibility sentinel.', + minimum: 0, + maximum: 30, + }, includeTests: { type: 'boolean', description: 'Include test-file symbols in traversal (default: false)', @@ -991,6 +1008,16 @@ export const REPO_SCOPED_TOOLS = new Set([ ]); for (const tool of GITNEXUS_TOOLS) { + // Advertises a closed schema; tools/call still fail-closes on the scrubbed key list. + // The unpublished handler aliases in tool-arguments.ts stay off this schema on + // purpose (#2175), and closing it strands no caller: every alias has an + // advertised counterpart reaching the same handler — `query` → `search_query` + // on query, `query` → `statement` on cypher, and `target` → `name` on group + // context, which local-backend maps to the group target (the group name comes + // from `repo: "@group"`, not from `name`; see test/unit/mcp/group-repo-routing). + // A schema-validating client therefore has a valid call for every tool, and + // advertising the aliases instead would re-break Claude Code on `query`. + tool.inputSchema.additionalProperties = false; if (!REPO_SCOPED_TOOLS.has(tool.name)) continue; if (tool.inputSchema.properties.branch) continue; // Optional — `required` is left unchanged so omitting `branch` keeps today's diff --git a/gitnexus/src/server/analyze-worker-core.ts b/gitnexus/src/server/analyze-worker-core.ts index 916cce42c..b51e0e759 100644 --- a/gitnexus/src/server/analyze-worker-core.ts +++ b/gitnexus/src/server/analyze-worker-core.ts @@ -22,7 +22,7 @@ import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; import { projectAnalyzeResultForIpc } from './analyze-worker-ipc.js'; // Value import (instanceof): index-lock is a lightweight storage primitive // (node:fs/net/crypto only), so this does NOT pull in run-analyze/repo-manager. -import { IndexLockTimeoutError } from '../storage/index-lock.js'; +import { IndexLockTimeoutError, isIndexLockGuardTimeout } from '../storage/index-lock.js'; export interface WorkerAnalysisDeps { runFullAnalysis: typeof import('../core/run-analyze.js').runFullAnalysis; @@ -84,9 +84,15 @@ export async function runWorkerAnalysis( // #2658 review M2: a lock-wait timeout is transient contention (another // analyze held the single-writer lock), not a broken build — tag it so the // parent can surface a retry signal instead of an opaque hard failure. + // An orphan guard needs quiesced recovery, not automatic retries. terminal = err instanceof IndexLockTimeoutError - ? { type: 'error', message, code: 'index-lock-timeout', retryable: true } + ? { + type: 'error', + message, + code: 'index-lock-timeout', + retryable: !isIndexLockGuardTimeout(err), + } : { type: 'error', message }; } diff --git a/gitnexus/src/server/analyze-worker-protocol.ts b/gitnexus/src/server/analyze-worker-protocol.ts index d573f5a5c..7b046bbe3 100644 --- a/gitnexus/src/server/analyze-worker-protocol.ts +++ b/gitnexus/src/server/analyze-worker-protocol.ts @@ -60,12 +60,17 @@ export interface ErrorMessage { /** * Machine-readable failure code for a parent that wants to branch instead of * only surfacing the string. `index-lock-timeout` (#2658 review M2) means - * another analyze held the single-writer lock past the wait ceiling — a - * transient, retryable condition, not a broken build. Absent for a generic - * failure. + * acquisition waited past a wait ceiling. Ordinary lock contention is + * transient (`retryable: true`). An orphan acquisition/reclaim guard + * (`IndexLockTimeoutError.guardPath` set) is not — retries re-hit the 30s + * cap and need quiesced recovery (RUNBOOK.md), so `retryable` is false. + * Absent for a generic failure. */ code?: 'index-lock-timeout'; - /** True when the failure is expected to clear on retry (e.g. lock contention). */ + /** + * True when the failure is expected to clear on retry (e.g. lock contention). + * False when `code` is `index-lock-timeout` because of guard contention. + */ retryable?: boolean; } diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 5a731ae0b..eb97e81e6 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -37,6 +37,7 @@ import { NODE_TABLES, type GraphNode, type GraphRelationship } from 'gitnexus-sh import { searchFTSFromLbug } from '../core/search/bm25-index.js'; import { hybridSearch } from '../core/search/hybrid-search.js'; import { ftsDegradedWarning } from '../core/search/fts-indexes.js'; +import { getFtsDisabledReason, type FtsDisabledReason } from '../core/search/fts-policy.js'; import { LocalBackend } from '../mcp/local/local-backend.js'; import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js'; import { fileURLToPath } from 'url'; @@ -65,6 +66,8 @@ import { GITHUB_TOKEN_HOSTS, } from './git-clone.js'; import { createAnalyzeUploadHandler } from './analyze-upload.js'; +import { checkStalenessAsync } from '../core/git-staleness.js'; +import { projectRepoDetail, projectRepoListEntry, resolveLastCommit } from './repo-projection.js'; // Shared with the CLI's `--branch` (via the analyze-config wrapper) so both // entry points accept the same refs. Imported from core — not cli/ — so // createServer does not close a cycle with cli/serve.ts. @@ -722,6 +725,28 @@ export const handleFileRequest = async ( } }; +async function loadFtsSession(storagePath: string): Promise<{ + meta: Awaited>; + ftsDisabledReason: FtsDisabledReason | undefined; + skipFts?: true; +}> { + const meta = await loadMeta(storagePath); + const ftsDisabledReason = getFtsDisabledReason(meta?.capabilities?.fts); + return { + meta, + ftsDisabledReason, + ...skipFtsOption(Boolean(ftsDisabledReason)), + }; +} + +function skipFtsOption(skipFts?: boolean): { skipFts?: true } { + return skipFts ? { skipFts: true } : {}; +} + +function readOnlyFtsOptions(skipFts?: true): { readOnly: true; skipFts?: true } { + return skipFts ? { readOnly: true, skipFts: true } : { readOnly: true }; +} + export const handleQueryRequest = async ( req: express.Request, res: express.Response, @@ -747,9 +772,12 @@ export const handleQueryRequest = async ( return; } const lbugPath = path.join(entry.storagePath, 'lbug'); - const result = await withLbugDb(lbugPath, () => executePrepared(cypher, queryParams ?? {}), { - readOnly: true, - }); + const { skipFts } = await loadFtsSession(entry.storagePath); + const result = await withLbugDb( + lbugPath, + () => executePrepared(cypher, queryParams ?? {}), + readOnlyFtsOptions(skipFts), + ); res.json({ result }); } catch (err: any) { if (isReadOnlyDbError(err)) { @@ -1000,18 +1028,24 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }); // List all registered repos - app.get('/api/repos', async (_req, res) => { + // Rate-limited (CodeQL js/missing-rate-limiting) because this route now spawns + // one `git rev-list` per registered repo to answer freshness: an unauthenticated + // GET that costs N subprocesses is worth the same 60 rpm/IP ceiling `/api/repo` + // already carries. Web callers hit this on connect/switch, never in a loop. + app.get('/api/repos', createRouteLimiter(), async (_req, res) => { try { const repos = await listRegisteredRepos(); + // Checked in parallel, for the reason `list_repos` already does it that + // way: each check spawns an async `git rev-list`, and the sequential + // variant took ~50s across 200 repos (#1363). Projecting inside the map + // keeps the entry and its own check together — an index-matched second + // array is the shape that silently mispairs them if either is reordered. res.json( - repos.map((r) => ({ - name: r.name, - path: r.path, - repoPath: r.path, - indexedAt: r.indexedAt, - lastCommit: r.lastCommit, - stats: r.stats, - })), + await Promise.all( + repos.map(async (r) => + projectRepoListEntry(r, await checkStalenessAsync(r.path, r.lastCommit)), + ), + ), ); } catch (err: any) { res.status(500).json({ error: err.message || 'Failed to list repos' }); @@ -1038,12 +1072,8 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => return; } const meta = await loadMeta(entry.storagePath); - res.json({ - name: entry.name, - repoPath: entry.path, - indexedAt: meta?.indexedAt ?? entry.indexedAt, - stats: meta?.stats ?? entry.stats ?? {}, - }); + const staleness = await checkStalenessAsync(entry.path, resolveLastCommit(entry, meta)); + res.json(projectRepoDetail(entry, meta, staleness)); } catch (err: any) { res.status(500).json({ error: err.message || 'Failed to get repo info' }); } @@ -1147,6 +1177,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const lbugPath = path.join(entry.storagePath, 'lbug'); const includeContent = req.query.includeContent === 'true'; const stream = req.query.stream === 'true'; + const { skipFts } = await loadFtsSession(entry.storagePath); if (stream) { const abortController = new AbortController(); @@ -1178,7 +1209,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => await withLbugDb( lbugPath, async () => streamGraphNdjson(res, includeContent, abortController.signal), - { readOnly: true }, + readOnlyFtsOptions(skipFts), ); if (!abortController.signal.aborted && !res.writableEnded) { res.end(); @@ -1191,9 +1222,11 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => return; } - const graph = await withLbugDb(lbugPath, async () => buildGraph(includeContent), { - readOnly: true, - }); + const graph = await withLbugDb( + lbugPath, + async () => buildGraph(includeContent), + readOnlyFtsOptions(skipFts), + ); res.json(graph); } catch (err: any) { if (err instanceof ClientDisconnectedError) { @@ -1234,6 +1267,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => } const lbugPath = path.join(entry.storagePath, 'lbug'); const parsedLimit = Number(req.body.limit ?? 10); + const { ftsDisabledReason, skipFts } = await loadFtsSession(entry.storagePath); const limit = Number.isFinite(parsedLimit) ? Math.max(1, Math.min(100, Math.trunc(parsedLimit))) : 10; @@ -1262,7 +1296,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => sources: ['semantic'], })); } else if (mode === 'bm25') { - const ftsResponse = await searchFTSFromLbug(query, limit); + const ftsResponse = await searchFTSFromLbug(query, limit, undefined, ftsDisabledReason); ftsAvailable = ftsResponse.ftsAvailable; searchResults = ftsResponse.results.map((r: any, i: number) => ({ ...r, @@ -1275,9 +1309,21 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => if (isEmbedderReady()) { const { semanticSearch: semSearch } = await import('../core/embeddings/embedding-pipeline.js'); - searchResults = await hybridSearch(query, limit, executeQuery, semSearch); + searchResults = await hybridSearch( + query, + limit, + executeQuery, + semSearch, + ftsDisabledReason, + ); + if (ftsDisabledReason) ftsAvailable = false; } else { - const ftsResponse = await searchFTSFromLbug(query, limit); + const ftsResponse = await searchFTSFromLbug( + query, + limit, + undefined, + ftsDisabledReason, + ); ftsAvailable = ftsResponse.ftsAvailable; searchResults = ftsResponse.results; } @@ -1371,11 +1417,11 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => return { searchResults: enriched, ftsAvailable }; }, - { readOnly: true }, + readOnlyFtsOptions(skipFts), ); const response: any = { results: results.searchResults ?? results }; if (results.ftsAvailable === false) { - response.warning = ftsDegradedWarning(); + response.warning = ftsDegradedWarning(undefined, ftsDisabledReason); } res.json(response); } catch (err: any) { @@ -1411,13 +1457,14 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // cut a stuck regex.test() when the wall-clock budget expires. const { regex, fileFilter, limit } = parseGrepQuery(req.query as Record); const repoRoot = path.resolve(entry.path); + const { skipFts } = await loadFtsSession(entry.storagePath); const lbugPath = path.join(entry.storagePath, 'lbug'); const fileRows = await withLbugDb( lbugPath, () => executeQuery(`MATCH (n:File) WHERE n.content IS NOT NULL RETURN n.filePath AS filePath`), - { readOnly: true }, + readOnlyFtsOptions(skipFts), ); const filePaths: string[] = []; @@ -1807,187 +1854,197 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => let partialRunDetail: AnalyzeJobPartialOutcome | undefined; try { const lbugPath = path.join(entry.storagePath, 'lbug'); - await withLbugDb(lbugPath, async () => { - const { runEmbeddingPipeline } = - await import('../core/embeddings/embedding-pipeline.js'); - const { resolveEmbeddingIdentity } = - await import('../core/embeddings/embedding-identity.js'); - const embeddingIdentity = resolveEmbeddingIdentity(); - let embeddingMeta = await loadMeta(entry.storagePath); - if (!embeddingMeta) { - throw new Error('Repository metadata is missing; run gitnexus analyze first'); - } - const priorCheckpoint = embeddingMeta.embeddingCheckpoint; - // The SAME decision the CLI's resume gate makes - // (core/embedding-checkpoint.ts). This route used to hard-throw on - // any identity mismatch and ignore `attempts` entirely, so a - // `'partial'` marker written by `gitnexus analyze` and resumed - // here hit exactly the permanent wedge `kind` exists to remove: - // two readers of one record disagreeing about the rule it encodes. - // No `force`/`--drop-embeddings` equivalent exists on this route, - // so the flag options go unset and `'discard'` is unreachable — - // it is folded into the abandon arm rather than given an invented - // flag. `maxAttempts` is left to the shared default. - const resume = priorCheckpoint - ? decideEmbeddingResume(priorCheckpoint, embeddingIdentity) - : undefined; - if (resume?.action === 'abort') throw new Error(resume.error); - if (resume?.action === 'abandon' || resume?.action === 'discard') { - logger.warn({ repo: entry.name }, resume.log); - } - const forceReembedNodeIds: ReadonlySet = - resume?.action === 'resume' ? resume.pendingNodeIds : new Set(); - const saveEmbeddingCheckpoint = async ( - checkpoint: { - nodesProcessed: number; - totalNodes: number; - chunksProcessed: number; - }, - pendingNodeIds: string[], - embeddings?: PersistedEmbeddingCount, - ): Promise => { - // tri-review NEW-2: re-read immediately before writing (mirrors - // the pattern in run-analyze.ts's --repair-fts stamp) instead of - // spreading the stale `embeddingMeta` snapshot captured once at - // job start. This job can run up to EMBED_TIMEOUT_MS (30 min); - // without a fresh read, a concurrent writer's update (e.g. a - // --repair-fts capability stamp) would be silently reverted on - // every checkpoint save for the job's whole lifetime. - const latestMeta = (await loadMeta(entry.storagePath)) ?? embeddingMeta; - // `stats.embeddings` only moves when the caller MEASURED the - // live count (the post-flush `onCheckpoint`). The window-start - // callback measures nothing and passes nothing: restating the - // old count there would re-publish a stale number and clobber - // what a preceding `onCheckpoint` just wrote (same split as - // run-analyze.ts's checkpoint writer). - embeddingMeta = withMeasuredEmbeddingCount( - { - ...latestMeta, - // In flight ⇒ `kind: 'interrupted'` (embedding-checkpoint.ts). - embeddingCheckpoint: mintInterruptedCheckpoint( - embeddingIdentity, - checkpoint, - pendingNodeIds, - ), + const ftsSession = await loadFtsSession(entry.storagePath); + let embeddingMeta = ftsSession.meta; + await withLbugDb( + lbugPath, + async () => { + const { runEmbeddingPipeline } = + await import('../core/embeddings/embedding-pipeline.js'); + const { resolveEmbeddingIdentity } = + await import('../core/embeddings/embedding-identity.js'); + const embeddingIdentity = resolveEmbeddingIdentity(); + if (!embeddingMeta) { + throw new Error('Repository metadata is missing; run gitnexus analyze first'); + } + const priorCheckpoint = embeddingMeta.embeddingCheckpoint; + // The SAME decision the CLI's resume gate makes + // (core/embedding-checkpoint.ts). This route used to hard-throw on + // any identity mismatch and ignore `attempts` entirely, so a + // `'partial'` marker written by `gitnexus analyze` and resumed + // here hit exactly the permanent wedge `kind` exists to remove: + // two readers of one record disagreeing about the rule it encodes. + // No `force`/`--drop-embeddings` equivalent exists on this route, + // so the flag options go unset and `'discard'` is unreachable — + // it is folded into the abandon arm rather than given an invented + // flag. `maxAttempts` is left to the shared default. + const resume = priorCheckpoint + ? decideEmbeddingResume(priorCheckpoint, embeddingIdentity) + : undefined; + if (resume?.action === 'abort') throw new Error(resume.error); + if (resume?.action === 'abandon' || resume?.action === 'discard') { + logger.warn({ repo: entry.name }, resume.log); + } + const forceReembedNodeIds: ReadonlySet = + resume?.action === 'resume' ? resume.pendingNodeIds : new Set(); + const saveEmbeddingCheckpoint = async ( + checkpoint: { + nodesProcessed: number; + totalNodes: number; + chunksProcessed: number; }, - embeddings, - ); - await saveMeta(entry.storagePath, embeddingMeta); - }; - /** - * Count the persisted rows, or report the answer never arrived. - * The TRI-STATE is carried to the fold rather than collapsed here: - * `unknown` is not 0, and only the fold knows what to carry - * forward instead (core/embedding-count.ts). - */ - const countPersistedEmbeddings = async (): Promise => { - const counted = await measurePersistedEmbeddingCount(executeQuery); - if (counted.kind === 'unknown') { - logger.warn( - { reason: counted.reason }, - '[embed] could not count persisted embeddings; leaving stats.embeddings untouched', + pendingNodeIds: string[], + embeddings?: PersistedEmbeddingCount, + ): Promise => { + // tri-review NEW-2: re-read immediately before writing (mirrors + // the pattern in run-analyze.ts's --repair-fts stamp) instead of + // spreading the stale `embeddingMeta` snapshot captured once at + // job start. This job can run up to EMBED_TIMEOUT_MS (30 min); + // without a fresh read, a concurrent writer's update (e.g. a + // --repair-fts capability stamp) would be silently reverted on + // every checkpoint save for the job's whole lifetime. + const latestMeta = (await loadMeta(entry.storagePath)) ?? embeddingMeta; + // `stats.embeddings` only moves when the caller MEASURED the + // live count (the post-flush `onCheckpoint`). The window-start + // callback measures nothing and passes nothing: restating the + // old count there would re-publish a stale number and clobber + // what a preceding `onCheckpoint` just wrote (same split as + // run-analyze.ts's checkpoint writer). + embeddingMeta = withMeasuredEmbeddingCount( + { + ...latestMeta, + // In flight ⇒ `kind: 'interrupted'` (embedding-checkpoint.ts). + embeddingCheckpoint: mintInterruptedCheckpoint( + embeddingIdentity, + checkpoint, + pendingNodeIds, + ), + }, + embeddings, + ); + await saveMeta(entry.storagePath, embeddingMeta); + }; + /** + * Count the persisted rows, or report the answer never arrived. + * The TRI-STATE is carried to the fold rather than collapsed here: + * `unknown` is not 0, and only the fold knows what to carry + * forward instead (core/embedding-count.ts). + */ + const countPersistedEmbeddings = async (): Promise => { + const counted = await measurePersistedEmbeddingCount(executeQuery); + if (counted.kind === 'unknown') { + logger.warn( + { reason: counted.reason }, + '[embed] could not count persisted embeddings; leaving stats.embeddings untouched', + ); + } + return counted; + }; + // Fetch existing content hashes for incremental embedding. + // Delegated to lbug-adapter which owns the DB query logic and legacy-fallback handling. + const { fetchExistingEmbeddingHashes } = + await import('../core/lbug/lbug-adapter.js'); + const existingEmbeddings = await fetchExistingEmbeddingHashes(executeQuery); + if (existingEmbeddings && existingEmbeddings.size > 0) { + console.log( + `[embed] ${existingEmbeddings.size} nodes already embedded — incremental run with content-hash comparison`, ); } - return counted; - }; - // Fetch existing content hashes for incremental embedding. - // Delegated to lbug-adapter which owns the DB query logic and legacy-fallback handling. - const { fetchExistingEmbeddingHashes } = await import('../core/lbug/lbug-adapter.js'); - const existingEmbeddings = await fetchExistingEmbeddingHashes(executeQuery); - if (existingEmbeddings && existingEmbeddings.size > 0) { - console.log( - `[embed] ${existingEmbeddings.size} nodes already embedded — incremental run with content-hash comparison`, - ); - } - const pipelineResult = await runEmbeddingPipeline( - executeQuery, - executeWithReusedStatement, - (p) => { - embedJobManager.updateJob(job.id, { - progress: { - // `ready` maps to 'finalizing', NOT 'complete' (#2790). - // The pipeline emits `ready`/100% unconditionally before - // returning — including when it dropped nodes to endpoint - // failures — and the route has not measured the index or - // decided the outcome yet, so 'complete' here would make - // the job record contradict itself (`status: 'analyzing'`, - // `progress.phase: 'complete'`). - phase: - p.phase === 'ready' - ? 'finalizing' - : p.phase === 'error' - ? 'failed' - : p.phase, - percent: p.percent, - message: - p.phase === 'loading-model' - ? 'Loading embedding model...' - : p.phase === 'embedding' - ? `Embedding nodes (${p.percent}%)...` - : p.phase === 'indexing' - ? 'Creating vector index...' - : p.phase === 'ready' - ? 'Finalizing embeddings...' - : `${p.phase} (${p.percent}%)`, + const pipelineResult = await runEmbeddingPipeline( + executeQuery, + executeWithReusedStatement, + (p) => { + embedJobManager.updateJob(job.id, { + progress: { + // `ready` maps to 'finalizing', NOT 'complete' (#2790). + // The pipeline emits `ready`/100% unconditionally before + // returning — including when it dropped nodes to endpoint + // failures — and the route has not measured the index or + // decided the outcome yet, so 'complete' here would make + // the job record contradict itself (`status: 'analyzing'`, + // `progress.phase: 'complete'`). + phase: + p.phase === 'ready' + ? 'finalizing' + : p.phase === 'error' + ? 'failed' + : p.phase, + percent: p.percent, + message: + p.phase === 'loading-model' + ? 'Loading embedding model...' + : p.phase === 'embedding' + ? `Embedding nodes (${p.percent}%)...` + : p.phase === 'indexing' + ? 'Creating vector index...' + : p.phase === 'ready' + ? 'Finalizing embeddings...' + : `${p.phase} (${p.percent}%)`, + }, + }); + }, + {}, // config: use defaults + undefined, // skipNodeIds + existingEmbeddings, + { + signal: embedController.signal, + forceReembedNodeIds, + onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { + await saveEmbeddingCheckpoint(checkpoint, nodeIds); + }, + onCheckpoint: async (checkpoint) => { + // Count AFTER the flush, so the number describes rows that + // are durable rather than rows still pending in the WAL. + await flushWAL(); + await saveEmbeddingCheckpoint( + checkpoint, + [], + await countPersistedEmbeddings(), + ); }, - }); - }, - {}, // config: use defaults - undefined, // skipNodeIds - existingEmbeddings, - { - signal: embedController.signal, - forceReembedNodeIds, - onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { - await saveEmbeddingCheckpoint(checkpoint, nodeIds); }, - onCheckpoint: async (checkpoint) => { - // Count AFTER the flush, so the number describes rows that - // are durable rather than rows still pending in the WAL. - await flushWAL(); - await saveEmbeddingCheckpoint(checkpoint, [], await countPersistedEmbeddings()); - }, - }, - ); + ); - // Flush WAL so subsequent /api/search requests see the new - // embeddings immediately (#1149). In the CLI path closeLbug() - // handles this during process exit, but the server keeps the - // connection open for other routes — a CHECKPOINT is enough. - await flushWAL(); - // Measure inside withLbugDb, after the flush and while the - // connection is still open — this is the route's only chance to - // stamp `stats.embeddings` (embed-run-outcome.ts). A partial run - // gets the same stamp: an honest count of a partial index is what - // makes it survivable. - const measuredEmbeddings = await countPersistedEmbeddings(); - // Same re-read-before-write reasoning as saveEmbeddingCheckpoint - // above — and the outcome decision reads it too: its - // `embeddingCheckpoint` is the marker this run's own mid-run - // writer saved, which is the only record of the work when the - // count query could not answer. - const finalMeta = (await loadMeta(entry.storagePath)) ?? embeddingMeta; - const finalizeContext: EmbedRunFinalizeContext = { - measuredEmbeddings: persistedEmbeddingCountOrUndefined(measuredEmbeddings), - onDisk: finalMeta, - // The marker the job STARTED from — `finalMeta`'s has since been - // overwritten by the in-flight writer, so only this one carries - // the `'partial'` attempt chain. - resumedFrom: priorCheckpoint, - }; - const outcome = resolveEmbedRunOutcome( - embeddingIdentity, - pipelineResult, - finalizeContext, - ); - partialRunError = outcome.error; - partialRunDetail = outcome.partial; - embeddingMeta = withMeasuredEmbeddingCount( - { ...finalMeta, embeddingCheckpoint: outcome.checkpoint }, - measuredEmbeddings, - ); - await saveMeta(entry.storagePath, embeddingMeta); - }); + // Flush WAL so subsequent /api/search requests see the new + // embeddings immediately (#1149). In the CLI path closeLbug() + // handles this during process exit, but the server keeps the + // connection open for other routes — a CHECKPOINT is enough. + await flushWAL(); + // Measure inside withLbugDb, after the flush and while the + // connection is still open — this is the route's only chance to + // stamp `stats.embeddings` (embed-run-outcome.ts). A partial run + // gets the same stamp: an honest count of a partial index is what + // makes it survivable. + const measuredEmbeddings = await countPersistedEmbeddings(); + // Same re-read-before-write reasoning as saveEmbeddingCheckpoint + // above — and the outcome decision reads it too: its + // `embeddingCheckpoint` is the marker this run's own mid-run + // writer saved, which is the only record of the work when the + // count query could not answer. + const finalMeta = (await loadMeta(entry.storagePath)) ?? embeddingMeta; + const finalizeContext: EmbedRunFinalizeContext = { + measuredEmbeddings: persistedEmbeddingCountOrUndefined(measuredEmbeddings), + onDisk: finalMeta, + // The marker the job STARTED from — `finalMeta`'s has since been + // overwritten by the in-flight writer, so only this one carries + // the `'partial'` attempt chain. + resumedFrom: priorCheckpoint, + }; + const outcome = resolveEmbedRunOutcome( + embeddingIdentity, + pipelineResult, + finalizeContext, + ); + partialRunError = outcome.error; + partialRunDetail = outcome.partial; + embeddingMeta = withMeasuredEmbeddingCount( + { ...finalMeta, embeddingCheckpoint: outcome.checkpoint }, + measuredEmbeddings, + ); + await saveMeta(entry.storagePath, embeddingMeta); + }, + skipFtsOption(ftsSession.skipFts), + ); // Don't overwrite 'failed' if the job was cancelled while the pipeline was running const current = embedJobManager.getJob(job.id); diff --git a/gitnexus/src/server/repo-projection.ts b/gitnexus/src/server/repo-projection.ts new file mode 100644 index 000000000..1084260c3 --- /dev/null +++ b/gitnexus/src/server/repo-projection.ts @@ -0,0 +1,80 @@ +/** + * Response projections for the `serve` repo routes. + * + * Extracted from the route bodies so the field list is assertable. Inline, the + * projections were only reachable by booting a server and indexing a real + * repository, which is how `branch` came to sit on `RegistryEntry` unexposed + * over HTTP while `gitnexus list` printed it, with no test to notice (#3226). + * + * These are pure: the caller resolves the registry entry, the on-disk metadata + * and the staleness check, and passes the results in. + */ +import { + stalenessPayload, + type StalenessInfo, + type StalenessPayload, +} from '../core/staleness-status.js'; +import type { RegistryEntry } from '../storage/repo-manager.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; + +/** + * Staleness through the shared {@link stalenessPayload} builder, so this route + * and MCP `list_repos` emit one shape for one fact (#3232 review, #3256). + * + * Absent when the index is current. Otherwise `staleness.status` says what git + * could establish: `behind` with the counted `commitsBehind`; `diverged` when + * HEAD has provably moved off the indexed commit but the history needed to + * count the gap is gone — the state a branch-pinned `url` clone reaches once + * git prunes the commit a failed re-index left behind; or `unknown` when the + * repository could not be measured at all. This is a listing a monitor reads, + * so `unknown` is included here, unlike on the hot read tools. + * + * All of it measures the index against the local working tree — the same thing + * `gitnexus status` and MCP `list_repos` measure — not against the remote. + */ +export const stalenessField = (info: StalenessInfo): { staleness?: StalenessPayload } => { + const staleness = stalenessPayload(info, { includeUnknown: true }); + return staleness ? { staleness } : {}; +}; + +/** One entry of `GET /api/repos`. */ +export const projectRepoListEntry = (entry: RegistryEntry, staleness: StalenessInfo) => ({ + name: entry.name, + path: entry.path, + repoPath: entry.path, + indexedAt: entry.indexedAt, + lastCommit: entry.lastCommit, + stats: entry.stats, + // The registry has carried these since #2106; #3199 made them load-bearing + // over HTTP, because a branch-pinned analyze now gets its own entry and the + // only other way to tell two entries apart is to parse the clone-directory + // slug — a layout detail, not an API contract. + branch: entry.branch, + branches: entry.branches, + ...stalenessField(staleness), +}); + +/** + * `GET /api/repo`. `meta ?? entry` throughout, for the reason `indexedAt` + * already did it: the on-disk metadata is the fresher record when the two + * disagree, and the entry is the fallback for a repo whose meta cannot be read. + */ +export const projectRepoDetail = ( + entry: RegistryEntry, + meta: RepoMeta | null | undefined, + staleness: StalenessInfo, +) => ({ + name: entry.name, + repoPath: entry.path, + indexedAt: meta?.indexedAt ?? entry.indexedAt, + stats: meta?.stats ?? entry.stats ?? {}, + lastCommit: meta?.lastCommit ?? entry.lastCommit, + branch: meta?.branch ?? entry.branch, + ...stalenessField(staleness), +}); + +/** The commit a `/api/repo` staleness check should be measured against. */ +export const resolveLastCommit = ( + entry: RegistryEntry, + meta: RepoMeta | null | undefined, +): string => meta?.lastCommit ?? entry.lastCommit; diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 792c1010b..b1b0adad0 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -750,10 +750,189 @@ export interface FileDiff { hunks: DiffHunk[]; } +/** `parseDiffHunks` plus how many `diff --git` headers could not be decoded. */ +export interface DiffHunkParseResult { + files: FileDiff[]; + unparsedGitHeaders: number; +} + +const DIFF_GIT_PREFIX = 'diff --git '; + +/** + * Decode one Git C-quoted token (`"a/\\344\\270\\255.png"`). Returns + * `undefined` when the quotes are unbalanced or a trailing escape is bare. + */ +function unquoteCStyleGitToken(quoted: string): string | undefined { + if (quoted.length < 2 || quoted[0] !== '"' || quoted[quoted.length - 1] !== '"') { + return undefined; + } + // Git C-quotes are byte-oriented: non-ASCII is `\nnn` octal of the UTF-8 + // code units, not JS UTF-16 characters. + const bytes: number[] = []; + const pushChar = (ch: string): void => { + const code = ch.charCodeAt(0); + if (code < 0x80) bytes.push(code); + else bytes.push(...new TextEncoder().encode(ch)); + }; + for (let i = 1; i < quoted.length - 1; i++) { + const ch = quoted[i]; + if (ch !== '\\') { + pushChar(ch); + continue; + } + const next = quoted[++i]; + if (next === undefined) return undefined; + switch (next) { + case '\\': + case '"': + pushChar(next); + break; + case 'n': + bytes.push(0x0a); + break; + case 't': + bytes.push(0x09); + break; + case 'r': + bytes.push(0x0d); + break; + case 'a': + bytes.push(0x07); + break; + case 'b': + bytes.push(0x08); + break; + case 'v': + bytes.push(0x0b); + break; + case 'f': + bytes.push(0x0c); + break; + default: { + if (next < '0' || next > '7') { + pushChar(next); + break; + } + let oct = next; + while (oct.length < 3 && i + 1 < quoted.length - 1) { + const digit = quoted[i + 1]; + if (digit < '0' || digit > '7') break; + oct += digit; + i++; + } + bytes.push(parseInt(oct, 8)); + } + } + } + return new TextDecoder('utf-8').decode(Uint8Array.from(bytes)); +} + +function takeCQuotedToken( + source: string, + start: number, +): { token: string; end: number } | undefined { + if (source[start] !== '"') return undefined; + for (let i = start + 1; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === '"') return { token: source.slice(start, i + 1), end: i + 1 }; + } + return undefined; +} + +function stripGitDstPrefix(raw: string): string | undefined { + return raw.startsWith('b/') ? raw.slice(2) : undefined; +} + +/** Unified-diff paths end at the first TAB (timestamp / empty terminator). */ +function stripUnifiedDiffTab(pathWithOptionalTab: string): string { + const tab = pathWithOptionalTab.indexOf('\t'); + return tab === -1 ? pathWithOptionalTab : pathWithOptionalTab.slice(0, tab); +} + +function decodeGitPathToken(raw: string): string | undefined { + const trimmed = stripUnifiedDiffTab(raw); + if (trimmed.startsWith('"')) return unquoteCStyleGitToken(trimmed); + return trimmed; +} + +/** + * Destination path from `diff --git a/ b/`. + * + * Same-path headers recover `name` from `a/${name} b/${name}` so a dest that + * itself contains ` b/` is not split at the last occurrence. C-quoted tokens + * (default `core.quotePath`) are decoded. Renames that the greedy split would + * mis-parse stay a best-effort dest; `rename to` / `+++` correct them. + */ +function takeGitHeaderPathToken( + source: string, + start: number, +): { path: string; end: number } | undefined { + if (start >= source.length) return undefined; + if (source[start] === '"') { + const tok = takeCQuotedToken(source, start); + if (!tok) return undefined; + const path = unquoteCStyleGitToken(tok.token); + if (path === undefined) return undefined; + return { path, end: tok.end }; + } + if (source.startsWith('b/', start)) { + return { path: stripUnifiedDiffTab(source.slice(start)), end: source.length }; + } + if (source.startsWith('a/', start)) { + for (let i = start + 2; i < source.length; i++) { + if (source.startsWith(' b/', i) || source.startsWith(' "', i)) { + return { path: source.slice(start, i), end: i }; + } + } + } + return undefined; +} + +function filePathFromGitHeader(line: string): string | undefined { + if (!line.startsWith(DIFF_GIT_PREFIX)) return undefined; + const rest = line.slice(DIFF_GIT_PREFIX.length); + + if (rest.startsWith('a/')) { + for (let i = 2; i < rest.length; i++) { + if (!rest.startsWith(' b/', i)) continue; + const nameA = rest.slice(2, i); + const nameB = rest.slice(i + 3); + if (nameA.length > 0 && nameA === nameB) return nameA; + } + } + + const src = takeGitHeaderPathToken(rest, 0); + if (!src) return undefined; + let i = src.end; + while (rest[i] === ' ') i++; + const dest = takeGitHeaderPathToken(rest, i); + return dest ? stripGitDstPrefix(dest.path) : undefined; +} + +function pathFromPlusPlusPlus(line: string): string | undefined { + if (!line.startsWith('+++ ')) return undefined; + const raw = decodeGitPathToken(line.slice(4)); + if (!raw || raw === '/dev/null') return undefined; + return stripGitDstPrefix(raw); +} + +function pathFromRenameTo(line: string): string | undefined { + if (!line.startsWith('rename to ')) return undefined; + return decodeGitPathToken(line.slice('rename to '.length)); +} + /** * Parse unified diff output (with -U0) into per-file hunk ranges. * Extracts the new-file line ranges from @@ hunk headers. * + * The `diff --git` header is also retained as a file entry. This matters for + * binary, rename-only, and mode-only changes, which have no `+++ b/` header. + * Such entries intentionally have no hunks: callers can count the changed + * path without pretending that a symbol line range was touched. + * * A pure deletion adds no new lines, and unified diff spells that empty range * as the line BEFORE it: `@@ -4,2 +3,0 @@` removed old lines 4–5 from between * new lines 3 and 4 (git emits `+0,0` when the deletion is at the head of the @@ -769,17 +948,64 @@ export interface FileDiff { * gap — the widening {@link coalesceHunks} is careful never to do. */ export function parseDiffHunks(diffOutput: string): FileDiff[] { + return parseDiffHunksResult(diffOutput).files; +} + +/** + * Same as {@link parseDiffHunks}, plus a count of `diff --git` lines that + * could not be decoded. `detect_changes` uses the count to fail closed + * (`partial` + `risk_level:'unknown'`) instead of attaching later hunks to a + * previous file. + */ +export function parseDiffHunksResult(diffOutput: string): DiffHunkParseResult { const files: FileDiff[] = []; let current: FileDiff | null = null; + let unparsedGitHeaders = 0; + // `+++` after the first `@@` of a file is hunk body (`+` plus source text + // that itself starts `++ …`), not another file header. + let inHunk = false; + // A deleted file has no new-side coordinates. Its indexed symbols still use + // the pre-delete file, so map those hunks with the old-side range instead. + let currentFileDeleted = false; for (const line of diffOutput.split('\n')) { - if (line.startsWith('+++ b/')) { - current = { filePath: line.slice(6), hunks: [] }; - files.push(current); + if (line.startsWith(DIFF_GIT_PREFIX)) { + // Drop the previous file first: an unparsed header must not leave + // `current` live for a later `@@` / quoted `+++` to steal. + current = null; + inHunk = false; + currentFileDeleted = false; + const filePath = filePathFromGitHeader(line); + if (filePath) { + current = { filePath, hunks: [] }; + files.push(current); + } else { + unparsedGitHeaders++; + } + } else if (line.startsWith('rename to ')) { + const filePath = pathFromRenameTo(line); + if (!filePath) continue; + if (current) current.filePath = filePath; + else { + current = { filePath, hunks: [] }; + files.push(current); + } + } else if (!inHunk && line === '+++ /dev/null') { + currentFileDeleted = true; + } else if (!inHunk && line.startsWith('+++ ')) { + const filePath = pathFromPlusPlusPlus(line); + if (!filePath) continue; + if (!current || current.filePath !== filePath) { + current = { filePath, hunks: [] }; + files.push(current); + } } else if (line.startsWith('@@') && current) { - const match = line.match(/@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/); + inHunk = true; + const match = line.match(/@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/); if (match) { - const start = parseInt(match[1], 10); - const count = match[2] !== undefined ? parseInt(match[2], 10) : 1; + const sideOffset = currentFileDeleted ? 1 : 3; + const start = parseInt(match[sideOffset], 10); + const rawCount = match[sideOffset + 1]; + const count = rawCount !== undefined ? parseInt(rawCount, 10) : 1; if (count > 0) { current.hunks.push({ startLine: start, endLine: start + count - 1 }); } else { @@ -791,7 +1017,7 @@ export function parseDiffHunks(diffOutput: string): FileDiff[] { } } } - return files; + return { files, unparsedGitHeaders }; } /** diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts index ba8e57225..19997379a 100644 --- a/gitnexus/src/storage/index-lock.ts +++ b/gitnexus/src/storage/index-lock.ts @@ -33,12 +33,13 @@ * - **file** (`O_EXCL` pidfile) — the portable fallback for macOS/BSD (no * abstract sockets; filesystem sockets don't release cleanly on death) and * for any environment where the socket backend can't bind. It uses pid- - * liveness staleness, an atomic rename-steal reclaim, bounded malformed-file - * handling, read-only tolerance, and a finite wait timeout (a reused pid can - * masquerade as live where process start-time isn't verifiable, so waiting is - * bounded rather than a hang). Its stale-takeover has an irreducible narrow - * race — inherent to file-based advisory locks — which is precisely why the - * socket backend is preferred; only a kernel primitive closes it. + * liveness staleness, a non-stealable acquisition/reclaim guard, bounded + * malformed-file handling, read-only tolerance, and a finite wait timeout (a + * reused pid can masquerade as live where process start-time isn't verifiable, so waiting is + * bounded rather than a hang). Every file acquisition participates in the + * guard, even for an empty slot. An orphan guard fails closed and requires + * quiesced manual recovery (RUNBOOK.md); socket locks avoid that tradeoff. + * Requires reliable local-filesystem O_EXCL and cooperating upgraded writers. * * Scope: cross-process, same logical index dir. The file backend never steals a * foreign-host lock (pid liveness is meaningless across hosts); the socket @@ -51,19 +52,20 @@ import { closeSync, readFileSync, unlinkSync, - renameSync, - existsSync, mkdirSync, readdirSync, realpathSync, + lstatSync, } from 'node:fs'; import net from 'node:net'; import path from 'node:path'; import os from 'node:os'; import { randomBytes, randomUUID, createHash } from 'node:crypto'; +import { isProcessAlive } from '../utils/process-identity.js'; const LOCK_FILENAME = 'analyze.lock'; const LOCK_RECORD_VERSION = 1 as const; +const lockGuardPath = (lockPath: string): string => `${lockPath}.guard`; /** Base poll interval while waiting for a live holder; jittered per attempt. */ const DEFAULT_POLL_MS = 250; @@ -77,19 +79,21 @@ const DIAGNOSTIC_INTERVAL_MS = 15_000; * GITNEXUS_INDEX_LOCK_TIMEOUT_MS (or set it ≤ 0 for unbounded). */ const DEFAULT_TIMEOUT_MS = 600_000; +/** Never wait indefinitely for a guard, even with an unbounded workload wait. */ +const GUARD_TIMEOUT_MS = 30_000; /** * How long a lock file must stay unreadable (empty/partial JSON) before we - * treat it as a crash orphan and reclaim it. Tolerates the microsecond - * create→write→close window of a *live* owner (see acquireIndexLock), so we - * never steal a lock that is a poll-interval away from being written. Scaled - * off the poll interval, floored at 1s. + * treat it as a crash orphan and reclaim it. This grace preserves legacy + * orphan handling; the guard, NOT elapsed time, excludes incomplete live + * creations. Scaled off the poll interval, floored at 1s. */ const malformedGraceMs = (pollMs: number): number => Math.max(1000, pollMs * 2); /** - * On-disk lock record. `token` proves ownership on release/steal; `startTime` - * (Linux only) defends against pid reuse; `invocationId` is a human-traceable - * id distinct from the security-irrelevant `token`. + * On-disk lock record. `token` proves ownership on release and guard/workload + * verification; `startTime` (Linux only) defends against pid reuse; + * `invocationId` is a human-traceable id distinct from the security-irrelevant + * `token`. */ export interface LockRecord { v: typeof LOCK_RECORD_VERSION; @@ -139,7 +143,9 @@ export interface AcquireOptions { * would otherwise block acquisition forever. Timing out is safe — it stops * *waiting*, never *steals* a possibly-live holder — and names the holder so * the caller can retry. Override (including to unbounded, value ≤ 0) via - * GITNEXUS_INDEX_LOCK_TIMEOUT_MS. + * GITNEXUS_INDEX_LOCK_TIMEOUT_MS. File acquisition/reclaim guard contention + * is separately capped at 30s, or the remaining timeout if shorter; an + * orphan guard is never automatically taken over. */ timeoutMs?: number; /** Base poll interval (ms); jittered. Default 250. */ @@ -158,21 +164,59 @@ export class IndexLockTimeoutError extends Error { * pid when this is false. */ readonly holderKnown: boolean; - constructor(holder: LockRecord, waitedMs: number, holderKnown = true) { - super( - holderKnown - ? `Timed out after ${waitedMs}ms waiting for another gitnexus analyze ` + - `(pid ${holder.pid} on ${holder.hostname}, invocation ${holder.invocationId}) ` + - `to release the index lock.` - : `Timed out after ${waitedMs}ms waiting for another gitnexus analyze ` + - `(holder identity unknown) to release the index lock.`, - ); + /** Present only for acquisition/reclaim guard contention, requiring quiesced recovery. */ + readonly guardPath?: string; + constructor(holder: LockRecord, waitedMs: number, holderKnown = true, guardPath?: string) { + super(formatIndexLockTimeoutMessage(holder, waitedMs, holderKnown, guardPath)); this.name = 'IndexLockTimeoutError'; this.holder = holder; this.holderKnown = holderKnown; + this.guardPath = guardPath; } } +export const isIndexLockGuardTimeout = ( + error: unknown, +): error is IndexLockTimeoutError & { guardPath: string } => + error instanceof IndexLockTimeoutError && error.guardPath !== undefined; + +/** Writers must refuse a handle that does not own the lock. */ +export const requireExclusiveIndexLock = (handle: IndexLockHandle, message: string): void => { + if (handle.lockFree) throw new Error(message); +}; + +const formatIndexLockTimeoutMessage = ( + holder: LockRecord, + waitedMs: number, + holderKnown: boolean, + guardPath?: string, +): string => { + if (guardPath !== undefined) { + return ( + `Timed out after ${waitedMs}ms waiting for acquisition/reclaim guard ${guardPath}. ` + + `Quiesce all relevant writers and prevent restart before manual recovery. ` + + `Never remove the guard while writers may run; see RUNBOOK.md for quiesced recovery.` + ); + } + if (holderKnown) { + return ( + `Timed out after ${waitedMs}ms waiting for another gitnexus analyze ` + + `(pid ${holder.pid} on ${holder.hostname}, invocation ${holder.invocationId}) ` + + `to release the index lock.` + ); + } + return ( + `Timed out after ${waitedMs}ms waiting for another gitnexus analyze ` + + `(holder identity unknown) to release the index lock.` + ); +}; + +const unverifiedGuardError = (guardPath: string): Error => + new Error( + `Cannot verify acquisition/reclaim guard ownership: ${guardPath}. ` + + 'Acquisition refused; see RUNBOOK.md for quiesced recovery.', + ); + const HOSTNAME = os.hostname(); /** Linux: field 22 of /proc//stat (starttime). null elsewhere / on error. */ @@ -193,16 +237,6 @@ const readProcStartTime = (pid: number): string | null => { } }; -/** true if the pid exists (signal 0). EPERM means it exists but isn't ours. */ -const pidAlive = (pid: number): boolean => { - try { - process.kill(pid, 0); - return true; - } catch (err) { - return (err as NodeJS.ErrnoException).code === 'EPERM'; - } -}; - const buildRecord = (): LockRecord => ({ v: LOCK_RECORD_VERSION, pid: process.pid, @@ -214,8 +248,14 @@ const buildRecord = (): LockRecord => ({ }); const readRecord = (lockPath: string): LockRecord | null => { + let raw: string; + try { + raw = readFileSync(lockPath, 'utf8'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw err; // An IO/permission failure is not evidence of a malformed orphan. + } try { - const raw = readFileSync(lockPath, 'utf8'); const parsed = JSON.parse(raw) as Partial; // `typeof NaN === 'number'`, so a bare number check lets NaN/0/-1/Infinity/ // fractional pids reach process.kill (#2658 review L4): a garbled or crafted @@ -223,10 +263,11 @@ const readRecord = (lockPath: string): LockRecord | null => { // for the full wait timeout. A real pid is a positive integer. if (!Number.isInteger(parsed.pid) || (parsed.pid as number) <= 0) return null; if (typeof parsed.token !== 'string') return null; + // Preserve the existing ownership format: unknown versions or missing + // ancillary metadata must not turn a live/foreign holder into an orphan. return parsed as LockRecord; } catch { - // Missing (won the race, file gone) or malformed/half-written → treat as - // "no readable holder"; the caller retries the O_EXCL create. + // Malformed/half-written; only reclaim under the guard after the grace. return null; } }; @@ -242,70 +283,12 @@ const readRecord = (lockPath: string): LockRecord | null => { */ const isStale = (holder: LockRecord): boolean => { if (holder.hostname !== HOSTNAME) return false; - if (!pidAlive(holder.pid)) return true; + if (!isProcessAlive(holder.pid)) return true; const now = readProcStartTime(holder.pid); if (holder.startTime && now && holder.startTime !== now) return true; // pid reused return false; }; -/** - * Reclaim a lock file we judged reclaimable — a dead holder (`expected` = its - * record) or a malformed/unreadable crash-orphan (`expected` = null) — moving - * the exact inode aside in ONE `rename` syscall to a token-unique name so two - * waiters reclaiming the same orphan can't both win (the loser's rename ENOENTs). - * - * CRITICAL (#2658 review): the reclaim must not act on a STALE judgment. The - * staleness decision (`isStale` / malformed-grace) happened a few syscalls ago; - * a live writer may have O_EXCL-created its own lock at `lockPath` since. Blindly - * renaming that live lock aside would delete it and admit a SECOND writer — the - * exact double-writer this lock exists to prevent (reproduced: ~18%/round under - * 4-way reclaim contention on the file backend). So: - * 1. re-read `lockPath` immediately BEFORE the rename and confirm it still holds - * exactly what we judged (same token, or still-unreadable) — shrinking the - * window to the single gap between this read and the rename; - * 2. after the rename, confirm what we ACTUALLY moved matches the judgment; if a - * live lock slipped into that residual gap, RESTORE it (rename back) so its - * holder is never displaced, and lose the reclaim. - * A concurrent creator whose fresh lock the restore overwrites is caught by the - * acquire loop's post-write read-back verify (see acquireViaFile), so it backs - * off rather than proceeding as a second writer. - * - * Returns true if we won the reclaim (caller retries the create), false if we - * lost the race or the judgment went stale (caller re-loops and re-reads). - */ -const matchesJudgment = (record: LockRecord | null, expected: LockRecord | null): boolean => - expected === null ? record === null : record?.token === expected.token; - -const stealLock = (lockPath: string, me: LockRecord, expected: LockRecord | null): boolean => { - // (1) Re-verify the judgment still holds right before we move anything. - if (!matchesJudgment(readRecord(lockPath), expected)) return false; - if (expected === null && !existsSync(lockPath)) return false; // malformed → but now vanished - - const aside = `${lockPath}.dead.${me.token}`; - try { - renameSync(lockPath, aside); - } catch (err) { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; // another stealer won - throw err; - } - // (2) Confirm what we moved is what we judged; if a live lock slipped into the - // read→rename gap, put it back — a live holder must never be displaced. - if (!matchesJudgment(readRecord(aside), expected)) { - try { - renameSync(aside, lockPath); // restore; an overwritten concurrent creator's read-back backs it off - } catch { - /* slot re-taken between our move and restore — leave it; we lost the reclaim */ - } - return false; - } - try { - unlinkSync(aside); // uniquely ours by token → safe; best-effort - } catch { - /* leftover .dead. is inert (not analyze.lock, not swept) — harmless */ - } - return true; -}; - /** * Placeholder holder for an {@link IndexLockTimeoutError} thrown while the lock * file exists but no valid record can be read (malformed/partial), or it keeps @@ -323,13 +306,91 @@ const unknownHolder = (): LockRecord => ({ acquiredAt: '', }); +/** Remaining wait before the next guard-create poll, or throws the matching timeout. */ +const remainingGuardCreateWaitMs = (args: { + now: number; + startedAt: number; + timeoutMs: number; + guardWaitSince: number; + permissionDeadline: number; + permissionError: unknown; + code: string | undefined; + err: unknown; + guardPath: string; + lastLiveHolder: LockRecord | null; +}): number => { + const workloadDeadline = args.startedAt + args.timeoutMs; + const guardDeadline = args.guardWaitSince + GUARD_TIMEOUT_MS; + const deadline = Math.min(workloadDeadline, guardDeadline, args.permissionDeadline); + if (args.now < deadline) return deadline - args.now; + if (args.permissionError && args.now >= args.permissionDeadline) throw args.permissionError; + if (args.code === 'EPERM') throw args.err; + if (args.now >= guardDeadline) { + throw new IndexLockTimeoutError( + unknownHolder(), + args.now - args.startedAt, + false, + args.guardPath, + ); + } + const remembered = args.lastLiveHolder ?? unknownHolder(); + throw new IndexLockTimeoutError( + remembered, + args.now - args.startedAt, + args.lastLiveHolder !== null, + ); +}; + +/** Drop this attempt's guard. A throw here discards the pending handle. */ +const releaseAcquisitionGuard = ( + guardPath: string, + me: LockRecord, + createdMain: boolean, + lockPath: string, +): void => { + try { + const guardRecord = readRecord(guardPath); + if (guardRecord && guardRecord.token !== me.token) { + throw unverifiedGuardError(guardPath); + } + if (guardRecord?.token === me.token) { + // Must complete before returning a workload handle or polling. + unlinkSync(guardPath); + return; + } + try { + lstatSync(guardPath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + throw unverifiedGuardError(guardPath); + } + throw err; + } + // Exists but unreadable: this process created the name via O_EXCL. + // Drop it so a failed metadata write cannot leave a permanent orphan, + // then refuse this attempt. + unlinkSync(guardPath); + throw unverifiedGuardError(guardPath); + } catch (guardError) { + // Roll back only the token-exact record this attempt created. + if (createdMain) { + try { + if (readRecord(lockPath)?.token === me.token) unlinkSync(lockPath); + } catch (cleanupError) { + throw new AggregateError( + [guardError, cleanupError], + `Guard and workload-lock cleanup failed: ${guardPath}. Acquisition refused; see RUNBOOK.md for quiesced recovery.`, + ); + } + } + throw guardError; + } +}; + /** - * Filesystem-create error codes we tolerate by proceeding lock-free: a - * read-only mount (EROFS) or a denied create (EACCES/EPERM). Such a filesystem - * rejects every index WRITE in the same directory too, so no concurrent writer - * can exist and the lock is moot — an already-indexed repo on a `:ro` mount - * must still reach its `alreadyUpToDate` fast path (#2658). A genuinely-needed - * write fails later exactly as it would have without the lock. + * Filesystem-create errors eligible for a read-only, non-owning handle when + * neither lock nor guard exists. Denied creation does NOT prove other writers + * lack access (ACLs may differ). Callers that write must reject lockFree handles. */ export const LOCK_UNWRITABLE_CODES: ReadonlySet = new Set(['EROFS', 'EACCES', 'EPERM']); export const isLockUnwritableCode = (code: string | undefined): boolean => @@ -345,6 +406,25 @@ const noopHandle = (record: LockRecord): IndexLockHandle => ({ release: () => {}, }); +const deniedCreateHandle = ( + lockPath: string, + record: LockRecord, + error: unknown, +): IndexLockHandle => { + // Do not turn an existing (even malformed/unreadable) owner into permission + // to proceed. lstat also sees dangling links; only ENOENT proves absence. + for (const candidate of [lockPath, lockGuardPath(lockPath)]) { + try { + lstatSync(candidate); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw err; + } + throw error; + } + return noopHandle(record); +}; + /** * Delete orphaned build/staging artifacts left in the lock directory by a * crashed prior writer. Safe precisely because we hold the exclusive lock: no @@ -404,39 +484,34 @@ const resolveTimeoutMs = (opt?: number): number => { const n = Number(env); return Number.isFinite(n) ? n : DEFAULT_TIMEOUT_MS; })(); + // Explicit NaN is a number, so it used to skip the env finite-check and + // poison every deadline (`startedAt + NaN`). Match the env fallback. + if (Number.isNaN(raw)) return DEFAULT_TIMEOUT_MS; return raw <= 0 ? Number.POSITIVE_INFINITY : raw; }; -/** - * Acquire the exclusive write lock for `lockDir` (the resolved index slot - * directory, e.g. `/.gitnexus` or `/.gitnexus/branches/`). - * - * Blocks until the lock is held (waiting only on live holders, stealing dead - * ones immediately), then sweeps orphaned staging files under the lock and - * returns a handle. Rejects with `IndexLockTimeoutError` if `timeoutMs` is - * exceeded while a live holder still holds the lock. - */ /** * File-based (O_EXCL pidfile) backend. The portable fallback used on platforms * without the socket backend (macOS/BSD) or when the OS socket lock is - * unavailable. Carries the pid-liveness staleness, atomic rename-steal reclaim, - * bounded malformed-file handling, and read-only tolerance. Its stale-takeover - * has an irreducible (narrow) race — see the module header — which is why the - * socket backend is preferred where available. + * unavailable. All inspection/reclaim/create/verify operations are serialized + * by a sibling O_EXCL guard. No waiter ever removes a guard, regardless of age, + * pid liveness, or malformed metadata. See RUNBOOK.md for orphan recovery. */ const acquireViaFile = async ( lockDir: string, me: LockRecord, opts: AcquireOptions, ): Promise => { + const lockPath = path.join(lockDir, LOCK_FILENAME); try { mkdirSync(lockDir, { recursive: true }); } catch (err) { - // Read-only / denied filesystem → proceed lock-free (see LOCK_UNWRITABLE_CODES). - if (isLockUnwritableCode((err as NodeJS.ErrnoException).code)) return noopHandle(me); + // Read-only / denied mkdir → lockFree only when neither lock nor guard exists. + if (isLockUnwritableCode((err as NodeJS.ErrnoException).code)) + return deniedCreateHandle(lockPath, me, err); throw err; } - const lockPath = path.join(lockDir, LOCK_FILENAME); + const guardPath = lockGuardPath(lockPath); const pollMs = opts.pollMs ?? DEFAULT_POLL_MS; const timeoutMs = resolveTimeoutMs(opts.timeoutMs); const startedAt = Date.now(); @@ -445,61 +520,168 @@ const acquireViaFile = async ( // When the lock file exists but has no readable record, the timestamp we // first observed it unreadable — used to reclaim a crash-orphan after a grace. let malformedSince: number | null = null; + let guardWaitSince: number | null = null; + let permissionWaitSince: number | null = null; + let permissionError: unknown; + // Last live workload holder observed while we held the inspect guard. Used + // when the overall wait budget expires during a brief peer inspect (EEXIST) + // so we do not mislabel ordinary contention as an orphan guard. + let lastLiveHolder: LockRecord | null = null; for (;;) { + const permissionDeadline = + permissionWaitSince === null + ? Number.POSITIVE_INFINITY + : permissionWaitSince + GUARD_TIMEOUT_MS; + if (Date.now() >= Math.min(startedAt + timeoutMs, permissionDeadline) && permissionError) { + throw permissionError; + } + let guardFd: number; try { - // O_WRONLY | O_CREAT | O_EXCL — the atomic arbiter of ownership. - const fd = openSync(lockPath, 'wx'); - try { - writeSync(fd, JSON.stringify(me)); - } finally { - closeSync(fd); - } - // Read-back verify (#2658 review L5): if this process stalled (a >graceMs - // GC pause) between the O_EXCL create of the *empty* file and the write - // above, a waiter could have reclaimed the empty file (renamed it aside) - // and O_EXCL-created its own lock at `lockPath`. Our write then landed on - // the renamed-aside inode, not `lockPath`. Confirm `lockPath` still carries - // our token before claiming ownership; if it was stolen, contend normally. - const confirmed = readRecord(lockPath); - if (!confirmed || confirmed.token !== me.token) continue; - return { - record: me, - release: () => { - const current = readRecord(lockPath); - if (current && current.token !== me.token) return; // no longer ours - try { - unlinkSync(lockPath); - } catch { - /* already gone */ - } - }, - }; + guardFd = openSync(guardPath, 'wx'); } catch (err) { const code = (err as NodeJS.ErrnoException).code; - if (code === 'EEXIST') { - // fall through to holder inspection / wait / reclaim below - } else if (isLockUnwritableCode(code)) { - return noopHandle(me); // read-only / denied → proceed lock-free - } else { - throw err; + // Windows can report EPERM while an unlinked guard is delete-pending. + // Retry under the same bounded budget, never interpret it as ownership. + if (code === 'EEXIST' || code === 'EPERM') { + const now = Date.now(); + guardWaitSince ??= now; + const remainingMs = remainingGuardCreateWaitMs({ + now, + startedAt, + timeoutMs, + guardWaitSince, + permissionDeadline, + permissionError, + code, + err, + guardPath, + lastLiveHolder, + }); + await sleep(jitteredDelay(pollMs, remainingMs, 0)); + continue; } + if (isLockUnwritableCode(code)) return deniedCreateHandle(lockPath, me, err); + throw err; + } + guardWaitSince = null; + // Metadata is diagnostic. Cleanup unlinks our token-exact record, or a + // self-created unreadable leftover (then refuses this attempt). A foreign + // token is never removed. + let holder: LockRecord | null = null; + let createdMain = false; + try { + try { + writeSync(guardFd, JSON.stringify(me)); + } finally { + closeSync(guardFd); + } + holder = readRecord(lockPath); + let tryCreate = false; + if (holder) { + malformedSince = null; + if (isStale(holder)) { + opts.log?.( + `Reclaiming stale index lock from dead analyze (pid ${holder.pid}, ` + + `invocation ${holder.invocationId}).`, + ); + unlinkSync(lockPath); + tryCreate = true; + holder = null; + } + } else { + tryCreate = true; + } + if (tryCreate) { + let fd: number | undefined; + try { + fd = openSync(lockPath, 'wx'); + createdMain = true; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'EEXIST') { + // Exclusive create is the presence check — do not existsSync first. + const current = readRecord(lockPath); + if (current === null) { + malformedSince ??= Date.now(); + if (Date.now() - malformedSince >= malformedGraceMs(pollMs)) { + opts.log?.( + 'Reclaiming a malformed/partial index lock file (no readable owner record).', + ); + unlinkSync(lockPath); + malformedSince = null; + } + } else { + holder = current; + } + } else if (code === 'EPERM') { + throw err; + } else if (isLockUnwritableCode(code)) { + return deniedCreateHandle(lockPath, me, err); + } else { + throw err; + } + } + if (createdMain && fd !== undefined) { + try { + writeSync(fd, JSON.stringify(me)); + } finally { + closeSync(fd); + } + if (readRecord(lockPath)?.token !== me.token) { + throw new Error(`Index lock verification failed: ${lockPath}`); + } + let released = false; + return { + record: me, + release: () => { + if (released) return; + released = true; + // No async boundary: a cooperating contender cannot replace a live + // owner's record before this one-shot unlink. Unknown is not ours. + try { + if (readRecord(lockPath)?.token !== me.token) return; + unlinkSync(lockPath); + } catch { + /* best-effort on exit; never retry against a successor */ + } + }, + }; + } + } + permissionWaitSince = null; + permissionError = undefined; + } catch (error) { + if (!createdMain && (error as NodeJS.ErrnoException).code === 'EPERM') { + // A releasing owner may leave the main file delete-pending on Windows. + // Release our guard in finally and retry; never reclaim an unreadable file. + permissionWaitSince ??= Date.now(); + permissionError = error; + holder = null; + if (Date.now() >= Math.min(startedAt + timeoutMs, permissionWaitSince + GUARD_TIMEOUT_MS)) { + throw error; + } + } else { + if (createdMain) { + try { + if (readRecord(lockPath)?.token === me.token) unlinkSync(lockPath); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `Workload-lock cleanup failed: ${lockPath}. Acquisition refused; see RUNBOOK.md for quiesced recovery.`, + ); + } + } + throw error; + } + } finally { + releaseAcquisitionGuard(guardPath, me, createdMain, lockPath); } - - const holder = readRecord(lockPath); const waited = Date.now() - startedAt; if (holder) { - malformedSince = null; - if (isStale(holder)) { - opts.log?.( - `Reclaiming stale index lock from dead analyze (pid ${holder.pid}, ` + - `invocation ${holder.invocationId}).`, - ); - stealLock(lockPath, me, holder); // reclaim ONLY this dead record; live locks are never stolen - continue; - } // Live holder → wait. + lastLiveHolder = holder; if (!announcedWait) { announcedWait = true; opts.onWaitStart?.(holder); @@ -521,27 +703,14 @@ const acquireViaFile = async ( continue; } - // holder === null: the lock file is either gone (vanished between the failed - // create and our read) or present-but-unreadable (a crash between the - // O_EXCL create and the record write, or a partial write). NEVER hot-loop - // here — both branches are bounded by sleep + timeout. - if (!existsSync(lockPath)) { - malformedSince = null; // genuinely vanished → the next create likely wins - if (waited >= timeoutMs) throw new IndexLockTimeoutError(unknownHolder(), waited, false); - await sleep(jitteredDelay(pollMs, timeoutMs, waited)); - continue; - } - // Malformed orphan present. Reclaim only after a grace, so a live owner's - // microsecond create→write window is never mistaken for a crash. - if (malformedSince === null) malformedSince = Date.now(); - if (Date.now() - malformedSince >= malformedGraceMs(pollMs)) { - opts.log?.('Reclaiming a malformed/partial index lock file (no readable owner record).'); - stealLock(lockPath, me, null); // reclaim ONLY while still unreadable; a live lock written since is left - malformedSince = null; - continue; - } if (waited >= timeoutMs) throw new IndexLockTimeoutError(unknownHolder(), waited, false); - await sleep(jitteredDelay(pollMs, timeoutMs, waited)); + const waitCeiling = Math.min( + timeoutMs, + permissionWaitSince === null + ? Number.POSITIVE_INFINITY + : permissionWaitSince + GUARD_TIMEOUT_MS - startedAt, + ); + await sleep(jitteredDelay(pollMs, waitCeiling, waited)); } }; @@ -736,10 +905,9 @@ export const acquireIndexLock = async ( } else { handle = await acquireViaFile(lockDir, me, opts); } - // Reclaim crashed-build staging orphans while we hold the lock. Best-effort: - // a read-only mount (no orphans reachable) just no-ops. + // Without ownership, a staging file may belong to an active writer. try { - sweepStagingArtifacts(lockDir, opts.log); + if (!handle.lockFree) sweepStagingArtifacts(lockDir, opts.log); } catch { /* best-effort */ } diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 8a22b5038..560c273e5 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -749,7 +749,21 @@ import { copyV8CacheIfPresent, tryLoadV8Cache, writeV8CacheFile } from './v8-sid // v97: Objective-C macro-marker preprocessing recognizes comment-prefixed // directives and rejects invalid numeric marker prefixes. A warm v96 cache can // replay error-recovered facts from the previous normalization behavior. -const SCHEMA_BUMP = 97; +// v98 (#3219): `ZIG_SCOPE_QUERY` gained three `@reference.value-ref` rules — +// bare call argument, qualified call argument (with `@reference.receiver`), and +// const-binding initialiser — so a Zig callable named in VALUE position now +// produces a `value-ref` entry in `ParsedFile.referenceSites` where it produced +// none before. These captures are PARSE-TIME facts, so a warm pre-v98 cache +// replays unchanged `.zig` files with zero value-ref sites, `--force` included +// (shards are content-addressed): `emitPropertyDispatchCalls` then emits no +// USES edge, `callableValueReferenceBoundaries` measures a real zero, and +// `impact` on a registered accessor republishes `epistemic: "exact"` — the exact +// #3399 defect this change exists to close, silently un-fixed. +// v99: ParsedImport retains declaredAtScope and export evidence changes in +// #3190. Old durable ParsedFiles lack the facts needed for scoped binding; +// invalidate both stores so warm indexing actually applies the correction. +// origin/main took 98 for #3219; 99 is the next free value. +const SCHEMA_BUMP = 99; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from @@ -882,6 +896,16 @@ export interface ParseCache { * Transient — never serialized to disk. */ usedKeys: Set; + /** + * Hashes this run decided it cannot vouch for — its durable generation could + * not be reset, or its chunk was worker-quarantined (#3204). `saveParseCache` + * refuses them, so neither a pre-existing `.v8` nor the chunk's durable + * directory survives into the next run. Kept separate from `usedKeys` + * because the orchestrator re-adds keys to that set after the parse phase + * (#2106 sibling fold), which would undo a deletion. + * Transient — never serialized to disk. + */ + staleKeys?: Set; /** * When set, chunk payloads are loaded from / flushed to sharded files on * demand instead of retaining every chunk in `entries` for the whole run @@ -1111,6 +1135,24 @@ export const persistParseCacheChunk = async ( cache.entries.set(chunkHash, slim); }; +/** + * Retire a chunk this run cannot vouch for — its durable ParsedFile generation + * could not be reset, or its chunk was worker-quarantined (#3204). + * + * `saveParseCache` refuses a stale key, so no pre-existing `.v8` is copied + * forward and the durable store — pruned to exactly the keys that save + * returns — drops the chunk in the same step. The two deletes matter because + * `loadParseCacheChunk` reads `entries` and `onDiskKeys` and does NOT consult + * `staleKeys`: without them a second lookup of the same hash inside this run + * would still serve the retired shard. + */ +export const markParseCacheChunkStale = (cache: ParseCache, chunkHash: string): void => { + cache.staleKeys ??= new Set(); + cache.staleKeys.add(chunkHash); + cache.entries.delete(chunkHash); + cache.onDiskKeys?.delete(chunkHash); +}; + const loadLegacyParseCache = async (storagePath: string): Promise => { const cachePath = getLegacyCachePath(storagePath); try { @@ -1196,7 +1238,14 @@ export const saveParseCache = async (storagePath: string, cache: ParseCache): Pr await fs.rm(tmpDir, { recursive: true, force: true }); await fs.mkdir(tmpDir, { recursive: true }); - const keys = [...cache.usedKeys].filter(isValidChunkCacheKey).sort(); + // A stale key is dropped here rather than at the failure site: the + // orchestrator folds sibling-branch keys back into `usedKeys` after the parse + // phase (#2106), so this is the last point that sees the final key set. The + // exclusion also reaches the durable store, which prunes to the keys this + // function returns — both stores drop the chunk together (#3204). + const keys = [...cache.usedKeys] + .filter((key) => isValidChunkCacheKey(key) && !cache.staleKeys?.has(key)) + .sort(); // Track hashes whose shard was actually written/copied this save. A hash can // be in `usedKeys` without a backing shard — its in-memory serialize threw, or // its on-disk copy failed/was-absent (e.g. a worker-quarantined chunk added to diff --git a/gitnexus/src/storage/parsedfile-store.ts b/gitnexus/src/storage/parsedfile-store.ts index 68ed2e06e..58c336c95 100644 --- a/gitnexus/src/storage/parsedfile-store.ts +++ b/gitnexus/src/storage/parsedfile-store.ts @@ -619,6 +619,22 @@ export const prepareDurableParsedFileChunk = async ( await fs.mkdir(dir, { recursive: true }); }; +/** + * Does this chunk still hold shards from a generation nobody cleared? + * + * Asked only after {@link prepareDurableParsedFileChunk} rejected, to tell its + * two failure modes apart (#3204). The `rm` failing leaves the previous + * generation in place, and a warm hit would union it with whatever this run's + * workers write — that chunk must be retired. The `rm` succeeding and the + * `mkdir` then failing leaves NO directory: the workers recreate it and write + * a clean generation, so retiring would throw away a good cache entry for + * nothing. Absent or empty ⇒ nothing to distrust. + */ +export const durableChunkHasStaleShards = async ( + durableDir: string, + chunkHash: string, +): Promise => (await listV8Shards(durableChunkDir(durableDir, chunkHash))).length > 0; + /** * Synchronous durable-shard writer for use INSIDE a parse worker, alongside * {@link persistParsedFileShardSync}. Writes the SAME bytes to a content-addressed @@ -728,8 +744,9 @@ export const loadDurableParsedFileIndex = async ( * Prune the durable store to `keepKeys` and rewrite its index. `keepKeys` must * be the parse cache's surviving on-disk keys (so the two stores stay coherent: * a chunk is "cached" iff BOTH its parse-cache shard and its durable shards - * exist; a quarantined chunk — no parse-cache shard — drops its durable subdir - * here and re-dispatches next run). Only chunks whose envelopes all validate + * exist; a chunk retired by `markParseCacheChunkStale` — worker-quarantined, or + * holding a durable generation that could not be reset — is absent from + * `keepKeys`, so it drops its durable subdir here and re-dispatches next run). Only chunks whose envelopes all validate * are indexed, together with their exact persisted path coverage (never vouch * for a missing/corrupt shard). The index write is tmp+rename atomic. */ @@ -745,6 +762,8 @@ export const pruneAndSaveDurableParsedFileStore = async ( return; // nothing written this run } const survivors: Record = {}; + const undeletable: string[] = []; + let firstRemoveError: unknown; for (const name of entries) { if (name === DURABLE_INDEX_FILENAME) continue; const full = path.join(durableDir, name); @@ -771,7 +790,29 @@ export const pruneAndSaveDurableParsedFileStore = async ( /* not a readable dir → drop below */ } } - await fs.rm(full, { recursive: true, force: true }); + // The causes that break `prepareDurableParsedFileChunk` — permissions, a + // locked file, a read-only mount — break this rm too (#3204). Dropping the + // entry from the index is what makes the chunk unreachable; losing the + // directory is a cleanup bonus. Never let one of them abort the loop and + // cost every remaining chunk its index entry. + try { + await fs.rm(full, { recursive: true, force: true }); + } catch (err) { + // `name` reached the drop branch precisely because it is not a live + // chunk key, so it may be any stray directory — report it as an entry. + undeletable.push(name); + firstRemoveError ??= err; + } + } + if (undeletable.length > 0) { + // One line per RUN, not per directory: a store-wide cause (read-only mount, + // wrong ownership) hits every non-survivor, and thousands of warns would + // bury the message that matters. + logger.warn( + { err: firstRemoveError, count: undeletable.length, firstEntry: undeletable[0] }, + 'parsedfile-cache: could not remove pruned durable chunk directories; ' + + 'they are excluded from the index and will be re-attempted next run', + ); } const idx: DurableParsedFileIndex = { version, entries: survivors }; const tmp = path.join(durableDir, `${DURABLE_INDEX_FILENAME}.tmp`); @@ -810,7 +851,20 @@ export const mergeStagedDurableParsedFileStore = async ( if (name === DURABLE_INDEX_FILENAME) continue; const from = path.join(stagedDir, name); const to = path.join(liveDir, name); - await replaceDurableChunkDir(from, to); + // Same reasoning as the prune's per-entry guard, on the loop that runs + // BEFORE it (#3204): this overlay targets the same live chunk directories, + // so the causes that break a reset break a replacement too. Letting one + // throw here would skip the prune entirely — the durable index would not be + // rewritten this run, and a retired chunk would keep its directory. + try { + await replaceDurableChunkDir(from, to); + } catch (err) { + logger.warn( + { err, entry: name }, + 'parsedfile-cache: could not publish a staged durable chunk; ' + + 'it stays uncached and will re-dispatch next run', + ); + } } await pruneAndSaveDurableParsedFileStore(liveDir, version, keepKeys); }; @@ -824,6 +878,12 @@ const replaceDurableChunkDir = async (from: string, to: string): Promise = /* dest exists, or the rename is cross-device */ } const backup = `${to}.replacing`; + // Deliberately NOT best-effort. If a non-empty backup survives, the + // `fs.rename(to, backup)` below cannot overwrite it and is swallowed as + // "dest was missing", so the `fs.cp` fallback would merge the staged + // generation INTO the live directory — manufacturing exactly the old+new + // union this fix exists to prevent. Let it throw; the caller's per-entry + // guard keeps one such chunk from costing the others their prune. await fs.rm(backup, { recursive: true, force: true }); let backedUp = false; try { @@ -847,6 +907,8 @@ const replaceDurableChunkDir = async (from: string, to: string): Promise = throw err; } if (backedUp) { - await fs.rm(backup, { recursive: true, force: true }); + // The new generation is already in place; an undeletable backup is litter, + // not a failure. The next prune re-attempts it. + await fs.rm(backup, { recursive: true, force: true }).catch(() => {}); } }; diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index c8d9ad487..b835db848 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -22,7 +22,12 @@ import { stripWindowsLongPathPrefix } from '../lib/utils.js'; import { writeFileAtomic } from './fs-atomic.js'; import { getGlobalDir } from './global-dir.js'; import { logger } from '../core/logger.js'; -import { acquireIndexLock, IndexLockTimeoutError, type IndexLockHandle } from './index-lock.js'; +import { + acquireIndexLock, + IndexLockTimeoutError, + requireExclusiveIndexLock, + type IndexLockHandle, +} from './index-lock.js'; import { branchSlug, BRANCHES_DIR, @@ -555,7 +560,8 @@ const REGISTRY_LOCK_TIMEOUT_MS = 5_000; * The registry is shared by every indexed repository, so per-index locks do * not protect this file. Reuse the cross-platform index lock primitive with a * registry-private lock namespace; the handle is kernel-owned on supported - * platforms and crash-reclaimable by the existing fallback. + * platforms. The file fallback reclaims dead workload holders; an orphan + * acquisition guard requires quiesced recovery (RUNBOOK.md). * * On timeout the transaction fails closed: continuing unlocked would reintroduce * the lost-update race this lock exists to prevent and can silently discard a @@ -582,6 +588,10 @@ const withRegistryLock = async (operation: () => Promise): Promise => { throw err; } try { + requireExclusiveIndexLock( + lock, + 'Cannot acquire the global registry lock; refusing an unlocked registry transaction.', + ); return await operation(); } finally { lock?.release(); diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index f65fc8ec1..3796eb561 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -28,6 +28,7 @@ import fs from 'fs/promises'; import path from 'path'; import type { UnresolvedReceiverSummary } from '../core/ingestion/scope-resolution/unresolved-receivers.js'; +import type { NameFallbackSummary } from '../core/ingestion/scope-resolution/name-fallback-summary.js'; import type { UndecidedSatisfactionSummary } from '../core/ingestion/scope-resolution/undecided-satisfaction.js'; import type { ScopeExtractionFailureSummary } from '../core/ingestion/scope-resolution/scope-extraction-failures.js'; @@ -163,23 +164,28 @@ export interface RepoMeta { * `'unavailable'` (#2841). Mirrors `AnalysisResult.ftsSkipReason` in * core/run-analyze.ts — the same discriminator that surface already * reports to the CLI, persisted rather than re-derived because the two - * causes need OPPOSITE handling on the next run: + * causes need distinct diagnostics and recovery handling: * * - `extension-unavailable` — the FTS extension could not load. Healable - * from outside the repo (install it), so the up-to-date fast path - * probes whether it loads now and re-analyzes when it does. + * from outside the repo (install it), then rebuild with --repair-fts. * - `build-failed` — the extension loaded fine and the index BUILD * failed (e.g. one un-tokenizable pre-existing row, #2544/#2546). * Deterministic: the same probe would "heal" it into a full * re-analysis that degrades identically and restamps, forever. Only * `--repair-fts` or a content change addresses it. + * - `disabled-by-flag` / `disabled-by-env` — deliberate opt-out. + * A later analyze without the opt-out rebuilds FTS at the same commit. * * Collapsing both into `status: 'unavailable'` is exactly what made that * loop reachable. ABSENT on indexes written before #2841 and on the * `--repair-fts` stamp (which writes `status: 'available'`); `undefined` * therefore reads as "cause unknown" and keeps the pre-#2841 behaviour. */ - skipReason?: 'extension-unavailable' | 'build-failed'; + skipReason?: + | 'extension-unavailable' + | 'build-failed' + | 'disabled-by-flag' + | 'disabled-by-env'; }; vectorSearch: { provider: string; @@ -311,6 +317,15 @@ export interface RepoMeta { * reads as absent, and both correctly mean "no hedge available from here". */ undecidedInterfaceSatisfaction?: UndecidedSatisfactionSummary; + /** + * Census of name-guessed call sites before edge coalescing, the impossible + * candidates the run refused, and the ambiguous + * `export *` names it declined to publish. Absent on indexes built before the + * census existed. The legacy key does not imply final edge counts: a precise + * site may prove a dependency shared with a guessed site. + * See `scope-resolution/name-fallback-summary.ts`. + */ + nameFallbackEdges?: NameFallbackSummary; /** * SHA-256 of every file's content at the time of the last successful * indexing run. The next run computes current hashes and diffs against diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index d665a7acf..4136cca42 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -33,6 +33,14 @@ export interface PipelineResult { * produced; graph edge semantics are unchanged. */ resolutionOutcomes: readonly ResolutionOutcome[]; + /** + * Caller node id → simple names of every callee it has a CALLS edge to, read + * through the streaming sink when one was active (the raw graph holds no + * streamed edge). Denominator for the name-fallback census + * (`countCallsByLanguage`), so a guess count can be read as a share of the + * call graph. Absent only when scope resolution did not run. + */ + resolvedCalleeNamesByCaller?: ReadonlyMap>; /** * Interfaces whose structural-satisfaction check could not be completed * (#2873). Empty for languages with no structural detection. diff --git a/gitnexus/test/fixtures/go-captures-golden/expected-captures.json b/gitnexus/test/fixtures/go-captures-golden/expected-captures.json index c8c5d6efe..0035a60c4 100644 --- a/gitnexus/test/fixtures/go-captures-golden/expected-captures.json +++ b/gitnexus/test/fixtures/go-captures-golden/expected-captures.json @@ -85,7 +85,7 @@ }, "go-constructor-type-inference/cmd/main.go": { "captureGroups": 19, - "digest": "4f2e8bb6dbca59bcb82fc2efd43899b7a92816af28811cfc1858f0ce7ba9f6fe" + "digest": "8dff73176d5890d7ede1d8b9cd9f8555518a1dcc11fbf02190beb474c0f9e47e" }, "go-constructor-type-inference/models/repo.go": { "captureGroups": 8, @@ -252,8 +252,8 @@ "digest": "ad4cfabcd53397ab664ba4bb05df4e1a803455b5c255a19e2ed283c6523a1660" }, "go-method-enrichment/app.go": { - "captureGroups": 15, - "digest": "35929a31f6921feb074f50385aa91879e93f25b88d7f50774fddcfbb7b4e8539" + "captureGroups": 16, + "digest": "b2ab0936ab28e22d0333119838c929ec6e3d460cc0be8d27aa4d178b9f6452d3" }, "go-mixed-chain/cmd/main.go": { "captureGroups": 22, diff --git a/gitnexus/test/fixtures/index-lock-child.mjs b/gitnexus/test/fixtures/index-lock-child.mjs index 9e64cca46..85c472bf9 100644 --- a/gitnexus/test/fixtures/index-lock-child.mjs +++ b/gitnexus/test/fixtures/index-lock-child.mjs @@ -10,10 +10,11 @@ * guarded by an O_EXCL sentinel create — if the sentinel already exists, * another process holds the lock at the same time, which is the exact * single-writer violation the test hunts. Hold briefly, remove the sentinel, - * release, exit 0. Exit 3 if the sentinel was already present (overlap). + * release, exit 0. Exit 3 only if the sentinel was already present (overlap), + * exit 4 for other sentinel-create failures, with stderr diagnostics. * Used by the multi-reclaimer test where ≥2 children reclaim one dead holder. */ -import { writeFileSync, openSync, closeSync, unlinkSync } from 'node:fs'; +import { writeFileSync, writeSync, openSync, closeSync, unlinkSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; // LOCK_MODULE is an absolute path. On Windows `import('C:\\…')` throws @@ -29,8 +30,16 @@ if (process.env.MODE === 'EXCLUSIVE') { let fd; try { fd = openSync(process.env.SENTINEL, 'wx'); - } catch { - process.exit(3); // overlap detected: two holders at once + } catch (error) { + // writeSync(2) flushes before process.exit; console.error on a piped + // stderr can be truncated when the child is spawned with stdio: pipe. + writeSync( + 2, + `Sentinel create failed: pid=${process.pid} path=${process.env.SENTINEL} ` + + `code=${error.code ?? 'unknown'} message=${error.message}\n`, + ); + lock.release(); + process.exit(error.code === 'EEXIST' ? 3 : 4); } closeSync(fd); // Hold the section briefly so concurrent reclaimers would collide here. diff --git a/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go b/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go index 9de68c3b0..c2734574a 100644 --- a/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go +++ b/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go @@ -1,9 +1,10 @@ -package main - -import "animal" +// This fixture exercises method enrichment and a valid same-package free call. +// A different package with a normal import cannot refer to Dog or Classify bare. +package animal func main() { dog := Dog{} sound := dog.Speak() category := Classify("dog") + _, _ = sound, category } diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/AppConfig.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/AppConfig.java new file mode 100644 index 000000000..2f06e41d4 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/AppConfig.java @@ -0,0 +1,12 @@ +package example; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AppConfig { + @Bean + public Handler handler() { + return null; + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Handler.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Handler.java new file mode 100644 index 000000000..2e2c8f480 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Handler.java @@ -0,0 +1,3 @@ +package example; + +public interface Handler {} diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/SpringContextUtil.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/SpringContextUtil.java new file mode 100644 index 000000000..045e01021 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/SpringContextUtil.java @@ -0,0 +1,9 @@ +package example; + +import java.util.Map; + +public class SpringContextUtil { + public static Map getBeans(Class type) { + return Map.of(); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Worker.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Worker.java new file mode 100644 index 000000000..eda02aca2 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Worker.java @@ -0,0 +1,7 @@ +package example; + +public class Worker { + public Worker() { + SpringContextUtil.getBeans(Handler.class); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig index 1cb77e2ee..15c44f34a 100644 --- a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig @@ -21,6 +21,7 @@ pub usingnamespace @import("mixin.zig"); pub const Interfaces = .{ @import("webapi/AbortController.zig"), @import("webapi/AbortSignal.zig"), + @import("webapi/Element.zig"), }; pub fn main() void { diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Element.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Element.zig new file mode 100644 index 000000000..02bdeed1e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Element.zig @@ -0,0 +1,224 @@ +// The JS-API binding-table idiom, as Lightpanda writes it (#3399). +// +// Every accessor below hands a Zig function to `bridge.accessor` AS A VALUE: +// the function is REGISTERED here, never called here. The eventual invocation +// runs through comptime reflection (`@call(.auto, func, args)` over a +// `func: anytype` field), which no static walk can follow — that terminal hop +// is out of scope. What was NOT acceptable is dropping the reference entirely: +// `impact` then reported the accessor as having only its two in-file callers +// and called that answer `exact`. +// +// A file-as-struct, like every webapi module in the real tree. +const Element = @This(); + +// A NAMESPACE-only module (no `@This()`), imported under a handle. The bridge +// table below registers one of its functions the same way it registers this +// file's own methods. +const dom_utils = @import("dom_utils.zig"); + +// A hub module that re-exports `dom_utils`' members without declaring any. +const hub = @import("hub.zig"); + +// A MODULE-LEVEL binding whose name collides with the container `Gauge.zig` +// declares, bound to something that is NOT a container. This file never imports +// `Gauge.zig`. `findClassBindingInScope` filters the scope chain by +// `isClassLike`, so it walks past this binding, and its workspace-wide +// qualified-name fallback answers with the other file's struct — while the +// shadow guard used to permit exactly this, treating the module scope as a floor +// it need not inspect. +// +// The name is bound by IMPORT rather than by a local `const Gauge: u8 = 3`, +// and that detail is the difference between a live case and a self-defeating +// one: a local declaration would also claim the workspace qualified name +// `Gauge`, leaving two candidates, and the fallback refuses to guess between +// two. An imported alias claims nothing, so the fallback stays unique and fires. +const Gauge = @import("dom_utils.zig").DEFAULT_NS; + +_namespace: u8 = 0, + +// ── Registered accessors ──────────────────────────────────────────────────── + +pub fn getNamespaceUri(self: *Element) u8 { + return self._namespace; +} + +// An ordinary in-file caller. The point of the defect is that the REGISTRATION +// was missing, not that the symbol looked like a leaf: a plausible-but-short +// caller list is exactly what makes `epistemic: "exact"` dangerous. +pub fn lookupNamespaceUri(self: *Element) u8 { + return self.getNamespaceUri(); +} + +// ── The control ───────────────────────────────────────────────────────────── + +// Called normally and registered NOWHERE. Its edges must not move: a change +// that hedges or re-links every method would be indistinguishable from one +// that models value references, and only the second is correct. +pub fn getTagNameLower(self: *Element) u8 { + return self._namespace; +} + +pub fn describe(self: *Element) u8 { + return self.getTagNameLower(); +} + +// ── Owner discrimination ──────────────────────────────────────────────────── + +// Shadowed below by a same-named sibling inside `JsApi`. The registration +// writes `Element.getLocalName`, so THIS is the one it must bind. +pub fn getLocalName(self: *Element) u8 { + return self._namespace; +} + +fn tick(self: *Element) u8 { + return self._namespace; +} + +// ── The binding table ─────────────────────────────────────────────────────── + +pub const JsApi = struct { + pub const bridge = Bridge(Element); + + // QUALIFIED value reference — the accessor names its container explicitly. + // This is the exact line from Element.zig:2296 that #3399 was filed over. + pub const namespaceURI = bridge.accessor(Element.getNamespaceUri, null, .{}); + + // BARE value reference to a sibling declared in this same container. + pub const tagName = bridge.accessor(_tagName, null, .{}); + + fn _tagName(self: *Element) u8 { + return self.getTagNameLower(); + } + + // A sibling with the SAME simple name as the file-struct method above. + // `walkScopeChain` gives a local binding precedence over the enclosing + // scope, so a registration resolved by TAIL NAME alone binds here — the + // wrong function, silently. Resolving `Element.getLocalName` through its + // written owner is what keeps them apart. + fn getLocalName(self: *Element) u8 { + return 0; + } + + pub const localName = bridge.accessor(Element.getLocalName, null, .{}); + + // A receiver this index cannot resolve. There is a file-level `tick`, and + // tail-name resolution would happily bind it even though the source says + // the function belongs to something else entirely. Declining is the only + // safe answer: a missing reference is recoverable, a confident wrong edge + // is not. + pub const ticker = bridge.accessor(unresolvable_ns.tick, null, .{}); + + // QUALIFIED value reference through a MODULE handle rather than a container. + // `dom_utils` is a namespace, not a class, so the class-owner lookup answers + // nothing here — and declining would be silent rather than safe: with no + // USES edge, `impact` on `compare` measures a real zero and reports `exact`, + // which is the claim this whole change exists to stop making. + pub const comparator = bridge.accessor(dom_utils.compare, null, .{}); + + // A namespace member that is NOT callable. Module receivers get the same + // callable gate as container receivers — a registration table full of + // constants must keep emitting nothing. + pub const defaultNs = bridge.accessor(dom_utils.DEFAULT_NS, null, .{}); + + // The receiver IS a known namespace import, but `dom_utils.zig` declares no + // `onlyOnDecoy` — so the namespace channel declines and the container + // channel runs, reaching `decoy.zig`'s same-named struct through the + // workspace-wide qualified-name index. A registration must NOT be minted + // there: the file wrote which module it meant. + pub const decoyed = bridge.accessor(dom_utils.onlyOnDecoy, null, .{}); + + // Through a HUB, whose published names are all imported ones. The CALL form + // resolves — `namespaceExportsIncludeImportedNames` is what makes a Zig hub + // work at all — so the REGISTRATION form has to resolve to the same def, or + // one name means two things depending on whether it is followed by `(`. + pub const scaled = bridge.accessor(hub.scale, null, .{}); + + // …and the callable gate still applies through the hub. + pub const hubNs = bridge.accessor(hub.DEFAULT_NS, null, .{}); + + // `Gauge` names this file's `const Gauge: u8`, not `Gauge.zig`'s container. + pub const level = bridge.accessor(Gauge.read, null, .{}); +}; + +// The CALL form of the same hub member, so the two are pinned side by side. +pub fn callsThroughTheHub(v: u8) u8 { + return hub.scale(v); +} + +// A LOCAL declaration shadowing the module handle. `dom_utils` here is a `u8` +// parameter with no member of its own; resolving `dom_utils.normalize` through +// the file-level import would attach the registration to a module the source +// did not name at this site — a wrong edge, the failure the same guard prevents +// on the member-CALL path. +pub fn shadowsTheModuleHandle(dom_utils: u8) u8 { + register(dom_utils.normalize); + return dom_utils; +} + +// A LOCAL declaration shadowing a CONTAINER name — the class-owner half of the +// same failure. `Ticker` here is a `u8` parameter, and this file neither +// declares nor imports the `Ticker` container that `Ticker.zig` defines. +// `findClassBindingInScope` filters the scope chain by `isClassLike`, so it +// walks straight past the parameter and its qualified-name fallback answers +// with a struct from a file this one never named. Resolving `Ticker.fire` +// through that is a confident edge to a function the source did not write. +pub fn shadowsAContainerName(Ticker: u8) u8 { + register(Ticker.fire); + return Ticker; +} + +// The positive half of the same guard: here the LOCAL declaration IS the +// container the registration names. A shadow test that only asked "is this name +// bound nearer than the module scope" would answer yes and decline — reading the +// declaration as its own shadow — and this whole class of local container would +// stop registering anything. +pub fn registersALocalContainer() u8 { + const Local = struct { + pub fn go() u8 { + return 3; + } + }; + return bridge.accessor(Local.go, null, .{}); +} + +// ── Const binding initialiser ─────────────────────────────────────────────── + +fn onReset(self: *Element) u8 { + return self._namespace; +} + +// A `const` whose initialiser IS a function value. Second value position, same +// class of drop. +pub const defaultHandler = onReset; + +// ── comptime anytype sink ─────────────────────────────────────────────────── + +var registered: ?*const fn (*Element) u8 = null; + +// Takes a callable by value into a `comptime … anytype` parameter and STORES +// it. Nothing in this file calls `f`. +pub fn register(comptime f: anytype) void { + registered = f; +} + +fn onTick(self: *Element) u8 { + return self._namespace; +} + +pub fn boot() void { + // `onTick` is never called in this file — it is handed over as a value and + // invoked later through `registered`. + register(onTick); +} + +fn Bridge(comptime T: type) type { + _ = T; + return struct { + pub fn accessor(comptime getter: anytype, comptime setter: anytype, comptime opts: anytype) u8 { + _ = getter; + _ = setter; + _ = opts; + return 0; + } + }; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Gauge.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Gauge.zig new file mode 100644 index 000000000..5b6071db4 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Gauge.zig @@ -0,0 +1,10 @@ +// The unique workspace definition of the name `Gauge`. `Element.zig` never +// imports it, and declares a module-local `const Gauge` of its own — see +// `readsThroughAShadowedContainerName` there. +const Gauge = @This(); + +_level: u8 = 0, + +pub fn read(self: *Gauge) u8 { + return self._level; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Ticker.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Ticker.zig new file mode 100644 index 000000000..58fb48aa5 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Ticker.zig @@ -0,0 +1,13 @@ +// A file-struct in a module `Element.zig` never imports. +// +// Its only job is to be the unique workspace definition of the name `Ticker`, +// so that `findClassBindingInScope`'s qualified-name fallback can reach it from +// a file that has no binding for that name at all. See the +// `shadowsAContainerName` case in `Element.zig`. +const Ticker = @This(); + +_ticks: u8 = 0, + +pub fn fire(self: *Ticker) u8 { + return self._ticks; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Widget.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Widget.zig new file mode 100644 index 000000000..595c3aab9 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Widget.zig @@ -0,0 +1,70 @@ +// A file-as-struct whose `@This()` alias is spelled `Self`, NOT `Widget`. +// +// This is the ordinary Zig idiom, and the case #3219 originally declined. The +// container this file mints is named after the FILE STEM (`Widget`), so before +// `bindZigThisAliases` the name `Self` meant nothing class-like here: a +// file-level `@This()` alias mints no Const at all (`isZigFileThisAlias` +// suppresses it so it cannot shadow the type for `w: *Widget`), and a +// container-level one mints a Variable that every `isClassLike` walk steps +// over. Every qualified reference through the alias — a CALL as much as a +// REGISTRATION — resolved to nothing. +// +// Counted on the corpora on hand when this was fixed: 73 of ghostty's 185 +// `@This()` files, 93 of tigerbeetle's 94 and 8 of mach's 42 spell the alias +// differently from the file stem, carrying 302 `Alias.member` references, 96 +// of them calls. +// +// `Element.zig` next door is the OTHER half of the control: it writes +// `const Element = @This();` in `Element.zig`, so its qualified references +// resolved through the stem binding and must keep resolving exactly as before. +const Self = @This(); + +_w: u8 = 0, + +pub fn width(self: *Self) u8 { + return self._w; +} + +// A qualified CALL through the alias — the explicit spelling Zig allows beside +// `self.width()`, and the shape `Alias.member` takes 96 times in the corpora +// above. +pub fn describeWidth(self: *Self) u8 { + return Self.width(self); +} + +pub const WidgetApi = struct { + pub const binder = Binder(Self); + // A qualified REGISTRATION through the alias: the #3399 shape, written the + // way most Zig files spell their own type. + pub const w = binder.accessor(Self.width, null, .{}); +}; + +// A NESTED container with its own differently-spelled alias. The file-level and +// container-level aliases take different code paths — one binds at the module +// scope against the file-struct, the other at the container's own scope — so +// both are exercised. +pub const Metrics = struct { + const Me = @This(); + + _n: u8 = 0, + + pub fn read(self: *Me) u8 { + return self._n; + } + + pub fn readTwice(self: *Me) u8 { + return Me.read(self) +% Me.read(self); + } +}; + +fn Binder(comptime T: type) type { + _ = T; + return struct { + pub fn accessor(comptime g: anytype, comptime s: anytype, comptime o: anytype) u8 { + _ = g; + _ = s; + _ = o; + return 0; + } + }; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/decoy.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/decoy.zig new file mode 100644 index 000000000..1197a40c8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/decoy.zig @@ -0,0 +1,24 @@ +// A container whose NAME collides with `Element.zig`'s `dom_utils` import +// handle, in a file `Element.zig` never imports. +// +// `findClassBindingInScope` does not stop at the scope chain: when its +// `isClassLike` walk misses — and a namespace import binds a Module, not a +// class — it falls back to `scopes.qualifiedNames`, a WORKSPACE-wide index, and +// answers with the unique def of that name. This struct is that unique def. A +// registration written `dom_utils.compare` in `Element.zig` must still bind +// `dom_utils.zig`'s function, not this one: the file said which module it meant. +pub const dom_utils = struct { + pub fn compare(a: u8, b: u8) u8 { + return if (a < b) a else b; + } + + // A callable that exists ONLY here. `dom_utils.zig` has no member of this + // name, so a registration written `dom_utils.onlyOnDecoy` declines in the + // namespace channel and falls through to the container channel — where the + // workspace-wide qualified-name fallback answers with this struct. The + // owner-shadow guard is the only thing standing between that and a + // confident edge into a file `Element.zig` never imported. + pub fn onlyOnDecoy(v: u8) u8 { + return v; + } +}; diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/dom_utils.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/dom_utils.zig new file mode 100644 index 000000000..e8972ee41 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/dom_utils.zig @@ -0,0 +1,24 @@ +// A NAMESPACE-only module: no `const X = @This()`, so this file declares no +// container symbol of its own. Its members are reachable only through an +// `@import` handle — the second kind of owner a qualified name can have, and +// the one `findClassBindingInScope` cannot answer for. +// +// Lightpanda's `libdom.zig` / `parser.zig` helpers are written exactly this +// way, and they are registered into the same bridge tables as the file-struct +// methods next door. + +pub const DEFAULT_NS: u8 = 7; + +pub fn compare(a: u8, b: u8) u8 { + return if (a > b) a else b; +} + +pub fn normalize(v: u8) u8 { + return v; +} + +// Republished by `hub.zig`, and by nothing else — so an assertion about the +// hub cannot be satisfied by an edge some other case emitted. +pub fn scale(v: u8) u8 { + return v +% 1; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/hub.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/hub.zig new file mode 100644 index 000000000..b99f60d9e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/hub.zig @@ -0,0 +1,10 @@ +// A HUB module: a file made only of re-exports, the shape +// `ScopeResolver.namespaceExportsIncludeImportedNames` exists for (ghostty's +// `src/terminal/`, tigerbeetle's `stdx`). It declares nothing of its own — +// every name it publishes is a name it imported. +// +// A consumer writes `hub.scale(x)` to CALL through it, and +// `bridge.accessor(hub.scale, …)` to REGISTER through it. Those are the same +// name resolved by the same rule, so they must not disagree. +pub const scale = @import("dom_utils.zig").scale; +pub const DEFAULT_NS = @import("dom_utils.zig").DEFAULT_NS; diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/README.md b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/README.md new file mode 100644 index 000000000..4ec85bc38 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/README.md @@ -0,0 +1,19 @@ +A Zig MONOREPO: several build packages, and no `build.zig` at the repo root. + +This is the layout the root-only config loader could not see at all. With no +root `build.zig` / `build.zig.zon`, `loadZigBuildConfig` answered `null`, and +every bare `@import("")` below went unresolved — so cross-file symbol +resolution in a repo like this degraded to relative imports only. + +The three packages are not interchangeable: + +- `core` declares module `core` and is imported by `app`. +- `app` declares module `app` and depends on `core` through a + `build.zig.zon` `.path = "../core"` — a spelling relative to the PACKAGE, + which has to be rebased to repo-relative `packages/core` before it can be + matched against indexed files. +- `tool` is the discriminating control. It binds the SAME alias `core` to its + OWN `src/core.zig`. A workspace that flattened every package's modules into + one repo-wide map would resolve `tool`'s `@import("core")` to `core`'s root + — a confident edge into a package `tool` never depends on. Only per-package + scoping (`zigPackageFor`, the `tsconfigFor` analogue) keeps them apart. diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig new file mode 100644 index 000000000..e237138a8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig @@ -0,0 +1,7 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const core_dep = b.dependency("core", .{}); + const app = b.addModule("app", .{ .root_source_file = b.path("src/main.zig") }); + app.addImport("core", core_dep.module("core")); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig.zon b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig.zon new file mode 100644 index 000000000..2b666b3d1 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig.zon @@ -0,0 +1,19 @@ +.{ + .name = "app", + .version = "0.1.0", + .dependencies = .{ + .core = .{ + .path = "../core", + }, + // ABSOLUTE, so it points outside the repository whichever package + // declares it. It sits in a NESTED package on purpose: that is the + // branch where the package prefix used to hide it. `packages/app/` + + // `/src` reads as the relative, in-repo `packages/app/src` — a real + // directory in this fixture — so the absolute dep silently became the + // package's own source tree instead of being rejected. + .escapes = .{ + .path = "/src", + }, + }, + .paths = .{""}, +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/main.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/main.zig new file mode 100644 index 000000000..26798d708 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/main.zig @@ -0,0 +1,18 @@ +// `core` is a PACKAGE dependency, declared by this package's build.zig.zon and +// wired in by its build.zig — not a relative path. Resolving it needs the +// package's own config, which lives two directories below the repo root. +const core = @import("core"); +const util = @import("util.zig"); + +pub fn run(attempt: u8) u8 { + return core.retryBudget(attempt); +} + +pub fn configured() u8 { + var cfg = core.Config{}; + return cfg.load(); +} + +pub fn delegated(attempt: u8) u8 { + return util.clamp(attempt); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/util.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/util.zig new file mode 100644 index 000000000..5387442cd --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/util.zig @@ -0,0 +1,8 @@ +// A second file of the same package reaching the same dependency: membership in +// a build module is the root plus what it reaches, so a file that is not itself +// a module root must resolve the alias too. +const core = @import("core"); + +pub fn clamp(attempt: u8) u8 { + return core.retryBudget(attempt); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/build.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/build.zig new file mode 100644 index 000000000..53ac256a7 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/build.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + _ = b.addModule("core", .{ .root_source_file = b.path("src/root.zig") }); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/src/root.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/src/root.zig new file mode 100644 index 000000000..b048ced4c --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/src/root.zig @@ -0,0 +1,13 @@ +pub const DEFAULT_RETRIES: u8 = 3; + +pub fn retryBudget(attempt: u8) u8 { + return DEFAULT_RETRIES - attempt; +} + +pub const Config = struct { + retries: u8 = DEFAULT_RETRIES, + + pub fn load(self: *Config) u8 { + return self.retries; + } +}; diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/build.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/build.zig new file mode 100644 index 000000000..8a3964bee --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/build.zig @@ -0,0 +1,8 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const tool = b.addModule("tool", .{ .root_source_file = b.path("src/main.zig") }); + // The SAME alias as `app` binds, pointing somewhere else entirely. + const own_core = b.createModule(.{ .root_source_file = b.path("src/core.zig") }); + tool.addImport("core", own_core); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/core.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/core.zig new file mode 100644 index 000000000..1616db72b --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/core.zig @@ -0,0 +1,5 @@ +// `tool`'s own `core`, unrelated to `packages/core`. Same module name, and that +// is the point: the name is only meaningful inside the package that binds it. +pub fn retryBudget(attempt: u8) u8 { + return attempt; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/main.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/main.zig new file mode 100644 index 000000000..979d76d7d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/main.zig @@ -0,0 +1,7 @@ +// Resolving this through a repo-wide module map would reach +// `packages/core/src/root.zig` — a package `tool` does not depend on. +const core = @import("core"); + +pub fn measure(attempt: u8) u8 { + return core.retryBudget(attempt); +} diff --git a/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts b/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts index d85744674..6e4416c02 100644 --- a/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts +++ b/gitnexus/test/integration/analyze-index-lock-concurrency.test.ts @@ -7,7 +7,7 @@ * after the child is SIGKILLed the kernel drops the binding and our next * acquire succeeds — the kernel-auto-release guarantee, no stale handling. * - Test 2 pins the FILE backend and races several children reclaiming one dead - * holder, asserting the atomic rename-steal never lets two into the critical + * holder, asserting the acquisition/reclaim guard never lets two into the critical * section at once. * * The child imports the BUILT module (dist/) and this process imports the @@ -59,6 +59,39 @@ afterEach(() => { }); describe('index lock across processes (#2658)', () => { + it.each(['EEXIST', 'ENOENT'])( + 'reports sentinel-create %s distinctly from other failures', + async (code) => { + const sentinel = + code === 'EEXIST' + ? path.join(dir, 'critical.sentinel') + : path.join(dir, 'missing-parent', 'critical.sentinel'); + if (code === 'EEXIST') writeFileSync(sentinel, 'occupied'); + child = spawn(process.execPath, [childScript], { + env: { + ...process.env, + LOCK_MODULE: lockModule, + LOCK_DIR: dir, + SENTINEL: sentinel, + MODE: 'EXCLUSIVE', + GITNEXUS_INDEX_LOCK_BACKEND: 'file', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr?.on('data', (chunk) => { + stderr += String(chunk); + }); + const exitCode = await new Promise((resolve, reject) => { + child!.once('error', reject); + child!.once('close', resolve); + }); + expect(exitCode, stderr).toBe(code === 'EEXIST' ? 3 : 4); + expect(stderr).toContain(`code=${code}`); + expect(stderr).toContain(sentinel); + }, + ); + it('excludes a second writer while held, then recovers after the holder is killed', async () => { if (!existsSync(lockModule)) { throw new Error( @@ -91,83 +124,77 @@ describe('index lock across processes (#2658)', () => { lock.release(); }, 90_000); - // The FILE backend is the DEFAULT only on macOS/BSD; Windows and Linux default - // to the race-free kernel lock (named pipe / abstract socket). This case FORCES - // the file backend to stress its rename-steal reclaim, so it runs where that - // backend is actually production (macOS — where the double-admit bug this - // guards lived and is now fixed) plus Linux. It is skipped on Windows, where - // the file backend is never the default; Windows' real lock (the named pipe) is - // covered by index-lock.test.ts on the Windows matrix and by the - // default-backend cross-process case above. - it.skipIf(process.platform === 'win32')( - 'lets multiple waiters reclaim one dead holder without ever admitting two writers', - async () => { - if (!existsSync(lockModule)) { - throw new Error( - `dist/storage/index-lock.js missing — run \`npm run build\` first ` + - `(or use \`npm run test:integration\`, which builds via pretest:integration).`, - ); - } - // This case targets the FILE backend's reclaim path specifically (the socket - // backend has no stale file to reclaim). Seed a stale lock owned by a dead, - // same-host holder — every child must reclaim it, and the reclaim must let - // exactly one at a time win so no two children are ever in their O_EXCL - // sentinel section together. - // - // The reclaim's rename-steal must NOT act on a stale staleness judgment: a - // waiter that judged the dead record must re-verify the file still holds it - // before renaming, or it will rename a live winner's freshly-created lock - // aside and admit a second writer (#2658 review — this reproduced at ~18% per - // round of 4-way contention before the judgment-verified steal). One round - // catches that regression only ~1-in-6 of the time, so loop several rounds to - // make it a reliable guard; with the fix every round is clean. - const sentinel = path.join(dir, 'critical.sentinel'); - const seedDeadHolder = (): void => { - writeFileSync( - path.join(dir, 'analyze.lock'), - JSON.stringify({ - v: 1, - pid: 999_999_999, - hostname: os.hostname(), - startTime: null, - token: 'dead-holder-token', - invocationId: 'dead-holder', - acquiredAt: new Date(0).toISOString(), - }), - ); - }; + // Force the portable fallback on every platform, including Windows where a + // socket-bind failure can select it. The default backend is covered above. + it('lets multiple waiters reclaim one dead holder without ever admitting two writers', async () => { + if (!existsSync(lockModule)) { + throw new Error( + `dist/storage/index-lock.js missing — run \`npm run build\` first ` + + `(or use \`npm run test:integration\`, which builds via pretest:integration).`, + ); + } + // This case targets the FILE backend's reclaim path specifically (the socket + // backend has no stale file to reclaim). Seed a stale lock owned by a dead, + // same-host holder — every child must reclaim it, and the reclaim must let + // exactly one at a time win so no two children are ever in their O_EXCL + // sentinel section together. + // + // Retain repeated real-process contention alongside the deterministic + // stale-read/rename/restore regression in index-lock-reclaim-guard.test.ts. + const sentinel = path.join(dir, 'critical.sentinel'); + const seedDeadHolder = (): void => { + writeFileSync( + path.join(dir, 'analyze.lock'), + JSON.stringify({ + v: 1, + pid: 999_999_999, + hostname: os.hostname(), + startTime: null, + token: 'dead-holder-token', + invocationId: 'dead-holder', + acquiredAt: new Date(0).toISOString(), + }), + ); + }; - const runChild = (): Promise<{ code: number | null; signal: NodeJS.Signals | null }> => - new Promise((resolve) => { - const c = spawn(process.execPath, [childScript], { - env: { - ...process.env, - LOCK_MODULE: lockModule, - LOCK_DIR: dir, - SENTINEL: sentinel, - MODE: 'EXCLUSIVE', - GITNEXUS_INDEX_LOCK_BACKEND: 'file', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - c.once('exit', (code, signal) => resolve({ code, signal })); + const runChild = (): Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + stderr: string; + }> => + new Promise((resolve, reject) => { + const c = spawn(process.execPath, [childScript], { + env: { + ...process.env, + LOCK_MODULE: lockModule, + LOCK_DIR: dir, + SENTINEL: sentinel, + MODE: 'EXCLUSIVE', + GITNEXUS_INDEX_LOCK_BACKEND: 'file', + }, + stdio: ['ignore', 'pipe', 'pipe'], }); + let stderr = ''; + c.stderr?.on('data', (chunk) => { + stderr += String(chunk); + }); + c.once('error', reject); + c.once('close', (code, signal) => resolve({ code, signal, stderr })); + }); - const ROUNDS = 8; - const KIDS = 5; - for (let round = 0; round < ROUNDS; round++) { - seedDeadHolder(); // the previous round's winner released (unlinked) the lock - const results = await Promise.all(Array.from({ length: KIDS }, () => runChild())); - // Every child acquired, ran its exclusive section, and exited cleanly (0). - // Exit 3 = it found the sentinel already present = two holders at once. - for (const r of results) { - expect(r.signal).toBeNull(); - expect(r.code).toBe(0); - } - // No leftover sentinel — the last holder cleaned up. - expect(existsSync(sentinel)).toBe(false); + const ROUNDS = 8; + const KIDS = 5; + for (let round = 0; round < ROUNDS; round++) { + seedDeadHolder(); // the previous round's winner released (unlinked) the lock + const results = await Promise.all(Array.from({ length: KIDS }, () => runChild())); + // Every child acquired, ran its exclusive section, and exited cleanly (0). + // Exit 3 = it found the sentinel already present = two holders at once. + for (const r of results) { + expect(r.signal, r.stderr).toBeNull(); + expect(r.code, r.stderr).toBe(0); } - }, - 60_000, - ); + // No leftover sentinel — the last holder cleaned up. + expect(existsSync(sentinel)).toBe(false); + } + }, 60_000); }); diff --git a/gitnexus/test/integration/cobol-import-index-reuse.test.ts b/gitnexus/test/integration/cobol-import-index-reuse.test.ts index bcde62b05..d9db7162c 100644 --- a/gitnexus/test/integration/cobol-import-index-reuse.test.ts +++ b/gitnexus/test/integration/cobol-import-index-reuse.test.ts @@ -118,11 +118,9 @@ describe('COBOL COPY resolution — index reuse across imports (#2908)', () => { // Source tier, reached only after every copybook missed. expect(resolveImportTarget('PAYROLL', FROM_FILE, files, undefined)).toBe('src/PAYROLL.cob'); expect(resolveImportTarget('TAXCALC', FROM_FILE, files, undefined)).toBe('src/TAXCALC.cobol'); - // `path.basename(fp, '.cpy')` will not strip `.CPY`, so the stem keeps it. - expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBe( - 'copybooks/Mixed.CPY', - ); - expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBeNull(); + // P1-A: raw extension stripped, so `Mixed.CPY` keys as `MIXED`. + expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBe('copybooks/Mixed.CPY'); + expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBeNull(); // Neither tier: a `.txt`, a file with no extension, and an absent member. expect(resolveImportTarget('NOEXT', FROM_FILE, files, undefined)).toBeNull(); expect(resolveImportTarget('ABSENT', FROM_FILE, files, undefined)).toBeNull(); diff --git a/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts index 283f68842..4d1d50802 100644 --- a/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts +++ b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts @@ -7,9 +7,11 @@ import { closeLbug, executeParameterized } from '../../../src/core/lbug/pool-ada import type { GroupConfig, RepoHandle } from '../../../src/core/group/types.js'; import { withTestLbugDB } from '../../helpers/test-indexed-db.js'; +// Graph Method.startLine is 0-based (tree-sitter row). `health()` is source line 5. const SEED = [ - `CREATE (:Method {id:'method:health', name:'health', filePath:'src/health.resolver.ts', startLine:5, endLine:5, content:'', description:''})`, - `CREATE (:Const {id:'const:health-document', name:'HealthDocument', filePath:'src/generated.ts', startLine:1, endLine:1, content:'', description:''})`, + `CREATE (:Method {id:'method:health', name:'health', filePath:'src/health.resolver.ts', startLine:4, endLine:4, content:'', description:''})`, + `CREATE (:Method {id:'method:save', name:'save', filePath:'src/health.resolver.ts', startLine:6, endLine:6, content:'', description:''})`, + `CREATE (:Const {id:'const:health-document', name:'HealthDocument', filePath:'src/generated.ts', startLine:0, endLine:0, content:'', description:''})`, ]; withTestLbugDB( @@ -140,6 +142,42 @@ withTestLbugDB( registrySpy.mockRestore(); } }); + + it('binds a decorated arrow-field provider at the wrapper startLine (#3201)', async () => { + providerRoot = path.join(handle.tmpHandle.dbPath, 'arrow-provider-repo'); + await fs.mkdir(path.join(providerRoot, 'src'), { recursive: true }); + await fs.writeFile( + path.join(providerRoot, 'src/health.resolver.ts'), + `import { Query, Mutation, Resolver } from '@nestjs/graphql'; +@Resolver() +class HealthResolver { + @Query() + health() { return 'ok'; } + + @Mutation() + save = async () => true; +}`, + 'utf8', + ); + const providerRepo: RepoHandle = { + id: handle.repoId, + path: 'api', + repoPath: providerRoot, + storagePath: handle.tmpHandle.dbPath, + }; + const execute = (query: string, params: Record = {}) => + executeParameterized(handle.repoId, query, params); + const contracts = await new GraphqlExtractor().extract(execute, providerRoot, providerRepo); + expect(contracts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contractId: 'graphql::mutation::save', + role: 'provider', + symbolUid: 'method:save', + }), + ]), + ); + }); }, { seed: SEED, poolAdapter: true }, ); diff --git a/gitnexus/test/integration/impact-callable-value-references.test.ts b/gitnexus/test/integration/impact-callable-value-references.test.ts new file mode 100644 index 000000000..6124d76d3 --- /dev/null +++ b/gitnexus/test/integration/impact-callable-value-references.test.ts @@ -0,0 +1,231 @@ +/** + * Integration test: a callable named in VALUE position makes `impact` a lower + * bound (#3399). + * + * The rejected behaviour, in one sentence: `Element.getNamespaceUri` — the DOM + * `Element.namespaceURI` accessor — is bound into a JS bridge table as + * `bridge.accessor(Element.getNamespaceUri, null, .{})`, and `impact` answered + * "2 callers, LOW risk, epistemic: exact". Everything about that answer except + * the number 2 was wrong, and `exact` is the part that made it unusable: it + * tells the reader not to look further. + * + * The seed below is the shape that matters, not the language. A registration + * edge (`USES`, reason `scope-resolution: value-ref`) says a function was + * handed somewhere as a value. Where the value goes next — a struct field, a + * registry lookup, comptime reflection — is not modelled, so no CALLS edge + * connects the eventual invocation back to the target. `tools.ts` defines + * `lower-bound` as "the walk provably missed callers", and that is exactly this + * situation. + * + * WHY the assertions are what they are: + * - `impactedCount` must NOT move. Hedging is not inventing callers; a fix + * that made the number go up would be a different (and unearned) claim. + * - a plain CALLS-only target in the SAME index must stay `exact`, or the + * hedge is noise sprayed over every query and carries no information. + * - the cause has its own slot rather than being folded into + * `dispatchBoundary`: an agent branching on the numbers would otherwise be + * told an interface boundary exists where there is none, and would draw the + * opposite conclusion about whether the gap is reducible. + */ +import { it, expect, beforeAll, vi } from 'vitest'; +import path from 'node:path'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; +import { VALUE_REF_EDGE_REASON } from '../../src/core/ingestion/scope-resolution/value-ref-edges.js'; + +/** + * Fail ONLY the value-reference probe, leaving every other query on the real + * database. Keyed on the bound `$reason` param rather than the query text, so + * it cannot accidentally match a different query that happens to mention USES. + */ +let failValueRefProbe = false; + +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + executeParameterized: (...args: any[]) => { + const params = args[2] as { reason?: string } | undefined; + if (failValueRefProbe && params?.reason === 'scope-resolution: value-ref') { + return Promise.reject(new Error('simulated: index unreadable')); + } + return (actual as any).executeParameterized(...args); + }, + }; +}); + +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + listRegisteredRepos: vi.fn().mockResolvedValue([]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), + }; +}); +const { listRegisteredRepos, saveMeta } = await import('../../src/storage/repo-manager.js'); + +const SEED = [ + // The registered accessor, with one ordinary in-file caller so the walk has + // something real to report. This mirrors Element.zig: `getNamespaceUri` genuinely + // has internal callers, and it is the REGISTRATION that the answer omits. + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.getNamespaceUri', name: 'getNamespaceUri', filePath: 'webapi/Element.zig', startLine: 439, endLine: 442, isExported: true, content: '', description: ''})`, + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.lookupPrefixForElement', name: 'lookupPrefixForElement', filePath: 'webapi/Element.zig', startLine: 490, endLine: 520, isExported: true, content: '', description: ''})`, + `MATCH (a:Method {id:'Method:webapi/Element.zig:Element.lookupPrefixForElement'}), (b:Method {id:'Method:webapi/Element.zig:Element.getNamespaceUri'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'scope-resolution: local-call', step:0}]->(b)`, + + // The binding table entry: `pub const namespaceURI = bridge.accessor(Element.getNamespaceUri, null, .{});` + // A registration, NOT an invocation — hence USES, not CALLS (Kythe `ref` vs + // `ref/call`; Joern METHOD_REF). + `CREATE (:Struct {id: 'Struct:webapi/Element.zig:Element.JsApi', name: 'JsApi', filePath: 'webapi/Element.zig', startLine: 2281, endLine: 2400, content: '', description: ''})`, + `MATCH (a:Struct {id:'Struct:webapi/Element.zig:Element.JsApi'}), (b:Method {id:'Method:webapi/Element.zig:Element.getNamespaceUri'}) CREATE (a)-[:CodeRelation {type:'USES', confidence:0.85, reason:'${VALUE_REF_EDGE_REASON}', step:0}]->(b)`, + + // Control: same file, same shape of caller, but nothing registers it as a + // value. This is `getTagNameLower` — it must come back `exact`. + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.getTagNameLower', name: 'getTagNameLower', filePath: 'webapi/Element.zig', startLine: 400, endLine: 410, isExported: true, content: '', description: ''})`, + `MATCH (a:Method {id:'Method:webapi/Element.zig:Element.lookupPrefixForElement'}), (b:Method {id:'Method:webapi/Element.zig:Element.getTagNameLower'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'scope-resolution: local-call', step:0}]->(b)`, + + // Second control: an ordinary USES edge that is NOT a value registration (a + // type reference). The probe keys on the reason, so this must not hedge — + // otherwise every type mention in the repo would downgrade its target. + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.getInnerText', name: 'getInnerText', filePath: 'webapi/Element.zig', startLine: 600, endLine: 610, isExported: true, content: '', description: ''})`, + `MATCH (a:Struct {id:'Struct:webapi/Element.zig:Element.JsApi'}), (b:Method {id:'Method:webapi/Element.zig:Element.getInnerText'}) CREATE (a)-[:CodeRelation {type:'USES', confidence:0.85, reason:'scope-resolution: type-reference', step:0}]->(b)`, + + // Third control: a registration whose invocation the analyzer DID synthesize. + // `emitPropertyDispatchCalls` sweep 2 connects `x.()` member calls to + // every function registered under `` and stamps those edges + // `property-dispatch`. Where that happened the walk followed the + // registration, so hedging would be noise over an answer that was actually + // computed. This is the JS/TS hook-table shape, not the Zig one — Zig has no + // object-literal key to dispatch through and so is never excluded. + `CREATE (:Function {id: 'Function:hooks/registry.js:onClick', name: 'onClick', filePath: 'hooks/registry.js', startLine: 5, endLine: 8, isExported: true, content: '', description: ''})`, + `CREATE (:Function {id: 'Function:hooks/registry.js:registerAll', name: 'registerAll', filePath: 'hooks/registry.js', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, + `CREATE (:Function {id: 'Function:hooks/consumer.js:runHandlers', name: 'runHandlers', filePath: 'hooks/consumer.js', startLine: 1, endLine: 6, isExported: true, content: '', description: ''})`, + `MATCH (a:Function {id:'Function:hooks/registry.js:registerAll'}), (b:Function {id:'Function:hooks/registry.js:onClick'}) CREATE (a)-[:CodeRelation {type:'USES', confidence:0.85, reason:'${VALUE_REF_EDGE_REASON}', step:0}]->(b)`, + `MATCH (a:Function {id:'Function:hooks/consumer.js:runHandlers'}), (b:Function {id:'Function:hooks/registry.js:onClick'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.7, reason:'property-dispatch', step:0}]->(b)`, +]; + +withTestLbugDB( + 'impact-callable-value-references', + (handle) => { + let backend: LocalBackend; + beforeAll(() => { + backend = (handle as any)._backend; + }); + + it('downgrades a registered accessor to lower-bound without inventing callers', async () => { + const result: any = await backend.callTool('impact', { + target: 'getNamespaceUri', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + // Pinned, not implied: hedging must not INVENT callers. The seed gives + // this symbol exactly two inbound edges — one CALLS, one value-ref USES — + // and both are traversed. If a later change made the hedge also widen the + // walk, every other assertion here would still pass. + expect(result.impactedCount).toBe(2); + // The registration is a real inbound edge, so it IS traversed and counted + // — but the call THROUGH the registered value is not, which is why the + // count still cannot be the whole story. + expect(result.epistemic).toBe('lower-bound'); + expect(result.causes.callableValueReferences).toBe(1); + // Its own slot: an agent must not read this as an interface boundary. + expect(result.causes.dispatchBoundary).toBe(0); + expect(result.boundaries.join(' ')).toContain('as a VALUE'); + }); + + it('leaves a symbol with only ordinary calls exact', async () => { + const result: any = await backend.callTool('impact', { + target: 'getTagNameLower', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('exact'); + }); + + it('does not hedge on a USES edge that is not a value registration', async () => { + const result: any = await backend.callTool('impact', { + target: 'getInnerText', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + // A type reference is a use, not an escape: nothing can be invoked + // through it, so the answer stays complete. + expect(result.epistemic).toBe('exact'); + }); + + it('hedges context() for the same reason it hedges impact()', async () => { + const result: any = await backend.callTool('context', { name: 'getNamespaceUri' }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('lower-bound'); + expect(result.causes.callableValueReferences).toBe(1); + }); + + it('stays exact when the analyzer already synthesized the dispatch', async () => { + // `onClick` is registered as a value AND reached by a synthesized + // property-dispatch CALLS edge. The walk did not "provably miss" that + // caller, so `lower-bound` would be wrong — and a signal that fires on + // every hook table in every JS codebase carries no information. + const result: any = await backend.callTool('impact', { + target: 'onClick', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('exact'); + }); + + it('hedges — never reports exact — when the probe itself cannot run', async () => { + // The failure mode this whole feature exists to remove is silence reading + // as certainty. A probe that threw has not established that the symbol is + // unregistered; swallowing the error into a zero would publish `exact` on + // the strength of a question that was never answered. + failValueRefProbe = true; + try { + const result: any = await backend.callTool('impact', { + target: 'getTagNameLower', // the control: exact when the probe works + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain('could not be run'); + // No count is claimed — the note is the signal, and inventing a + // magnitude from a failed query would be the same error inverted. + expect(result.causes.callableValueReferences).toBe(0); + } finally { + failValueRefProbe = false; + } + }); + + it('stays exact downstream — a reference INTO a symbol says nothing about what it reaches', async () => { + const result: any = await backend.callTool('impact', { + target: 'getNamespaceUri', + direction: 'downstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('exact'); + }); + }, + { + seed: SEED, + poolAdapter: true, + afterSetup: async (h) => { + // Without a completeness receipt EVERY answer is hedged ("scope-extraction + // completeness was not recorded"), which would make the controls below + // pass for the wrong reason and prove nothing about this probe. + // `saveMeta` is the only writer production uses. + await saveMeta(path.dirname(h.dbPath), { scopeExtractionReceipt: 1 } as any); + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: h.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc123', + stats: { files: 1, nodes: 5, communities: 0, processes: 0 }, + }, + ] as any); + const backend = new LocalBackend(); + await backend.init(); + (h as any)._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/mcp/update-notice.test.ts b/gitnexus/test/integration/mcp/update-notice.test.ts index fee79fc1d..a1380af39 100644 --- a/gitnexus/test/integration/mcp/update-notice.test.ts +++ b/gitnexus/test/integration/mcp/update-notice.test.ts @@ -51,6 +51,7 @@ function mockBackend() { : { ok: true }, ), listRepos: vi.fn().mockResolvedValue([]), + countRepos: vi.fn().mockResolvedValue(0), resolveRepo: vi .fn() .mockResolvedValue({ name: 'test', repoPath: '/tmp/test', lastCommit: 'abc' }), diff --git a/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts b/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts index 949f0d75e..e662603ea 100644 --- a/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts +++ b/gitnexus/test/integration/parse-impl-quarantine-cache-skip.test.ts @@ -78,6 +78,7 @@ import { fileContentHash, packParseCacheChunks, } from '../../src/storage/parse-cache.js'; +import type { ParseCache } from '../../src/storage/parse-cache.js'; import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; /** @@ -257,7 +258,7 @@ describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex f const expectedChunkHash = hashPacks(scanned).poison; - const parseCache = { + const parseCache: ParseCache = { version: 'test', entries: new Map(), usedKeys: new Set(), @@ -323,6 +324,9 @@ describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex f // against a fresh-quarantine pool. expect(parseCache.entries.has(expectedChunkHash)).toBe(false); expect(parseCache.usedKeys.has(expectedChunkHash)).toBe(true); + // #3204: `usedKeys` alone would let `saveParseCache` copy a pre-existing + // shard forward, so the skipped chunk is also retired from the save. + expect(parseCache.staleKeys?.has(expectedChunkHash)).toBe(true); for (const hash of hashPacks(scanned).others) { expect(parseCache.entries.has(hash)).toBe(true); } @@ -338,7 +342,7 @@ describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex f })); const expectedChunkHash = hashPacks(scanned).poison; - const parseCache = { + const parseCache: ParseCache = { version: 'test', entries: new Map(), usedKeys: new Set(), diff --git a/gitnexus/test/integration/resolvers/barrel-dir-index-wildcard.test.ts b/gitnexus/test/integration/resolvers/barrel-dir-index-wildcard.test.ts new file mode 100644 index 000000000..276015b25 --- /dev/null +++ b/gitnexus/test/integration/resolvers/barrel-dir-index-wildcard.test.ts @@ -0,0 +1,136 @@ +/** + * C2 — the grafana `Button` shape: a workspace barrel `export *`s a components + * index, which re-exports NAMED bindings (with inline `type` modifiers) from a + * DIRECTORY index, which `export *`s the real file, whose `Button` is a + * `React.forwardRef` const. Measured on grafana@871af0720: `Button` resolved 8 + * of 475 ledger entries while siblings through plain hops resolved at scale. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import path from 'path'; +import fs from 'node:fs'; +import os from 'node:os'; +import { + getRelationships, + getResolutionOutcomes, + runPipelineFromRepo, + writeFixtureRepo, + type PipelineResult, +} from './helpers.js'; + +describe('named re-export through a directory index that wildcards (grafana Button shape)', () => { + let result: PipelineResult; + let repoDir: string | undefined; + + beforeAll(async () => { + repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-c2-dir-index-')); + writeFixtureRepo(repoDir, { + 'package.json': '{ "name": "root", "private": true, "workspaces": ["packages/*"] }\n', + 'packages/ui/package.json': + '{ "name": "@x/ui", "version": "1.0.0", "main": "src/index.ts" }\n', + 'packages/ui/src/index.ts': `export * from './components';\nexport * from './themes';\n`, + // Inline `type` modifiers on the same statement as value re-exports. + 'packages/ui/src/components/index.ts': `export { Stack } from './Layout/Stack'; +export { Button, LinkButton, type ButtonVariant, ButtonGroup, type ButtonProps, clearButtonStyles } from './Button'; +`, + // Directory index: wildcard + one named re-export. + 'packages/ui/src/components/Button/index.ts': `export * from './Button';\nexport { ButtonGroup } from './ButtonGroup';\n`, + 'packages/ui/src/components/Button/Button.tsx': `import React from 'react'; +export type ButtonVariant = 'primary' | 'secondary'; +export interface ButtonProps { variant?: ButtonVariant; label: string } +export const Button = React.forwardRef((props, ref) => { + return null; +}); +export const LinkButton = React.forwardRef((props, ref) => { + return null; +}); +export const clearButtonStyles = (theme: string) => { + return theme; +}; +`, + 'packages/ui/src/components/Button/ButtonGroup.tsx': `export function ButtonGroup(children: string) { + return children; +} +`, + 'packages/ui/src/components/Layout/Stack.tsx': `export function Stack(children: string) { + return children; +} +`, + 'packages/ui/src/themes/index.ts': `export * from './hooks';\n`, + 'packages/ui/src/themes/hooks/index.ts': `export * from './useStyles2';\n`, + 'packages/ui/src/themes/hooks/useStyles2.ts': `export function useStyles2(fn: (t: string) => string) { + return fn('theme'); +} +`, + 'packages/app/package.json': + '{ "name": "@x/app", "version": "1.0.0", "main": "src/main.tsx", "dependencies": { "@x/ui": "1.0.0" } }\n', + 'packages/app/tsconfig.json': `{ "compilerOptions": { "jsx": "react-jsx" } }\n`, + 'packages/app/src/main.tsx': `import { Button, LinkButton, ButtonGroup, clearButtonStyles, Stack, useStyles2 } from '@x/ui'; + +export function render() { + const styles = useStyles2((t) => t); + clearButtonStyles(styles); + ButtonGroup('x'); + Stack('y'); + const a =