diff --git a/.claude/skills/gitnexus-impact-analysis/SKILL.md b/.claude/skills/gitnexus-impact-analysis/SKILL.md index 45eb7ce87..0b81795de 100644 --- a/.claude/skills/gitnexus-impact-analysis/SKILL.md +++ b/.claude/skills/gitnexus-impact-analysis/SKILL.md @@ -17,22 +17,23 @@ description: "Use when the user wants to know what will break if they change som ## Workflow ``` -1. impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) or `node .gitnexus/run.cjs impact "X" --direction upstream --repo .` 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. detect_changes() → Map current git changes to affected flows +3. detect_changes({scope: "all"}) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` 4. Assess risk and report to user ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If `.gitnexus/run.cjs` is missing, replace `node .gitnexus/run.cjs` with `npx gitnexus` in the fallback commands. ## Checklist ``` -- [ ] impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) or CLI fallback to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] detect_changes() for pre-commit check +- [ ] detect_changes({scope: "all"}) or CLI fallback for pre-commit check - [ ] Assess risk level and report to user ``` @@ -55,7 +56,7 @@ description: "Use when the user wants to know what will break if they change som ## Tools -**impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: ``` impact({ @@ -73,10 +74,10 @@ impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis. If MCP is unavailable, use `node .gitnexus/run.cjs detect-changes --scope all --repo .` instead: ``` -detect_changes({scope: "staged"}) +detect_changes({scope: "all"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -86,7 +87,7 @@ detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) or `node .gitnexus/run.cjs impact "validateUser" --direction upstream --repo .` → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/.claude/skills/gitnexus-review/SKILL.md b/.claude/skills/gitnexus-review/SKILL.md index 9eabc426c..7b47cc3f6 100644 --- a/.claude/skills/gitnexus-review/SKILL.md +++ b/.claude/skills/gitnexus-review/SKILL.md @@ -120,10 +120,17 @@ and do not claim a complete graph-backed review. review surface: when the diff changes what gets emitted or persisted, verify every schema/version constant gating caches, incremental writebacks, and fingerprint baselines was bumped or regenerated — in - GitNexus itself, for example: `INCREMENTAL_SCHEMA_VERSION` (the - incremental write set covers only changed files, so new cross-file edges - never reach an existing index without the bump), the parse-store - `SCHEMA_BUMP`, and both bench fingerprint sets. + GitNexus itself, for example: graph DDL needs no manual bump, because + `SCHEMA_FINGERPRINT` (`gitnexus/src/core/lbug/schema.ts`) is derived + from `NODE_SCHEMA_QUERIES` + `REL_SCHEMA_QUERIES` and moves on its own; + the check there is whether the diff changed any string in those arrays, + and, if it added a new DDL array, whether that array was folded into the + fingerprint. The hand-maintained ritual still applies where no + declarative artifact describes the invalidated set: the parse-store + `SCHEMA_BUMP` and both bench fingerprint sets still need an explicit + bump, re-checked against the base branch right before merge. Semantic + changes that leave the DDL untouched are outside the fingerprint; they + rely on the analyzer runner-identity receipt in the index metadata. ## Expert lenses diff --git a/.github/workflows/build-tree-sitter-prebuilds.yml b/.github/workflows/build-tree-sitter-prebuilds.yml index 06e2b4c9f..d992c5634 100644 --- a/.github/workflows/build-tree-sitter-prebuilds.yml +++ b/.github/workflows/build-tree-sitter-prebuilds.yml @@ -414,7 +414,13 @@ jobs: # prebuilds/-/.node. ( cd "$pkgdir" && npx --no-install prebuildify --napi --strip ) - out=$(find "$pkgdir/prebuilds" -name '*.node' -print -quit) + # `|| true` so the `test -n` below is the thing that reports a missing + # prebuild. `rm -rf` above deletes the directory, so a prebuildify + # run that emits nothing without failing leaves `find` searching a + # path that no longer exists — it exits 1 and `-e` would kill the step + # before the `::error::` line, which is exactly the case that line + # exists to explain. + out=$(find "$pkgdir/prebuilds" -name '*.node' -print -quit || true) test -n "$out" || { echo "::error::prebuildify produced no .node"; exit 1; } produced=$(basename "$(dirname "$out")") [ "$produced" = "$PLATFORM_ARCH" ] || { echo "::error::built $produced, expected $PLATFORM_ARCH"; exit 1; } diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 6a55254ca..f1afd43b7 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -256,21 +256,37 @@ jobs: fi } + # ── Helper: first matching file, tolerating an absent root ── + # `coverage-merge` (ci-tests.yml) is `needs: tests` with no + # `if: always()`, so a failing shard skips it and the `test-reports` + # artifact is never uploaded. A bare `find` on the missing directory + # exits 1; `-o pipefail` carries that through `| head -1` and `-e` + # then killed this step — silently, because stderr is discarded and + # stdout is redirected to $GITHUB_OUTPUT. That skipped "Comment on + # PR" and failed the run precisely when a PR had failing tests, which + # is when the report matters most. Degrade to "" instead so the + # coverage-unavailable fallback below can do its job. + find_first() { + local root=$1 name=$2 + [ -d "$root" ] || return 0 + find "$root" -name "$name" -type f 2>/dev/null | head -1 || true + } + # ── Read coverage reports ── - UNIT_SUMMARY=$(find "$DIR/test-reports" -name "coverage-summary.json" -type f 2>/dev/null | head -1) + UNIT_SUMMARY=$(find_first "$DIR/test-reports" "coverage-summary.json") read_cov "U" "$UNIT_SUMMARY" # ── Read base branch coverage (main) ── BASE_SUMMARY="" if [ "$BASE_FOUND" = "true" ] && [ -n "$BASE_DIR" ]; then - BASE_SUMMARY=$(find "$BASE_DIR/base" -name "coverage-summary.json" -type f 2>/dev/null | head -1) + BASE_SUMMARY=$(find_first "$BASE_DIR/base" "coverage-summary.json") fi read_cov "B" "$BASE_SUMMARY" # ── Locate test results ── - RESULTS_FILE=$(find "$DIR/test-reports" -name "test-results.json" -type f 2>/dev/null | head -1) - WEB_RESULTS_FILE=$(find "$DIR/test-reports" -name "web-test-results.json" -type f 2>/dev/null | head -1) + RESULTS_FILE=$(find_first "$DIR/test-reports" "test-results.json") + WEB_RESULTS_FILE=$(find_first "$DIR/test-reports" "web-test-results.json") sum_results() { local file=$1 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 18ac7fd89..87d2c4377 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -500,6 +500,14 @@ jobs: run: node --import tsx bench/callable-value-flow/measure.mjs --check working-directory: gitnexus + - name: C++ qualified-namespace resolution guards (#2788) + # Build-free: asserts resolveCppQualifiedNamespaceMember resolves an + # unchanged symbol set (fingerprint) and that per-call-site cost stays + # independent of corpus size. Rationale and history: see the header of + # bench/cpp-qualified-ns/measure.mjs. + run: node --import tsx bench/cpp-qualified-ns/measure.mjs --check + working-directory: gitnexus + - name: Receiver-resolution drop guards # NOT build-free: this one runs the real pipeline, so it needs dist/ # (the setup action above builds). ~2m15s. @@ -509,9 +517,19 @@ jobs: # recorder gates on the receiver's punctuation, not on what the # reference is, so property reads would inflate it by ~20%. The shape # arm asserts the state of each receiver spelling by EDGE PRESENCE, - # which is the only arm that can see the shapes the recorder is blind - # to (`?.`, explicit type args, `repos[0]`): those emit no edge AND no - # drop, so fixing them moves the count by zero. + # which is the only arm that can see shapes the recorder is blind to: + # they emit no edge AND no drop, so fixing them moves the count by zero. + # + # `repos[0]` is no longer among them (#2766): Case 0's gate now accepts + # a minted receiver chain instead of testing the receiver's punctuation, + # so subscript receivers record a drop and ARE countable. 13 shapes moved + # INVISIBLE -> VISIBLE that way. `?.` and explicit type args remain + # invisible on some languages, so the shape arm still earns its keep. + # + # The check is EXACT-MATCH, which is strictly stronger than a ratchet: + # the count cannot rise without a deliberate rebaseline, and the + # rebaseline path demands the movement be explained. No separate + # drop-ratchet gate is needed on top of this. run: node --import tsx bench/receiver-resolution/measure.mjs --check working-directory: gitnexus @@ -556,12 +574,17 @@ jobs: working-directory: gitnexus - name: Cross-language pipeline benchmarks (GITNEXUS_BENCH, serial) + # cpp-adl-benchmark.test.ts is not a `*-pipeline-benchmark.test.ts` but + # belongs here for the same reason: it is skipIf-gated on GITNEXUS_BENCH, + # so it had never run in CI and the PR #1990 ADL emit-scaling guard it + # holds was dead. ~45s of test time. env: GITNEXUS_BENCH: '1' run: >- npx vitest run --no-file-parallelism test/integration/cobol-pipeline-benchmark.test.ts test/integration/csharp-pipeline-benchmark.test.ts + test/integration/cpp-adl-benchmark.test.ts test/integration/instance-ownership-pipeline-benchmark.test.ts test/integration/spring-bean-resource-benchmark.test.ts test/integration/rust-pipeline-benchmark.test.ts diff --git a/AGENTS.md b/AGENTS.md index 2e03061e2..b34ddef32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,30 +111,31 @@ mirror. `gitnexus/test/unit/shipped-skills-sync.test.ts` guards the copies. Toke # GitNexus — Code Intelligence -This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **GitNexus** (248612 symbols, 565510 relationships, 918 execution flows). Use GitNexus graph tools to understand code, assess impact, and navigate safely. > Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: `detect_changes({scope: "compare", base_ref: "main"})`. +- **MUST run impact analysis before editing.** Use `impact({target: "symbolName", direction: "upstream"})` (MCP) or `node .gitnexus/run.cjs impact "symbolName" --direction upstream --repo .` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis. For unified PDG impact, add `mode: "pdg"` with optional `line: ` — it returns statement-level `affectedStatements` over CDG + REACHING_DEF and inter-procedural symbols in `interproceduralByDepth`/`byDepth`; no-layer/degraded PDG results are UNKNOWN-risk notes (`--pdg` layer). CLI equivalent: `node .gitnexus/run.cjs impact "symbolName" --direction upstream --mode pdg --line --repo .`. +- **MUST analyze graph changes before committing.** Use `detect_changes({scope: "all"})` (MCP) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` (CLI fallback). For regression review: `detect_changes({scope: "compare", base_ref: "main"})` or `node .gitnexus/run.cjs detect-changes --scope compare --base-ref "main" --repo .`. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. - When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. - When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. - For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). +- For control/data dependence, `pdg_query({mode: "controls", target: "fileOrSymbol"})` answers "under what condition does X run?" (CDG, incl. guard clauses) and `pdg_query({mode: "flows", target, variable})` traces "where does variable Y flow?" (REACHING_DEF). `--pdg` layer. ## Never Do -- NEVER edit a function, class, or method without first running `impact` on it. +- NEVER edit a function, class, or method before MCP/CLI impact analysis. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. - NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. -- NEVER commit changes without running `detect_changes()` to check affected scope. +- NEVER commit before MCP/CLI graph change analysis. ## Resources | Resource | Use for | -|----------|---------| +| --- | --- | | `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | | `gitnexus://repo/GitNexus/clusters` | All functional areas | | `gitnexus://repo/GitNexus/processes` | All execution flows | @@ -143,7 +144,7 @@ This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relati ## CLI | Task | Read this skill file | -|------|---------------------| +| --- | --- | | Understand architecture / "How does X work?" | `.claude/skills/gitnexus-exploring/SKILL.md` | | Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus-impact-analysis/SKILL.md` | | Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus-debugging/SKILL.md` | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 86ad6c426..42160627e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -4,18 +4,18 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## Repository layout -| Path | Role | -|------|------| -| `gitnexus/` | npm package `gitnexus`: CLI, MCP server (stdio), HTTP API, ingestion pipeline, LadybugDB graph, embeddings. | -| `gitnexus-web/` | Vite + React thin client: graph explorer + AI chat. All queries via `gitnexus serve` HTTP API. | -| `gitnexus-shared/` | Shared TypeScript types and constants (consumed by CLI and Web). | -| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Agent skills and plugin metadata. | -| `eval/` | Evaluation harnesses for benchmarking tool usage. | -| `.github/` | CI workflows + composite actions (`setup-gitnexus/`, `setup-gitnexus-web/`). | +| Path | Role | +| --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `gitnexus/` | npm package `gitnexus`: CLI, MCP server (stdio), HTTP API, ingestion pipeline, LadybugDB graph, embeddings. | +| `gitnexus-web/` | Vite + React thin client: graph explorer + AI chat. All queries via `gitnexus serve` HTTP API. | +| `gitnexus-shared/` | Shared TypeScript types and constants (consumed by CLI and Web). | +| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Agent skills and plugin metadata. | +| `eval/` | Evaluation harnesses for benchmarking tool usage. | +| `.github/` | CI workflows + composite actions (`setup-gitnexus/`, `setup-gitnexus-web/`). | ## End-to-end flow: index → graph → tools -1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 15 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. +1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). The default DAG of 19 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. 2. **Persistence** — `repo-manager.ts` (paths, registry, LadybugDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches). @@ -28,53 +28,53 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## MCP tools -| Tool | Purpose | -|------|---------| -| `list_repos` | Discover indexed repos | -| `query` | Hybrid BM25 + vector search over the graph | -| `cypher` | Ad hoc Cypher against the schema | -| `context` | Callers, callees, processes for one symbol | -| `impact` | Blast radius (upstream/downstream) with risk summary | -| `detect_changes` | Map git diffs to affected symbols and processes | -| `rename` | Graph-assisted multi-file rename with `dry_run` preview | -| `api_impact` | Pre-change impact report for an API route handler | -| `trace` | Shortest directed path between two symbols (call + class-member edges); group-aware (`repo: "@"`) for cross-repo traces | -| `route_map` | API route → handler → consumer mappings | -| `tool_map` | MCP/RPC tool definitions and handlers | -| `shape_check` | Response shape vs consumer property access mismatches | -| `explain` | Persisted taint findings (source→sink data flows) — needs `analyze --pdg` | -| `pdg_query` | Control/data dependence — CDG (`mode: controls`) / REACHING_DEF (`mode: flows`) — needs `analyze --pdg` | -| `group_list` | List repo groups or details for one group | -| `group_sync` | Rebuild group Contract Registry (`contracts.json`) and bridge graph | +| Tool | Purpose | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `list_repos` | Discover indexed repos | +| `query` | Hybrid BM25 + vector search over the graph | +| `cypher` | Ad hoc Cypher against the schema | +| `context` | Callers, callees, processes for one symbol | +| `impact` | Blast radius (upstream/downstream) with risk summary | +| `detect_changes` | Map git diffs to affected symbols and processes | +| `rename` | Graph-assisted multi-file rename with `dry_run` preview | +| `api_impact` | Pre-change impact report for an API route handler | +| `trace` | Shortest directed path between two symbols (call + class-member edges); group-aware (`repo: "@"`) for cross-repo traces | +| `route_map` | API route → handler → consumer mappings | +| `tool_map` | MCP/RPC tool definitions and handlers | +| `shape_check` | Response shape vs consumer property access mismatches | +| `explain` | Persisted taint findings (source→sink data flows) — needs `analyze --pdg` | +| `pdg_query` | Control/data dependence — CDG (`mode: controls`) / REACHING_DEF (`mode: flows`) — needs `analyze --pdg` | +| `group_list` | List repo groups or details for one group | +| `group_sync` | Rebuild group Contract Registry (`contracts.json`) and bridge graph | `query`, `context`, and `impact` are group-aware: pass `repo: "@"` (or `"@/"` to scope to one member) plus optional `service: ""`. Group-mode `query` merges per-repo results via Reciprocal Rank Fusion; group-mode `impact` runs the local walk in the chosen member and fans out across boundaries via the Contract Bridge (`gitnexus/src/core/group/cross-impact.ts`). `trace` is also group-aware via `repo: "@"` — but, unlike the others, it resolves `from`/`to` across **all** members (a `@/` suffix is advisory for trace, not a scope); pass `from_uid`/`to_uid` to disambiguate a symbol name that occurs in more than one member. Group-mode `trace` (`gitnexus/src/core/group/cross-trace.ts`) stitches a path that crosses repositories: it resolves `from`/`to` across all members, and when they live in different repos it joins the home-repo segment to the target-repo segment over a single `ContractLink` boundary (an HTTP consumer→provider link, joined on `Contract.symbolUid`), reported as a `CONTRACT_LINK` hop in `crossings[]`. The crossing is clamped to one boundary (`MAX_SUPPORTED_CROSS_DEPTH`, shared with cross-impact); deeper `crossDepth` is reported via `notes[]`. With `pdg: true` (experimental, opt-in), each boundary-adjacent segment is enriched with its intra-procedural REACHING_DEF data-flow when that repo was indexed with `--pdg` (reusing the same anchored `flows` query as `pdg_query`); data flow never crosses the repo boundary, and a missing PDG layer degrades to call-level hops with a note. Two stores meet only at the `symbolUid` grain — the per-repo PDG/call graph and the group bridge — so this is the documented join; full cross-program (SDG-like) data flow across the boundary remains deferred (see `docs/plans/2026-06-18-002-feat-unified-pdg-impact-evaluation-plan.md`). The previously-planned `group_query`, `group_context`, `group_impact`, `group_contracts`, `group_status` MCP tools are intentionally not introduced — group-level state is exposed via resources instead: -| Resource URI | Purpose | -|--------------|---------| +| Resource URI | Purpose | +| ----------------------------------- | -------------------------------------------------------- | | `gitnexus://group/{name}/contracts` | Contract Registry (provider/consumer rows + cross-links) | -| `gitnexus://group/{name}/status` | Per-member index + Contract Registry staleness | +| `gitnexus://group/{name}/status` | Per-member index + Contract Registry staleness | ## Where to change what -| Concern | Start in | -|---------|----------| -| CLI commands/flags | `src/cli/` (`index.ts`, per-command modules) | -| Parsing/graph construction | `src/core/ingestion/pipeline-phases/` + `pipeline.ts` | -| Graph schema/DB | `src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`) | -| MCP tools/resources | `src/mcp/server.ts`, `tools.ts`, `resources.ts` | -| Cross-repo groups (sync, contracts, `@` routing) | `src/core/group/` (`service.ts`, `cross-impact.ts`, `sync.ts`, `bridge-db.ts`) | -| Search ranking | `src/core/search/` (BM25, hybrid fusion) | -| Embeddings | `src/core/embeddings/` + `src/core/run-analyze.ts` | -| Wiki generation | `src/core/wiki/` | -| Language support | `src/core/ingestion/languages/` + `tree-sitter-queries.ts` + `gitnexus-shared/src/languages.ts` | -| Import resolution | `src/core/ingestion/import-processor.ts` + `import-resolvers/configs/` + `model/resolution-context.ts` | -| Call resolution/inheritance/MRO | `src/core/ingestion/scope-resolution/` (pipeline, passes, graph-bridge) | -| Type extraction | `src/core/ingestion/type-extractors/` | -| Worker pool | `src/core/ingestion/workers/` | -| Web UI | `gitnexus-web/src/` | -| CI | `.github/workflows/*.yml`, `.github/actions/` | +| Concern | Start in | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| CLI commands/flags | `src/cli/` (`index.ts`, per-command modules) | +| Parsing/graph construction | `src/core/ingestion/pipeline-phases/` + `pipeline.ts` | +| Graph schema/DB | `src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`) | +| MCP tools/resources | `src/mcp/server.ts`, `tools.ts`, `resources.ts` | +| Cross-repo groups (sync, contracts, `@` routing) | `src/core/group/` (`service.ts`, `cross-impact.ts`, `sync.ts`, `bridge-db.ts`) | +| Search ranking | `src/core/search/` (BM25, hybrid fusion) | +| Embeddings | `src/core/embeddings/` + `src/core/run-analyze.ts` | +| Wiki generation | `src/core/wiki/` | +| Language support | `src/core/ingestion/languages/` + `tree-sitter-queries.ts` + `gitnexus-shared/src/languages.ts` | +| Import resolution | `src/core/ingestion/import-processor.ts` + `import-resolvers/configs/` + `model/resolution-context.ts` | +| Call resolution/inheritance/MRO | `src/core/ingestion/scope-resolution/` (pipeline, passes, graph-bridge) | +| Type extraction | `src/core/ingestion/type-extractors/` | +| Worker pool | `src/core/ingestion/workers/` | +| Web UI | `gitnexus-web/src/` | +| CI | `.github/workflows/*.yml`, `.github/actions/` | > Paths above are relative to `gitnexus/` unless they start with `gitnexus-web/` or `.github/`. @@ -82,30 +82,35 @@ Group-mode `trace` (`gitnexus/src/core/group/cross-trace.ts`) stitches a path th ## Pipeline Phase DAG -15 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. +19 default phases are defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. `--pdg` adds `taintSummaries` and `callSummaries` (21 total). ``` -scan → structure → [markdown, cobol] → parse → [routes, tools, orm] - → crossFile → scopeResolution → pruneLocalSymbols → mro → di → communities → processes +scan → structure → [springConfig, markdown, cobol] → parse → [routes, tools, orm] + → crossFile → scopeResolution → [springAutoConfiguration, springAop] + → pruneLocalSymbols → mro → springAopInheritance → di → communities → processes ``` -| Phase | File | Deps | Output | -|-------|------|------|--------| -| `scan` | `scan.ts` | (root) | File paths + sizes | -| `structure` | `structure.ts` | `scan` | File/Folder nodes, CONTAINS edges, `allPathSet` | -| `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx | -| `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) | -| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries | -| `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators) | -| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges | -| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) | -| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order | -| `scopeResolution` | `scope-resolution/pipeline/phase.ts` | `parse`, `crossFile`, `structure` | Binding/reference + inheritance edges; disposes BindingAccumulator | -| `pruneLocalSymbols` | `prune-local-symbols.ts` | `scopeResolution` | Drops inert block-local `Const`/`Variable`/`Static` nodes (only a `File→DEFINES` edge) post-resolution | -| `mro` | `mro.ts` | `crossFile`, `scopeResolution`, `pruneLocalSymbols`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | -| `di` | `di.ts` | `mro` | INJECTS edges from consumer Classes or factory Methods to provider Classes/declaration CodeElements (framework-neutral DI resolution; per-language matchers registered in `di-extractors/`) | -| `communities` | `communities.ts` | `mro`, `pruneLocalSymbols`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | -| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `pruneLocalSymbols`, `structure` | Process nodes + STEP_IN_PROCESS edges | +| Phase | File | Deps | Output | +| ------------------------- | -------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scan` | `scan.ts` | (root) | File paths + sizes | +| `structure` | `structure.ts` | `scan` | File/Folder nodes, CONTAINS edges, `allPathSet` | +| `springConfig` | `spring-config.ts` | `structure` | Spring configuration-property nodes and metadata | +| `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx | +| `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) | +| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries | +| `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators) | +| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges | +| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) | +| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order | +| `scopeResolution` | `scope-resolution/pipeline/phase.ts` | `parse`, `crossFile`, `structure` | Binding/reference + inheritance edges; disposes BindingAccumulator | +| `springAutoConfiguration` | `spring-auto-configuration.ts` | `structure`, `scopeResolution` | DECLARES and CONDITIONAL_ON metadata for Spring configuration candidates | +| `springAop` | `spring-aop.ts` | `scopeResolution` | Direct declarative/advice ADVISED_BY edges and pointcut evidence | +| `pruneLocalSymbols` | `prune-local-symbols.ts` | `scopeResolution` | Drops inert block-local `Const`/`Variable`/`Static` nodes (only a `File→DEFINES` edge) post-resolution | +| `mro` | `mro.ts` | `crossFile`, `scopeResolution`, `pruneLocalSymbols`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | +| `springAopInheritance` | `spring-aop.ts` | `springAop`, `mro` | Propagates declarative behavior through class/interface inheritance decisions | +| `di` | `di.ts` | `mro` | INJECTS edges from consumer Classes or factory Methods to provider Classes/declaration CodeElements (framework-neutral DI resolution; per-language matchers registered in `di-extractors/`) | +| `communities` | `communities.ts` | `mro`, `pruneLocalSymbols`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | +| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `pruneLocalSymbols`, `structure` | Process nodes + STEP_IN_PROCESS edges | **Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `types.ts`, `runner.ts`, `index.ts`. @@ -124,6 +129,7 @@ scan → structure → [markdown, cobol] → parse → [routes, tools, orm] 4. **Timing** — per-phase `durationMs` in `PhaseResult`, dev-mode console logging. **Design patterns:** + - **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output. - **Typed phase access** — `getPhaseOutput(deps, 'name')` for type-safe upstream results. - **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership. @@ -141,7 +147,9 @@ import type { PipelinePhase, PhaseResult } from './types.js'; import { getPhaseOutput } from './types.js'; import type { ParseOutput } from './parse.js'; -export interface MyPhaseOutput { /* ... */ } +export interface MyPhaseOutput { + /* ... */ +} export const myPhase: PipelinePhase = { name: 'myPhase', @@ -149,7 +157,9 @@ export const myPhase: PipelinePhase = { async execute(ctx, deps) { const { allPaths } = getPhaseOutput(deps, 'parse'); // ... write to ctx.graph ... - return { /* typed output */ }; + return { + /* typed output */ + }; }, }; ``` @@ -224,11 +234,19 @@ Property-key dispatch remains a separate conservative fallback. Its per-key fan- Standalone (regex-based) providers such as COBOL participate via `ScopeResolver.scopeResolutionEdgeMode: 'callable-flow-only'`: `runScopeResolution` runs for them, but every ordinary emission path — heritage, interface implementations, receiver-bound, free-call fallback, reference/import edges, post-resolution hooks — is gated off, so their legacy phase (e.g. `cobolPhase`) remains the sole owner of structural edges and the callable solver's `CALLS` are purely additive. A callable-flow-only provider whose files emitted no callable facts exits early, before finalize, keeping the opt-in proportional to source scanning. +### Receiver chains and the drop census (#2766) + +A compound receiver (`svc.getUser().address.save()`) is captured as a compact string on `ReferenceSite.receiverChain`. `utils/receiver-chain-codec.ts` is the ONE encoder/decoder — capture emitters, the scope-resolution fold, and the durable ParsedFile store all import it rather than hand-rolling the format. + +Wire format is **v2**: `2|||…`, one-character version prefix, then base-first steps, each a one-character kind sigil plus the member name (`c` = call, `f` = field). `a` (await) and `i` (index) are **name-free** and encode as a bare sigil — an awaited call's name already lives on its `c` step, and a subscript key is a value, not a lookup-able identifier. The version went 1 → 2 when those two kinds were added, and a decoder REFUSES a foreign version rather than decoding the prefix it understands: a chain missing its await/index hop decodes cleanly as a different, shorter chain and would type the receiver against the wrong member. The format is unescaped (`|` and `~` cannot occur in an identifier), so an unencodable name is refused rather than escaped, and the payload is capped at `MAX_RECEIVER_CHAIN_BYTES` / `MAX_CHAIN_DEPTH` steps. Because these strings live in the incremental parse cache and the durable ParsedFile store, a format change requires a `PARSE_CACHE_VERSION` schema bump — a stale cache would otherwise replay v1 chains this build discards. + +Receivers the resolver could not type are not silently dropped. Each records a `ResolutionOutcome` (`scope-resolution/resolution-outcome.ts`) carrying the receiver's *shape* (`classifyReceiverShape`: `chain-call` / `chain-field` / `chain-mixed` / `chain-unwrap` / `no-chain` — the bench censuses these) and its *origin* (`in-program` / `external` / `unknown`). `scope-resolution/unresolved-receivers.ts` aggregates them per member name into the index-persisted `unresolvedReceiverMembers` summary, keeping in-program and external counts under separate keys. Only in-program drops make a count short: an external-rooted call (`System.out.println`, `fetch(...)`) has no in-graph node an edge could have reached, so it is reported but does not hedge. `impact` / `context` read that summary and publish `epistemic: 'exact' | 'lower-bound'`, prose `boundaries`, and the machine-readable `causes` split (`EpistemicCauses` in `mcp/local/local-backend.ts`). + ### Optional CFG/PDG emission (`--pdg`, #2081–#2086) On a `--pdg` run the parse worker builds a per-function control-flow graph from the tree-sitter AST (`LanguageProvider.cfgVisitor`; TypeScript/JavaScript today) and serializes it onto `ParsedFile.cfgSideChannel` as plain data. Scope-resolution then emits the program-dependence layers from that side-channel **inside Phase 4 of `runScopeResolution`, while the disk-backed ParsedFile store is still live** — the only window where the worker-built CFGs are loaded (the store is cleared right after the phase returns). A standalone post-`mro` phase would read an empty store, so the emit deliberately lives in-phase, mirroring the `applyCaptureSideChannel` pattern. The opt-in is off by default (graph byte-identical), folded into the parse-cache key (a pdg-off warm cache is never reused on a `--pdg` run), and each layer is bounded by a per-function edge cap that logs any dropped edges. All layers are `BasicBlock → BasicBlock` edges in the single `CodeRelation` table, keyed by `type`; there is **no** `Function → BasicBlock` edge — the symbol↔block join is reconstructed from the BasicBlock id prefix + line span. The layers build on each other: -- **M1 — CFG** (#2081): `BasicBlock` nodes + `CFG` edges. Edge *kind* (`seq`/`cond-true`/`loop-back`/…) rides the `reason` column (CFG is one `CodeRelation` type, not one per kind). +- **M1 — CFG** (#2081): `BasicBlock` nodes + `CFG` edges. Edge _kind_ (`seq`/`cond-true`/`loop-back`/…) rides the `reason` column (CFG is one `CodeRelation` type, not one per kind). - **M2 — REACHING_DEF** (#2082): GEN/KILL def→use data dependence from a pure fixpoint solver; the variable name rides `reason`. - **M3/M4 — TAINTED / SANITIZES / TAINT_PATH** (#2083–#2084): intra- and inter-procedural taint (source→sink) — the `explain` tool's data. - **M5 — CDG** (#2085): Ferrante control dependence over a Cooper–Harvey–Kennedy post-dominator tree (the EXIT-rooted reverse CFG); branch sense (`'T'`/`'F'`) rides `reason`. A CFG whose EXIT is unreachable from some block is skipped for CDG (post-dominance would be unsound) while its CFG/REACHING_DEF layers are kept. @@ -241,25 +259,26 @@ See `core/ingestion/cfg/` (emit + the pure CFG / post-dominator / control-depend Single interface a language implements to plug into the pipeline. Contract fully documented in `scope-resolution/contract/scope-resolver.ts`. -| Hook | Purpose | -|------|---------| -| `languageProvider` | Base `LanguageProvider` (tree-sitter query, `emitScopeCaptures`, import/binding interpreters, hooks) | -| `populateOwners(parsed)` | Fill deferred `ownerId` fields on method defs (captures can't always know the owning class at parse time) | -| `buildMro(graph, parsed, nodeLookup)` | Produce `mroByClassDefId: Map` — C3, Ruby-mixin, or first-wins per language | -| `resolveImportTarget(target, fromFile, allFiles)` | `(rawImportPath, sourceFile) → targetFilePath` (PEP-328 for Python, etc.) | -| `isNamespaceImport(parsedImport, targetFile, fromFile)` | Optionally reclassify a resolved named import as a namespace handle when the imported symbol is itself a module | -| `mergeBindings(existing, incoming, scopeId)` | Shadowing / LEGB precedence | -| `arityCompatibility` | Provider consumed by registry during `MethodRegistry.lookup` Step 2 | -| `importEdgeReason` | Confidence-tier string for IMPORTS edge reason field | -| `propagatesReturnTypesAcrossImports?` | Opt out of cross-file return-type propagation (default on) | -| `fieldFallbackOnMethodLookup?` | Statically-typed languages turn this OFF — the heuristic over-connects (default on) | -| `unwrapCollectionAccessor?` | Property-style collection views (`data.Values` on Dictionary-like receivers) — default off | -| `collapseMemberCallsByCallerTarget?` | One CALLS edge per (caller, target) instead of per-site — default off | -| `populateNamespaceSiblings?` | Cross-file implicit visibility (compiler-implicit namespace sharing) — default off; ctx carries `treeCache` | -| `hoistTypeBindingsToModule?` | Walk up to Module scope when looking up a method's return-type typeBinding — default off; enable only when bindings are stored at module level | -| `hasFileLocalCallableLinkage?` | Precise internal-linkage predicate used only when joining callable declarations/prototypes to cross-file definitions; C/C++ use it for `static` free functions | -| `constructorCallTargetsClass?` | A constructor-form call `Type(...)` links to the Class def rather than its explicit Constructor def — default off; Swift and Dart opt in | -| `constructionSyntax?` | How the language spells construction, so an INLINE constructor receiver (`Service(db).m()`, `new Service(db).m()`, `Service.new.m()`) can be typed — `bare` / `keyword` / `selector`; default off, opt in per language only where measured to be needed (#2708) | +| Hook | Purpose | +| ------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `languageProvider` | Base `LanguageProvider` (tree-sitter query, `emitScopeCaptures`, import/binding interpreters, hooks) | +| `populateOwners(parsed)` | Fill deferred `ownerId` fields on method defs (captures can't always know the owning class at parse time) | +| `buildMro(graph, parsed, nodeLookup)` | Produce `mroByClassDefId: Map` — C3, Ruby-mixin, or first-wins per language | +| `resolveImportTarget(target, fromFile, allFiles)` | `(rawImportPath, sourceFile) → targetFilePath` (PEP-328 for Python, etc.) | +| `isNamespaceImport(parsedImport, targetFile, fromFile)` | Optionally reclassify a resolved named import as a namespace handle when the imported symbol is itself a module | +| `mergeBindings(existing, incoming, scopeId)` | Shadowing / LEGB precedence | +| `arityCompatibility` | Provider consumed by registry during `MethodRegistry.lookup` Step 2 | +| `importEdgeReason` | Confidence-tier string for IMPORTS edge reason field | +| `propagatesReturnTypesAcrossImports?` | Opt out of cross-file return-type propagation (default on) | +| `fieldFallbackOnMethodLookup?` | Statically-typed languages turn this OFF — the heuristic over-connects (default on) | +| `elementTypeOf?` | `(containerType, via: {kind:'index'} \| {kind:'accessor',name}) → elementType \| undefined` — element type of a container, reached by subscript (`repos[0]`) or by a property-style collection view (`data.Values`). ONE hook for both routes (it replaced the split `unwrapCollectionAccessor` / `unwrapCollectionElement`, where implementing one silently answered nothing for the other). Consulted only where the source actually performed the access — never as a general type-name normalizer | +| `stripTypePreservingDecoration?` | `(typeName) → strippedName \| undefined` — strip ONE layer of TYPE-PRESERVING decoration (pointer, reference, `const`, nullable, borrow, sigil) so a receiver declared `*Host` still finds the `Host` binding (#2766). Never a container: unwrapping `Repo[]` here would fold `repos.find(x)` to `Repo.find` — that is `elementTypeOf`'s job, and only after a real subscript. Consulted only after every undecorated lookup fails, and only by receiver-chain base/step resolution — default off | +| `collapseMemberCallsByCallerTarget?` | One CALLS edge per (caller, target) instead of per-site — default off | +| `populateNamespaceSiblings?` | Cross-file implicit visibility (compiler-implicit namespace sharing) — default off; ctx carries `treeCache` | +| `hoistTypeBindingsToModule?` | Walk up to Module scope when looking up a method's return-type typeBinding — default off; enable only when bindings are stored at module level | +| `hasFileLocalCallableLinkage?` | Precise internal-linkage predicate used only when joining callable declarations/prototypes to cross-file definitions; C/C++ use it for `static` free functions | +| `constructorCallTargetsClass?` | A constructor-form call `Type(...)` links to the Class def rather than its explicit Constructor def — default off; Swift and Dart opt in | +| `constructionSyntax?` | How the language spells construction, so an INLINE constructor receiver (`Service(db).m()`, `new Service(db).m()`, `Service.new.m()`) can be typed — `bare` / `keyword` / `selector`; default off, opt in per language only where measured to be needed (#2708) | ### Per-language registration @@ -270,21 +289,21 @@ CI auto-discovers the set via `tsx`. No workflow edit required. ### Code references -| Module | Purpose | -|--------|---------| -| `scope-resolution/contract/scope-resolver.ts` | `ScopeResolver` interface + shared types | -| `scope-resolution/pipeline/run.ts` | Generic orchestrator | -| `scope-resolution/pipeline/phase.ts` | Pipeline-phase wrapper (deps: `parse`, `structure`) | -| `scope-resolution/pipeline/registry.ts` | `SCOPE_RESOLVERS` map | -| `scope-resolution/passes/*.ts` | Reference-resolution passes (receiver-bound, free-call fallback, compound-receiver, MRO, cross-file return-type propagation) | -| `scope-resolution/graph-bridge/*.ts` | CLI-local translation from resolved references → `KnowledgeGraph` edges | -| `scope-resolution/scope/*.ts` | Generic scope-chain walkers + namespace targets | -| `scope-resolution/workspace-index.ts` | Build-once O(1) lookup index | -| `languages/python/index.ts` | Python `ScopeResolver` hooks + known-limitation docs | -| `languages/python/captures.ts` | `emitPythonScopeCaptures` (honors cross-phase Tree cache) | -| `languages/csharp/index.ts` | C# `ScopeResolver` hooks + known-limitation docs | -| `languages/csharp/captures.ts` | `emitCsharpScopeCaptures` (honors cross-phase Tree cache) | -| `languages/csharp/namespace-siblings.ts` | Cross-file implicit-namespace visibility hook (reads `treeCache`) | +| Module | Purpose | +| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `scope-resolution/contract/scope-resolver.ts` | `ScopeResolver` interface + shared types | +| `scope-resolution/pipeline/run.ts` | Generic orchestrator | +| `scope-resolution/pipeline/phase.ts` | Pipeline-phase wrapper (deps: `parse`, `structure`) | +| `scope-resolution/pipeline/registry.ts` | `SCOPE_RESOLVERS` map | +| `scope-resolution/passes/*.ts` | Reference-resolution passes (receiver-bound, free-call fallback, compound-receiver, MRO, cross-file return-type propagation) | +| `scope-resolution/graph-bridge/*.ts` | CLI-local translation from resolved references → `KnowledgeGraph` edges | +| `scope-resolution/scope/*.ts` | Generic scope-chain walkers + namespace targets | +| `scope-resolution/workspace-index.ts` | Build-once O(1) lookup index | +| `languages/python/index.ts` | Python `ScopeResolver` hooks + known-limitation docs | +| `languages/python/captures.ts` | `emitPythonScopeCaptures` (honors cross-phase Tree cache) | +| `languages/csharp/index.ts` | C# `ScopeResolver` hooks + known-limitation docs | +| `languages/csharp/captures.ts` | `emitCsharpScopeCaptures` (honors cross-phase Tree cache) | +| `languages/csharp/namespace-siblings.ts` | Cross-file implicit-namespace visibility hook (reads `treeCache`) | ### Performance notes @@ -314,15 +333,15 @@ CI auto-discovers the set via `tsx`. No workflow edit required. Each language implements `LanguageProvider` (`language-provider.ts`). Key fields: -| Field | Purpose | -|-------|---------| -| `id`, `extensions` | Language identity and file matching | -| `treeSitterQueries` | S-expression queries for AST extraction | -| `importSemantics` | `named` / `wildcard-leaf` / `wildcard-transitive` / `namespace` | -| `importResolver` | Language-specific path → file resolution | -| `exportChecker` | Public/exported symbol detection | -| `typeConfig` | Type annotation extraction rules | -| `mroStrategy` | `first-wins` / `c3` / `none` | +| Field | Purpose | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id`, `extensions` | Language identity and file matching | +| `treeSitterQueries` | S-expression queries for AST extraction | +| `importSemantics` | `named` / `wildcard-leaf` / `wildcard-transitive` / `namespace` | +| `importResolver` | Language-specific path → file resolution | +| `exportChecker` | Public/exported symbol detection | +| `typeConfig` | Type annotation extraction rules | +| `mroStrategy` | `first-wins` / `c3` / `none` | | `descriptionExtractor` | Optional hook returning a symbol's doc-comment text as its `description`; feeds the embedding metadata header so doc-only terms are semantically searchable (issue #2270). Most languages register `createLeadingDocDescriptionExtractor` (shared, language-neutral; per-language comment/wrapper config passed at the call site) | 16 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. @@ -337,22 +356,23 @@ Per-language import resolution uses the **configs + factory** pattern (like call Unified 3-tier algorithm (`model/resolution-context.ts`), per-language `importSemantics` controls which tier activates: -| Tier | Confidence | Mechanism | -|------|-----------|-----------| -| 1 — same-file | 0.95 | Symbol table for caller's file | -| 2 — import-scoped | 0.9 | `NamedImportMap` chains (named) or all files in `importMap` (wildcard) | -| 3 — global | 0.5 | O(1) index lookups: class, impl, callable. Fallback only | +| Tier | Confidence | Mechanism | +| ----------------- | ---------- | ---------------------------------------------------------------------- | +| 1 — same-file | 0.95 | Symbol table for caller's file | +| 2 — import-scoped | 0.9 | `NamedImportMap` chains (named) or all files in `importMap` (wildcard) | +| 3 — global | 0.5 | O(1) index lookups: class, impl, callable. Fallback only | -| Import strategy | Languages | Behavior | -|----------------|-----------|----------| -| `named` | TS, JS, Java, C#, Rust, PHP, Kotlin | Only explicitly imported names visible | -| `wildcard-leaf` | Go, Ruby, Swift, Dart | Whole-package import, no transitive re-exports | -| `wildcard-transitive` | C, C++ | `#include` closure chains through re-exports | -| `namespace` | Python | Module aliases resolved at call site | +| Import strategy | Languages | Behavior | +| --------------------- | ----------------------------------- | ---------------------------------------------- | +| `named` | TS, JS, Java, C#, Rust, PHP, Kotlin | Only explicitly imported names visible | +| `wildcard-leaf` | Go, Ruby, Swift, Dart | Whole-package import, no transitive re-exports | +| `wildcard-transitive` | C, C++ | `#include` closure chains through re-exports | +| `namespace` | Python | Module aliases resolved at call site | ### Chunked parse-and-resolve `parse` processes files in ~20 MB byte-budget chunks to bound memory. Per chunk: + 1. Worker pool dispatches files (the sole parse path — there is no sequential fallback; `skipWorkers`, `--workers 0`, and `GITNEXUS_WORKER_POOL_SIZE=0` are rejected with an actionable error) 2. Each worker: detect language → load grammar → run queries → return unified `ParseWorkerResult` 3. Synthesize wildcard bindings (`wildcard-synthesis.ts`) @@ -363,11 +383,12 @@ Inheritance edges are emitted later, by the scope-resolution phase (`preEmitInhe Workers: `workers/worker-pool.ts`, `workers/parse-worker.ts`. -**Worker-serialized ParsedFiles (#2038).** To index very large repos (e.g. the Linux kernel) without OOM, the worker pool is the *sole* parse path and workers serialize each file's `ParsedFile` (plus its capture side-channel) in parallel, streaming them to scope-resolution through a disk-backed store. Scope-resolution consumes the pre-extracted artifact instead of re-parsing every file on the main thread — tree-sitter's native input buffers are not GC-reclaimable, so the former main-thread re-parse leaked native memory until the process died. Pool creation is lazy / cache-miss-gated, so a warm all-cache-hit run replays cached worker output without spawning a worker (hence `usedWorkerPool` can be false even when the repo has parseable files). +**Worker-serialized ParsedFiles (#2038).** To index very large repos (e.g. the Linux kernel) without OOM, the worker pool is the _sole_ parse path and workers serialize each file's `ParsedFile` (plus its capture side-channel) in parallel, streaming them to scope-resolution through a disk-backed store. Scope-resolution consumes the pre-extracted artifact instead of re-parsing every file on the main thread — tree-sitter's native input buffers are not GC-reclaimable, so the former main-thread re-parse leaked native memory until the process died. Pool creation is lazy / cache-miss-gated, so a warm all-cache-hit run replays cached worker output without spawning a worker (hence `usedWorkerPool` can be false even when the repo has parseable files). ### Inheritance and MRO Inheritance is captured by the `@reference.inherits` tag and emitted by the scope-resolution phase: `preEmitInheritanceEdges` resolves each base in scope, then `emitHeritageEdges` writes the `EXTENDS`/`IMPLEMENTS` edges. The phase then computes method resolution order via each `ScopeResolver`'s `buildMro` hook, feeding a `MethodDispatchIndex` used for owner-scoped lookups. Per-language strategy: + - **`first-wins`** — Java, C#, C++, TS, Ruby, Go - **`c3`** — Python (C3 linearization) - **`ruby-mixin`** — Ruby (mixin-aware linearization) @@ -419,7 +440,7 @@ Defined in `lbug/schema.ts`. Separate node tables per type, single `CodeRelation **Node tables:** File, Folder, Function, Class, Interface, Method, Constructor, CodeElement, Struct, Enum, Macro, Typedef, Union, Namespace, Trait, Impl, TypeAlias, Const, Static, Property, Record, Delegate, Annotation, Template, Module, Community, Process, Route, Tool, Section, Embedding. -**Relation types** (`CodeRelation.type`): CONTAINS, DEFINES, CALLS, IMPORTS, EXTENDS, IMPLEMENTS, HAS_METHOD, HAS_PROPERTY, ACCESSES, METHOD_OVERRIDES, METHOD_IMPLEMENTS, MEMBER_OF, STEP_IN_PROCESS, HANDLES_ROUTE, FETCHES, HANDLES_TOOL, ENTRY_POINT_OF. +**Relation types** (`CodeRelation.type`): CONTAINS, DEFINES, CALLS, IMPORTS, INHERITS, EXTENDS, IMPLEMENTS, USES, DECORATES, HAS_METHOD, HAS_PROPERTY, ACCESSES, METHOD_OVERRIDES, METHOD_IMPLEMENTS, MEMBER_OF, STEP_IN_PROCESS, HANDLES_ROUTE, FETCHES, HANDLES_TOOL, ENTRY_POINT_OF, WRAPS, QUERIES, INJECTS, CONDITIONAL_ON, DECLARES, ADVISED_BY, BINDS_EVENT_HANDLER, EMITS_EVENT. **Optional `--pdg` additions** (off by default, opt-in via `gitnexus analyze --pdg`; see _Optional CFG/PDG emission_ above): a `BasicBlock` node table, plus the PDG relation types `CFG`, `REACHING_DEF`, `CDG`, `TAINTED`, `SANITIZES`, and `TAINT_PATH` on the same `CodeRelation` table. These are deliberately kept out of the default `VALID_RELATION_TYPES` / web graph schema — query them via `cypher`, `explain`, or `pdg_query`. @@ -447,12 +468,12 @@ Node IDs use arity suffix (`#`): `Method:file:Class.method#1` vs `#2 **METHOD_IMPLEMENTS confidence tiering:** -| Match quality | Confidence | -|---|---| -| Exact parameter types match | 1.0 | -| Arity match, types unavailable | 1.0 | -| Variadic vs fixed | 0.7 | -| Insufficient info | 0.7 | +| Match quality | Confidence | +| ------------------------------ | ---------- | +| Exact parameter types match | 1.0 | +| Arity match, types unavailable | 1.0 | +| Variadic vs fixed | 0.7 | +| Insufficient info | 0.7 | ## Related docs diff --git a/CLAUDE.md b/CLAUDE.md index cede8bee9..69c77a423 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,30 +62,31 @@ See the ` … ` block in **[AGENTS.m # GitNexus — Code Intelligence -This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **GitNexus** (248612 symbols, 565510 relationships, 918 execution flows). Use GitNexus graph tools to understand code, assess impact, and navigate safely. > Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939). ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: `detect_changes({scope: "compare", base_ref: "main"})`. +- **MUST run impact analysis before editing.** Use `impact({target: "symbolName", direction: "upstream"})` (MCP) or `node .gitnexus/run.cjs impact "symbolName" --direction upstream --repo .` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis. For unified PDG impact, add `mode: "pdg"` with optional `line: ` — it returns statement-level `affectedStatements` over CDG + REACHING_DEF and inter-procedural symbols in `interproceduralByDepth`/`byDepth`; no-layer/degraded PDG results are UNKNOWN-risk notes (`--pdg` layer). CLI equivalent: `node .gitnexus/run.cjs impact "symbolName" --direction upstream --mode pdg --line --repo .`. +- **MUST analyze graph changes before committing.** Use `detect_changes({scope: "all"})` (MCP) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` (CLI fallback). For regression review: `detect_changes({scope: "compare", base_ref: "main"})` or `node .gitnexus/run.cjs detect-changes --scope compare --base-ref "main" --repo .`. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. - When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. - When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. - For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). +- For control/data dependence, `pdg_query({mode: "controls", target: "fileOrSymbol"})` answers "under what condition does X run?" (CDG, incl. guard clauses) and `pdg_query({mode: "flows", target, variable})` traces "where does variable Y flow?" (REACHING_DEF). `--pdg` layer. ## Never Do -- NEVER edit a function, class, or method without first running `impact` on it. +- NEVER edit a function, class, or method before MCP/CLI impact analysis. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. - NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. -- NEVER commit changes without running `detect_changes()` to check affected scope. +- NEVER commit before MCP/CLI graph change analysis. ## Resources | Resource | Use for | -|----------|---------| +| --- | --- | | `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | | `gitnexus://repo/GitNexus/clusters` | All functional areas | | `gitnexus://repo/GitNexus/processes` | All execution flows | @@ -94,7 +95,7 @@ This project is indexed by GitNexus as **GitNexus** (20319 symbols, 54304 relati ## CLI | Task | Read this skill file | -|------|---------------------| +| --- | --- | | Understand architecture / "How does X work?" | `.claude/skills/gitnexus-exploring/SKILL.md` | | Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus-impact-analysis/SKILL.md` | | Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus-debugging/SKILL.md` | diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 9005219fd..3fe36a875 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -44,7 +44,13 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Trigger:** Semantic search quality drops; `stats.embeddings` in the index metadata (`gitnexus.json` / legacy `meta.json`) is 0 after refresh. - **Do:** Re-run `npx gitnexus analyze --embeddings` to regenerate. Check the analyze log for a `Warning: could not load cached embeddings` line — if present, the cache restore failed (corrupt DB / schema mismatch) and the rebuild had nothing to preserve. If you intentionally passed `--drop-embeddings`, this is expected. -- **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; the only ways to end up at zero are an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. +- **Why:** Plain `analyze` preserves prior vectors by re-inserting them after the rebuild; ways to end up at zero include an explicit `--drop-embeddings`, a cache-load failure (now logged), or a model/dimension change that invalidates the cache — but zero is no longer the only embedding-loss signature to watch for; see the Sign below for the non-zero, partial-failure case. A dirty-recovery run that cannot move the crashed WAL aside now either discards it (logged: forensics lost, embeddings still preserved) or fails fast with a lock error naming the holder — it never silently zeroes embeddings. + +### Analyze finishes but embeddings are incomplete (partial embedding index) + +- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["embedding-checkpoint-pending"]` (or the human-readable "Index incomplete reasons" line); `stats.embeddings` is honest and **non-zero**, and the preceding analyze log showed a `Warning: N node(s) lost their embeddings to embedding-endpoint failures` line (#2790). +- **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. +- **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal. ### MCP lists no repos diff --git a/MIGRATION.md b/MIGRATION.md index ccc94fc35..9c6c1de2d 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -117,3 +117,122 @@ repo as never analyzed. The `meta.json` mirror will remain until a future major version. Removal will be announced in this file and in the changelog before it happens. + +## Ambiguous responses report the true match count (PR #2796, issue #2787) + +The MCP symbol resolver returns at most 20 candidate rows. Every ambiguous +response used to take its count from that capped window, so a name with 92 +matches (`constructor`, in this repo's own index) reported 20. The same PR +pinned the window with an `ORDER BY`, which turned that undercount from +flaky into stable — and a stable wrong number reads as authoritative. + +Three consumer-visible changes follow: + +- **`impact`'s `totalCandidates` changed meaning.** It was the length of the + capped 20-row window; it is now the true `COUNT(*)` of matching symbols. + Callers using `totalCandidates === candidates.length` as a "not truncated" + proxy will now see the two diverge. This is a bug fix — the old number was + wrong — but it is still a value change on a published field. +- **`totalCandidates` and `candidatesTruncated` are new on other tools.** + They now also appear on `context`, `trace`, the `explain` / `pdg_query` + block-anchor path, and on `rename` (which returns `context`'s ambiguous + payload verbatim). `candidatesTruncated: true` is present only when + `candidates[]` is shorter than `totalCandidates` — absent otherwise, never + `false`. +- **The `message` template gained a `(showing M)` suffix.** It follows the + total — `Found 92 symbols matching 'constructor' (showing 20). …` — and + appears only when the returned window is smaller than the total. `impact` + uses the longer `(showing M of N)` form. + +### Do I need to migrate? + +**Only if you read `totalCandidates` or parse `message`.** The last two +changes are purely additive — no field was removed or renamed and +`candidates[]` keeps its shape — so PR #888's "no existing field has changed. +No migration required for `context` callers" still holds for `context`. + +- Reading `totalCandidates` on `impact`: it is a true total now. Detect a + shortened window with `candidatesTruncated` (or `totalCandidates > + candidates.length`) rather than by comparing it to an array length. +- Parsing `message` for a count: the total is still the first number, but a + `(showing M)` parenthetical may now follow it. Prefer the structured + `totalCandidates` field over the string. + +### What happens on re-index? + +Nothing — this is an MCP-surface change only. The graph schema, indexer, +and stored data are untouched. + +## `schemaVersion` → `schemaFingerprint` (issue #2798) + +The field that decides whether an existing index can be reused changed in +`.gitnexus/gitnexus.json` (and in each `branches//gitnexus.json`): +`schemaVersion?: number` has been removed and `schemaFingerprint?: string` +added. The new value is a 12-character digest of the graph DDL this build +creates, so it *describes* the schema an index's tables were actually built +from rather than asserting a number about it. + +An absent fingerprint is treated as a mismatch, and that is the whole +backward-compatibility story: every index written by an earlier GitNexus +carries no fingerprint, so it is rebuilt exactly once. + +### Do I need to migrate? + +**No.** There is nothing to run, edit, or pass. The first `analyze` after +upgrading logs one line — + +``` +index schema changed (built by an unidentified GitNexus build, this build is ); forcing a full re-analyze so the database is recreated from the current schema. +``` + +— and then performs that full re-analyze itself. The same run stamps the +fingerprint, and every run after it takes the normal incremental path again. + +### What happens on re-index? + +One automatic full re-analyze, once per index. Nothing else changes; the +resulting graph is what the current build would have produced anyway. + +The scope of that one-time cost is worth knowing before you hit it. It is +per **index**, not per machine or per repository — branch-scoped index slots +(#2106) each keep their own `gitnexus.json`, so every slot pays for itself +the first time it is analyzed after the upgrade. On a very large repository +a full re-analyze is substantial, not a blip; plan the first post-upgrade +run accordingly. + +### Why a digest instead of a version number? + +`schemaVersion` was hand-incremented, and it had to predict something a +number cannot know: whether the DDL an on-disk database was created from +matches this build's. It collided with `main` eight times, twice *exactly* — +and an exact clash was the quiet failure. Two builds stamp the same number +over different DDL, the strict `===` reuse gate reads the index as current, +the `CREATE … TABLE` statements are skipped as "already exists", and edges +whose endpoint pair the live database cannot persist are dropped. A wrong +graph, with no error anywhere. + +A derived digest cannot fail that way: two builds agree exactly when their +DDL agrees, so concurrent branches never need renumbering and a mismatch is +always a real mismatch. The retired ladder's per-version rationale (v2 +`BasicBlock.callees` through v35's generated relation cross-product) now +lives only in git history: +`git show 561f913a3:gitnexus/src/storage/repo-manager.ts`. + +### What about rollback? + +Downgrading to an older GitNexus is safe. The older binary looks for +`schemaVersion`, does not find one, treats the index as pre-versioning, and +forces its own full rebuild — the same one-time cost in the other direction, +never a stale or mismatched graph. + +### What if I alternate between an old and a new binary? + +Every switch forces a rebuild. The end-of-run metadata is written as a fresh +object literal rather than merged over the previous file, so a new build's +write drops `schemaVersion` and an old build's write drops +`schemaFingerprint` — neither field survives the other's run, and each binary +then finds its own gate unsatisfied. This hits anyone running a pinned +`npx gitnexus@` alongside a local build, or an editor hook still on +an older release. It is a cost, not a correctness problem: each run rebuilds +against its own schema, and the graph it serves is correct for the binary +that produced it. Pin one version per index to avoid the churn. diff --git a/RUNBOOK.md b/RUNBOOK.md index cec107401..e34d20c2b 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -56,7 +56,15 @@ npx gitnexus list npx gitnexus analyze --embeddings ``` -**Important:** If you already had embeddings, **always** pass `--embeddings` on later analyzes, or they can be dropped. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none). +**Important:** If you already had embeddings, a plain `npx gitnexus analyze` **preserves** them (Non-negotiable 5 in [GUARDRAILS.md](GUARDRAILS.md)) — pass `--embeddings` when you also want vectors generated for new or changed nodes, and `--drop-embeddings` only for a deliberate wipe. See `stats.embeddings` in `.gitnexus/gitnexus.json` (or its legacy `meta.json` mirror; 0 means none) — but that figure isn't always freshly measured: if a run's embedding-count query can't answer, it carries the previous run's number forward instead of writing a wrong zero. For a certified read, check `capabilities.vectorSearch.status` instead — it reads `unavailable` (never a stale count) whenever GitNexus can't vouch for the live vector index. + +**Partial embedding index (analyze exits 0, but some nodes never got embedded):** A long run against a flaky embedding endpoint can finish successfully while a bounded number of sub-batches still fail. Affected nodes are dropped to zero rows (never left half-written) and recorded as a pending `embeddingCheckpoint`; `npx gitnexus status` then reports `incompleteReasons: ["embedding-checkpoint-pending"]`. Recovery is a plain: + +```bash +npx gitnexus analyze +``` + +No `--embeddings` flag needed — a retained checkpoint forces embedding generation for the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. **Large repos:** Analyze may skip or limit embedding work when node counts are very high; watch CLI output. diff --git a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md index 45eb7ce87..0b81795de 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md @@ -17,22 +17,23 @@ description: "Use when the user wants to know what will break if they change som ## Workflow ``` -1. impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) or `node .gitnexus/run.cjs impact "X" --direction upstream --repo .` 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. detect_changes() → Map current git changes to affected flows +3. detect_changes({scope: "all"}) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` 4. Assess risk and report to user ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If `.gitnexus/run.cjs` is missing, replace `node .gitnexus/run.cjs` with `npx gitnexus` in the fallback commands. ## Checklist ``` -- [ ] impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) or CLI fallback to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] detect_changes() for pre-commit check +- [ ] detect_changes({scope: "all"}) or CLI fallback for pre-commit check - [ ] Assess risk level and report to user ``` @@ -55,7 +56,7 @@ description: "Use when the user wants to know what will break if they change som ## Tools -**impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: ``` impact({ @@ -73,10 +74,10 @@ impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis. If MCP is unavailable, use `node .gitnexus/run.cjs detect-changes --scope all --repo .` instead: ``` -detect_changes({scope: "staged"}) +detect_changes({scope: "all"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -86,7 +87,7 @@ detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) or `node .gitnexus/run.cjs impact "validateUser" --direction upstream --repo .` → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/gitnexus-claude-plugin/skills/gitnexus-review/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-review/SKILL.md index 9eabc426c..7b47cc3f6 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-review/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-review/SKILL.md @@ -120,10 +120,17 @@ and do not claim a complete graph-backed review. review surface: when the diff changes what gets emitted or persisted, verify every schema/version constant gating caches, incremental writebacks, and fingerprint baselines was bumped or regenerated — in - GitNexus itself, for example: `INCREMENTAL_SCHEMA_VERSION` (the - incremental write set covers only changed files, so new cross-file edges - never reach an existing index without the bump), the parse-store - `SCHEMA_BUMP`, and both bench fingerprint sets. + GitNexus itself, for example: graph DDL needs no manual bump, because + `SCHEMA_FINGERPRINT` (`gitnexus/src/core/lbug/schema.ts`) is derived + from `NODE_SCHEMA_QUERIES` + `REL_SCHEMA_QUERIES` and moves on its own; + the check there is whether the diff changed any string in those arrays, + and, if it added a new DDL array, whether that array was folded into the + fingerprint. The hand-maintained ritual still applies where no + declarative artifact describes the invalidated set: the parse-store + `SCHEMA_BUMP` and both bench fingerprint sets still need an explicit + bump, re-checked against the base branch right before merge. Semantic + changes that leave the DDL untouched are outside the fingerprint; they + rely on the analyzer runner-identity receipt in the index metadata. ## Expert lenses diff --git a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md index 139b897e4..8f9f1d1e7 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md @@ -16,22 +16,23 @@ description: Analyze blast radius before making code changes ## Workflow ``` -1. impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) or `node .gitnexus/run.cjs impact "X" --direction upstream --repo .` 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. detect_changes() → Map current git changes to affected flows +3. detect_changes({scope: "all"}) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` 4. Assess risk and report to user ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If `.gitnexus/run.cjs` is missing, replace `node .gitnexus/run.cjs` with `npx gitnexus` in the fallback commands. ## Checklist ``` -- [ ] impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) or CLI fallback to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] detect_changes() for pre-commit check +- [ ] detect_changes({scope: "all"}) or CLI fallback for pre-commit check - [ ] Assess risk level and report to user ``` @@ -54,7 +55,7 @@ description: Analyze blast radius before making code changes ## Tools -**impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: ``` impact({ target: "validateUser", @@ -71,9 +72,9 @@ impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis. If MCP is unavailable, use `node .gitnexus/run.cjs detect-changes --scope all --repo .` instead: ``` -detect_changes({scope: "staged"}) +detect_changes({scope: "all"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -83,7 +84,7 @@ detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) or `node .gitnexus/run.cjs impact "validateUser" --direction upstream --repo .` → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/gitnexus-cursor-integration/skills/gitnexus-review/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-review/SKILL.md index 9eabc426c..7b47cc3f6 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-review/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-review/SKILL.md @@ -120,10 +120,17 @@ and do not claim a complete graph-backed review. review surface: when the diff changes what gets emitted or persisted, verify every schema/version constant gating caches, incremental writebacks, and fingerprint baselines was bumped or regenerated — in - GitNexus itself, for example: `INCREMENTAL_SCHEMA_VERSION` (the - incremental write set covers only changed files, so new cross-file edges - never reach an existing index without the bump), the parse-store - `SCHEMA_BUMP`, and both bench fingerprint sets. + GitNexus itself, for example: graph DDL needs no manual bump, because + `SCHEMA_FINGERPRINT` (`gitnexus/src/core/lbug/schema.ts`) is derived + from `NODE_SCHEMA_QUERIES` + `REL_SCHEMA_QUERIES` and moves on its own; + the check there is whether the diff changed any string in those arrays, + and, if it added a new DDL array, whether that array was folded into the + fingerprint. The hand-maintained ritual still applies where no + declarative artifact describes the invalidated set: the parse-store + `SCHEMA_BUMP` and both bench fingerprint sets still need an explicit + bump, re-checked against the base branch right before merge. Semantic + changes that leave the DDL untouched are outside the fingerprint; they + rely on the analyzer runner-identity receipt in the index metadata. ## Expert lenses diff --git a/gitnexus-shared/src/graph/types.ts b/gitnexus-shared/src/graph/types.ts index cf6d7252b..d9d916e0d 100644 --- a/gitnexus-shared/src/graph/types.ts +++ b/gitnexus-shared/src/graph/types.ts @@ -153,6 +153,12 @@ export type RelationshipType = * semantics belong in `reason` so the relationship can be reused by other * metadata-driven systems. */ | 'DECLARES' + /** Framework advice relationship. Source = the class-like/Method whose behavior + * is intercepted; target = either the concrete advice Method or a synthetic + * CodeElement describing a declarative interceptor (transaction, cache, or + * method security). Runtime activation remains explicitly unknown in the + * relationship reason; this edge records statically visible advice only. */ + | 'ADVISED_BY' /** Vue component event system: a handler function in a parent component is * bound to an event emitted by a child component (`@event="handlerFn"`). * Source = handler Function/Method node in the parent. diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index 0ac74b1ed..284e94268 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -190,6 +190,7 @@ export { ResilientFetchExhaustedError, RETRY_AFTER_CAP_MS, parseRetryAfter, + isTerminalNetworkError, } from './integrations/resilient-fetch.js'; export type { ResilientFetchOptions } from './integrations/resilient-fetch.js'; diff --git a/gitnexus-shared/src/integrations/resilient-fetch.ts b/gitnexus-shared/src/integrations/resilient-fetch.ts index 828dea731..e126744f8 100644 --- a/gitnexus-shared/src/integrations/resilient-fetch.ts +++ b/gitnexus-shared/src/integrations/resilient-fetch.ts @@ -81,6 +81,25 @@ type Outcome = | { kind: 'terminal-network'; err: unknown } // TimeoutError or AbortError: no retry, breaker neutral | { kind: 'retryable-network'; err: unknown }; // DNS, ECONNRESET, etc. +/** + * The network errors `resilientFetch` treats as terminal — never retried, and + * routed through the breaker's neutral path. + * + * Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`) and + * caller-driven aborts (`AbortController.abort()` → `AbortError`) qualify: + * retrying against an already-aborted signal would fail again immediately, and + * neither outcome reflects backend health. + * + * Exported because callers that hook into the retry loop (a `fetchImpl` that + * inspects or re-wraps its own throws) have to agree with {@link + * classifyOutcome} about which errors are terminal. Sharing this predicate is + * what makes that agreement structural instead of a hand-copied condition that + * can drift. + */ +export function isTerminalNetworkError(err: unknown): err is DOMException { + return err instanceof DOMException && (err.name === 'TimeoutError' || err.name === 'AbortError'); +} + /** Exported for unit tests. */ export function classifyOutcome( result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response }, @@ -88,15 +107,7 @@ export function classifyOutcome( retryAfterCapMs = RETRY_AFTER_CAP_MS, ): Outcome { if (result.kind === 'error') { - // Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`) - // and caller-driven aborts (`AbortController.abort()` → `AbortError`) - // are terminal: retrying against an already-aborted signal would - // fail again immediately, and neither outcome reflects backend - // health. They route through the breaker's neutral path. - if ( - result.err instanceof DOMException && - (result.err.name === 'TimeoutError' || result.err.name === 'AbortError') - ) { + if (isTerminalNetworkError(result.err)) { return { kind: 'terminal-network', err: result.err }; } return { kind: 'retryable-network', err: result.err }; diff --git a/gitnexus-shared/src/lbug/schema-constants.ts b/gitnexus-shared/src/lbug/schema-constants.ts index ff3f4535b..350aa273d 100644 --- a/gitnexus-shared/src/lbug/schema-constants.ts +++ b/gitnexus-shared/src/lbug/schema-constants.ts @@ -72,6 +72,7 @@ export const REL_TYPES = [ 'INJECTS', 'CONDITIONAL_ON', 'DECLARES', + 'ADVISED_BY', // Taint/PDG substrate (issue #2080) — reserved edge types, emitted by no // phase yet (CFG → M1, REACHING_DEF → M2, TAINTED/SANITIZES/TAINT_PATH → // M3/M4). REACHING_DEF's variable name rides the relation's `reason` column. diff --git a/gitnexus-shared/src/scope-resolution/reference-site.ts b/gitnexus-shared/src/scope-resolution/reference-site.ts index e40e0a511..e8a58e1b2 100644 --- a/gitnexus-shared/src/scope-resolution/reference-site.ts +++ b/gitnexus-shared/src/scope-resolution/reference-site.ts @@ -138,6 +138,30 @@ export interface ReferenceSite { * majority of sites — the field costs nothing where it is not needed. */ readonly receiverChain?: string; + /** + * This site sits in CALLEE position: it is the expression being invoked by an + * enclosing call, not a value the program otherwise consumes. Only ever set on + * `kind: 'read'` sites, and only by languages whose member-read capture also + * matches the callee of a member call (`obj.f()` yields both a `call` site on + * `f` and a `read` site on `obj.f`). + * + * It is a POSITION FACT, not a decision. Whether that read is redundant + * depends on what the tail resolves to, which the capture layer cannot know: + * + * - tail is a METHOD → the read duplicates the call's own edge and must be + * suppressed (an `ACCESSES → m` beside a `CALLS → m` + * at the same position is a phantom). + * - tail is a FIELD → the read is GENUINE. `h.dep.Work()` where + * `Work func() error` selects a func-typed field and + * then calls the value it holds; deleting the read + * erases the only evidence that the field was used + * (callback/hook structs, hand-rolled mocks). + * + * The suppression is therefore applied at edge emission, where the resolved + * target's kind is known — see `tryEmitEdge`. Absent on every site that is not + * in callee position, so nothing changes for languages that never set it. + */ + readonly inCalleePosition?: boolean; } /** @@ -153,4 +177,16 @@ export interface ReferenceSite { * (`extractMixedChain`) walks a tree-sitter AST and so must stay in the * analyzer, but the shape it yields crosses into resolution. */ -export type MixedChainStep = { kind: 'field' | 'call'; name: string }; +/** + * One hop in a receiver chain. + * + * `field` and `call` carry the member name they reach. `await` and `index` are + * NAME-FREE: the call step already holds the method name for an awaited call, + * and a subscript has no member name at all — an index expression's key is a + * value, not an identifier the resolver could look up. The codec encodes them + * as a bare sigil and rejects any trailing characters, so the encoder's + * non-empty-name guard stays live for exactly the two kinds it was written for. + */ +export type MixedChainStep = + | { kind: 'field' | 'call'; name: string } + | { kind: 'await' | 'index'; name?: undefined }; diff --git a/gitnexus-shared/src/scope-resolution/types.ts b/gitnexus-shared/src/scope-resolution/types.ts index dbefe60d7..dcd074400 100644 --- a/gitnexus-shared/src/scope-resolution/types.ts +++ b/gitnexus-shared/src/scope-resolution/types.ts @@ -264,8 +264,24 @@ export type ParsedImport = export interface ParsedTypeBinding { /** The name being bound (parameter name, `self`, assignment LHS, …). */ readonly boundName: string; - /** The raw type name as written in source (`'User'`, `'models.User'`, …). */ + /** The type name AFTER this provider's normalization (`'User'`, + * `'models.User'`, …) — see `TypeRef.rawName`. */ readonly rawTypeName: string; + /** + * Optional override for `TypeRef.declaredSpelling`, for a grammar that does + * not keep the whole written type under `@type-binding.type`. + * + * The scope extractor derives the spelling from that capture by default, + * which is right for every language whose type node spans the annotation. + * C++ is the exception: `User* repos` parses with the `*` on the DECLARATOR, + * so the type capture is a bare `User` and the container-ness the index step + * needs is nowhere in the captures the extractor reads. A provider that can + * reconstruct it exactly sets it here. + * + * Leave undefined otherwise — the extractor's derivation is preferred to a + * per-language reimplementation of it. + */ + readonly declaredSpelling?: string; readonly source: TypeRef['source']; } @@ -370,8 +386,36 @@ export interface BindingRef { * re-exports, and nested modules. Generics deferred to V2 via `typeArgs`. */ export interface TypeRef { - /** The name as written in source (e.g., `'User'`, `'models.User'`, `'List'`). */ + /** + * The type name AFTER the language's capture-time normalization — NOT + * necessarily what the source says. Every provider's `interpretTypeBinding` + * reduces the annotation before it gets here: TypeScript runs + * `stripGeneric` + `stripArraySuffix` to a FIXED POINT (`User[][]` → `User`), + * Go's `normalizeGoTypeName` drops `[]` and `map[K]`, C#/Python/Kotlin/Rust + * strip their single-arg collection wrappers. What survives is the name a + * class lookup can use (`'User'`, `'models.User'`, `'List'`). + * + * A consumer that needs the CONTAINER, not the element, must read + * `declaredSpelling` — see below. + */ readonly rawName: string; + /** + * The annotation exactly as written, kept ONLY when `rawName` is not it. + * + * `rawName` alone cannot distinguish `repos: User[]` (a container the capture + * layer already reduced, so the position IS the element) from `grid: Grid` + * (an ordinary class the source happened to subscript). Both arrive as a bare + * class name that resolves. An index step reading only `rawName` therefore had + * no choice but to guess, and guessing "already reduced" typed `grid[0]` as + * `Grid` — a confidently WRONG owner for the next member. + * + * Absent when the provider's normalization was a no-op (nothing was lost, so + * `rawName` is already the written spelling), and absent for TypeRefs + * synthesized outside the capture path (a `this` receiver binding, a + * propagated return type). Consumers must treat absence as "no container + * evidence" and decline, never as "not a container". + */ + readonly declaredSpelling?: string; /** Anchor for resolving `rawName` — the scope where the annotation/inference was written. */ readonly declaredAtScope: ScopeId; readonly source: diff --git a/gitnexus/bench/cpp-qualified-ns/baselines.json b/gitnexus/bench/cpp-qualified-ns/baselines.json new file mode 100644 index 000000000..51ce5405d --- /dev/null +++ b/gitnexus/bench/cpp-qualified-ns/baselines.json @@ -0,0 +1,7 @@ +{ + "_comment": "Baselines for bench/cpp-qualified-ns/measure.mjs --check (#2788). `fingerprint` is a sha256 over every `receiver::member(arity|argumentTypes) -> outcome` the synthetic corpus resolves at the LARGE scale (hit nodeId, `` per #1564, or ``); it is a CORRECTNESS gate, so drift means C++ qualified `ns::member()` lookup started resolving a different symbol set and must be explained, never re-baselined to make CI green. THAT RULE IS UNCHANGED and applies to every future edit of inline-namespaces.ts. `scaling_budget` is a timing gate and carries deliberate headroom for shared CI runners.", + "_rebaseline_2788_review": "This fingerprint was moved ONCE, deliberately, during review of #2788 — because the bench CORPUS was expanded, not because a check failed. Do not read it as precedent. What changed: (1) receivers now mirror production — ~1 in 5 name a declared namespace, ~4 in 5 are plain identifiers naming none (`obj0`, `Widget3`, `buf12`). The previous corpus drew every receiver from `ns_${…}`, so the receiver lookup NEVER missed, while Case 1.5 in scope-resolution/passes/receiver-bound-calls.ts is reached by every plain-identifier receiver call and misses on the overwhelming majority. (2) A namespace reopened across two files (C++ namespaces are open — the cross-file merge property). (3) A same-name inline nest `namespace ns { inline namespace ns { … } }`, which is the only shape that observes `gatherQualifiedNsMember`'s `visited` dedup. (4) A member declared at BOTH the namespace level and in an inline child, selected apart by argument type, pinning both collection sources by nodeId. (5) Call sites carrying a real `Callsite`, without which narrowOverloadCandidates / cppConversionRank / isOverloadAmbiguousAfterNormalization were outside the fingerprinted surface entirely. Measured effect, same patched resolver, old bench vs new: removing the `visited` dedup — old PASS with a byte-identical fingerprint, new FAIL (fingerprint 1e6c51b9… != aba39c34…); resetting `visited` per root instead of across roots — old PASS, new FAIL on the same fingerprint. A PURE reorder of a candidate list still passes both, and correctly so: the resolver's return contract is order-blind by construction (see QualifiedNsMemberIndex's doc comment), so there is no behaviour there to gate.", + "fingerprint": "aba39c342ce536006bebded8b32260dc7807487be91f5c7ee9548f9e9283f9c9", + "scaling_budget": 1.8, + "_scaling_note": "(t_large/t_small)/(1600/400). ~1.0 is linear. OBSERVED BAND: 1.28-1.45 over ten unloaded runs on a 24-core dev box. The band this file previously claimed — 0.93-1.21 — did not reproduce and was an artifact: the small arm then measured ~1.7 ms, small enough that timer granularity and JIT warm-up, not scaling, set the number (the same ten-run sweep of that bench spanned 1.11-1.40). CALLS_PER_FILE is now sized so the small arm lands at ~14 ms; that halves the unloaded spread (0.30 -> 0.16) and costs ~2.0 s of wall time for the whole bench. The residual above 1.0 is real and not a defect: at LARGE the index and corpus are 4x the working set, so per-call-site locality is worse (~85 ns/site vs ~63 ns) while the algorithm stays linear. TRIAGE: a scaling failure is a TIMING signal — RE-RUN IT on an idle machine before investigating. Runner contention dominates everything above: pinned to 2 CPUs against 2 spinners the identical binary produced 1.16-2.18, i.e. a spurious FAIL, and the sibling bench/callable-value-flow drifts out of its own documented band the same way. The fingerprint arm is the opposite — it is deterministic; a re-run never changes it and must never be used to wish it away. Floor check: a per-call-site workspace rescan reintroduced ONLY on the receiver-bucket-absent path (the most plausible way #2788 returns) measures 4.538 at these same 400/1600 file scales — 812x slower on the small arm, 2850x on the large — while leaving the fingerprint byte-identical. The old always-hits corpus scored that same patch 1.279 and printed PASS. Resolution is timed alone; the fingerprint's outcome strings are built in a separate untimed pass because their allocation cost grows with the corpus and would otherwise show up as scaling." +} diff --git a/gitnexus/bench/cpp-qualified-ns/measure.mjs b/gitnexus/bench/cpp-qualified-ns/measure.mjs new file mode 100644 index 000000000..6ffbc18a0 --- /dev/null +++ b/gitnexus/bench/cpp-qualified-ns/measure.mjs @@ -0,0 +1,496 @@ +/** + * Build-free scaling + identity bench for `resolveCppQualifiedNamespaceMember`, + * the C++ qualified `ns::member()` receiver resolver (issue #2788). + * + * Before #2788 this function re-scanned EVERY parsed file — rebuilding a + * per-file `scopesById` map each time — once per qualified call site, so the + * scope-resolution emit phase cost O(callsites × scopes). On a 1,473-file C++ + * repo that was 25.3 min of a 33-min analyze, with 75% of total self-time in + * this one function. It is the same bug #1990 had already fixed in the sibling + * ADL path (`pickCppAdlCandidates` → `AdlCandidateIndex`). #1990 DID ship a + * scaling gate for that path — `test/integration/cpp-adl-benchmark.test.ts`, + * which asserts `emitRatio < fileRatio^1.5` — but it could never have caught + * this one, for two independent reasons: its corpus asserts + * `callsResolved === 0`, i.e. it generates only UNRESOLVED ADL sites, so it + * never drives the qualified-receiver path at all; and it is + * `describe.skipIf(!BENCH_ENABLED)` while the single CI step that sets + * `GITNEXUS_BENCH=1` names its test files explicitly and, until this PR wired + * it in, listed neither C++ bench — so it had never executed in CI. Even now + * that it runs, the `callsResolved === 0` half stands: it still cannot reach + * this path. Hence this bench, in an always-on step: a per-call-site workspace + * scan must not be reintroduced silently. + * + * For a synthetic corpus at two scales it reports: + * - `elapsed_ms` per scale (fastest of REPS, see `fastest`) for resolving + * every call site once, INCLUDING the one-time index build — that build is + * the work the per-site scan was traded for, so hiding it would let an + * index that is itself quadratic pass; + * - a scaling ratio `(t_large/t_small)/(LARGE/SMALL)`: ~1.0 linear, + * ~4.x quadratic at this scale gap; + * - a sha256 fingerprint over every `receiver::member(callsite) → outcome` + * the corpus resolves, as the correctness gate. A fingerprint change means + * qualified lookup started resolving different symbols — a behaviour + * change, never a performance one. + * + * A gate only covers the code path its corpus drives. Two properties below are + * therefore load-bearing and must not be "simplified" away: + * + * 1. **The receiver mix is production-shaped: ~1 in 5 receivers names a + * namespace, the other ~4 name nothing.** Case 1.5 in + * `scope-resolution/passes/receiver-bound-calls.ts` is reached by EVERY + * plain-identifier receiver call — `obj.size()`, `Widget::make()`, + * `buf.data()` — so in real source the overwhelming majority of calls into + * this resolver are receiver MISSES, not member misses inside a resolved + * receiver. An earlier revision of this bench drew every receiver from + * `ns_${…}`, i.e. always a namespace the corpus declared, so the receiver + * lookup never missed. Measured consequence: a "defensive full rescan when + * the receiver bucket is absent" regression — the single most plausible + * way this bug returns — scored 1.332 against the 1.8 budget and printed + * PASS, while costing 507× on a production-shaped corpus. + * 2. **The corpus contains every structural shape whose loss the fingerprint + * is supposed to catch** (see `buildCorpus`), including a batch of sites + * that pass a real `Callsite`. Without those, `narrowOverloadCandidates` / + * `cppConversionRank` / `isOverloadAmbiguousAfterNormalization` are not in + * the fingerprinted surface at all, and behaviour-only regressions there + * re-fingerprint byte-identically. + * + * Build-free: imports the `.ts` hotpath through tsx + * (`node --import tsx bench/cpp-qualified-ns/measure.mjs`). + * + * Without args: prints the JSON report. + * With `--check`: asserts the fingerprint == the committed baseline AND the + * scaling ratio is within budget; exits non-zero on drift/regression. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; + +import { + clearCppInlineNamespaces, + markCppInlineNamespaceRange, + populateCppInlineNamespaceScopes, + resolveCppQualifiedNamespaceMember, +} from '../../src/core/ingestion/languages/cpp/inline-namespaces.ts'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const SMALL = 400; +const LARGE = 1600; +/** Sized so the SMALL arm measures in the tens of ms rather than ~1.7 ms. + * Sub-2 ms samples are dominated by timer granularity and scheduler noise on + * a shared runner, which is what made the ratio drift out of its documented + * band under load; see `_scaling_note` in baselines.json. */ +const CALLS_PER_FILE = 480; +const REPS = 7; +const WARMUP = 3; + +/** 1 in N receivers names a declared namespace; the rest name nothing. Header + * property 1 is why this ratio, and not an always-hits corpus. */ +const NS_RECEIVER_IN = 5; + +const NO_SCOPES = {}; + +/** + * Deterministic 32-bit avalanche (murmur3 finalizer). Stands in for + * `Math.random()` — the corpus, the receiver mix and therefore the fingerprint + * must be byte-reproducible across machines and Node versions. + */ +function mix(n) { + let x = n >>> 0; + x = Math.imul(x ^ (x >>> 16), 0x85ebca6b) >>> 0; + x = Math.imul(x ^ (x >>> 13), 0xc2b2ae35) >>> 0; + return (x ^ (x >>> 16)) >>> 0; +} + +/** + * Deterministic synthetic corpus — no randomness, so the fingerprint is stable. + * + * Per file `f`, three top-level namespaces. Every shape here exists because + * some behaviour of `resolveCppQualifiedNamespaceMember` is unobservable + * without it; dropping one silently un-gates that behaviour. + * + * namespace ns_f { // ABI-versioning idiom (std::__1) + * void own0(); void own1(); // direct members → hit + * void both(int); // ALSO declared in v1 below + * void over(int); // overload set spanning levels + * inline namespace v1 { // transitively visible + * void inl0(); // → hit + * void dup(); + * void both(double); // the inline-child twin of `both` + * void over(int, int); void over(double); + * void same(int); + * } + * inline namespace v2 { + * void dup(); // two inline children → ambiguous + * void same(int); // identical signature → ambiguous + * } + * namespace detail { void hidden0(); } // NOT inline → invisible → miss + * } + * + * namespace twin_f { inline namespace twin_f { void twinned(); } } + * namespace shared_{f>>1} { void part{f&1}(); } + * + * What each shape gates: + * - `own0` / `inl0` / `hidden0` / `nosuch`: the three outcome classes (hit + * from the namespace's own defs, hit through an inline child, miss), each + * a different exit from the resolver. + * - `dup` across v1 and v2: `'ambiguous'` (#1564). + * - `twin_f`: the same-name inline nest — the only shape that observes + * `gatherQualifiedNsMember`'s `visited` dedup, without which the one + * `twinned` is collected twice and a resolved def flips to `'ambiguous'` + * (that function's comment explains why both scopes land on one receiver). + * - `shared_g` declared by files 2g and 2g+1: C++ namespaces are open, so one + * receiver's members are spread over however many files reopen it. The + * legacy per-call-site scan got this for free; the index has to merge + * across the whole `parsedFiles` array. `part0` and `part1` are declared in + * DIFFERENT files and both must resolve. + * - `both` at the namespace level and in the inline child: pins BOTH + * collection sources by nodeId, via the two `both` call sites that select + * between them on argument type. Drop own-def collection and the `int` + * probe moves; drop inline-child descent and the `double` probe moves. A + * pure REORDER of the two stays invisible, and correctly so: the return + * contract is order-blind — see `QualifiedNsMemberIndex.rootsByReceiver`. + * - `over` / `same` with a real `Callsite`: see `NS_PROBES`. + */ +function buildCorpus(fileCount) { + const parsedFiles = []; + for (let f = 0; f < fileCount; f++) { + const filePath = `src/file${f}.cpp`; + const scopes = []; + const inlineRanges = []; + let line = 1; + /** Push one Namespace scope with a range unique within this file, so + * `populateCppInlineNamespaceScopes` marks exactly the intended scopes. */ + const scope = (id, parent, ownedDefs, isInline = false) => { + const range = { startLine: line, startCol: 0, endLine: line + 1, endCol: 0 }; + line += 2; + scopes.push({ id, kind: 'Namespace', parent, ownedDefs, range }); + if (isInline) inlineRanges.push(range); + return id; + }; + const ns = (qualifiedName) => ({ + nodeId: `def:${filePath}#${qualifiedName}`, + type: 'Namespace', + qualifiedName, + }); + /** A callable def. `parameterTypes` are what makes overloads distinguishable + * both to `narrowOverloadCandidates` and — via the nodeId, exactly as the + * real C++ node keys do it — to the fingerprint. */ + const fn = (qualifiedName, parameterTypes) => + parameterTypes === undefined + ? { nodeId: `def:${filePath}#${qualifiedName}`, type: 'Function', qualifiedName } + : { + nodeId: `def:${filePath}#${qualifiedName}(${parameterTypes.join(',')})`, + type: 'Function', + qualifiedName, + parameterTypes, + parameterCount: parameterTypes.length, + requiredParameterCount: parameterTypes.length, + }; + + const nsId = scope(`sc:${f}:ns`, null, [ + ns(`ns_${f}`), + fn(`ns_${f}.own0`), + fn(`ns_${f}.own1`), + fn(`ns_${f}.both`, ['int']), + fn(`ns_${f}.over`, ['int']), + ]); + scope( + `sc:${f}:v1`, + nsId, + [ + ns(`ns_${f}.v1`), + fn(`ns_${f}.v1.inl0`), + fn(`ns_${f}.v1.dup`), + fn(`ns_${f}.v1.both`, ['double']), + fn(`ns_${f}.v1.over`, ['int', 'int']), + fn(`ns_${f}.v1.over`, ['double']), + fn(`ns_${f}.v1.same`, ['int']), + ], + true, + ); + scope( + `sc:${f}:v2`, + nsId, + [ns(`ns_${f}.v2`), fn(`ns_${f}.v2.dup`), fn(`ns_${f}.v2.same`, ['int'])], + true, + ); + scope(`sc:${f}:detail`, nsId, [ns(`ns_${f}.detail`), fn(`ns_${f}.detail.hidden0`)]); + + const twinId = scope(`sc:${f}:twin`, null, [ns(`twin_${f}`)]); + scope( + `sc:${f}:twin@inner`, + twinId, + [ns(`twin_${f}.twin_${f}`), fn(`twin_${f}.twin_${f}.twinned`)], + true, + ); + + const group = f >> 1; + scope(`sc:${f}:shared`, null, [ns(`shared_${group}`), fn(`shared_${group}.part${f & 1}`)]); + + parsedFiles.push({ filePath, scopes, inlineRanges }); + } + return parsedFiles; +} + +/** Capture-time inline marking + `populateOwners`-time scope-id resolution, in + * the same order the pipeline runs them. Must re-run after every + * `clearCppInlineNamespaces`, which drops both the marks and the index. */ +function populateInlineState(parsedFiles) { + clearCppInlineNamespaces(); + for (const parsed of parsedFiles) { + for (const range of parsed.inlineRanges) markCppInlineNamespaceRange(parsed.filePath, range); + populateCppInlineNamespaceScopes(parsed); + } +} + +/** + * Namespace-receiver probes: `[family, member, callsite]`. Drawn for the ~1 in + * `NS_RECEIVER_IN` call sites whose receiver actually names a namespace. + * + * The tail entries pass a real `Callsite`, which is the only way any of + * `narrowOverloadCandidates`, `cppConversionRank` or + * `isOverloadAmbiguousAfterNormalization` is reached — the resolver threads + * `callsite?.arity` / `callsite?.argumentTypes` into narrowing, and with no + * callsite those filters are pass-throughs. Each one is chosen to land on a + * DIFFERENT exit, so the fingerprint pins the whole narrowing ladder: + * - `over(int)` → exact-type filter, unique survivor (ns level) + * - `over(int,int)` → arity filter, unique survivor (inline child) + * - `over(double)` → exact-type filter, unique survivor (inline child) + * - `over(char)` → no exact match, `cppConversionRank` dominance + * picks `over(int)` (promotion 1) over + * `over(double)` (standard conversion 2) + * - `over(braced-init)` → conversion ranking rejects every candidate and + * `CPP_CONVERSION_ONLY_ARG_TYPE_PREFIXES` turns + * that into an empty set → `undefined` + * - `over` with arity 9 → arity filter empties an all-known-bounds set, + * the authoritative-empty branch → `undefined` + * - `same(int)` → two identical signatures survive narrowing → + * `isOverloadAmbiguousAfterNormalization` → `'ambiguous'` + * - `both(int)`/`both(double)` → select the namespace-level def and the + * inline-child def respectively, pinning both + * collection sources by nodeId + */ +const NS_PROBES = [ + ['ns', 'own0', undefined], + ['ns', 'own1', undefined], + ['ns', 'inl0', undefined], + ['ns', 'dup', undefined], + ['ns', 'hidden0', undefined], + ['ns', 'nosuch', undefined], + ['ns', 'both', undefined], + ['twin', 'twinned', undefined], + ['twin', 'nosuch', undefined], + ['shared', 'part0', undefined], + ['shared', 'part1', undefined], + ['ns', 'over', { arity: 1, argumentTypes: ['int'] }], + ['ns', 'over', { arity: 2, argumentTypes: ['int', 'int'] }], + ['ns', 'over', { arity: 1, argumentTypes: ['double'] }], + ['ns', 'over', { arity: 1, argumentTypes: ['char'] }], + ['ns', 'over', { arity: 1, argumentTypes: ['braced-init:int:3'] }], + ['ns', 'over', { arity: 9, argumentTypes: [] }], + ['ns', 'same', { arity: 1, argumentTypes: ['int'] }], + ['ns', 'both', { arity: 1, argumentTypes: ['int'] }], + ['ns', 'both', { arity: 1, argumentTypes: ['double'] }], +]; + +/** Members asked of the non-namespace receivers. Real-source member names, so + * the miss is a receiver miss and not a member miss. */ +const MISS_MEMBERS = ['size', 'begin', 'data', 'reset', 'own0', 'dup']; + +/** A plain identifier naming NO namespace in the corpus — a local, a type, a + * buffer. The ~4-in-5 majority of header property 1. */ +function missReceiver(key) { + const shape = key % 3; + if (shape === 0) return `obj${key % 97}`; + if (shape === 1) return `Widget${key % 31}`; + return `buf${key % 197}`; +} + +/** + * The call sites: `[receiver, member, callsite]`, deterministic, with the + * production receiver mix (~1 in `NS_RECEIVER_IN` names a namespace). + * + * Two independently mixed keys per site so the receiver class (`a`) and the + * probe choice (`b`) do not correlate — deriving both from one linear key made + * `key % NS_RECEIVER_IN === 0` imply `key % NS_PROBES.length ∈ {0, 5}`, which + * silently reduced the probe set to two entries. + */ +function callSites(fileCount) { + const sharedGroups = Math.ceil(fileCount / 2); + const sites = []; + let nsReceiverSites = 0; + /** The declared-namespace receiver a probe family asks for. */ + const nsReceiver = (family, key) => { + if (family === 'twin') return `twin_${key % fileCount}`; + if (family === 'shared') return `shared_${key % sharedGroups}`; + return `ns_${key % fileCount}`; + }; + const pushNs = (probe, key) => { + sites.push([nsReceiver(probe[0], key), probe[1], probe[2]]); + nsReceiverSites++; + }; + // Coverage prelude: every probe at least once at BOTH scales, so the + // fingerprinted outcome set never depends on how the mixer happens to spread. + for (let p = 0; p < NS_PROBES.length; p++) pushNs(NS_PROBES[p], p); + for (let f = 0; f < fileCount; f++) { + for (let c = 0; c < CALLS_PER_FILE; c++) { + const a = mix(f * 65599 + c); + const b = mix(a ^ 0x9e3779b9); + if (a % NS_RECEIVER_IN === 0) pushNs(NS_PROBES[b % NS_PROBES.length], b); + else sites.push([missReceiver(b), MISS_MEMBERS[a % MISS_MEMBERS.length], undefined]); + } + } + return { sites, nsReceiverSites }; +} + +/** The timed loop: resolution only. The outcome strings the fingerprint needs + * are built in a separate untimed pass (`outcomesOf`), so their allocation + * cost — which grows with the corpus and would inflate the scaling ratio on + * its own — never lands in the measurement. `sink` keeps the calls live. */ +function resolveAll(parsedFiles, sites) { + let sink = 0; + for (const [receiver, member, callsite] of sites) { + const hit = resolveCppQualifiedNamespaceMember( + receiver, + member, + parsedFiles, + NO_SCOPES, + callsite, + ); + if (hit !== undefined) sink++; + } + return sink; +} + +/** Fingerprint key for one site. The callsite is part of the key: `ns::over` + * resolves to a different def per arity/argument-type, and collapsing those + * onto one key would drop the whole narrowing ladder from the gate. */ +function siteKey(receiver, member, callsite) { + const args = + callsite === undefined ? '' : `${callsite.arity}|${callsite.argumentTypes.join(',')}`; + return `${receiver}::${member}(${args})`; +} + +/** Untimed identity pass, one resolve per DISTINCT `siteKey`. On a fixed corpus + * the resolver is a pure function of `(receiver, member, callsite)`, so a + * repeated site can only re-derive what the first occurrence already put in + * the Set — the same argument that makes collecting into a Set correct makes + * skipping the repeat correct. That is nearly the whole pass: the 192k/768k + * sites carry only 2,330/3,470 distinct outcomes. */ +function outcomesOf(parsedFiles, sites) { + const outcomes = new Set(); + const seen = new Set(); + for (const [receiver, member, callsite] of sites) { + const key = siteKey(receiver, member, callsite); + if (seen.has(key)) continue; + seen.add(key); + const hit = resolveCppQualifiedNamespaceMember( + receiver, + member, + parsedFiles, + NO_SCOPES, + callsite, + ); + outcomes.add( + `${key}\u0000${hit === undefined ? '' : hit === 'ambiguous' ? '' : hit.nodeId}`, + ); + } + return outcomes; +} + +/** + * MIN, not median — same rationale as bench/callable-value-flow: both scales + * are timed in one process and every error source (scheduler preemption, GC, a + * noisy neighbour on a shared CI runner) is additive, so the fastest observed + * run is the closest estimate of the uncontended cost and keeps the derived + * ratio comparable across machines. + */ +function fastest(values) { + return Math.min(...values); +} + +/** Time one full pass: index build (lazy, on the first call) + every call + * site. The corpus state is reset OUTSIDE the timer so the reset's own + * O(files) cost never lands in the measurement. */ +function timeResolution(parsedFiles, sites) { + for (let w = 0; w < WARMUP; w++) { + populateInlineState(parsedFiles); + resolveAll(parsedFiles, sites); + } + const samples = []; + for (let r = 0; r < REPS; r++) { + populateInlineState(parsedFiles); + const t0 = performance.now(); + resolveAll(parsedFiles, sites); + samples.push(performance.now() - t0); + } + return { ms: fastest(samples), outcomes: outcomesOf(parsedFiles, sites) }; +} + +function fingerprint(outcomes) { + return crypto + .createHash('sha256') + .update([...outcomes].sort().join('\n')) + .digest('hex'); +} + +const scales = {}; +for (const [name, fileCount] of [ + ['small', SMALL], + ['large', LARGE], +]) { + const parsedFiles = buildCorpus(fileCount); + const { sites, nsReceiverSites } = callSites(fileCount); + const { ms, outcomes } = timeResolution(parsedFiles, sites); + scales[name] = { + files: fileCount, + call_sites: sites.length, + // Reported, not asserted: `ns_receiver_sites` evidences header property 1's + // mix and `distinct_outcomes` the fingerprinted surface's size — a corpus + // edit collapsing either still yields a "valid" fingerprint over far less. + ns_receiver_sites: nsReceiverSites, + distinct_outcomes: outcomes.size, + ms: Number(ms.toFixed(3)), + fingerprint: fingerprint(outcomes), + }; +} + +const scalingRatio = scales.large.ms / scales.small.ms / (LARGE / SMALL); + +const report = { + small: scales.small, + large: scales.large, + scaling_ratio: Number(scalingRatio.toFixed(3)), + fingerprint: scales.large.fingerprint, +}; + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); +const failures = []; +if (report.fingerprint !== baseline.fingerprint) { + failures.push( + `fingerprint drift: ${report.fingerprint} != ${baseline.fingerprint} — qualified ` + + `namespace lookup resolved a DIFFERENT symbol set. This is a behaviour change, not a perf one.`, + ); +} +if (report.scaling_ratio > baseline.scaling_budget) { + failures.push( + `scaling ${report.scaling_ratio} > budget ${baseline.scaling_budget} — per-call-site cost ` + + `now grows with corpus size again (#2788). Timing arm: re-run on an idle machine before ` + + `investigating (see _scaling_note in baselines.json); the fingerprint arm never warrants a re-run.`, + ); +} + +console.log(JSON.stringify(report, null, 2)); +if (failures.length > 0) { + console.error(`[cpp-qualified-ns --check] FAIL\n - ${failures.join('\n - ')}`); + process.exit(1); +} +console.log('[cpp-qualified-ns --check] PASS'); diff --git a/gitnexus/bench/python-scope/baseline-fingerprint.txt b/gitnexus/bench/python-scope/baseline-fingerprint.txt index b27ff7a41..49a146016 100644 --- a/gitnexus/bench/python-scope/baseline-fingerprint.txt +++ b/gitnexus/bench/python-scope/baseline-fingerprint.txt @@ -1 +1 @@ -b57c5479f158c41a6328fa8d61c234aaec88fc41780252d2fbd7baca21491aa2 +a0da3e7c00f603e4bdad91a376b3fc181577a73c2ca1719ab7449d3463c671e0 diff --git a/gitnexus/bench/receiver-resolution/BASELINE.md b/gitnexus/bench/receiver-resolution/BASELINE.md index 49ac4979b..f612258bf 100644 --- a/gitnexus/bench/receiver-resolution/BASELINE.md +++ b/gitnexus/bench/receiver-resolution/BASELINE.md @@ -1,5 +1,430 @@ # Receiver-resolution baseline +> **`baseline.json` is the source of truth for every number.** It is what +> `measure.mjs --check` enforces byte-exactly. This file is a lab notebook: +> each section records what was measured AT THAT UNIT and why it changed the +> plan. A figure here that disagrees with `baseline.json` is a superseded +> snapshot, not a live claim — sections carry a snapshot marker where that has +> already happened. Never quote a count from this file into code, a gate, or a +> commit message; read it from `baseline.json`. + +## Receiver ORIGIN — three quarters of the hedge was the program boundary + +The drop count was measuring two different things and reporting both as +uncertainty. Dumping all 102 call drops with source context settles it: + +| Origin | Count | Is anything lost? | +| ------------ | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `external` | **44** | **No.** `System.out.println`, `fetch(...)`, `os.environ.setdefault`, `document.body.appendChild`, `.stream()`. The callee is not in the graph — there is no node an edge could point at. | +| `in-program` | 36 | Yes in principle — but see below. | +| `unknown` | 22 | Yes. Casts, ternaries, `globalThis.x ??= []`, and everything the classifier will not guess about. | + +> **These numbers moved once, in review, and the movement is the point.** They +> were first measured as 76 / 20 / 6, when `external` was the FALLTHROUGH: any +> base whose type did not resolve was called external. Review reproduced two +> triggers where that published `epistemic: 'exact'` over a real in-program loss +> — a Go pointer receiver (`*Host`, whose lookup was missing the decoration +> stripper) and any base with no type binding at all, including this branch's own +> `droppedCall(svc)` fixture. `external` is now a POSITIVE determination via +> `LanguageProvider.isBuiltInName`, and everything unproven is `unknown`, which +> still hedges. So `external` fell 76 -> 44 and the difference went to +> `in-program` (+16, the drops that really were ours) and `unknown` (+16, the +> drops we decline to characterize). Total call drops is unchanged at 102 — +> this is re-bucketing, not resolution. +> +> A controlled A/B over the Java built-in set (off vs on, same tree) reads +> 7/36/59 vs 44/36/22: `in-program` is byte-identical across the toggle, so +> naming built-ins reclassified nothing the index can demonstrate is ours. + +**A compiler resolves `System.out.println` against the JDK.** Lacking the JDK, +the honest statement is _"this call leaves the analyzed program"_ — not _"this +analysis is incomplete"_. Those are different epistemic states, and collapsing +them is what made `impact` report a lower bound on essentially every real +codebase, which is what teaches readers to ignore the signal. + +`ResolutionOutcome.receiverOrigin` now records which one applies, and +`summarizeUnresolvedReceivers` skips `external`. `unknown` still counts — +assuming a completeness we cannot demonstrate is the unsafe direction. + +### How origin is decided + +By the receiver base's **declared type**, not its name. A first cut asked +whether the base was a local, which marked `inputs.stream()` in-program: +`inputs` is a local, but its type `List` is JDK, so the target is +external. Asking whether the base's _type_ is one this index contains moved 28 +sites to the correct bucket. + +### What the remaining in-program drops actually are + +Mostly **not** product defects. `user.Address.Save()` resolves cleanly in +isolation — the `csharp-deep-field-chain` fixture alone emits both expected +edges with **zero** drops. It drops in the count arm only because the corpus is +~200 independent mini-projects in one directory and **55 files define +`Address`**, so the resolver correctly declines on ambiguity rather than picking +one. That is right behaviour measured on an unrepresentative corpus. + +The genuinely untypeable population is the `unknown` bucket — and those are the real +targets for type resolution, because a cast _gives_ you the type +(`((Box) obj).open()`) and a ternary needs a join of its branch types. +They were previously invisible under the stdlib calls the old fallthrough swept +into `external`. + +`callDropsByOrigin` is now part of the gated projection, so this split cannot +drift silently. + +--- + +## Phantom callee read sites — a duplicate-edge bug the U8 test missed + +Go's `@reference.read` pattern matches **every** `selector_expression`, with no +call-position exclusion. So `h.dep.Work()` minted **three** reference sites: + +| site | kind | name | what it is | +| ---- | ------ | ------ | ------------------------------------------------------------- | +| S1 | `call` | `Work` | the member call | +| S2 | `read` | `Work` | **phantom** — the callee `h.dep.Work`, already captured by S1 | +| S3 | `read` | `dep` | the genuine field read | + +S2 resolved through `findOwnedMember`, which prefers methods over fields, and +emitted an `ACCESSES` edge to the **method** duplicating S1's `CALLS` edge at the +same position. + +**The U8 assertion passed by accident.** It asserted `RunSamePackage → Work` was +absent from `ACCESSES`, and it was — but only because that row has a _pointer_ +receiver whose text-cascade head lookup failed for an unrelated reason. The +value-receiver twin was emitting the bad edge the whole time: + +``` +ACCESSES RunFromValueReceiver -> DoWork:Method <- phantom, shipped +ACCESSES RunLocal -> DoWork:Method <- phantom, shipped +``` + +First fix, at capture: drop the match outright, on the rule _"a selector in +function position is never a read."_ **That rule is false, and review caught +it.** In Go a func-typed struct field IS read and then called indirectly — +`h.dep.Work()` where `Work func() error` — and `isCalleeOfMemberCall` cannot +tell a method from a func-valued field, because the AST shape is identical. +Dropping at capture therefore deleted the only `ACCESSES` evidence for callback +structs, hook structs and hand-rolled mocks (`mock.DoFunc`, `opts.OnEvent`). + +Second fix, and the one that shipped: **split the decision across the two layers +that each hold half of it.** Capture records the POSITION as a fact +(`@reference.callee-position` → `ReferenceSite.inCalleePosition`) — only the AST +knows it, and it is gone by resolution time. Emit makes the DECISION from the +resolved target's kind — only resolution knows whether the tail is a method or a +field, and it may be declared in another package. Neither layer can answer alone. +The suppression is language-neutral in `graph-bridge/edges.ts` and keys on the +canonical `CALL_TARGET_TYPES`, so `Macro` and `Delegate` targets are covered too. + +A method _value_ (`f := h.dep.Work`) is not in function position and is +untouched. The assertion is backed by an exact-set check over the whole fixture — +now carrying target KINDS, so it catches both a new phantom and a deleted +genuine read. + +### What the numbers say + +- `callDrops` **unchanged at 102** — no call was lost, in either fix. +- `read` drops went **27 → 22** under the capture-time drop, then **22 → 27** + again once the marker replaced it. The round trip is the finding: those five + sites are genuine field reads, and the first fix was scoring their deletion as + an improvement. +- `totalDropsAllKinds` **124 → 129**, the same five sites. +- One drop reclassified `chain-field` → `chain-unwrap`. The phantom and the real + call share a site key, so the phantom's field-shaped chain was previously the + one recorded. The census now describes the actual dropped call. + +Caught by three review agents dispatched at the A1 regression; the phantom was +the mechanism, not the global-normalization story the first revert note asserted. +The func-field regression it introduced was then caught by two more, on the +tri-review of #2782 — which is the argument for the exact-set-with-kinds +assertion over the targeted one that passed by accident the first time. + +--- + +## U9 (part 2) — no drop ratchet is needed; the gate is already stronger + +The plan's R10 set a ZERO supported-shape drop target, and review correctly +found that it contradicts R12: a site whose normalized name matches more than +one class MUST decline, a decline records a drop, and simple names collide +routinely in large Go and Java codebases. The proposed fix was a ratchet — the +count may not rise above the value measured after the last unit. + +Neither is needed. `measure.mjs --check` already asserts **exact match** against +the committed baseline, which is strictly stronger than a ratchet: the count +cannot rise _or_ fall without a deliberate `--update-baseline`, and that path +prints an instruction to explain the movement in the commit message. A ratchet +would be a weakening. + +So R10 as written (zero) was wrong, and the ratchet proposed to repair it is +redundant. The existing gate stands, now also covering `callDropsByShape` since +the shape census joined the gated projection. + +**Deferred and NOT done: the `impact` risk-cutoff recalibration.** Review flagged +that added edges push symbols toward the absolute cutoffs (`directCount >= 30`, +`impacted.length >= 200`), so edits read HIGHER risk without being more +dangerous, and agents warning on HIGH/CRITICAL escalate more often. That is real, +but measuring it honestly needs a before/after risk distribution over a corpus +large enough for those thresholds to bind — the committed fixtures are nowhere +near 200 impacted symbols. Recording it as owed rather than inventing a number +from fixtures that cannot exercise the cutoffs. + +--- + +## U6 — the depth cap does NOT limit resolution. Measured, not raised. + +The premise was that a chain deeper than `MAX_CHAIN_DEPTH` (3) is discarded +whole rather than truncated, so a 4-hop builder chain "contributes nothing at +all". The first half is true; the second is not. + +`fourHopChain` was added to the TypeScript corpus as a declared extra +specifically to make the question answerable — without a chain longer than the +cap, raising the cap measures nothing: + +```ts +root.getSvc().getUser().address.getCity().save(); +// ^step1 ^step2 ^step3 ^step4 receiver of `save` = 4 steps +``` + +| Cap | Chain minted? | Cell state | +| --- | ---------------------------------------------------- | ------------ | +| 3 | **none** (confirmed by probing the emitter directly) | **RESOLVES** | +| 4 | `2\|root\|cgetSvc\|cgetUser\|faddress\|cgetCity` | RESOLVES | + +The site resolves at BOTH depths. At 3 it resolves through the text cascade, +which owns the fallback path and runs to its own +`COMPOUND_RECEIVER_MAX_DEPTH` of 8. + +**So the cap bounds which chains are typed structurally, not which calls +resolve.** Raising it moves work from the cascade to the fold without changing a +single edge — measured across the whole matrix: totals identical at 3 and 4, +`callDrops` 102 at both. + +Left at 3. The fixture is committed so the next person to reach for this number +inherits the measurement instead of the intuition. + +What DID need fixing: `unwrapTransparentReceiver` shared `MAX_CHAIN_DEPTH` as +its iteration bound. The two answer unrelated questions — how many chain hops do +we type, versus how many redundant parens might someone write — so raising the +chain cap would have silently widened the paren peel as a side effect. That +coupling got worse when the await/subscript work added a peel call at loop +entry. Now `MAX_TRANSPARENT_WRAPPER_DEPTH`, its own constant. + +--- + +## U9 — the epistemic hedge has TWO producers, and only one is a defect + +`impact` reports `epistemic: 'lower-bound'` for two independent reasons that were +previously indistinguishable in the output: + +| Cause | Unit | What it means | Is it a defect? | +| ------------------ | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `receiverTyping` | call sites | Call sites dropped because the analyzer could not type the receiver | **Yes** — a resolver gap. This is the population this whole series targets. | +| `dispatchBoundary` | symbols | The symbol sits behind an interface with real consumers or 2+ implementations; the number is the implementations plus interface-level consumers behind it | **No** — callers binding through DI or dynamic dispatch are genuinely untraceable statically. A compiler refuses here too. | +| `externalBoundary` | call sites | The call left the indexed program (`System.out.println`, `fetch(...)`) | **No**, and not even a shortfall — there is no in-graph node an edge could have reached. An `epistemic: 'exact'` result can carry it. | + +Both collapsed into one enum plus prose, so a consumer — especially a coding +agent gating its own edits on the result — could tell THAT a count was short but +not WHY, and could not branch on the difference. Worse, it made "the hedge should +stop appearing" unfalsifiable: with no way to see which producer fired, there was +no way to check whether fixing receiver typing had done anything. + +`impact` and `context` now carry a structured `causes: { receiverTyping, +dispatchBoundary, externalBoundary }` alongside the prose. Every field counts +MISSING THINGS, never notes: there is one note per symbol name (and one per +boundary node) but each reports N of something, so counting notes published `1` +next to prose reading "2 call sites", and a consumer branching on the number +would have read a different magnitude than the human reading the text. The same +rule applies to `dispatchBoundary`, which counts the implementations plus +interface-level consumers behind the boundary rather than the boundary sentences +— one sentence can describe an interface with 40 implementations. Its unit is +SYMBOLS rather than call sites because per-site multiplicity is not retained on +those edges (consumers are counted `DISTINCT`, and +`collapseMemberCallsByCallerTarget` languages emit one CALLS edge per +caller/target pair); the units are stated per field on `EpistemicCauses` so a +consumer knows which it is holding. + +**Only the `receiverTyping` producer is addressed by this series.** The dispatch +boundary is untouched and will keep firing for interface-dispatched symbols — +which is correct. Any claim that the hedge has "stopped appearing" has to be read +per-producer, and that is now possible. + +Measured on the #2766 reproduction: `WithTx` went from `impactedCount: 0` with a +`lower-bound` hedge to `impactedCount: 1` with `epistemic: exact`. The hedge is +gone there because its cause is gone, not because it was suppressed. + +--- + +## U10 — recorded drops, censused by receiver shape + +`ResolutionOutcome`'s suppressed variant now carries `receiverShape`, set by the +emitting case from the site's ENCODED CHAIN — the compact string the capture +emitters mint by walking the real AST. Never re-derived from the source line: +doing that would mean regex-classifying the number that gates this work, the +same textual-shape dispatch the structural-receiver line exists to remove. +Diagnostic only, so the persisted `RepoMeta.unresolvedReceiverMembers` artifact +is unchanged. + +Census of the call drops on the committed fixture corpus, **as measured at U10** +— it predates the phantom-read fix documented above, which reclassified one drop +`chain-field` → `chain-unwrap`. `callDropsByShape` in `baseline.json` is current: + +| Shape | Count | Share | +| ------------------------------------------------------------- | ----- | ----- | +| `chain-field` — every step a field (`h.repo.save()`) | 60 | 59% | +| `chain-call` — every step a call (`svc.getUser().save()`) | 27 | 27% | +| `no-chain` — no chain minted; the walk found no nameable base | 12 | 12% | +| `chain-mixed` — interleaved (`svc.getUser().addr.save()`) | 2 | 2% | + +Two decisions come out of it. + +**The `.java` bucket is not one defect.** Its 49 call drops split 30 field-chain +/ 14 call-chain / 5 no-chain, so the open question of whether Java's largest- +single-bucket status hides a single cause is answered: it does not. It is the +same population as everywhere else, just more of it. + +**Field-receiver chains are where the remaining value is.** At 59% of the U10 +census they dominate, and they are precisely the shape U1 fixed for Go. The same +defect class in java, csharp, cpp, php, py and rust is the largest addressable +population the count arm can see. (This paragraph used to quote a per-extension × +per-shape split from the U10 run. `baseline.json` carries `callDropsByExtension` +and `callDropsByShape` but not their cross-product, so that split has to be +re-derived from a fresh run rather than read off the committed baseline.) + +**What this census CANNOT justify.** Await-wrapped and subscript receivers barely +appear, because the committed fixture corpus contains almost no such sites — not +because they are rare in real code. At U10 `indexElement` was a gap in every +language in the shape arm, so U5's population was real but structurally invisible +to the count arm. (It no longer is uniform — the subscript route resolves in +several languages now; read the current per-language state from `indexElement` in +`baseline.json`, not from this paragraph.) The durable point: any decision to fund +or drop U4 and U5 has to be read off the SHAPE arm, because reading it off this +census confuses "absent from these fixtures" with "does not happen". + +## U2 — shape matrix expanded to a canonical axis + +The shape arm was three languages with an ad-hoc shape list each. It is now a +**canonical 10-shape axis** (`SHAPE_IDS`) that every language must answer for, +with two states added so a hole cannot masquerade as a measurement: + +- `N/A` — the grammar does not admit this spelling. **A reason is required.** An + omitted cell and a genuinely inapplicable cell look identical in a diff + otherwise, which is how coverage rots. +- `GRAMMAR-UNAVAILABLE` — the parser could not be loaded, so nothing was + measured. Neither passes nor fails the gate, and `drift` skips it on **both** + sides so the gate cannot fail for the environment it ran in. `tree-sitter-dart`, + `-kotlin` and `-swift` are vendored _optional_ grammars: absent when a run sets + `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`, and soft-failing when no vendored prebuild + matches the host (the set covers darwin/linux arm64+x64 and win32-arm64 — a + win32-x64 or musl host has none). **All 14 load on a glibc linux-x64 host, so + this state has no producer in the committed baseline** — it guards the + skip-flag and unsupported-host cases rather than a condition seen here. + +`assertMatrixComplete` throws when a language omits a cell, declares an unknown +id, or writes an `N/A` with no reason. Languages may declare `extraShapeIds` for +diagnostics the canonical axis cannot express (PHP's annotated/unannotated +return-type pair, C++'s pointer/value base pair) — an extra must be declared, so +it stays a deliberate diagnostic rather than a typo'd canonical id. + +**Vue and COBOL** are language-level `N/A` rows: their emitters never call +`synthesizeReceiverChainCapture`, so there is nothing to measure — but the +language axis now obeys the same no-omitted-cells rule as the shape axis. + +### What the first expanded run found + +Three results that redirected the plan they were built to serve. **Snapshot: the +first U2 run, before any of the fixes below landed** — these cells state the +problem, and several have since flipped (`baseline.json` is current): + +**Go — the root cause, isolated to one cell.** Three rows vary receiver +decoration and field decoration independently: + +| Cell | Receiver | Field | State | +| ----------------------- | ----------- | ----------- | --------------- | +| `fieldReceiverCall` | value | value | RESOLVES | +| `decoratedFieldType` | value | **pointer** | RESOLVES | +| `decoratedReceiverBase` | **pointer** | value | **VISIBLE-GAP** | + +Only the pointer _receiver_ fails. Go already normalizes field type bindings +through `normalizeGoTypeName`, so the step lookup is sound and the defect is +entirely the base — `synthesizeGoReceiverBinding` stores `typeNode.text` raw, so +`func (h *Host)` binds `h` to the literal `*Host`, which +`findClassBindingInScope` cannot resolve. + +**PHP — the sigil hypothesis is dead.** The two rows differ only in whether the +called method declares a return type: + +| Cell | Return type | State | +| --------------------------------------------- | ------------- | ------------- | +| `arrowCallChain` — `$svc->getUser()->save()` | unannotated | INVISIBLE-GAP | +| `plainChain` — `$svc->getUserTyped()->save()` | **annotated** | **RESOLVES** | + +Same chain, same `->`, same base. PHP chains resolve when the return type is +declared; the `$` sigil is not involved. `decoratedFieldType` (`?User $repo`) +also resolves, so PHP nullable field types already work. + +**C++ — the base already resolves, but `this->` field receivers do not.** +`pointerArrowChain` and `valueDotChain` both RESOLVE, so a decorated C++ base is +not a gap. But `this->repo.save()` and `this->repo->save()` are both +INVISIBLE-GAP — a distinct defect, not a decoration one. + +**Rust — the decorated receiver is NOT a gap.** `&mut self` resolves, so Go is +the only language whose method receiver decoration defeats the lookup. Rust's +gap is the field: `Box` is INVISIBLE-GAP. + +### The decoration cells, across all 14 + +The rows U1 exists to fix. Everything else is a different defect. **Snapshot: as +measured at U2, i.e. BEFORE U1 landed** — it is the statement of the problem, not +of the current state. Go's `decoratedReceiverBase` and TypeScript's +`decoratedFieldType` have since moved; `baseline.json` has the live cells. + +| Language | `decoratedReceiverBase` | `decoratedFieldType` | +| ------------------------- | ------------------------- | ---------------------------------- | +| go | **VISIBLE-GAP** (`*Host`) | RESOLVES | +| rust | RESOLVES (`&mut self`) | **INVISIBLE-GAP** (`Box`) | +| typescript | N/A | **INVISIBLE-GAP** (`User \| null`) | +| csharp | N/A | **VISIBLE-GAP** (`User?`) | +| swift | N/A | **INVISIBLE-GAP** (`User?`) | +| cpp | N/A | **INVISIBLE-GAP** (`User*`) | +| python, php, kotlin, dart | N/A | RESOLVES | +| java, c, javascript, ruby | N/A | N/A | + +So U1's measured scope is **Go's receiver base**, plus the field-type gap in +**Rust, TypeScript, C#, Swift and C++** — and _not_ PHP, Python, Kotlin, Dart or +Java, whose decoration handling already works or does not exist. Five of the +seven hooks the plan speculatively listed were aimed at languages that need +none; three languages that do need one were not on the list at all. + +### Other gaps this run surfaced, not in the plan + +- **Swift resolves almost nothing.** `plainChain`, `plainDeepChain`, + `optionalChain` and `nonNullAssert` are all INVISIBLE-GAP, while + `fieldReceiverCall` resolves. Chained receivers are essentially unsupported. +- **Ruby chains are VISIBLE-GAPs** (`plainChain`, `plainDeepChain`, + `optionalChain`) and `fieldReceiverCall` on `@repo` is INVISIBLE. +- **C++ `this->` field receivers** are INVISIBLE-GAP in both the value and + pointer form. +- **C# has four gaps** beyond the field one: `optionalChain`, `nonNullAssert`, + `awaitParen`, `explicitTypeArgs`. +- **Dart `await` already resolves** — the only language where `awaitParen` is + green, which makes it the reference for U4's unwrap direction. +- **`indexElement` was INVISIBLE-GAP in all 14** at U2 — uniform, and exactly what + U5 targets. (Superseded: several languages resolve it now; see `baseline.json`.) + +### Coverage status + +All 14 languages measured, plus `vue` and `cobol` as language-level `N/A` rows. +The cell tally recorded at U2 was 164 cells / 42 RESOLVES / 22 VISIBLE-GAP / 31 +INVISIBLE-GAP / 69 N/A / 0 GRAMMAR-UNAVAILABLE — a snapshot, superseded by every +unit since (the axis also gained TypeScript's declared `fourHopChain` extra). +Count the states off `baseline.json` rather than quoting this line. + +The **count arm did not move when the shape axis was expanded** — shape fixtures +are built in temp directories and never touch the committed corpus, so expanding +the shape axis moves the shape arm only. + +--- + > **Updated after U10** (structural receiver typing wired into Case 0). Three > TypeScript shapes flipped to `RESOLVES` — `svc?.getUser().save()`, > `svc!.getUser().save()`, `svc.getTyped().save()` — and the call-drop @@ -11,12 +436,18 @@ > change as "no improvement" and stopped the series. Nothing regressed: no edge > was lost and no new drop appeared. > -> Still gaps after U10, both genuine: -> - `(await svc.getUserAsync()).save()` — `extractMixedChain` reaches -> `await …`, which is not a chain node, so no chain is minted. Remains a -> VISIBLE-GAP and is now the call-kind fixture in the drop-recorder test. -> - `repos[0].save()` — Case 0's punctuation gate never fires for a subscript -> receiver, so it stays INVISIBLE. +> Two gaps remained open **at U10**, both genuine at the time: +> +> - `(await svc.getUserAsync()).save()` — `extractMixedChain` reached `await …`, +> which is not a chain node, so no chain was minted. It was a VISIBLE-GAP and is +> the call-kind fixture in the drop-recorder test. +> - `repos[0].save()` — Case 0's punctuation gate never fired for a subscript +> receiver, so it was INVISIBLE. +> +> Both were subsequently closed for TypeScript by the `await`/`index` step kinds +> (wire format v2) and by Case 0's third gate arm, which admits any site carrying +> a minted chain regardless of receiver punctuation. Per-language state is in +> `baseline.json` — `awaitParen` and `indexElement`. > > The tables below are the pre-U10 measurement, kept as the reference point. @@ -27,15 +458,15 @@ A/B produced by reverting ONLY the fold wiring (`compound-receiver.ts` + emission — and therefore the persisted bytes — is identical in both arms and the delta isolates the fold. Build + both caches wiped before every run (KTD4). -| Metric | Control | Treatment | Δ | Threshold | Verdict | -|---|---|---|---|---|---| -| scope-resolution wall-clock, median of 3 | 25470.0 ms | 25687.9 ms | +0.86% | ≤ +3% | **PASS** | -| wall-clock, slowest of 3 | 25520.0 ms | 25832.6 ms | +1.22% | ≤ +5% p95 | **PASS** | -| serialized bytes per emitting site | — | **35.2 B** | — | ≤ 48 B | **PASS** | -| persisted store growth | 1 234 600 B | 1 235 340 B | **+0.0599%** | ≤ 3% | **PASS** | -| retained chain payload | — | 740 B | — | ≤ 6 MB | **PASS** | -| call drops (no regression) | 99 | 99 | 0 | no new drops | **PASS** | -| peak RSS | — | — | — | ≤ +2% | **NOT RESOLVABLE** | +| Metric | Control | Treatment | Δ | Threshold | Verdict | +| ---------------------------------------- | ----------- | ----------- | ------------ | ------------ | ------------------ | +| scope-resolution wall-clock, median of 3 | 25470.0 ms | 25687.9 ms | +0.86% | ≤ +3% | **PASS** | +| wall-clock, slowest of 3 | 25520.0 ms | 25832.6 ms | +1.22% | ≤ +5% p95 | **PASS** | +| serialized bytes per emitting site | — | **35.2 B** | — | ≤ 48 B | **PASS** | +| persisted store growth | 1 234 600 B | 1 235 340 B | **+0.0599%** | ≤ 3% | **PASS** | +| retained chain payload | — | 740 B | — | ≤ 6 MB | **PASS** | +| call drops (no regression) | 99 | 99 | 0 | no new drops | **PASS** | +| peak RSS | — | — | — | ≤ +2% | **NOT RESOLVABLE** | **The 35.2 B result confirms KTD7 by measurement rather than by assertion.** The 48-byte threshold was set deliberately so the object encoding (~71 B predicted) @@ -43,7 +474,7 @@ fails and the compact string (~35 B predicted) passes. Measured: 35.2 B, including the JSON key and quotes. The encoding decision is now evidence-backed. **Peak RSS: the threshold is below this instrument's resolution, so it is -reported as unresolvable rather than as a pass or a fail.** Three *independent* +reported as unresolvable rather than as a pass or a fail.** Three _independent_ treatment runs with the code held constant gave 414.9 / 436.6 / 436.9 MB — a 5.3% spread, wider than the ±2% being tested. (An earlier pair of 3-reps-in-one- process runs read 536 vs 551 MB and looked like a +2.77% regression; that was @@ -82,26 +513,32 @@ Two consecutive runs were byte-identical, not merely within noise. ## Count arm — `test/fixtures/lang-resolution` -| Metric | Value | -|---|---| -| **Call drops (the gate number)** | **99** | -| Total drops, all site kinds | 124 | -| Split by site kind | `call: 99`, `read: 25` | +**Snapshot: the U7-era measurement (commit `f87b2cbe`), kept as the reference +point for the A/B above.** The gate enforces `countArm` in `baseline.json`, which +has moved since — read the live call-drop number, site-kind split, and +per-extension breakdown from there. -Call drops by extension: +| Metric | Value at U7 | +| -------------------------------- | ---------------------- | +| **Call drops (the gate number)** | **99** | +| Total drops, all site kinds | 124 | +| Split by site kind | `call: 99`, `read: 25` | -| ext | n | ext | n | ext | n | -|---|---|---|---|---|---| -| `.java` | 49 | `.py` | 5 | `.rs` | 3 | -| `.cs` | 8 | `.go` | 5 | `.kt` | 3 | -| `.ts` | 7 | `.cpp` | 5 | `.rb` | 2 | -| `.tsx` | 6 | `.php` | 4 | `.js` | 1 | -| | | | | `.swift` | 1 | +Call drops by extension, at U7: -**Why the split matters (KTD6 defect 1, now measured).** 25 of the 124 drops — 20% — -are property *reads*, not lost calls. Case 0's recorder gates on the receiver's +| ext | n | ext | n | ext | n | +| ------- | --- | ------ | --- | -------- | --- | +| `.java` | 49 | `.py` | 5 | `.rs` | 3 | +| `.cs` | 8 | `.go` | 5 | `.kt` | 3 | +| `.ts` | 7 | `.cpp` | 5 | `.rb` | 2 | +| `.tsx` | 6 | `.php` | 4 | `.js` | 1 | +| | | | | `.swift` | 1 | + +**Why the split matters (KTD6 defect 1, now measured).** About a fifth of the +drops are property _reads_, not lost calls (25 of 124 at U7; `bySiteKind` in +`baseline.json` is current). Case 0's recorder gates on the receiver's punctuation, not on what the reference is, so `d.source.kind` lands in the same -bucket as a dropped method call. Gating on the unsplit 124 would have measured a +bucket as a dropped method call. Gating on the unsplit total would have measured a population one fifth of which this work does not target. ## Shape arm @@ -110,19 +547,24 @@ population one fifth of which this work does not target. name-keyed fallback onto a same-named member reads as `RESOLVES`, so a shape whose receiver has no well-defined type is not a usable control. -| Language | Shape | State | siteKind | -|---|---|---|---| -| TypeScript | `svc.getUser().save()` | RESOLVES | — | -| TypeScript | `svc.getUser().address.save()` | RESOLVES | — | -| TypeScript | `svc?.getUser().save()` | **INVISIBLE-GAP** | — | -| TypeScript | `svc!.getUser().save()` | VISIBLE-GAP | `call` | -| TypeScript | `(await svc.getUserAsync()).save()` | VISIBLE-GAP | `call` | -| TypeScript | `svc.getTyped().save()` | **INVISIBLE-GAP** | — | -| TypeScript | `repos[0].save()` | **INVISIBLE-GAP** | — | -| PHP | `$svc->getUser()->save()` | VISIBLE-GAP | `call` | -| PHP | `$this->repo->save()` (typed property) | RESOLVES | — | -| C++ | `svc->getUser()->save()` | **INVISIBLE-GAP** | — | -| C++ | `svc2.getUser()->save()` | RESOLVES | — | +**Snapshot: the pre-U10 measurement over three languages**, kept because it is the +evidence that the shape arm moves when the count arm does not. Superseded twice — +by U8's rollout table above and by the canonical shape axis in `baseline.json`. +The three TypeScript rows marked as gaps here (`?.`, `!`, ``) all resolve now. + +| Language | Shape | State at pre-U10 | siteKind | +| ---------- | -------------------------------------- | ----------------- | -------- | +| TypeScript | `svc.getUser().save()` | RESOLVES | — | +| TypeScript | `svc.getUser().address.save()` | RESOLVES | — | +| TypeScript | `svc?.getUser().save()` | **INVISIBLE-GAP** | — | +| TypeScript | `svc!.getUser().save()` | VISIBLE-GAP | `call` | +| TypeScript | `(await svc.getUserAsync()).save()` | VISIBLE-GAP | `call` | +| TypeScript | `svc.getTyped().save()` | **INVISIBLE-GAP** | — | +| TypeScript | `repos[0].save()` | **INVISIBLE-GAP** | — | +| PHP | `$svc->getUser()->save()` | VISIBLE-GAP | `call` | +| PHP | `$this->repo->save()` (typed property) | RESOLVES | — | +| C++ | `svc->getUser()->save()` | **INVISIBLE-GAP** | — | +| C++ | `svc2.getUser()->save()` | RESOLVES | — | ## Corrections to the plan, forced by measurement @@ -142,11 +584,11 @@ receiver has no well-defined type is not a usable control. `@reference.call.member`, `@reference.name`, and crucially `@reference.receiver`: - | Shape | `@reference.receiver` | - |---|---| - | `svc?.getUser().save()` | `svc?.getUser()` | + | Shape | `@reference.receiver` | + | ----------------------------- | ---------------------- | + | `svc?.getUser().save()` | `svc?.getUser()` | | `svc.getTyped().save()` | `svc.getTyped()` | - | `repos[0].save()` | `repos[0]` | + | `repos[0].save()` | `repos[0]` | So a `ReferenceSite` exists for every one of them, and hanging a `receiverChain` field on `ReferenceSite` is a viable carrier for all of them. @@ -155,7 +597,7 @@ receiver has no well-defined type is not a usable control. The drop suppression is therefore downstream of capture. For `repos[0]` the cause is known and matches the plan: the receiver has neither `.` nor `(`, so Case 0's gate never fires. For `?.` and `` the receiver text satisfies the - gate, so Case 0 *does* run and one of two things happens — the site was marked + gate, so Case 0 _does_ run and one of two things happens — the site was marked in `handledSites` by another case, or `resolveCompoundReceiverClass` returned a class on which the member was then not found, leaving `compoundReceiverUnresolved` false. Those are materially different defects and @@ -163,13 +605,13 @@ receiver has no well-defined type is not a usable control. establish, since the second would mean the recorder under-reports by mis-attribution rather than by a gate. - *(An earlier revision of this file asserted that these shapes produce no + _(An earlier revision of this file asserted that these shapes produce no reference site at all. That was inferred from edge-and-drop absence and is - disproven by the capture dump above.)* + disproven by the capture dump above.)_ 3. **KTD6 defect 2 overstates the PHP blindness.** The claim is that Case 0's C-family punctuation test means PHP `->` receivers "never record a drop at - all". Measured, `$svc->getUser()->save()` *is* recorded, because its receiver + all". Measured, `$svc->getUser()->save()` _is_ recorded, because its receiver text `$svc->getUser()` contains `(` and satisfies the gate. And the plan's own example, `$this->repo->save()`, does not need recording — with a typed property it resolves. The genuine PHP gap is the call chain, and it is already visible. @@ -182,12 +624,26 @@ receiver has no well-defined type is not a usable control. ## Known blind spots Every count here is a lower bound on a known-biased population, and any later delta -must be read against the same bias. +must be read against the same bias. Kept in sync with `KNOWN_BLIND` in +`measure.mjs`, which prints these on every run. -- Case 0's gate is a C-family punctuation test (`.` or `(`), so a receiver that is a - plain property path (`$this->repo`, `a::b`) never reaches the recorder. -- `repos[0].save()` has neither `.` nor `(` in its receiver — same result. -- `?.` and explicit type arguments produce no reference site at all. +- Case 0 is reached by a receiver-TEXT punctuation test (`.` or `(`) **or** by a + minted receiver chain. A receiver spelled without that punctuation — a subscript + `repos[0]`, a PHP `->` / `::` property path — therefore reaches the recorder only + where its emitter mints a chain. Where no chain is minted, the call still + vanishes with the instrument blind to it. +- A drop is recorded only while `compoundReceiverUnresolved` stays true. When the + cascade TYPES the receiver but then finds no member on it, the flag is false and + no drop is recorded even though no edge was emitted. So an absent drop is not + evidence a site resolved — the recorder can under-report by mis-attribution, not + only by a gate. (This is what moved PHP's `arrowCallChain` from VISIBLE-GAP to + INVISIBLE-GAP when its fixture parameter was typed; see U8 below.) +- Retracted, and left here because it was quoted for several units: the earlier + claim that `?.` and explicit type arguments _"produce no reference site at all"_. + They do — the capture dump under "Corrections to the plan" §2 shows a full call + match with `@reference.receiver` for all three of `svc?.getUser()`, + `svc.getTyped()` and `repos[0]`. The absence was of an EDGE and of a DROP, + never of a site. ## U8 — per-language rollout @@ -200,15 +656,15 @@ match, an absent receiver, or a chain with no nameable base all leave the match untouched — so inserting the call before every `out.push(grouped)` is safe even in the emitters that have three or four such paths. -| Language | Shape | Before | After | -|---|---|---|---| -| TypeScript | `svc?.getUser().save()` | INVISIBLE-GAP | **RESOLVES** | -| TypeScript | `svc!.getUser().save()` | VISIBLE-GAP | **RESOLVES** | -| TypeScript | `svc.getTyped().save()` | INVISIBLE-GAP | **RESOLVES** | -| C++ | `svc->getUser()->save()` | INVISIBLE-GAP | **RESOLVES** | -| C++ | `svc2.getUser()->save()` (control) | RESOLVES | RESOLVES | -| PHP | `$svc->getUser()->save()` | VISIBLE-GAP | INVISIBLE-GAP | -| PHP | `$this->repo->save()` (control) | RESOLVES | RESOLVES | +| Language | Shape | Before | After | +| ---------- | ---------------------------------- | ------------- | ------------- | +| TypeScript | `svc?.getUser().save()` | INVISIBLE-GAP | **RESOLVES** | +| TypeScript | `svc!.getUser().save()` | VISIBLE-GAP | **RESOLVES** | +| TypeScript | `svc.getTyped().save()` | INVISIBLE-GAP | **RESOLVES** | +| C++ | `svc->getUser()->save()` | INVISIBLE-GAP | **RESOLVES** | +| C++ | `svc2.getUser()->save()` (control) | RESOLVES | RESOLVES | +| PHP | `$svc->getUser()->save()` | VISIBLE-GAP | INVISIBLE-GAP | +| PHP | `$this->repo->save()` (control) | RESOLVES | RESOLVES | The C++ row is the one the plan flagged as having **no fixture anywhere** — `cpp-chain-call/` uses the value `.` form, which already worked. It now has one, @@ -225,6 +681,10 @@ the emitter: name=save chain=1|$svc|cgetUser recv=$svc.getUser() ``` +The leading `1` is the **v1** wire prefix current when this dump was taken; the +codec is at v2 now (`2|$svc|cgetUser`), and a v2 decoder refuses a v1 payload by +design — do not copy this literal into a fixture. + The chain is minted correctly. The residual is that the fold's base, `$svc`, does not bind in the PHP resolver, so the fold returns `undefined` and the site falls through to the text cascade. That is PHP binding-key work, not a diff --git a/gitnexus/bench/receiver-resolution/baseline.json b/gitnexus/bench/receiver-resolution/baseline.json index 904bc658b..31d2933ef 100644 --- a/gitnexus/bench/receiver-resolution/baseline.json +++ b/gitnexus/bench/receiver-resolution/baseline.json @@ -1,28 +1,208 @@ { "shapeArm": { + "vue": { + "plainChain": "N/A", + "plainDeepChain": "N/A", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "N/A", + "indexElement": "N/A", + "fieldReceiverCall": "N/A", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "N/A" + }, + "cobol": { + "plainChain": "N/A", + "plainDeepChain": "N/A", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "N/A", + "indexElement": "N/A", + "fieldReceiverCall": "N/A", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "N/A" + }, "typescript": { "plainChain": "RESOLVES", "plainDeepChain": "RESOLVES", "optionalChain": "RESOLVES", "nonNullAssert": "RESOLVES", - "awaitParen": "VISIBLE-GAP", + "awaitParen": "RESOLVES", "explicitTypeArgs": "RESOLVES", - "indexElement": "INVISIBLE-GAP" + "indexElement": "RESOLVES", + "fourHopChain": "RESOLVES", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "RESOLVES" }, "php": { "arrowCallChain": "INVISIBLE-GAP", - "arrowPropertyPath": "RESOLVES" + "arrowPropertyPath": "RESOLVES", + "plainChain": "RESOLVES", + "plainDeepChain": "INVISIBLE-GAP", + "optionalChain": "INVISIBLE-GAP", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "N/A", + "indexElement": "VISIBLE-GAP", + "fieldReceiverCall": "N/A", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "RESOLVES" }, "cpp": { "pointerArrowChain": "RESOLVES", - "valueDotChain": "RESOLVES" + "valueDotChain": "RESOLVES", + "plainChain": "N/A", + "plainDeepChain": "RESOLVES", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "VISIBLE-GAP", + "indexElement": "RESOLVES", + "fieldReceiverCall": "INVISIBLE-GAP", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "INVISIBLE-GAP" + }, + "go": { + "plainChain": "RESOLVES", + "plainDeepChain": "RESOLVES", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "N/A", + "indexElement": "RESOLVES", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "RESOLVES", + "decoratedFieldType": "RESOLVES" + }, + "javascript": { + "plainChain": "VISIBLE-GAP", + "plainDeepChain": "VISIBLE-GAP", + "optionalChain": "VISIBLE-GAP", + "nonNullAssert": "N/A", + "awaitParen": "VISIBLE-GAP", + "explicitTypeArgs": "N/A", + "indexElement": "VISIBLE-GAP", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "N/A" + }, + "python": { + "plainChain": "RESOLVES", + "plainDeepChain": "RESOLVES", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "RESOLVES", + "explicitTypeArgs": "N/A", + "indexElement": "RESOLVES", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "RESOLVES" + }, + "java": { + "plainChain": "RESOLVES", + "plainDeepChain": "RESOLVES", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "RESOLVES", + "indexElement": "VISIBLE-GAP", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "N/A" + }, + "csharp": { + "plainChain": "RESOLVES", + "plainDeepChain": "RESOLVES", + "optionalChain": "INVISIBLE-GAP", + "nonNullAssert": "VISIBLE-GAP", + "awaitParen": "RESOLVES", + "explicitTypeArgs": "VISIBLE-GAP", + "indexElement": "VISIBLE-GAP", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "VISIBLE-GAP" + }, + "ruby": { + "plainChain": "VISIBLE-GAP", + "plainDeepChain": "VISIBLE-GAP", + "optionalChain": "VISIBLE-GAP", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "N/A", + "indexElement": "INVISIBLE-GAP", + "fieldReceiverCall": "INVISIBLE-GAP", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "N/A" + }, + "rust": { + "plainChain": "RESOLVES", + "plainDeepChain": "RESOLVES", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "INVISIBLE-GAP", + "indexElement": "RESOLVES", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "RESOLVES", + "decoratedFieldType": "INVISIBLE-GAP" + }, + "c": { + "plainChain": "N/A", + "plainDeepChain": "VISIBLE-GAP", + "optionalChain": "N/A", + "nonNullAssert": "N/A", + "awaitParen": "N/A", + "explicitTypeArgs": "N/A", + "indexElement": "VISIBLE-GAP", + "fieldReceiverCall": "INVISIBLE-GAP", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "N/A" + }, + "kotlin": { + "plainChain": "RESOLVES", + "plainDeepChain": "RESOLVES", + "optionalChain": "RESOLVES", + "nonNullAssert": "VISIBLE-GAP", + "awaitParen": "RESOLVES", + "explicitTypeArgs": "RESOLVES", + "indexElement": "RESOLVES", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "RESOLVES" + }, + "swift": { + "plainChain": "INVISIBLE-GAP", + "plainDeepChain": "INVISIBLE-GAP", + "optionalChain": "INVISIBLE-GAP", + "nonNullAssert": "INVISIBLE-GAP", + "awaitParen": "VISIBLE-GAP", + "explicitTypeArgs": "N/A", + "indexElement": "INVISIBLE-GAP", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "INVISIBLE-GAP" + }, + "dart": { + "plainChain": "VISIBLE-GAP", + "plainDeepChain": "VISIBLE-GAP", + "optionalChain": "VISIBLE-GAP", + "nonNullAssert": "VISIBLE-GAP", + "awaitParen": "RESOLVES", + "explicitTypeArgs": "N/A", + "indexElement": "INVISIBLE-GAP", + "fieldReceiverCall": "RESOLVES", + "decoratedReceiverBase": "N/A", + "decoratedFieldType": "RESOLVES" } }, "countArm": { - "callDrops": 101, - "totalDropsAllKinds": 128, + "callDrops": 102, + "totalDropsAllKinds": 129, "bySiteKind": { - "call": 101, + "call": 102, "read": 27 }, "callDropsByExtension": { @@ -34,11 +214,23 @@ ".py": 5, ".go": 5, ".php": 4, + ".kt": 4, ".rs": 3, - ".kt": 3, ".rb": 2, ".js": 1, ".swift": 1 + }, + "callDropsByShape": { + "chain-field": 60, + "chain-call": 27, + "no-chain": 12, + "chain-mixed": 2, + "chain-unwrap": 1 + }, + "callDropsByOrigin": { + "external": 44, + "in-program": 36, + "unknown": 22 } } } diff --git a/gitnexus/bench/receiver-resolution/measure.mjs b/gitnexus/bench/receiver-resolution/measure.mjs index 7a279ea87..8cfb8e9f5 100644 --- a/gitnexus/bench/receiver-resolution/measure.mjs +++ b/gitnexus/bench/receiver-resolution/measure.mjs @@ -7,8 +7,9 @@ * * TWO ARMS, because neither one alone is trustworthy: * - * 1. SHAPE ARM (`--shapes`, default). A fixed corpus of receiver spellings, - * each classified by what the graph actually contains: + * 1. SHAPE ARM (`--shapes`, default). A fixed matrix of receiver spellings — + * every SHAPE_ID against every language — each cell classified by what the + * graph actually contains: * * RESOLVES edge emitted. A test written against this shape starts * green and proves nothing. Note this states only that an @@ -20,13 +21,35 @@ * Measurable by the count arm below. * INVISIBLE-GAP no edge, and NO drop was recorded. The call is lost and * the instrument cannot see it. + * N/A the language's grammar does not admit this spelling. + * Carries a REQUIRED reason — an omitted cell and a + * genuinely inapplicable one look identical in a diff + * otherwise, and that is how coverage silently rots. + * GRAMMAR- the language's parser could not be loaded, so nothing + * UNAVAILABLE about it was measured. Neither passes nor fails the + * gate. Dart, Kotlin and Swift are vendored OPTIONAL + * grammars: they are skipped entirely by + * GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1, and they soft-fail + * when no vendored prebuild matches the host (the set + * covers darwin/linux arm64+x64 and win32-arm64 — a + * win32-x64 or musl host has none). Without this state an + * unloaded grammar emits no edges and every one of its + * cells reads as a resolution gap indistinguishable from a + * real regression. + * NOTE: all 14 load on a glibc linux-x64 host, so this + * state has no producer in the committed baseline. It + * guards the skip-flag and unsupported-host cases. * - * The third state is why this arm exists. Case 0's recorder is reached only - * when the receiver text contains `.` or `(` AND the capture layer produced - * a reference site at all. Measured on this corpus, `svc?.getUser().save()`, - * `svc.getTyped().save()` and `repos[0].save()` are all INVISIBLE — - * so fixing them moves the count arm by exactly zero. Gating solely on a - * drop count would read a working fix as "no improvement". + * INVISIBLE-GAP is why this arm exists, and the series proved it: at the + * time this arm was built `svc?.getUser().save()`, `svc.getTyped().save()` + * and `repos[0].save()` were all INVISIBLE, so fixing them moved the count + * arm by exactly zero and a gate reading only the drop count would have + * scored a working change as "no improvement". They resolve for TypeScript + * now; the current per-shape state is `baseline.json`, never this comment. + * + * COMPLETENESS IS ENFORCED: every language must declare a cell for every + * SHAPE_ID, and every N/A must carry a reason. A missing cell fails the run + * rather than being silently skipped. * * 2. COUNT ARM (`--corpus `). Runs the real pipeline over a repo and * reports drops SPLIT BY SITE KIND. The gate number is `call` only: the @@ -38,10 +61,16 @@ * is a lower bound on a KNOWN-BIASED population and any delta measured later * must be read against the same bias: * - * - Case 0's gate is a C-family punctuation test, so PHP `$this->repo->save()` - * and `::` receivers never record a drop. - * - `repos[0].save()` has neither `.` nor `(` in its receiver, same result. - * - `?.` and explicit type arguments produce no reference site to begin with. + * - Case 0 is reached by a receiver-TEXT punctuation test (`.` or `(`) OR by a + * minted receiver chain, so a receiver spelled without that punctuation — a + * subscript, PHP `->`/`::` — records a drop only where its emitter mints a + * chain. Where none is minted the call still vanishes unrecorded. + * - A drop is recorded only while `compoundReceiverUnresolved` stays true: if + * the cascade types the receiver but finds no member, no drop is recorded + * even though no edge was emitted, so an absent drop is not proof of + * resolution. (Superseded claim, kept because it was quoted for several + * units: `?.` and explicit type arguments DO produce a reference site — + * what was absent was the edge and the drop, never the site.) * * MEASUREMENT HYGIENE — a run that skips this is void: * @@ -74,21 +103,65 @@ const DEFAULT_CORPUS = path.resolve(__dirname, '..', '..', 'test', 'fixtures', ' // Shape corpus // --------------------------------------------------------------------------- +/** + * The canonical shape axis. Every language corpus must declare a cell for every + * id here — a body to measure, or `na` with a reason. `assertMatrixComplete` + * enforces it, so adding a shape forces every language to answer for it. + */ +const SHAPE_IDS = [ + 'plainChain', + 'plainDeepChain', + 'optionalChain', + 'nonNullAssert', + 'awaitParen', + 'explicitTypeArgs', + 'indexElement', + 'fieldReceiverCall', + 'decoratedReceiverBase', + 'decoratedFieldType', +]; + +/** + * Languages that carry no receiver-chain emission at all, recorded as + * language-level rows so the language axis obeys the same no-omitted-cells rule + * as the shape axis. Their capture emitters do not call + * `synthesizeReceiverChainCapture`, so there is nothing to measure — but an + * absent row and an inapplicable row must not look alike in a diff. + */ +const NA_LANGUAGES = { + vue: 'SFC templates delegate to the TypeScript/JavaScript emitters; no receiver chain is minted for the .vue file itself.', + cobol: 'No method-call receiver syntax; the emitter mints no receiver chains.', +}; + /** * Each entry is one receiver spelling. `entry` is the function that contains * it; `member` is the method it should reach. Classification asks only two * questions of the result — is there a CALLS edge from `entry` to `member`, * and was a drop recorded on that line — so it never guesses from an id shape. + * + * An entry with `na` declares the shape inapplicable to this language and is + * never compiled into the fixture. */ const CORPORA = [ { lang: 'typescript', ext: '.ts', + // `fourHopChain` exists to make MAX_CHAIN_DEPTH answerable. Without a chain + // longer than the cap, raising the cap measures nothing and the question + // "what does depth N buy?" has no instrument behind it. + extraShapeIds: ['fourHopChain'], support: { - 'models.ts': `export class Address { + 'models.ts': `export class City { save(): void {} } +export class Address { + save(): void {} + getCity(): City { + return new City(); + } +} + export class User { name: string = ''; address: Address = new Address(); @@ -129,29 +202,99 @@ export class Service { }, { id: 'explicitTypeArgs', member: 'save', body: 'svc.getTyped().save();', note: 'PF4' }, { id: 'indexElement', member: 'save', body: 'repos[0].save();', note: 'PF5' }, + { + // FOUR STEPS in the receiver of `save`: getSvc, getUser, address, + // getCity. One more than the pre-U6 cap of 3, so the chain is DISCARDED + // WHOLE at depth 3 and types end to end at depth 4 — the whole basis + // for choosing the number. (An earlier version of this fixture had only + // three steps and therefore resolved at both depths, measuring nothing.) + id: 'fourHopChain', + member: 'save', + body: 'root.getSvc().getUser().address.getCity().save();', + raw: `class Root { + getSvc(): Service { + return new Service(); + } +} + +export function fourHopChain(root: Root): void { + root.getSvc().getUser().address.getCity().save(); +} +`, + }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'this.repo.save();', + raw: `class TsHolder { + repo: User = new User(); + fieldReceiverCall(): void { + this.repo.save(); + } +} +`, + note: 'control — undecorated field receiver', + }, + { + id: 'decoratedReceiverBase', + na: 'TypeScript `this` carries no type decoration — there is no pointer/reference spelling on a method receiver.', + }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'this.repo!.save();', + raw: `class TsNullableHolder { + repo: User | null = null; + decoratedFieldType(): void { + this.repo!.save(); + } +} +`, + note: 'union-with-null field type', + }, ], }, { lang: 'php', ext: '.php', + // `arrowCallChain` / `arrowPropertyPath` discriminate an unannotated return + // type from a property-path receiver — neither is expressible on the + // canonical axis, and both are load-bearing for the PHP diagnosis. + extraShapeIds: ['arrowCallChain', 'arrowPropertyPath'], support: { 'models.php': ` `function ${entry}(Service $svc) {\n ${body}\n}\n`, + // that is really a fixture defect. `$repos` is typed as an array for the + // subscript row for the same reason. + wrap: (entry, body) => `function ${entry}(Service $svc, array $repos) {\n ${body}\n}\n`, shapes: [ { id: 'arrowCallChain', @@ -177,27 +320,84 @@ class Service { `, note: 'PF6-control — property-path receiver, no `.` and no `(`', }, + { + id: 'plainChain', + member: 'save', + body: '$svc->getUserTyped()->save();', + note: 'discriminator for arrowCallChain — same chain, ANNOTATED return type', + }, + { + id: 'plainDeepChain', + member: 'save', + body: '$svc->getUser()->address->save();', + note: 'deep arrow chain', + }, + { id: 'optionalChain', member: 'save', body: '$svc?->getUser()->save();', note: 'nullsafe' }, + { + id: 'nonNullAssert', + na: 'PHP has no non-null assertion operator.', + }, + { id: 'awaitParen', na: 'PHP has no await expression in the core language.' }, + { + id: 'explicitTypeArgs', + na: 'PHP has no generic type-argument syntax at a call site.', + }, + { id: 'indexElement', member: 'save', body: '$repos[0]->save();', note: 'array subscript' }, + { + id: 'fieldReceiverCall', + na: 'Covered by `arrowPropertyPath` above — the same undecorated field-receiver shape.', + }, + { + id: 'decoratedReceiverBase', + na: 'PHP `$this` carries no type decoration on a method receiver.', + }, + { + id: 'decoratedFieldType', + member: 'save', + body: '$this->repo->save();', + raw: `class NullableHolder { + public ?User $repo; + public function decoratedFieldType() { + $this->repo->save(); + } +} +`, + note: 'nullable field type', + }, ], }, { lang: 'cpp', ext: '.cpp', + // `pointerArrowChain` / `valueDotChain` discriminate a `->` base from a `.` + // base on the same chain — the pair is why PF7 was attributable to the base + // rather than to C++ chaining in general. + extraShapeIds: ['pointerArrowChain', 'valueDotChain'], support: { 'models.h': `#pragma once +class Address { +public: + void save(); +}; + class User { public: + Address addr; void save(); }; class Service { public: User* getUser(); + template + T* getTyped(); }; `, }, header: `#include "models.h"\n`, - wrap: (entry, body) => `void ${entry}(Service* svc, Service svc2) {\n ${body}\n}\n`, + wrap: (entry, body) => + `void ${entry}(Service* svc, Service svc2, User* repos) {\n ${body}\n}\n`, shapes: [ { id: 'pointerArrowChain', @@ -215,10 +415,812 @@ public: body: 'svc2.getUser()->save();', note: 'PF7-control — value `.` base resolves', }, + { + id: 'plainChain', + na: 'C++ spells a chained call through `->` or `.`; both are measured above as pointerArrowChain and valueDotChain.', + }, + { id: 'plainDeepChain', member: 'save', body: 'svc->getUser()->addr.save();' }, + { id: 'optionalChain', na: 'C++ has no optional-chaining operator.' }, + { id: 'nonNullAssert', na: 'C++ has no non-null assertion operator.' }, + { + id: 'awaitParen', + na: 'co_await is a coroutine feature the emitter does not model as an async container.', + }, + { id: 'explicitTypeArgs', member: 'save', body: 'svc->getTyped()->save();' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save();' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'this->repo.save();', + raw: `struct CppHolder { + User repo; + void fieldReceiverCall() { + this->repo.save(); + } +}; +`, + note: 'control — undecorated field receiver', + }, + { + id: 'decoratedReceiverBase', + na: 'C++ `this` is implicitly a pointer with no written decoration; the pointer base is measured by pointerArrowChain.', + }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'this->repo->save();', + raw: `struct CppPtrHolder { + User* repo; + void decoratedFieldType() { + this->repo->save(); + } +}; +`, + note: 'pointer field type', + }, + ], + }, + { + // Go is the language whose root cause this plan proves: a method with a + // POINTER receiver binds its receiver to the literal string `*Host`, which + // findClassBindingInScope cannot resolve. The three field-receiver rows + // below isolate that — they vary receiver decoration and field decoration + // independently, so a gap can be attributed to one or the other rather than + // to "Go field receivers" as a whole. + lang: 'go', + ext: '.go', + support: { + 'models.go': `package main + +type Address struct{} + +func (a *Address) Save() {} + +type User struct { + Address *Address +} + +func (u *User) Save() {} + +type Service struct{} + +func (s *Service) GetUser() *User { return &User{} } + +type Tgt struct{} + +func (t *Tgt) PtrM() {} + +func (t Tgt) ValM() {} +`, + }, + header: `package main\n`, + wrap: (entry, body) => `func ${entry}(svc *Service, repos []*User) {\n\t${body}\n}\n`, + shapes: [ + { id: 'plainChain', member: 'Save', body: 'svc.GetUser().Save()', note: 'control' }, + { + id: 'plainDeepChain', + member: 'Save', + body: 'svc.GetUser().Address.Save()', + note: 'control', + }, + { id: 'optionalChain', na: 'Go has no optional-chaining operator.' }, + { id: 'nonNullAssert', na: 'Go has no non-null assertion operator.' }, + { id: 'awaitParen', na: 'Go has no await expression; concurrency is channel-based.' }, + { + id: 'explicitTypeArgs', + na: 'Go methods cannot declare type parameters, so there is no call-site type-argument spelling on a method.', + }, + { id: 'indexElement', member: 'Save', body: 'repos[0].Save()', note: 'slice subscript' }, + { + // VALUE receiver + VALUE field. Neither side carries decoration, so this + // is the control that isolates the two rows below. + id: 'fieldReceiverCall', + member: 'ValM', + body: 'h.f.ValM()', + raw: `type HostVV struct{ f Tgt } + +func (h HostVV) fieldReceiverCall() { + h.f.ValM() +} +`, + note: 'control — value receiver, value field', + }, + { + // POINTER receiver + VALUE field. The receiver binds as `*HostPV` and + // the class lookup misses every branch (no dot, so the dotted-tail + // fallback never fires). THE root cause this plan fixes. + id: 'decoratedReceiverBase', + member: 'ValM', + body: 'h.f.ValM()', + raw: `type HostPV struct{ f Tgt } + +func (h *HostPV) decoratedReceiverBase() { + h.f.ValM() +} +`, + note: 'pointer receiver, value field — isolates the BASE', + }, + { + // VALUE receiver + POINTER field. Go already normalizes field type + // bindings through normalizeGoTypeName, so this resolves today and + // proves the step lookup is NOT the defect. + id: 'decoratedFieldType', + member: 'PtrM', + body: 'h.f.PtrM()', + raw: `type HostVP struct{ f *Tgt } + +func (h HostVP) decoratedFieldType() { + h.f.PtrM() +} +`, + note: 'value receiver, pointer field — isolates the STEP', + }, + ], + }, + { + lang: 'javascript', + ext: '.js', + support: { + 'models.js': `export class Address { + save() {} +} + +export class User { + constructor() { + this.address = new Address(); + } + save() {} +} + +export class Service { + getUser() { + return new User(); + } + async getUserAsync() { + return new User(); + } +} +`, + }, + header: `import { Service, User } from './models.js';\n`, + wrap: (entry, body) => `export async function ${entry}(svc, repos) {\n ${body}\n}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.getUser().save();' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.getUser().address.save();' }, + { id: 'optionalChain', member: 'save', body: 'svc?.getUser().save();' }, + { id: 'nonNullAssert', na: 'JavaScript has no non-null assertion operator.' }, + { id: 'awaitParen', member: 'save', body: '(await svc.getUserAsync()).save();' }, + { id: 'explicitTypeArgs', na: 'JavaScript has no type-argument syntax.' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save();' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'this.repo.save();', + raw: `class JsHolder { + constructor() { + this.repo = new User(); + } + fieldReceiverCall() { + this.repo.save(); + } +} +`, + }, + { + id: 'decoratedReceiverBase', + na: 'JavaScript method receivers carry no type decoration.', + }, + { + id: 'decoratedFieldType', + na: 'JavaScript fields carry no declared type, so there is no decoration to strip.', + }, + ], + }, + { + lang: 'python', + ext: '.py', + support: { + 'models.py': `class Address: + def save(self) -> None: + pass + + +class User: + def __init__(self) -> None: + self.address: Address = Address() + + def save(self) -> None: + pass + + +class Service: + def get_user(self) -> User: + return User() + + async def get_user_async(self) -> User: + return User() +`, + }, + header: `from typing import List, Optional\nfrom models import Address, Service, User\n`, + wrap: (entry, body) => `def ${entry}(svc: Service, repos: List[User]) -> None:\n ${body}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.get_user().save()' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.get_user().address.save()' }, + { id: 'optionalChain', na: 'Python has no optional-chaining operator.' }, + { id: 'nonNullAssert', na: 'Python has no non-null assertion operator.' }, + { + id: 'awaitParen', + member: 'save', + body: '(await svc.get_user_async()).save()', + raw: `async def awaitParen(svc: Service) -> None: + (await svc.get_user_async()).save() +`, + note: 'annotated coroutine — the declared return type IS the awaited type', + }, + { id: 'explicitTypeArgs', na: 'Python has no call-site type-argument syntax.' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save()' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'self.repo.save()', + raw: `class PyHolder: + def __init__(self) -> None: + self.repo: User = User() + + def fieldReceiverCall(self) -> None: + self.repo.save() +`, + }, + { id: 'decoratedReceiverBase', na: 'Python `self` carries no type decoration.' }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'self.repo.save()', + raw: `class PyOptionalHolder: + def __init__(self) -> None: + self.repo: Optional[User] = None + + def decoratedFieldType(self) -> None: + self.repo.save() +`, + note: 'Optional[...] field annotation', + }, + ], + }, + { + lang: 'java', + ext: '.java', + support: { + 'Address.java': `public class Address { + public void save() {} +} +`, + 'User.java': `public class User { + public Address address; + public void save() {} +} +`, + 'Service.java': `import java.util.List; + +public class Service { + public User getUser() { return new User(); } + public User getTyped() { return new User(); } +} +`, + }, + header: `import java.util.List;\n`, + // Java has no free functions, so each shape gets its own wrapper class. The + // METHOD is named for the shape, which is what classification matches on. + wrap: (entry, body) => + `class Cap_${entry} {\n void ${entry}(Service svc, User[] repos) {\n ${body}\n }\n}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.getUser().save();' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.getUser().address.save();' }, + { id: 'optionalChain', na: 'Java has no optional-chaining operator.' }, + { id: 'nonNullAssert', na: 'Java has no non-null assertion operator.' }, + { id: 'awaitParen', na: 'Java has no await expression in the core language.' }, + { id: 'explicitTypeArgs', member: 'save', body: 'svc.getTyped().save();' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save();' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'this.repo.save();', + raw: `class JavaHolder { + User repo; + void fieldReceiverCall() { + this.repo.save(); + } +} +`, + }, + { id: 'decoratedReceiverBase', na: 'Java `this` carries no type decoration.' }, + { + id: 'decoratedFieldType', + na: "Java's only field-type decorations are containers (arrays, generics), which change the member set and are measured by indexElement rather than by a type-preserving strip.", + }, + ], + }, + { + lang: 'csharp', + ext: '.cs', + support: { + 'Models.cs': `using System.Threading.Tasks; + +public class Address { + public void Save() {} +} + +public class User { + public Address Address; + public void Save() {} +} + +public class Service { + public User GetUser() { return new User(); } + public async Task GetUserAsync() { return new User(); } + public User GetTyped() { return new User(); } +} +`, + }, + header: `using System.Threading.Tasks;\n`, + wrap: (entry, body) => + `class Cap_${entry} {\n async Task ${entry}(Service svc, User[] repos) {\n ${body}\n }\n}\n`, + shapes: [ + { id: 'plainChain', member: 'Save', body: 'svc.GetUser().Save();' }, + { id: 'plainDeepChain', member: 'Save', body: 'svc.GetUser().Address.Save();' }, + { id: 'optionalChain', member: 'Save', body: 'svc?.GetUser().Save();' }, + { id: 'nonNullAssert', member: 'Save', body: 'svc!.GetUser().Save();' }, + { id: 'awaitParen', member: 'Save', body: '(await svc.GetUserAsync()).Save();' }, + { id: 'explicitTypeArgs', member: 'Save', body: 'svc.GetTyped().Save();' }, + { id: 'indexElement', member: 'Save', body: 'repos[0].Save();' }, + { + id: 'fieldReceiverCall', + member: 'Save', + body: 'this.repo.Save();', + raw: `class CsHolder { + User repo; + void fieldReceiverCall() { + this.repo.Save(); + } +} +`, + }, + { id: 'decoratedReceiverBase', na: 'C# `this` carries no type decoration.' }, + { + id: 'decoratedFieldType', + member: 'Save', + body: 'this.repo.Save();', + raw: `class CsNullableHolder { + User? repo; + void decoratedFieldType() { + this.repo.Save(); + } +} +`, + note: 'nullable reference-type field', + }, + ], + }, + { + lang: 'ruby', + ext: '.rb', + support: { + 'models.rb': `class Address + def save; end +end + +class User + attr_reader :address + def initialize + @address = Address.new + end + def save; end +end + +class Service + def get_user + User.new + end +end +`, + }, + header: `require_relative 'models'\n`, + wrap: (entry, body) => `def ${entry}(svc, repos)\n ${body}\nend\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.get_user.save' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.get_user.address.save' }, + { id: 'optionalChain', member: 'save', body: 'svc&.get_user.save', note: 'safe navigation' }, + { id: 'nonNullAssert', na: 'Ruby has no non-null assertion operator.' }, + { id: 'awaitParen', na: 'Ruby has no await expression in the core language.' }, + { id: 'explicitTypeArgs', na: 'Ruby has no type-argument syntax.' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: '@repo.save', + raw: `class RbHolder + def initialize + @repo = User.new + end + + def fieldReceiverCall + @repo.save + end +end +`, + }, + { id: 'decoratedReceiverBase', na: 'Ruby method receivers carry no type decoration.' }, + { + id: 'decoratedFieldType', + na: 'Ruby instance variables carry no declared type, so there is no decoration to strip.', + }, + ], + }, + { + lang: 'rust', + ext: '.rs', + support: { + 'models.rs': `pub struct Address; + +impl Address { + pub fn save(&self) {} +} + +pub struct User { + pub address: Address, +} + +impl User { + pub fn save(&self) {} +} + +pub struct Service; + +impl Service { + pub fn get_user(&self) -> User { + User { address: Address } + } + pub fn get_typed(&self) -> User { + User { address: Address } + } +} +`, + }, + header: `mod models;\nuse models::{Address, Service, User};\n`, + wrap: (entry, body) => `fn ${entry}(svc: &Service, repos: &Vec) {\n ${body}\n}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.get_user().save();' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.get_user().address.save();' }, + { + id: 'optionalChain', + na: 'Rust `?` is error propagation, not optional chaining on a receiver.', + }, + { id: 'nonNullAssert', na: 'Rust has no non-null assertion operator.' }, + { + id: 'awaitParen', + na: 'Rust `async fn` declares the AWAITED type; `.await` is postfix and mints no wrapper the emitter models as an async container.', + }, + { id: 'explicitTypeArgs', member: 'save', body: 'svc.get_typed::().save();' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save();' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'self.repo.save();', + raw: `struct RsHolder { + repo: User, +} + +impl RsHolder { + fn fieldReceiverCall(&self) { + self.repo.save(); + } +} +`, + }, + { + // Rust is the one language besides Go whose method receiver carries + // written decoration: `&self` / `&mut self` rather than a bare `self`. + id: 'decoratedReceiverBase', + member: 'save', + body: 'self.repo.save();', + raw: `struct RsMutHolder { + repo: User, +} + +impl RsMutHolder { + fn decoratedReceiverBase(&mut self) { + self.repo.save(); + } +} +`, + note: '&mut self receiver', + }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'self.repo.save();', + raw: `struct RsBoxHolder { + repo: Box, +} + +impl RsBoxHolder { + fn decoratedFieldType(&self) { + self.repo.save(); + } +} +`, + note: 'Box — deref-transparent smart pointer', + }, + ], + }, + { + lang: 'c', + ext: '.c', + support: { + 'models.h': `#pragma once + +typedef struct Address { + void (*save)(void); +} Address; + +typedef struct User { + Address* address; + void (*save)(void); +} User; +`, + }, + header: `#include "models.h"\n`, + wrap: (entry, body) => `void ${entry}(User* u, User* repos) {\n ${body}\n}\n`, + // C has no method dispatch. The only receiver-shaped construct is a call + // through a struct function-pointer field, so the chain shapes that assume a + // method on a returned value are inapplicable rather than gaps. + shapes: [ + { + id: 'plainChain', + na: 'C has no method call on a returned value; dispatch is through struct function-pointer fields only.', + }, + { id: 'plainDeepChain', member: 'save', body: 'u->address->save();' }, + { id: 'optionalChain', na: 'C has no optional-chaining operator.' }, + { id: 'nonNullAssert', na: 'C has no non-null assertion operator.' }, + { id: 'awaitParen', na: 'C has no await expression.' }, + { id: 'explicitTypeArgs', na: 'C has no generics.' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save();' }, + { id: 'fieldReceiverCall', member: 'save', body: 'u->save();' }, + { + id: 'decoratedReceiverBase', + na: 'C has no method receiver, so there is no receiver decoration.', + }, + { + id: 'decoratedFieldType', + na: 'A C struct field is a function pointer, not a typed receiver whose decoration could be stripped.', + }, + ], + }, + { + // Kotlin, Swift and Dart are vendored OPTIONAL grammars shipping only + // darwin-arm64 prebuilds. On any other host these rows come back + // GRAMMAR-UNAVAILABLE and are skipped by the gate in both directions, so the + // cells below are the contract for a host that CAN load them rather than a + // measurement this machine produced. + lang: 'kotlin', + ext: '.kt', + support: { + 'models.kt': `class Address { + fun save() {} +} + +class User { + val address: Address = Address() + fun save() {} +} + +class Service { + fun getUser(): User = User() + suspend fun getUserAsync(): User = User() + fun getTyped(): User = User() +} +`, + }, + header: '', + wrap: (entry, body) => + `suspend fun ${entry}(svc: Service, repos: List) {\n ${body}\n}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.getUser().save()' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.getUser().address.save()' }, + { id: 'optionalChain', member: 'save', body: 'svc?.getUser()?.save()' }, + { id: 'nonNullAssert', member: 'save', body: 'svc!!.getUser().save()' }, + { id: 'awaitParen', member: 'save', body: '(svc.getUserAsync()).save()' }, + { id: 'explicitTypeArgs', member: 'save', body: 'svc.getTyped().save()' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save()' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'this.repo.save()', + raw: `class KtHolder { + val repo: User = User() + fun fieldReceiverCall() { + this.repo.save() + } +} +`, + }, + { id: 'decoratedReceiverBase', na: 'Kotlin `this` carries no type decoration.' }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'this.repo?.save()', + raw: `class KtNullableHolder { + val repo: User? = null + fun decoratedFieldType() { + this.repo?.save() + } +} +`, + note: 'nullable field type', + }, + ], + }, + { + lang: 'swift', + ext: '.swift', + support: { + 'models.swift': `class Address { + func save() {} +} + +class User { + var address: Address = Address() + func save() {} +} + +class Service { + func getUser() -> User { return User() } + func getUserAsync() async -> User { return User() } +} +`, + }, + header: '', + wrap: (entry, body) => `func ${entry}(svc: Service, repos: [User]) async {\n ${body}\n}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.getUser().save()' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.getUser().address.save()' }, + { id: 'optionalChain', member: 'save', body: 'svc.getUser().address?.save()' }, + { id: 'nonNullAssert', member: 'save', body: 'svc.getUser().address!.save()' }, + { id: 'awaitParen', member: 'save', body: '(await svc.getUserAsync()).save()' }, + { + id: 'explicitTypeArgs', + na: 'Swift infers generic parameters; there is no call-site type-argument spelling on a method.', + }, + { id: 'indexElement', member: 'save', body: 'repos[0].save()' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'self.repo.save()', + raw: `class SwHolder { + var repo: User = User() + func fieldReceiverCall() { + self.repo.save() + } +} +`, + }, + { id: 'decoratedReceiverBase', na: 'Swift `self` carries no type decoration.' }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'self.repo?.save()', + raw: `class SwOptionalHolder { + var repo: User? = nil + func decoratedFieldType() { + self.repo?.save() + } +} +`, + note: 'optional field type', + }, + ], + }, + { + lang: 'dart', + ext: '.dart', + support: { + 'models.dart': `class Address { + void save() {} +} + +class User { + Address address = Address(); + void save() {} +} + +class Service { + User getUser() => User(); + Future getUserAsync() async => User(); +} +`, + }, + header: `import 'models.dart';\n`, + wrap: (entry, body) => + `Future ${entry}(Service svc, List repos) async {\n ${body}\n}\n`, + shapes: [ + { id: 'plainChain', member: 'save', body: 'svc.getUser().save();' }, + { id: 'plainDeepChain', member: 'save', body: 'svc.getUser().address.save();' }, + { id: 'optionalChain', member: 'save', body: 'svc.getUser().address?.save();' }, + { id: 'nonNullAssert', member: 'save', body: 'svc.getUser().address!.save();' }, + { id: 'awaitParen', member: 'save', body: '(await svc.getUserAsync()).save();' }, + { id: 'explicitTypeArgs', na: 'Dart infers generic parameters at a method call site.' }, + { id: 'indexElement', member: 'save', body: 'repos[0].save();' }, + { + id: 'fieldReceiverCall', + member: 'save', + body: 'this.repo.save();', + raw: `class DartHolder { + User repo = User(); + void fieldReceiverCall() { + this.repo.save(); + } +} +`, + }, + { id: 'decoratedReceiverBase', na: 'Dart `this` carries no type decoration.' }, + { + id: 'decoratedFieldType', + member: 'save', + body: 'this.repo?.save();', + raw: `class DartNullableHolder { + User? repo; + void decoratedFieldType() { + this.repo?.save(); + } +} +`, + note: 'nullable field type', + }, ], }, ]; +/** + * Every corpus declares a cell for every SHAPE_ID, every `na` carries a reason, + * and no id is declared twice or unknown. Throws rather than warning: a matrix + * with a hole reports as if the hole were measured, which is the exact failure + * the N/A-with-reason rule exists to prevent. + */ +function assertMatrixComplete(corpora) { + const problems = []; + for (const corpus of corpora) { + // A language may carry EXTRA cells beyond the canonical axis when they + // discriminate something the axis cannot — PHP's annotated/unannotated + // return-type pair, C++'s pointer/value base pair. They must be declared, so + // an extra stays a deliberate diagnostic rather than a typo'd canonical id. + const known = new Set([...SHAPE_IDS, ...(corpus.extraShapeIds ?? [])]); + const seen = new Set(); + for (const shape of corpus.shapes) { + if (!known.has(shape.id)) { + problems.push( + `${corpus.lang}: shape id "${shape.id}" is neither canonical nor in extraShapeIds`, + ); + } + if (seen.has(shape.id)) problems.push(`${corpus.lang}: duplicate shape id "${shape.id}"`); + seen.add(shape.id); + if (shape.na !== undefined && String(shape.na).trim() === '') { + problems.push(`${corpus.lang}.${shape.id}: N/A with no reason`); + } + if (shape.na === undefined && shape.body === undefined) { + problems.push(`${corpus.lang}.${shape.id}: neither a body nor an N/A reason`); + } + } + for (const id of SHAPE_IDS) { + if (!seen.has(id)) problems.push(`${corpus.lang}: missing cell for "${id}"`); + } + } + if (problems.length > 0) { + throw new Error( + `[receiver-resolution] shape matrix is incomplete:\n ${problems.join('\n ')}`, + ); + } +} + function classify(corpus, result) { const calls = []; for (const rel of result.graph.iterRelationships()) { @@ -233,6 +1235,9 @@ function classify(corpus, result) { ); return corpus.shapes.map((shape) => { + if (shape.na !== undefined) { + return { shape: null, id: shape.id, note: shape.na, state: 'N/A', siteKind: null }; + } const hasEdge = calls.some((call) => call.from === shape.id && call.to === shape.member); // A drop belongs to this shape when it names the shape's member and sits on // the shape's own line — matched on the generated source, not on an id. @@ -249,8 +1254,48 @@ function classify(corpus, result) { }); } +/** Every cell for a language whose parser could not be loaded. Neither passes + * nor fails the gate — an unbuilt optional grammar is a fact about the host, + * not a resolution result. */ +function grammarUnavailableRow(corpus, reason) { + return corpus.shapes.map((shape) => ({ + shape: shape.body ?? null, + id: shape.id, + note: reason, + state: 'GRAMMAR-UNAVAILABLE', + siteKind: null, + })); +} + +/** A parse failure caused by a missing vendored grammar, as opposed to a real + * pipeline error. The vendored loader throws with the grammar package name, and + * a language whose grammar never loaded produces a graph with no nodes at all. */ +function isGrammarLoadFailure(error) { + const text = `${error?.message ?? ''}`; + return ( + text.includes('tree-sitter-') && + (text.includes('Cannot find module') || + text.includes('No native build') || + text.includes('not supported on this platform') || + text.includes('prebuild')) + ); +} + async function runShapeArm() { + assertMatrixComplete(CORPORA); const results = []; + for (const [lang, reason] of Object.entries(NA_LANGUAGES)) { + results.push({ + language: lang, + shapes: SHAPE_IDS.map((id) => ({ + shape: null, + id, + note: reason, + state: 'N/A', + siteKind: null, + })), + }); + } for (const corpus of CORPORA) { const root = fs.mkdtempSync(path.join(os.tmpdir(), `gn-recv-${corpus.lang}-`)); try { @@ -263,6 +1308,8 @@ async function runShapeArm() { const lineOfShape = new Map(); let text = corpus.header; for (const shape of corpus.shapes) { + // N/A cells are declarations about the language, not code to measure. + if (shape.na !== undefined) continue; // A shape whose receiver needs surrounding structure (a class with a // property, say) supplies `raw`; everything else is wrapped in a plain // function. Either way the statement itself is `body`, and its offset @@ -275,7 +1322,27 @@ async function runShapeArm() { } fs.writeFileSync(path.join(root, `main${corpus.ext}`), text, 'utf8'); - const result = await runPipelineFromRepo(root, () => {}); + let result; + try { + result = await runPipelineFromRepo(root, () => {}); + } catch (error) { + if (!isGrammarLoadFailure(error)) throw error; + results.push({ + language: corpus.lang, + shapes: grammarUnavailableRow(corpus, `parser failed to load: ${error.message}`), + }); + continue; + } + // A grammar that soft-fails produces no parse and therefore no nodes. + // Reporting that as ten resolution gaps would be indistinguishable from a + // real regression, so it gets its own state. + if ([...result.graph.iterNodes()].length === 0) { + results.push({ + language: corpus.lang, + shapes: grammarUnavailableRow(corpus, 'parser produced no nodes — grammar unavailable'), + }); + continue; + } // Attach the owning shape to each drop by line before classifying. for (const outcome of result.resolutionOutcomes ?? []) { for (const [id, line] of lineOfShape) { @@ -303,12 +1370,32 @@ async function runCountArm(repoPath) { const byKind = new Map(); const byExtension = new Map(); const callDropsByExtension = new Map(); + // Shape census over the CALL drops only, so the answer to "which kind of + // receiver are we losing?" is not diluted by property reads and writes. + const callDropsByShape = new Map(); + // THE number that matters now. A drop whose receiver is rooted outside the + // indexed program (`System.out.println`, `fetch(...)`) reaches code with no + // node to point an edge at — nothing was lost. Only `in-program` and + // `unknown` drops represent a real resolver gap, and only those make an + // `impact` count a lower bound. Measured on this corpus: 76 external, 20 + // in-program, 6 unknown — i.e. three quarters of the raw count was the + // program boundary rather than uncertainty about the program. + const callDropsByOrigin = new Map(); + const callDropsByShapeAndExt = new Map(); for (const drop of drops) { const kind = drop.siteKind ?? '<>'; byKind.set(kind, (byKind.get(kind) ?? 0) + 1); const ext = path.extname(drop.filePath); byExtension.set(ext, (byExtension.get(ext) ?? 0) + 1); - if (kind === 'call') callDropsByExtension.set(ext, (callDropsByExtension.get(ext) ?? 0) + 1); + if (kind === 'call') { + callDropsByExtension.set(ext, (callDropsByExtension.get(ext) ?? 0) + 1); + const shape = drop.receiverShape ?? '<>'; + callDropsByShape.set(shape, (callDropsByShape.get(shape) ?? 0) + 1); + const origin = drop.receiverOrigin ?? '<>'; + callDropsByOrigin.set(origin, (callDropsByOrigin.get(origin) ?? 0) + 1); + const pair = `${ext} ${shape}`; + callDropsByShapeAndExt.set(pair, (callDropsByShapeAndExt.get(pair) ?? 0) + 1); + } } const sortDesc = (map) => Object.fromEntries([...map.entries()].sort((a, b) => b[1] - a[1])); @@ -319,6 +1406,9 @@ async function runCountArm(repoPath) { totalDropsAllKinds: drops.length, bySiteKind: sortDesc(byKind), callDropsByExtension: sortDesc(callDropsByExtension), + callDropsByShape: sortDesc(callDropsByShape), + callDropsByOrigin: sortDesc(callDropsByOrigin), + callDropsByShapeAndExtension: sortDesc(callDropsByShapeAndExt), allDropsByExtension: sortDesc(byExtension), }; } @@ -399,9 +1489,8 @@ async function runPerfArm(repoPath, reps) { // --------------------------------------------------------------------------- const KNOWN_BLIND = [ - "Case 0's gate is a C-family punctuation test (`.` or `(`), so PHP `->` and `::` receivers record no drop.", - '`repos[0].save()` has neither `.` nor `(` in its receiver — same result.', - '`?.` and explicit type arguments produce no reference site at all, so no drop is recorded.', + 'Case 0 is reached by a receiver-TEXT punctuation test (`.` or `(`) OR by a minted receiver chain. A receiver spelled without that punctuation — a subscript, PHP `->`/`::` — therefore records a drop only where its emitter mints a chain; where none is minted the call still vanishes with the recorder blind to it.', + 'A drop is recorded only while `compoundReceiverUnresolved` stays true. When the cascade TYPES the receiver but then finds no member on it, the flag is false and no drop is recorded even though no edge was emitted — so an absent drop is not evidence that a site resolved.', 'Every count is therefore a LOWER BOUND on a known-biased population. A later delta must be read against the same bias.', ]; @@ -434,11 +1523,22 @@ function projection(output) { totalDropsAllKinds: output.countArm.totalDropsAllKinds, bySiteKind: output.countArm.bySiteKind, callDropsByExtension: output.countArm.callDropsByExtension, + callDropsByShape: output.countArm.callDropsByShape, + callDropsByOrigin: output.countArm.callDropsByOrigin, }, }; } -/** Every leaf whose value differs, as `dotted.path: expected -> actual`. */ +/** Every leaf whose value differs, as `dotted.path: expected -> actual`. + * + * `GRAMMAR-UNAVAILABLE` is skipped on EITHER side. That state is a fact about + * the host, not about the code: the optional grammars are absent when a run + * sets `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` or when no vendored prebuild matches + * the platform, so the same commit legitimately measures those cells on one + * runner and cannot on another. Gating on them would fail CI for the + * environment it ran in rather than for a regression — the opposite of what the + * state was added to prevent. Skipping both directions is what makes "neither + * passes nor fails the gate" true. */ function drift(expected, actual, prefix = '') { const out = []; const keys = new Set([...Object.keys(expected ?? {}), ...Object.keys(actual ?? {})]); @@ -446,6 +1546,7 @@ function drift(expected, actual, prefix = '') { const want = expected?.[key]; const got = actual?.[key]; const at = prefix === '' ? key : `${prefix}.${key}`; + if (want === 'GRAMMAR-UNAVAILABLE' || got === 'GRAMMAR-UNAVAILABLE') continue; if (want !== null && typeof want === 'object') out.push(...drift(want, got ?? {}, at)); else if (want !== got) out.push(`${at}: ${JSON.stringify(want)} -> ${JSON.stringify(got)}`); } @@ -455,11 +1556,26 @@ function drift(expected, actual, prefix = '') { const args = process.argv.slice(2); const corpusIndex = args.indexOf('--corpus'); const check = args.includes('--check'); +const updateBaseline = args.includes('--update-baseline'); +// `--update-baseline` defaults the corpus for the same reason `--check` does: +// the two must measure the SAME thing or the gate can never be satisfied. It +// previously did not, so a bare `--update-baseline` skipped the count arm and +// then crashed dereferencing `output.countArm` in `projection` — which is the +// command the `--check` failure message tells you to run. Writing a baseline +// with no count arm would have been worse: the gate would go green while +// silently measuring nothing. const corpusPath = - corpusIndex === -1 ? (check ? DEFAULT_CORPUS : undefined) : path.resolve(args[corpusIndex + 1]); + corpusIndex === -1 + ? check || updateBaseline + ? DEFAULT_CORPUS + : undefined + : path.resolve(args[corpusIndex + 1]); const output = { knownBlind: KNOWN_BLIND }; -if (corpusPath === undefined || check || args.includes('--shapes')) { +// `--update-baseline` needs BOTH arms, because `projection` writes both and +// `--check` compares both. Running one arm would write a baseline missing the +// other, and the next `--check` would then crash rather than fail a comparison. +if (corpusPath === undefined || check || updateBaseline || args.includes('--shapes')) { output.shapeArm = await runShapeArm(); } if (corpusPath !== undefined) { @@ -471,7 +1587,7 @@ if (perfIndex !== -1) { output.perfArm = await runPerfArm(corpusPath ?? DEFAULT_CORPUS, Number.isFinite(reps) ? reps : 3); } -if (args.includes('--update-baseline')) { +if (updateBaseline) { fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(projection(output), null, 2)}\n`, 'utf8'); console.error(`[receiver-resolution] wrote ${BASELINE_PATH}`); } else if (check) { diff --git a/gitnexus/bench/schema-pairs/README.md b/gitnexus/bench/schema-pairs/README.md new file mode 100644 index 000000000..29594b9d7 --- /dev/null +++ b/gitnexus/bench/schema-pairs/README.md @@ -0,0 +1,100 @@ +# Schema pair-set bench (#2793) + +What a bigger `CodeRelation` FROM/TO pair set costs at query time, measured +against a real `@ladybugdb/core` database. + +```bash +# from gitnexus/ +node --import tsx bench/schema-pairs/measure.mjs # print one JSON line per size + a summary +node --import tsx bench/schema-pairs/measure.mjs --check # gate vs baselines.json +``` + +## Why it exists + +`src/core/lbug/schema.ts` generates its relation pairs from two cross products, +and declines to add a third one **on the strength of a number** — roughly 1.04× +at 450 declared pairs, 1.6× at 786, 2.1× at 1024. That measurement used to live +in a scratch directory, so nobody proposing a third rule could re-run it. This +harness is that measurement, committed — and it reproduces those figures. + +Run it before widening a rule, and quote the new ratio in the review. + +Observed on the reference box, **four runs** (ratios vs the 332-pair list): + +| pairs | untyped | typed (floor) | +| ----- | ---------- | ------------- | +| 332 | 1.00× | 1.00× | +| 450 | 0.93–1.05× | 0.98–1.17× | +| 641 | 1.22–1.43× | 1.11–1.23× | +| 786 | 1.52–1.75× | 1.19–1.31× | +| 1024 | 2.03–2.34× | 1.31–1.57× | + +Production's 450 came out _faster_ than 332 on three of the four runs, so at this +size the pair count is inside run-to-run noise. Everything past ~640 is not. +**Quote the range, not a single run** — one run is not evidence here. + +## What it measures + +For each pair-set size it builds a fresh database with all 32 node tables, a +`CodeRelation` table declaring exactly that many FROM/TO pairs, and **identical +data**, then times two query shapes over 40 anchors × 15 reps (median): + +- **`untyped_ms_`** — `MATCH (a {id: $id})-[r:CodeRelation]->(b)`. Neither + endpoint is labelled, so LadybugDB must treat every declared pair as a + candidate. This is the shape `impact`, `context` and `detect_changes` issue + when they walk out from one node id, and the only one whose plan depends on + how many pairs the table declares. +- **`typed_ms_`** — `MATCH (a:Function {…})-[r]->(b:Function)`, the lower + bound. Both endpoints labelled prunes the plan to a single pair, so this was + expected to be flat in the pair count. **It is not** — up to 1.17× at 450 and + 1.57× at 1024 — so a declared-but-unused pair costs something even when the + planner never considers it. `typed_ratio_*` is therefore the floor, not a noise + control; the real cost of widening sits between it and `ratio_*`. A run where + `typed_ratio` moves _more_ than `ratio` is noise-dominated and should be + rerun. +- **`ratio_`** — `untyped_ms_ / untyped_ms_332`. `ratio_450` is the + figure `schema.ts` quotes. + +### Sizes + +The pair set is a prefix of a fixed 32×32 (`NODE_TABLES`²) enumeration, so each +size is a strict superset of the smaller ones. The four pairs the synthetic data +uses are pinned to the front, so **the same rows are reachable by the same query +at every size** — the only variable is how many unused pairs are declared. The +harness fails if the row counts ever differ across sizes. + +| size | what it is | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 332 | the pre-#2792 hand-written list — the reference for every ratio | +| 450 | production today (two cross products + 72 hand-declared pairs) | +| 641 | the third cross product `schema.ts` defers (`DEFINITION_ANCHOR_LABELS × {CodeElement, Section, Typedef, Union, Namespace, Impl, TypeAlias, Static, Template}`), which would leave ~29 hand-declared lines | +| 786 | the size an earlier revision of that comment attributed to the third rule — it is 641; kept as a measured waypoint | +| 1024 | the full cross product, the ceiling | + +## Correctness gate + +Before timing anything, the harness round-trips the **real** `SCHEMA_QUERIES` +through a real database and asserts that `CALL SHOW_CONNECTION('CodeRelation')` +reports exactly the pairs `parseRelationSchemaPairs` finds in `RELATION_SCHEMA`. + +No magic number is baked in: the invariant is that the DDL LadybugDB _accepted_ +carries the pair set our own parser believes it declares. The absolute count is +reported as `declared_pairs`. A pair declared twice would not reach this check at +all — LadybugDB rejects the `CREATE REL TABLE` outright, which is why a duplicate +kills every `analyze` rather than one repository's. + +## What it does NOT measure + +- **Ingest / `COPY` cost.** Pair-set size also multiplies the number of per-pair + CSVs the emitter routes to (`src/core/lbug/rel-pair-routing.ts`); that cost is + covered by `bench/emit-persistence`. +- **At-scale absolute numbers.** Row counts here are small and deliberately + constant. The ratios are the signal; the milliseconds are box-specific. + +## Regenerating the baseline + +`baselines.json` holds one budget, `ratio_450_budget` — the ceiling on what +production's own pair count may cost relative to the 332-pair hand-list it +replaced. Re-run without `--check` **several times** and copy the top of the +observed `ratio_450` range plus headroom — the spread between runs on this box +is wider than the effect being measured at 450, so a single run cannot set it. diff --git a/gitnexus/bench/schema-pairs/baselines.json b/gitnexus/bench/schema-pairs/baselines.json new file mode 100644 index 000000000..41a12d485 --- /dev/null +++ b/gitnexus/bench/schema-pairs/baselines.json @@ -0,0 +1,4 @@ +{ + "_comment": "ratio_450_budget — ceiling on what production's 450-pair set may cost on untyped-endpoint anchored queries, relative to the 332-pair hand-list it replaced. Observed 0.94x and 1.05x across two runs on the reference box (i.e. inside run-to-run noise; it came out faster than 332 once). The budget carries headroom for that spread — compare typed_ratio_450 (1.10-1.17x) for this box's floor. Raise it only with a measured range, never a single run.", + "ratio_450_budget": 1.3 +} diff --git a/gitnexus/bench/schema-pairs/measure.mjs b/gitnexus/bench/schema-pairs/measure.mjs new file mode 100644 index 000000000..59fe92db7 --- /dev/null +++ b/gitnexus/bench/schema-pairs/measure.mjs @@ -0,0 +1,357 @@ +/** + * What a bigger `CodeRelation` FROM/TO pair set costs at query time (#2793). + * + * `src/core/lbug/schema.ts` declares its relation pairs from two cross products + * plus a small hand-written remainder, and it justifies NOT adding a third cross + * product with a number: anchored queries cost ~1.04× at 450 declared pairs but + * 1.6× at 786 and 2.1× at 1024. That measurement previously lived in a scratch + * directory, so the claim could not be re-checked when someone proposed + * widening a rule. This is it, committed. + * + * WHAT IT MEASURES. Against a real `@ladybugdb/core` database, with byte-identical + * DATA at every size, it times the query shape whose plan actually depends on the + * declared pair set: + * + * MATCH (a {id: $id})-[r:CodeRelation]->(b) RETURN b.id + * + * Neither endpoint is labelled, so LadybugDB must consider every declared + * FROM/TO pair as a candidate — this is the shape `impact`, `context` and + * `detect_changes` all issue when they walk out from one node id. + * + * A LABEL-typed query (`MATCH (a:Function)-[r]->(b:Function)`) is measured + * alongside it as the LOWER BOUND. Its plan prunes to a single pair, so it was + * expected to be flat in the pair count — it is NOT. Measured here it reaches + * 1.17× at 450 and 1.57× at 1024 against the same 332-pair reference, i.e. a + * declared-but-unused pair costs something even when the planner never + * considers it (per-pair catalog/storage overhead the query pays regardless). + * So `typed_ratio_*` is not a noise control: it is the floor, and the true cost + * of a wider pair set lies between it and `ratio_*`. Treat any run where + * `typed_ratio` moves MORE than `ratio` as noise-dominated. + * + * SIZES. The pair set is a prefix of a fixed 32×32 (`NODE_TABLES`²) enumeration + * so every size is a strict SUPERSET of the smaller ones, and the four pairs the + * data actually uses are pinned first — so the same rows are reachable by the + * same query at every size, and the only variable is how many UNUSED pairs the + * table declares: + * - 332 — the pre-#2792 hand-written list (the historical baseline); + * - 450 — production today (two cross products + 72 hand-declared); + * - 641 — the third cross product schema.ts defers + * (`DEFINITION_ANCHOR_LABELS × {CodeElement, Section, Typedef, Union, + * Namespace, Impl, TypeAlias, Static, Template}`), which would leave + * only ~29 hand-declared lines; + * - 786 — the size an earlier revision of that comment attributed to the + * third rule (it is 641; 786 is kept as a measured waypoint); + * - 1024 — the full cross product, the ceiling. + * + * Ratios are reported against 332, the smallest size — `ratio_450` is the + * number schema.ts quotes. + * + * CORRECTNESS GATE. Before timing anything it round-trips the REAL + * `SCHEMA_QUERIES` through a real database and asserts that + * `CALL SHOW_CONNECTION('CodeRelation')` reports exactly the pairs + * `parseRelationSchemaPairs` finds in `RELATION_SCHEMA`. That is the invariant + * that matters and it needs no magic number: it proves the DDL LadybugDB + * ACCEPTED carries the pair set our own parser believes it declares. (A + * duplicated FROM/TO would not even get this far — LadybugDB rejects the + * `CREATE REL TABLE` outright, which is why that failure kills every `analyze`.) + * The absolute count is reported as `declared_pairs` for the record. + * + * Build-free: imports the `.ts` sources through tsx. + * + * node --import tsx bench/schema-pairs/measure.mjs # print JSON lines + * node --import tsx bench/schema-pairs/measure.mjs --check # gate vs baselines.json + * + * `--check` fails if the correctness gate breaks, or if `ratio_450` exceeds its + * budget — i.e. if production's own pair count starts costing materially more + * than the hand-written list it replaced. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { NODE_TABLES } from 'gitnexus-shared'; +import { + NODE_SCHEMA_QUERIES, + RELATION_SCHEMA, + REL_TABLE_NAME, +} from '../../src/core/lbug/schema.ts'; +import { parseRelationSchemaPairs } from '../../src/core/lbug/rel-pair-routing.ts'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const lbug = (await import('@ladybugdb/core')).default; + +// ---- sizes + the pair enumeration every size is a prefix of ---- + +const SIZES = [332, 450, 641, 786, 1024]; +const REFERENCE_SIZE = 332; // ratios are relative to this +const PRODUCTION_SIZE = 450; // the size schema.ts ships + +// The four pairs the synthetic data uses. Pinned to the FRONT of the +// enumeration so they are declared at every size — otherwise a smaller pair set +// would simply carry fewer rows and the comparison would measure data volume, +// not pair-set size. +const DATA_PAIRS = [ + ['File', 'Function'], + ['Function', 'Function'], + ['Function', 'Class'], + ['Class', 'Method'], +]; + +const pairKey = ([from, to]) => `${from}|${to}`; + +// NODE_TABLES² in declaration order, data pairs first, deduped. 32² = 1024. +const PAIR_UNIVERSE = (() => { + const seen = new Set(DATA_PAIRS.map(pairKey)); + const all = [...DATA_PAIRS]; + for (const from of NODE_TABLES) { + for (const to of NODE_TABLES) { + const key = `${from}|${to}`; + if (seen.has(key)) continue; + seen.add(key); + all.push([from, to]); + } + } + return all; +})(); + +if (PAIR_UNIVERSE.length !== NODE_TABLES.length ** 2) { + throw new Error( + `bench: pair universe is ${PAIR_UNIVERSE.length}, expected ${NODE_TABLES.length ** 2} ` + + `(NODE_TABLES changed — update SIZES, the 1024 ceiling is no longer the ceiling)`, + ); +} +for (const size of SIZES) { + if (size > PAIR_UNIVERSE.length) { + throw new Error(`bench: size ${size} exceeds the ${PAIR_UNIVERSE.length}-pair universe`); + } +} + +const relTableDdlFor = (size) => { + const pairs = PAIR_UNIVERSE.slice(0, size).map(([from, to]) => ` FROM \`${from}\` TO \`${to}\``); + return `CREATE REL TABLE ${REL_TABLE_NAME} (\n${pairs.join(',\n')},\n type STRING,\n confidence DOUBLE,\n reason STRING,\n step INT32\n)`; +}; + +// ---- synthetic data (identical at every size) ---- + +const FILES = 20; +const FNS_PER_FILE = 8; +const CLASSES = 40; +const METHODS_PER_CLASS = 4; +const CALLS_PER_FN = 3; +const REPS = 15; // median over reps +const ANCHORS = 40; // distinct anchor ids queried per rep + +// Batched with UNWIND rather than one statement per row: per-statement overhead +// dwarfs the insert itself here, and load time is not what this bench measures. +function dataStatements() { + const stmts = []; + const fnIds = []; + const classIds = []; + const methodIds = []; + const fileIds = []; + for (let f = 0; f < FILES; f++) fileIds.push(`file-${f}`); + for (let f = 0; f < FILES; f++) { + for (let i = 0; i < FNS_PER_FILE; i++) fnIds.push(`fn-${f}-${i}`); + } + for (let c = 0; c < CLASSES; c++) { + classIds.push(`cls-${c}`); + for (let m = 0; m < METHODS_PER_CLASS; m++) methodIds.push(`m-${c}-${m}`); + } + + const nodeBatch = (label, ids) => + `UNWIND [${ids.map((id) => `{id: '${id}'}`).join(', ')}] AS r ` + + `CREATE (:\`${label}\` {id: r.id, name: r.id, filePath: 'bench.ts'})`; + stmts.push(nodeBatch('File', fileIds)); + stmts.push(nodeBatch('Function', fnIds)); + stmts.push(nodeBatch('Class', classIds)); + stmts.push(nodeBatch('Method', methodIds)); + + const relBatch = (fromLabel, toLabel, type, edges) => + `UNWIND [${edges.map(([f, t]) => `{f: '${f}', t: '${t}'}`).join(', ')}] AS e ` + + `MATCH (a:\`${fromLabel}\` {id: e.f}), (b:\`${toLabel}\` {id: e.t}) ` + + `CREATE (a)-[:${REL_TABLE_NAME} {type: '${type}', confidence: 1.0, reason: 'bench', step: 0}]->(b)`; + + const contains = []; + for (let f = 0; f < FILES; f++) { + for (let i = 0; i < FNS_PER_FILE; i++) contains.push([`file-${f}`, `fn-${f}-${i}`]); + } + stmts.push(relBatch('File', 'Function', 'CONTAINS', contains)); + + // Function→Function calls: each fn calls the next CALLS_PER_FN, wrapping. + const calls = []; + for (let i = 0; i < fnIds.length; i++) { + for (let k = 1; k <= CALLS_PER_FN; k++) calls.push([fnIds[i], fnIds[(i + k) % fnIds.length]]); + } + stmts.push(relBatch('Function', 'Function', 'CALLS', calls)); + + const uses = fnIds.map((id, i) => [id, classIds[i % classIds.length]]); + stmts.push(relBatch('Function', 'Class', 'USES', uses)); + + const hasMethod = []; + for (let c = 0; c < CLASSES; c++) { + for (let m = 0; m < METHODS_PER_CLASS; m++) hasMethod.push([`cls-${c}`, `m-${c}-${m}`]); + } + stmts.push(relBatch('Class', 'Method', 'HAS_METHOD', hasMethod)); + + // Anchors: functions, which have out-edges on two distinct declared pairs. + return { stmts, anchors: fnIds.slice(0, ANCHORS) }; +} + +const { stmts: DATA_STATEMENTS, anchors: ANCHOR_IDS } = dataStatements(); + +// ---- timing ---- + +const median = (xs) => { + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +}; + +const withDb = async (fn) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-bench-pairs-')); + const db = new lbug.Database(path.join(dir, 'db')); + const conn = new lbug.Connection(db); + try { + return await fn(conn); + } finally { + await conn.close().catch(() => {}); + await db.close?.().catch?.(() => {}); + fs.rmSync(dir, { recursive: true, force: true }); + } +}; + +async function runAll(conn, statements) { + for (const s of statements) await conn.query(s); +} + +// The measured shape: BOTH endpoints untyped, anchored by id. LadybugDB must +// consider every declared FROM/TO pair as a candidate. +const UNTYPED_QUERY = (id) => + `MATCH (a {id: '${id}'})-[r:${REL_TABLE_NAME}]->(b) RETURN b.id AS id, r.type AS type`; +// The lower bound: both endpoints labelled, so the planner prunes to one pair. +// Still not flat in the pair count (see the header) — an unused declared pair +// costs something even when the plan never touches it. +const TYPED_QUERY = (id) => + `MATCH (a:Function {id: '${id}'})-[r:${REL_TABLE_NAME}]->(b:Function) RETURN b.id AS id`; + +async function timeQueries(conn, build) { + // Warm: run the whole anchor sweep once uncounted (plan cache + page cache). + for (const id of ANCHOR_IDS) await (await conn.query(build(id))).getAll(); + const samples = []; + let rows = 0; + for (let rep = 0; rep < REPS; rep++) { + const start = process.hrtime.bigint(); + let n = 0; + for (const id of ANCHOR_IDS) n += (await (await conn.query(build(id))).getAll()).length; + samples.push(Number(process.hrtime.bigint() - start) / 1e6); + rows = n; + } + return { ms: median(samples), rows }; +} + +async function measureSize(size) { + return withDb(async (conn) => { + for (const q of NODE_SCHEMA_QUERIES) await conn.query(q); + await conn.query(relTableDdlFor(size)); + await runAll(conn, DATA_STATEMENTS); + const untyped = await timeQueries(conn, UNTYPED_QUERY); + const typed = await timeQueries(conn, TYPED_QUERY); + return { + pairs: size, + untyped_ms: Number(untyped.ms.toFixed(3)), + untyped_rows: untyped.rows, + typed_ms: Number(typed.ms.toFixed(3)), + typed_rows: typed.rows, + }; + }); +} + +// ---- correctness gate: the REAL schema, round-tripped ---- + +async function verifyRealSchema() { + return withDb(async (conn) => { + for (const q of NODE_SCHEMA_QUERIES) await conn.query(q); + // If RELATION_SCHEMA declared a pair twice, LadybugDB rejects this outright + // — the failure mode that kills every `analyze`, not just one repo's. + await conn.query(RELATION_SCHEMA); + const res = await conn.query(`CALL SHOW_CONNECTION('${REL_TABLE_NAME}') RETURN *`); + const rows = await res.getAll(); + const actual = new Set( + rows.map( + (r) => + `${r['source table name'] ?? r.source}|${r['destination table name'] ?? r.destination}`, + ), + ); + const expected = parseRelationSchemaPairs(RELATION_SCHEMA); + const missing = [...expected].filter((p) => !actual.has(p)).sort(); + const extra = [...actual].filter((p) => !expected.has(p)).sort(); + return { declared_pairs: expected.size, db_pairs: actual.size, missing, extra }; + }); +} + +// ---- run ---- + +const CHECK = process.argv.includes('--check'); +const failures = []; + +const verified = await verifyRealSchema(); +if (verified.missing.length > 0 || verified.extra.length > 0) { + failures.push( + `RELATION_SCHEMA round-trip mismatch: ${verified.missing.length} pair(s) parsed but absent ` + + `from SHOW_CONNECTION (${verified.missing.slice(0, 5).join(', ')}), ${verified.extra.length} ` + + `present in the DB but unparsed (${verified.extra.slice(0, 5).join(', ')})`, + ); +} + +const results = []; +for (const size of SIZES) results.push(await measureSize(size)); + +const reference = results.find((r) => r.pairs === REFERENCE_SIZE); +const summary = { + ...verified, + missing: undefined, + extra: undefined, + reference_pairs: REFERENCE_SIZE, +}; +for (const r of results) { + summary[`untyped_ms_${r.pairs}`] = r.untyped_ms; + summary[`typed_ms_${r.pairs}`] = r.typed_ms; + summary[`ratio_${r.pairs}`] = Number((r.untyped_ms / reference.untyped_ms).toFixed(3)); + summary[`typed_ratio_${r.pairs}`] = Number((r.typed_ms / reference.typed_ms).toFixed(3)); +} + +// Row counts must be identical at every size — otherwise the sizes are not +// carrying the same data and the ratios mean nothing. +const rowShapes = new Set(results.map((r) => `${r.untyped_rows}/${r.typed_rows}`)); +if (rowShapes.size !== 1) { + failures.push( + `row counts differ across pair-set sizes (${[...rowShapes].join(' vs ')}) — the data pins ` + + `in DATA_PAIRS are not holding, so the ratios compare different graphs`, + ); +} + +if (!CHECK) { + for (const r of results) process.stdout.write(JSON.stringify(r) + '\n'); + process.stdout.write(JSON.stringify(summary) + '\n'); +} else { + const baselines = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8')); + const budget = baselines[`ratio_${PRODUCTION_SIZE}_budget`]; + if (budget !== undefined && summary[`ratio_${PRODUCTION_SIZE}`] >= budget) { + failures.push( + `production pair set (${PRODUCTION_SIZE}) costs ${summary[`ratio_${PRODUCTION_SIZE}`]}× vs ` + + `${REFERENCE_SIZE} pairs, >= budget ${budget} (untyped ${reference.untyped_ms}ms -> ` + + `${summary[`untyped_ms_${PRODUCTION_SIZE}`]}ms; typed control ` + + `${summary[`typed_ratio_${PRODUCTION_SIZE}`]}×)`, + ); + } + process.stdout.write(JSON.stringify(summary) + '\n'); +} + +if (failures.length > 0) { + for (const f of failures) process.stderr.write(`[schema-pairs] FAIL: ${f}\n`); + process.exit(1); +} +if (CHECK) process.stderr.write(`[schema-pairs --check] PASS (${results.length} sizes)\n`); diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index d0ef387ca..84a235786 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -1,18 +1,24 @@ { "_comment": "Per-language baselines for bench/scope-capture/measure.mjs --check. fingerprint = order-independent sha256 over the lang-resolution/-* fixture corpus + a 20-entity synthetic source (correctness gate; re-baseline intentionally on a legitimate capture change). scaling_budget = max allowed (t800/t250)/(800/250); ~1.0 is linear, ~3.2 is quadratic. The synthetic source is now HERITAGE-BEARING for every language (each Entity extends/implements/embeds/uses-trait/conforms-to a shared base) so the #1951 @reference.inherits synth is gated at scale, not just the base capture loop. All languages thread the tree-sitter captured node instead of re-deriving it with findNodeAtRange(tree.rootNode,...) per match, so all are linear (go #1915, python #1918, ruby/php/rust/csharp #1951, java #1956).", "go": { - "fingerprint": "5d6c59c2f2c0dd937c53bf5d736e0f8376b2899a381e488a33aec23524823efb", + "fingerprint": "e47302079e17a5e73711bbed5416557b49327cb67e4932008700ec6b8fb468b3", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 3d4e32e7490c830516126e28931827949baa3594cb521f7a3d8dcfed95b6018a -> 57b3c55135af8d2af33b9a7c4bf89796a7bee5b5822b402a2dea91af7232cf4a; scaling 1.058 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: provider-owned callable assignment/copy/formal/argument/invoke facts with invocation/constructor-result suppression. Prior 09ecd94911b830f52fa8807560abcbd79f163d02a2072870c1a59297e9a326e1 -> 3d4e32e7490c830516126e28931827949baa3594cb521f7a3d8dcfed95b6018a; scaling 1.039 < 1.5.", "_rebaselined": "#1976: F33 generic composite literal constructor inference adds generic_type captures in composite_literal patterns; fingerprint drift expected.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 57b3c55135af8d2af33b9a7c4bf89796a7bee5b5822b402a2dea91af7232cf4a -> 5d6c59c2f2c0dd937c53bf5d736e0f8376b2899a381e488a33aec23524823efb." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 57b3c55135af8d2af33b9a7c4bf89796a7bee5b5822b402a2dea91af7232cf4a -> 5d6c59c2f2c0dd937c53bf5d736e0f8376b2899a381e488a33aec23524823efb.", + "_rebaselined_2766_go_pointer_receiver_fixture": "#2766: added test/fixtures/lang-resolution/go-pointer-receiver-field-chain/ (2 Go files) as the committed regression fixture for pointer-receiver base resolution. Go fixture_count 100 -> 102. Prior 5d6c59c2f2c0dd937c53bf5d736e0f8376b2899a381e488a33aec23524823efb -> 8cba537ff211fab3bac5fb4456cd1ffba14d6a2db75c40acae28ab8bf29f3d2e. FIXTURE-CORPUS GROWTH, NOT A CAPTURE CHANGE: the accompanying fix is a resolution-time lookup fallback (stripTypePreservingDecoration) and cannot move capture output; go was the ONLY language whose fingerprint drifted, and every other language matched its baseline on the same run.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 8cba537ff211fab3bac5fb4456cd1ffba14d6a2db75c40acae28ab8bf29f3d2e -> 8162272bb897b0b89472c406321cf8d88a5ae4ea83ea9e3c45f8e817041bff9f.", + "_rebaselined_2766_await_subscript_emission": "#2766: extractMixedChain now walks THROUGH await and subscript nodes and peels transparent wrappers at loop entry, so sites whose receiver is `repos[0]` or `(await f())` mint a receiver chain where they previously minted none. EMISSION CHANGE: more sites carry `@reference.receiver-chain`; no existing chain changed shape. Only go and kotlin drifted of 15 \u2014 the two whose fixture corpora contain such receivers. Prior 8162272bb897b0b89472c406321cf8d88a5ae4ea83ea9e3c45f8e817041bff9f -> c9c908f441e3be12fad2448120ed3ea35dc235a12b3f63b0ec532ffdae11d9e9.", + "_rebaselined_2766_phantom_callee_read_site": "#2766: Go's `@reference.read` pattern matches EVERY selector_expression, so a member call `h.dep.Work()` minted THREE sites \u2014 the call, the genuine `h.dep` field read, and a PHANTOM read on the callee `h.dep.Work`. The phantom resolved through findOwnedMember (which prefers methods over fields) and emitted an ACCESSES edge to the METHOD duplicating the CALLS edge at the same position; visible today on any receiver the text cascade can type (`RunFromValueReceiver -> DoWork`). The emitter now drops a read match whose selector is in FUNCTION position. FEWER capture matches for Go, no other language affected \u2014 go was the only fingerprint of 15 that moved. A method VALUE (`f := h.dep.Work`) is not in function position and is untouched. Prior c9c908f441e3be12fad2448120ed3ea35dc235a12b3f63b0ec532ffdae11d9e9 -> 7bb524a32a2eed57a15b454e3a33480e92a496c683e6856ef02179693c0e02e3.", + "_rebaselined_2766_callee_position_marker": "#2766 review fix: a call's callee selector is no longer DROPPED at capture. An earlier commit on this branch dropped it outright, which also deleted the genuine field read on a func-typed struct field (`h.dep.Work()` where `Work func() error`) - callback/hook/mock structs lost their only ACCESSES evidence. The match is now emitted carrying `@reference.callee-position`, and the phantom is suppressed at EMIT by the resolved target's kind instead. Go only: the other 14 languages' fingerprints are byte-identical, which is the check that this is not a cross-language capture change. Prior 7bb524a32a2eed57a15b454e3a33480e92a496c683e6856ef02179693c0e02e3 -> e47302079e17a5e73711bbed5416557b49327cb67e4932008700ec6b8fb468b3; scaling 1.001 < 1.5; fixtures 102 (unchanged), capture_groups_fp 2103." }, "cobol": { - "fingerprint": "d45bb091b0893d0de4fae2486b31ba21719c9377bf35a0908fd3a36fa1c3bf4e", + "fingerprint": "c8c00b56a7da24e04080eb885714fbbf45e3903324f0cf9df0754f5b5a92e3aa", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: COBOL procedure-pointer callable flow facts; multi-topic extraction now consumes each grouped scope/declaration match once instead of requiring a duplicate declaration-only match. Prior 68ee0e95eb9f86f2d92ca35f730f4c2d4d83abc1b5241ae767ff3437780ec8d1 -> d45bb091b0893d0de4fae2486b31ba21719c9377bf35a0908fd3a36fa1c3bf4e; scaling 0.853 < 1.5.", - "_note": "Updated for F17-F23 fixes (P2: TIMES guard, ADD GIVING, SQL AS alias). See PR #1959." + "_note": "Updated for F17-F23 fixes (P2: TIMES guard, ADD GIVING, SQL AS alias). See PR #1959.", + "_rebaselined_2793_declaratives": "PR #2793: corpus-only re-baseline. `cobol-declaratives` was added to test/fixtures/lang-resolution to reproduce the `Namespace\u2192Record` analyze abort (DECLARATIVES / USE AFTER STANDARD ERROR ON ), and this bench globs `lang-resolution/cobol-*`, so the corpus grew 14 -> 15 files. Verified capture-neutral: with that one fixture moved aside the fingerprint is byte-identical to the prior d45bb091b0893d0de4fae2486b31ba21719c9377bf35a0908fd3a36fa1c3bf4e. No COBOL capture code changed in that PR. Scaling 0.677 < 1.5." }, "c": { "fingerprint": "3418cded9f7072152f68992f0a426f43ae7d9d553579a47075fc0cab185848a5", @@ -25,7 +31,7 @@ "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression)." }, "cpp": { - "fingerprint": "7e27aea46f3e17f33c41babbe0ddd982d1ab5920f143864763e0a1c6aef882a5", + "fingerprint": "856d02f3f9d22cb973877211100aee8e052d4bc545922f78704b1a21ce49ddcc", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature/cv metadata. Prior dde874d2c30bda9f634f9799281a66de800cad9f76cf65e7c31839e2ae9da9ff -> 57860dd2a8d4b06c6d2dd0d854c08b781faee3da8f2b6c42ba0c68a9f70e5ccb; scaling 1.090 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: C++ overload-aware function/reference/member-pointer flow facts with invocation/constructor-result suppression. Prior 3a503a1513e7eede3f7a223dcce0896c06d15bdfa920445224c9025848c0d710 -> dde874d2c30bda9f634f9799281a66de800cad9f76cf65e7c31839e2ae9da9ff; scaling 1.034 < 1.5.", @@ -37,20 +43,22 @@ "_note": "#1975: + cpp-out-of-line-class fixture, fixture_count 263->265. #1990: + cpp-adl-ns-plus-hidden-friend-same-name fixture (ADL hidden-friend + namespace-callable merge parity test). Pure fixture-corpus drift \u2014 no scope-extractor change; existing fixtures' captures byte-identical. fixture_count 265->267. #1995: + cpp-union-nested-tail-collision and cpp-anon-ns-tail-collision fixtures \u2014 pure fixture-corpus drift; fixture_count 270->272, fingerprint 538e8be->d63ded6. #1993: + cpp-cross-namespace-same-tail fixture \u2014 pure fixture-corpus drift; fixture_count 272->273, fingerprint d63ded6->6d6207ae. #2077 review follow-up: cpp-member-lattice adds cross-file, qualified-base, nested-template, inherited-using, this-receiver, and non-virtual-override regressions; fixture_count 274->275. Capture scaling remains linear (1.134 < 1.5). #1899: braced-init call arguments emit a conservative parameter-type capture; fixture_count 277, scaling remains linear (1.141 < 1.5).", "_rebaselined_2522_review_fixes": "PR #2522 review fixes: outermost-chain passing modes; ->* ERROR-recovery role order; member-store visibility. Prior 57860dd2a8d4b06c6d2dd0d854c08b781faee3da8f2b6c42ba0c68a9f70e5ccb -> f29bc3f7b1622954d6f6b7647bc9cf6c7a2629ffcc0fe00ac7918e4925876b65; scaling ratio re-verified within budget.", "_rebaselined_2522_prototype_value_cells": "Plain function/method prototypes no longer index as callable value cells (only pointer/parenthesized variable declarators do) \u2014 removes the spurious indirect-invoke facts that leaked phantom CALLS past two-phase suppression. Prior f29bc3f7b1622954d6f6b7647bc9cf6c7a2629ffcc0fe00ac7918e4925876b65 -> a70625bb0a9ef74e760d9d79cc5557485d0f0d3fb935e8a22a0c9556c65b5bb1; scaling re-verified within budget.", - "_rebaselined_receiver_chain_2747": "#2747: additionally adds the `cpp-receiver-chain-arrow` fixture, the behavioural proof for a `->` BASE receiver (`svc->getUser()->save()`) that the rollout fixed and that `cpp-chain-call/` could never catch because it uses the value `.` form. Prior a70625bb0a9ef74e760d9d79cc5557485d0f0d3fb935e8a22a0c9556c65b5bb1 -> 7e27aea46f3e17f33c41babbe0ddd982d1ab5920f143864763e0a1c6aef882a5." + "_rebaselined_receiver_chain_2747": "#2747: additionally adds the `cpp-receiver-chain-arrow` fixture, the behavioural proof for a `->` BASE receiver (`svc->getUser()->save()`) that the rollout fixed and that `cpp-chain-call/` could never catch because it uses the value `.` form. Prior a70625bb0a9ef74e760d9d79cc5557485d0f0d3fb935e8a22a0c9556c65b5bb1 -> 7e27aea46f3e17f33c41babbe0ddd982d1ab5920f143864763e0a1c6aef882a5.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 7e27aea46f3e17f33c41babbe0ddd982d1ab5920f143864763e0a1c6aef882a5 -> 856d02f3f9d22cb973877211100aee8e052d4bc545922f78704b1a21ce49ddcc." }, "csharp": { "_rebaselined": "#1956 synth-widening: + csharp-qualified-base fixture; the synth now walks record_declaration + struct_declaration base_lists and handles alias_qualified_name (matching the #1940 legacy leg), so record/struct heritage now emits. csharp-record-base gains a record inherits capture. (record->record SAME-namespace EXTENDS is a separate registry resolution gap, tracked as follow-up.) Linear (~1.00). (Earlier #1956: heritage-bearing scale source.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged. | #1924 F16: record primary-constructor base bindings now exclude constructor arguments; capture fingerprint changes, scaling remains linear. | #2036 review follow-up: csharp-record-base now exercises primary-constructor base dispatch end to end; +2 capture groups, scaling remains linear.", - "fingerprint": "8a282254b93b3ef2ff34c2fdba819ebc95c53c4fcb09942cbad99f96d3687855", + "fingerprint": "476d98a7cc659951c315d63319c8077bbcf0e5f3ec12d32ed773992a1f3a2adc", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior f31544530924748f9aa37d11cec570bc10c3ddf9d9b237e6df7a17623fd2bb3a -> 75cf380209fa7d1a8a3ec873be1a9424b4e5173be0b08234c2291e8521a9b3c1; scaling 1.061 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: C# method-group/delegate callable flow facts with invocation-result suppression. Prior 2bb5bc8c19cb8eb08c9590545ad8a1968a7152951f7e12746e2d7901d542fed9 -> f31544530924748f9aa37d11cec570bc10c3ddf9d9b237e6df7a17623fd2bb3a; scaling 1.115 < 1.5.", "_note": "#2046: F35 qualified-constructor captures now emit @reference.qualified-name + a simple-name @reference.name on `new Ns.Foo()`/`new A.B.Foo()`; namespace_declaration/file_scoped_namespace_declaration now emit @declaration.namespace name captures (feeding the non-destructive namespacePrefix sidecar for `new B.Foo()` same-tail disambiguation). + csharp-interface-only-base and csharp-namespace-qualified-ctor fixtures. Pure capture-additive + fixture-corpus drift; scaling stays linear (~1.11).", "_rebaselined_2563_instance_ownership": "#2563: csharp-using-static adds same-file ownership, local-function, overload, partial-class, and cross-namespace same-name coverage. Prior 75cf380209fa7d1a8a3ec873be1a9424b4e5173be0b08234c2291e8521a9b3c1 -> e05dc27456bde8175948586c9e7689033a378fa40e9ca4ce78cce41fbea0f2f8; scaling 1.058 < 1.5.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 05a85bae70cf9c94f42459c843cfc36e3e81c872e5dcc7d77bc42fbc390f4bfe -> 8a282254b93b3ef2ff34c2fdba819ebc95c53c4fcb09942cbad99f96d3687855." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 05a85bae70cf9c94f42459c843cfc36e3e81c872e5dcc7d77bc42fbc390f4bfe -> 8a282254b93b3ef2ff34c2fdba819ebc95c53c4fcb09942cbad99f96d3687855.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 8a282254b93b3ef2ff34c2fdba819ebc95c53c4fcb09942cbad99f96d3687855 -> 476d98a7cc659951c315d63319c8077bbcf0e5f3ec12d32ed773992a1f3a2adc." }, "rust": { - "fingerprint": "83812d82f0e2c3eb552f3246381ca3dd5ccd6783d63aba3325f1343e7772280c", + "fingerprint": "6174889b8c98e0af430fa54c268dc781989ca9a8172d690eebae37a95f77e809", "scaling_budget": 1.5, "_rebaselined_mod_node_identity_2745_review": "#2745 review: added rust-2742-mod-members, rust-2742-nested-mods and rust-2742-type-vs-module under lang-resolution for the container/owner-edge fix, nested inline modules, and the imported-type-vs-module precedence. emitRustScopeCaptures is unchanged \u2014 verified by removing ONLY those three fixture dirs and re-running, which reproduces the prior fingerprint exactly, so the shift is purely corpus growth (fixture_count 196 -> 202, capture_groups_fp 3432 -> 3556). Prior 90fda086a4e13aa069a5981f63ed58ab1c71f1ed3da5e1480a080e1992b0d3e5 -> 05acbaca48427e0d9e0793bcd0ce4057712d3716b5e7868189c12e05ef8dd300; scaling 1.022 local / 1.057 CI < 1.5. NOTE for the next fixture author: a new rust-* fixture drifts BOTH this bench baseline and the rust-captures-golden snapshot. Updating only the golden is how this reached CI red.", "_rebaselined_dyn_trait_object_2604": "#2604: RUST_SCOPE_QUERY now captures function_signature_item (abstract trait methods, no body) as a scope + declaration, so a &dyn Trait receiver can dispatch a CALLS edge to the trait's own method. Additive capture shift across every bench fixture with a required trait method. Prior df369c5a5f8de7753fc8bab8b4108ef5081750974ea5085ba9a867675ac9eb29 -> f7742f65f14d7d6590df7f16303fc3cc9dc0c233cd80bf90c98b084933cd3846; scaling 1.033 < 1.5.", @@ -61,35 +69,39 @@ "_rebaselined_import_disambiguation_2514": "#2514: added rust-import-* and rust-dup-* fixtures under lang-resolution for the range-binding ambiguity latch + import-disambiguated resolution (for-loops / struct destructuring across explicit/aliased/glob use imports). emitRustScopeCaptures is unchanged; the corpus fingerprint shifts purely because the fixture set grew (130 -> 174). Prior f7742f65f14d7d6590df7f16303fc3cc9dc0c233cd80bf90c98b084933cd3846 -> 655aed01cf1b6b84fa0c64d48dfb2526ecb67f47d90f0a91edabacd269a212db; scaling 1.06 < 1.5.", "_rebaselined_self_type_binding_2714": "#2714: a Rust `Self` type binding now records the enclosing impl's type instead of the literal 'Self'. `let fresh = Self { .. }` inside `impl User` binds `fresh: User`; recorded verbatim it bound `fresh: Self`, which resolves to nothing. The type-env channel already substituted this (type-extractors/rust.ts findEnclosingImplType); the scope-resolution channel did not, so the two disagreed. The gap was invisible while lookupCore Step 1 still walked the lexical chain for NAMED receivers \u2014 the impl scope binds the method by name, so fresh.validate() resolved by accident \u2014 and became a lost CALLS edge when #2714 stopped that walk. Only the rust fingerprint moves; the other 14 languages are byte-identical.", "_rebaselined_module_tree_2730": "#2730 + #2741 review: RUST_SCOPE_QUERY captures mod_item as @declaration.namespace (a Rust module is an item, mirroring the C++ namespace_definition capture) and tags scoped call sites with @reference.qualified-name so the written path survives to resolution. Both are additive captures: every bench fixture holding a mod block or a Foo::bar() call gains groups, and the corpus also grew by the rust-2730-* fixtures added for the fix and its review (workspace-crates, type-qualified, gaps, samename-wrapper, crate-layout). Prior 7f1240b38457468f06b7931e0c2c578f218f922774d0dc7e2ee6ef3b08d4d689 -> 90fda086a4e13aa069a5981f63ed58ab1c71f1ed3da5e1480a080e1992b0d3e5; scaling 1.061 < 1.5; fixture_count 196. Only the rust fingerprint moves; the other 14 languages are byte-identical. The earlier revision of this note cited 655aed01... as the prior value, which was two rebaselines stale (it predates #2604 and #2714); the CI gate compares live fingerprints, not this prose, so nothing caught it.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 05acbaca48427e0d9e0793bcd0ce4057712d3716b5e7868189c12e05ef8dd300 -> 83812d82f0e2c3eb552f3246381ca3dd5ccd6783d63aba3325f1343e7772280c." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 05acbaca48427e0d9e0793bcd0ce4057712d3716b5e7868189c12e05ef8dd300 -> 83812d82f0e2c3eb552f3246381ca3dd5ccd6783d63aba3325f1343e7772280c.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 83812d82f0e2c3eb552f3246381ca3dd5ccd6783d63aba3325f1343e7772280c -> 6174889b8c98e0af430fa54c268dc781989ca9a8172d690eebae37a95f77e809." }, "php": { - "fingerprint": "3745662053c76b6ae0a84a29aad319626ed5ccb88f7b9376c2680d3dc6502e28", + "fingerprint": "b213a872342da2d866b04681dede988770e4d3dfdc0d6e9f62212ec5b59cdc2c", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior df7b1565f9115d66b1ae32e4a408d651afb2521b14e5ca615f3be426c29af618 -> 4a688fa5a7016546f7f3c6d44de023608ae80c5b0e3670c16f6e61b3632608fd; scaling 1.078 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: PHP first-class callable and variable-invocation flow facts with invocation-result suppression. Prior 31c9e3f3cb7094a2bf9021cf9db859036e002f8b44605cd993b470fc600e97cb -> df7b1565f9115d66b1ae32e4a408d651afb2521b14e5ca615f3be426c29af618; scaling 1.074 < 1.5.", "_rebaselined": "#1956: heritage-bearing scale source (class extends Base + use trait); both forms gated at scale; linear (~1.04). | #2481/#2482: PHP imports carry a symbol-kind capture so function/constant imports resolve by declaring file; capture shape changes, scaling remains linear (~1.04).", "_note": "PR #1931: F53 import multi-clause, F54 enum_case, F55 anonymous_class \u2014 fixture count 138\u2192140, fingerprint drift expected.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 4a688fa5a7016546f7f3c6d44de023608ae80c5b0e3670c16f6e61b3632608fd -> 3745662053c76b6ae0a84a29aad319626ed5ccb88f7b9376c2680d3dc6502e28." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 4a688fa5a7016546f7f3c6d44de023608ae80c5b0e3670c16f6e61b3632608fd -> 3745662053c76b6ae0a84a29aad319626ed5ccb88f7b9376c2680d3dc6502e28.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 3745662053c76b6ae0a84a29aad319626ed5ccb88f7b9376c2680d3dc6502e28 -> b213a872342da2d866b04681dede988770e4d3dfdc0d6e9f62212ec5b59cdc2c." }, "ruby": { - "fingerprint": "fc81941b0a921074fa80dc448284de9a23bd07358ddc84d4894797cc08c3fe83", + "fingerprint": "1c8c9c4b54036fa24c2a81e39ea530e938645c856d369075e5f437da78218c57", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior cff273ae6cb7232c977d9241581834a2a2fa8bcf6369f7bd8f2471cd4419a6ef -> bf50ec6a53c8c91680dc6feac63a8956e78b1059249232dc25a0cfed25f31236; scaling 1.103 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Ruby Method/Proc callable flow facts with invocation/constructor-result suppression. Prior b5ea93bb3d0469c3821a8c70f5d5991c6f326e41097c119ad691154301dcc753 -> cff273ae6cb7232c977d9241581834a2a2fa8bcf6369f7bd8f2471cd4419a6ef; scaling 1.086 < 1.5.", "_rebaselined": "#1956 synth-widening: + ruby-qualified-base fixture; synth now reduces a scope_resolution superclass (class C < Mod::Super) to its trailing constant (matching the #1940 legacy leg), at parity. Linear (~1.03). (Earlier #1956: heritage-bearing scale source.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged.", "_note": "F62: + scope_resolution class/module declaration captures \u2014 fixture count 78\u219281, fingerprint drift expected. #1975: + ruby-tail-collision fixture (Foo::Bar vs Baz::Bar stay distinct nodes) \u2014 pure fixture-corpus drift, scope-extractor captures unchanged; 81\u219282. #1991: + ruby-nested-mixin-tail-collision fixture (85\u219286). Recomputed on the #942 merge (fixture-comment rewording shifts capture byte-positions, capture LOGIC unchanged): bf6b13a -> b5ea93bb.", "_rebaselined_2522_review_fixes": "PR #2522 review fixes: bare identifiers are calls, not callable references (bareNamesAreCalls). Prior bf50ec6a53c8c91680dc6feac63a8956e78b1059249232dc25a0cfed25f31236 -> 070e4e11502442998ddf4048c2981cf1b2b735a87362ff854c5d14d71f98f4e2; scaling ratio re-verified within budget.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior fea3edf82f521995147874b7f6c5f9e2eb88efdebf6365668f3260e913f0b558 -> fc81941b0a921074fa80dc448284de9a23bd07358ddc84d4894797cc08c3fe83." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior fea3edf82f521995147874b7f6c5f9e2eb88efdebf6365668f3260e913f0b558 -> fc81941b0a921074fa80dc448284de9a23bd07358ddc84d4894797cc08c3fe83.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior fc81941b0a921074fa80dc448284de9a23bd07358ddc84d4894797cc08c3fe83 -> 1c8c9c4b54036fa24c2a81e39ea530e938645c856d369075e5f437da78218c57." }, "swift": { - "fingerprint": "a6fca5f052ae5ec635b56051e28a168c864a988b2221a3279ddd69807378ba0b", + "fingerprint": "2f04ae960123cf50138a49fabdc5a146c2963170cecf5755c552b23c9055a9e7", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 5f923c6604d825d12b249f31c155b0f4d13a8379d532e5dde64a0f9b15cf4725 -> 7687ee2466e16020a12440a03fbda53e63aa05f94b4481f6133c09867a0d560d; scaling 1.042 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Swift function-value callable flow facts with invocation-result suppression. Prior 180ac68e780bdf6f9089d53f51cbb9a66aed3e7774631cc3fcbaae5020213998 -> 5f923c6604d825d12b249f31c155b0f4d13a8379d532e5dde64a0f9b15cf4725; scaling 1.043 < 1.5.", "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression).", "_rebaselined_2522_review_fixes": "PR #2522 review fixes: assignment target:/result: fields join the shared fallback. Prior 7687ee2466e16020a12440a03fbda53e63aa05f94b4481f6133c09867a0d560d -> 115c5da807e36bb12fdeba28e44f2b6484ef322ff26c19fa0f191febaf774248; scaling ratio re-verified within budget.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 115c5da807e36bb12fdeba28e44f2b6484ef322ff26c19fa0f191febaf774248 -> a6fca5f052ae5ec635b56051e28a168c864a988b2221a3279ddd69807378ba0b." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 115c5da807e36bb12fdeba28e44f2b6484ef322ff26c19fa0f191febaf774248 -> a6fca5f052ae5ec635b56051e28a168c864a988b2221a3279ddd69807378ba0b.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior a6fca5f052ae5ec635b56051e28a168c864a988b2221a3279ddd69807378ba0b -> 2f04ae960123cf50138a49fabdc5a146c2963170cecf5755c552b23c9055a9e7." }, "dart": { "fingerprint": "ba93c90dcd341259e8e088816bc8c76ad27882419f665e35c056dc22fa54cf73", @@ -102,7 +114,7 @@ "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0." }, "java": { - "fingerprint": "310adbc2e0827b5ac749acaa981cd12d256fc5b7cbc5592c5bee219e92abf9ee", + "fingerprint": "a9943355e945e03ddb87c800f4cc1f62b3d04feefb3ec64c258d8e0bb3b3fcd9", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata; same-name lexical regions use an O(ancestor-depth) ID-set lookup. Prior d5c59d7dc9e206637515d5aea1163f7c1cdd76410c38c5fe6143d13d19677d6a -> 004a3592998dca1193bd1429a8284513725de7764f2a3eceedaaa984cfd763b4; scaling 0.992 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Java method-reference/SAM callable flow facts with invocation-result suppression. Prior 062d754764aaa8a6772fb90875c710502a63e3e7a300e633942381ed914faada -> d5c59d7dc9e206637515d5aea1163f7c1cdd76410c38c5fe6143d13d19677d6a; scaling 1.074 < 1.5.", @@ -114,16 +126,18 @@ "_rebaselined_2564_record_capture": "PR for #2564: JAVA_QUERIES gained a (record_declaration name: (identifier) @name) @definition.record capture, previously entirely missing (record_declaration had no structure-phase capture at all, unlike class/interface/enum) - a record's methods existed as ownerless Method nodes with no HAS_METHOD edge. Two new java-* fixtures (java-record-methods, java-new-expr-chain-call) joined the corpus. Prior 975b68aaac6d06094260fb0c67f9b1bc03692ba7220669d192aca9dccd5fc0ca -> 85fc7af9c3c1bceac76cb4f27214410b04967682a2eaa7e468e26efd1f4e2537; scaling 1.059 < 1.5.", "_rebaselined_2561_enum_constant_receiver": "PR for #2561: synthesizeJavaAnonymousClassDeclarations now emits a class-scope @type-binding.annotation/name/type per enum constant (constant simple name -> its E$N synthesized class when bodied, else the host enum) so E.CONST.method() resolves through the existing compound-receiver chain walk. Two drivers of the drift, both in the java-enum-constant-body fixture (this bench's corpus IS test/fixtures/lang-resolution): (1) one extra type-binding match per enum_constant from the capture change; (2) review follow-up added a body-less Plain.java enum + EnumConst.dispatchToConstant/dispatchInherited methods (bodied-override, inherited-via-MRO, and body-less dispatch call sites). The review's fail-safe hardening (bodied constant binds ONLY to E$N, never the host enum, when name synthesis fails on a malformed tree) is output-neutral on this well-formed corpus (verified: fingerprint identical with and without it). Prior 85fc7af9c3c1bceac76cb4f27214410b04967682a2eaa7e468e26efd1f4e2537 -> d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686; scaling < 1.5.", "_rebaselined_2562_local_classes": "#2562: Java block-local classes, enums, records, and interfaces use source-type-relative JLS 13.1 Host$NLocal identities with javac-compatible per-(host, simple-name) numbering; anonymous numbering remains separate. Lexical aliases begin at each declaration and end with its immediate block. Expanded java-local-class-naming fixtures cover declaration order, disjoint blocks, initializers, lambdas, local type kinds, and recursive local/member/anonymous host chains. Prior d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686 -> 6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197; scaling 1.204 < 1.5.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197 -> 310adbc2e0827b5ac749acaa981cd12d256fc5b7cbc5592c5bee219e92abf9ee." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197 -> 310adbc2e0827b5ac749acaa981cd12d256fc5b7cbc5592c5bee219e92abf9ee.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 310adbc2e0827b5ac749acaa981cd12d256fc5b7cbc5592c5bee219e92abf9ee -> a9943355e945e03ddb87c800f4cc1f62b3d04feefb3ec64c258d8e0bb3b3fcd9." }, "java-local-types": { - "fingerprint": "3ca67847ea2b9a71b0a41e09f943767e5a2d3a113d3e203499ee364e37f40236", + "fingerprint": "8c50bbc83dff4f7f5abd06078aa6abc6b64af05fddb17ee826b5f3df3d346633", "scaling_budget": 1.5, "_added": "#2562 performance follow-up: co-scales same-host, same-name local classes and anonymous classes to gate JLS binary-name ordinal allocation. Precomputed per-sequence ordinals reduce the focused 100->800 workload from 176->6655ms to 141->752ms; normalized 250->800 scaling is 1.054.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior a9ad88de21ca6747a923260dbdf677fb74a004abbf9d57781f745e3a9027530b -> 3ca67847ea2b9a71b0a41e09f943767e5a2d3a113d3e203499ee364e37f40236." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior a9ad88de21ca6747a923260dbdf677fb74a004abbf9d57781f745e3a9027530b -> 3ca67847ea2b9a71b0a41e09f943767e5a2d3a113d3e203499ee364e37f40236.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 3ca67847ea2b9a71b0a41e09f943767e5a2d3a113d3e203499ee364e37f40236 -> 8c50bbc83dff4f7f5abd06078aa6abc6b64af05fddb17ee826b5f3df3d346633." }, "typescript": { - "fingerprint": "9e112415f1169f08576826c12ea1d137d1994e34b44c45986c9ffee83b8b4edc", + "fingerprint": "cdefe88d3c275f31953216c676ef32c7bf5727d56b9c3840b81ee6bf85749dff", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 27f937bfb47d4bded316ea3c785ff659c8cd88a5761d928f113477a08c802c78 -> e05446620c5b80b7aae291cfdf32f693580fada2ae687124769b04a0c03bfe63; scaling 0.983 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: lexical callable bindings, direct-callee argument metadata, and invocation-result suppression. Prior db5933cc6760234ed7d495123410feba6de243646d583f20d43032b9459f81fd -> 27f937bfb47d4bded316ea3c785ff659c8cd88a5761d928f113477a08c802c78; scaling 0.975 < 1.5.", @@ -133,10 +147,11 @@ "_rebaselined_2522": "#2522 intentional @reference.value-ref/property-key capture additions. GitHub Actions run 29553361660 job 87800394279: prior 3f44a4a6892698df2d145c8ff2812c3b318807648983c88aca28fbd694f172f9 -> 25de86fd3377132c4e35d3d98f4f94a58e0cfeb7c22948a8ea3be4e793be74fd; scaling ratio 0.987 < 1.5.", "_rebaselined_2550_instance_model": "PR #2549 (#2545/#2551): object literals emit @scope.object (was unscoped, then @scope.block during development). Prior e05446620c5b80b7aae291cfdf32f693580fada2ae687124769b04a0c03bfe63 -> 3280b13d3f9378ab23eee31c2edc779b5a9ae1e7bb510c23a24855b44406d2f4; scaling 0.981 < 1.5.", "_rebaselined_receiver_owner_2701": "#2701: every non-arrow function form now carries a `@receiver-owner.this` marker on the same node as `@scope.function`, so a scope that BINDS its own `this` can stop the receiver walk (`Scope.ownsReceivers`). Verified before re-baselining by diffing the capture-name histogram over this same fixture corpus against 1d3088173f6f93827641b476d614d5d15cd4f3ea: the ONLY delta is @receiver-owner.this (typescript +143, javascript +32) \u2014 every other capture count is byte-identical, so no existing capture moved. Prior 3280b13d3f9378ab23eee31c2edc779b5a9ae1e7bb510c23a24855b44406d2f4 -> 281e95484203b481094729ca249ef0423c41273eac35e424cdfd032a0dac7699.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior cad25be9f81d6e021ebae8dcb166bc0af3a1ba8021f1506f6ca93fd4c2649000 -> 9e112415f1169f08576826c12ea1d137d1994e34b44c45986c9ffee83b8b4edc." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior cad25be9f81d6e021ebae8dcb166bc0af3a1ba8021f1506f6ca93fd4c2649000 -> 9e112415f1169f08576826c12ea1d137d1994e34b44c45986c9ffee83b8b4edc.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 9e112415f1169f08576826c12ea1d137d1994e34b44c45986c9ffee83b8b4edc -> cdefe88d3c275f31953216c676ef32c7bf5727d56b9c3840b81ee6bf85749dff." }, "javascript": { - "fingerprint": "83344b7cba093702f4528eeee44e438809c229d43b12e69ed288812ce7ffc7bc", + "fingerprint": "806f70ad3cce5fc849f6d06a08ace8a95f92a1ea84a2418fddabb1eef5846594", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3 -> 479927409bbdd9852a36172c8260aa56df260e99129a7a9c20a0d1903dd5538b; scaling 1.050 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: lexical callable bindings, direct-callee argument metadata, and invocation-result suppression. Prior 917a9cd975ba035bdad71fdb70cd72eeddec58c25797e5a1addfa6172808a55c -> b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3; scaling 1.093 < 1.5.", @@ -146,10 +161,11 @@ "_rebaselined_2522": "#2522 intentional @reference.value-ref/property-key capture additions. GitHub Actions run 29553361660 job 87800394279: prior d72f03c6c502235d2d4b74d66baa5c7d361f040d7a1b72e84acad61210d05ae8 -> 5567dd47e7ba29821a518c4a9852adc3b774e25ef3e7a6e2b3ecb7b59ddab73c; scaling ratio 1.031 < 1.5.", "_rebaselined_2550_instance_model": "PR #2549 (#2545/#2551): object literals emit @scope.object. Prior 479927409bbdd9852a36172c8260aa56df260e99129a7a9c20a0d1903dd5538b -> f1ccf42a36895c8e34dcb724286f247d469835f2dcbb23ad3347190adc7fde1c; scaling 1.096 < 1.5.", "_rebaselined_receiver_owner_2701": "#2701: every non-arrow function form now carries a `@receiver-owner.this` marker on the same node as `@scope.function`, so a scope that BINDS its own `this` can stop the receiver walk (`Scope.ownsReceivers`). Verified before re-baselining by diffing the capture-name histogram over this same fixture corpus against 1d3088173f6f93827641b476d614d5d15cd4f3ea: the ONLY delta is @receiver-owner.this (typescript +143, javascript +32) \u2014 every other capture count is byte-identical, so no existing capture moved. Prior f1ccf42a36895c8e34dcb724286f247d469835f2dcbb23ad3347190adc7fde1c -> 90601494695b834d3a9af7ac4844eac603f4f432809a05554cc59de0674a4354.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 1c71ef628eb75a3b111afa8c2a7c351c16a7f5aab9fac2f098f82b2866312aa8 -> 83344b7cba093702f4528eeee44e438809c229d43b12e69ed288812ce7ffc7bc." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 1c71ef628eb75a3b111afa8c2a7c351c16a7f5aab9fac2f098f82b2866312aa8 -> 83344b7cba093702f4528eeee44e438809c229d43b12e69ed288812ce7ffc7bc.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 83344b7cba093702f4528eeee44e438809c229d43b12e69ed288812ce7ffc7bc -> 806f70ad3cce5fc849f6d06a08ace8a95f92a1ea84a2418fddabb1eef5846594." }, "kotlin": { - "fingerprint": "d3c4d2fa0d82d248a2299cfc888b067187ad1faf2c87a97f93c6ed835eefc3f1", + "fingerprint": "efd5dbf80ffcd3bab2834d1010f6fe2b239dcc5d58229938dea9cff8d0f380f2", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior bddba25d5a88152bbbee8d70e82c944b5302accb4b625df782adb1d4f7a7ac12 -> e856951c2a779163d555dadc8e1bf59304a86caed78ac1f450d9caa2b50f63d1; scaling 1.090 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Kotlin callable-reference flow facts with invocation-result suppression. Prior 4900431791f2b9280009deb2b82659c26ead8aa6fb8731190a7c505dec5a9041 -> bddba25d5a88152bbbee8d70e82c944b5302accb4b625df782adb1d4f7a7ac12; scaling 0.880 < 1.5.", @@ -159,6 +175,8 @@ "_rebaselined_2522_review_fixes": "PR #2522 review fixes: fieldless assignment nodes decomposed positionally. Prior e856951c2a779163d555dadc8e1bf59304a86caed78ac1f450d9caa2b50f63d1 -> 4b31f46cfb004ba769a96feeb06ae4ef109c77410f54e7aaab4a688df599b112; scaling ratio re-verified within budget.", "_rebaselined_2550_instance_model": "PR #2549 (#2545): anonymous object expressions (object_literal) emit @scope.class, and the kotlin-object-literal-scope fixture joined the corpus. Prior 4b31f46cfb004ba769a96feeb06ae4ef109c77410f54e7aaab4a688df599b112 -> a6fce0dff00e88d41d85023eaf3f35016b5217c7e5225f24a598e4c70bb63091; scaling 0.951 < 1.5.", "_rebaselined_2563_instance_ownership": "#2563: kotlin-instance-ownership adds unrelated, inherited, outer-instance, and anonymous-object coverage. Prior a6fce0dff00e88d41d85023eaf3f35016b5217c7e5225f24a598e4c70bb63091 -> 9f159f8810d342ef1c821f466efd6920dad9a190f06000056e6cd2815861b195; scaling 1.257 < 1.5.", - "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 9f159f8810d342ef1c821f466efd6920dad9a190f06000056e6cd2815861b195 -> d3c4d2fa0d82d248a2299cfc888b067187ad1faf2c87a97f93c6ed835eefc3f1." + "_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 9f159f8810d342ef1c821f466efd6920dad9a190f06000056e6cd2815861b195 -> d3c4d2fa0d82d248a2299cfc888b067187ad1faf2c87a97f93c6ed835eefc3f1.", + "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior d3c4d2fa0d82d248a2299cfc888b067187ad1faf2c87a97f93c6ed835eefc3f1 -> c1f0cc9058ab11b7cd6fc8b440deb6db2b2f530f2eb21178923e68a3d0796c4b.", + "_rebaselined_2766_await_subscript_emission": "#2766: extractMixedChain now walks THROUGH await and subscript nodes and peels transparent wrappers at loop entry, so sites whose receiver is `repos[0]` or `(await f())` mint a receiver chain where they previously minted none. EMISSION CHANGE: more sites carry `@reference.receiver-chain`; no existing chain changed shape. Only go and kotlin drifted of 15 \u2014 the two whose fixture corpora contain such receivers. Prior c1f0cc9058ab11b7cd6fc8b440deb6db2b2f530f2eb21178923e68a3d0796c4b -> efd5dbf80ffcd3bab2834d1010f6fe2b239dcc5d58229938dea9cff8d0f380f2." } } diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 6ecb49d33..98555cd48 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -2333,15 +2333,15 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/busboy": { diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts new file mode 100644 index 000000000..bf5100436 --- /dev/null +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -0,0 +1,136 @@ +/** + * Weight-aware partitioning for the cross-platform test matrix. + * + * WHY THIS EXISTS. `run-cross-platform.ts` used to hand vitest the whole file + * list plus `--shard=i/n`, and vitest partitions by file COUNT. Runtime on this + * suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 361 s + * and `worker-pool` 221 s, while most files are under a second — so a + * count-split routinely put several of the heaviest suites on one shard. That + * is #2449, and this file's sibling header has documented the symptom ("the + * heaviest spawn suites can cluster on one shard") since the watchdog was first + * raised from 15 to 20 minutes. + * + * It went from a latent hazard to a red matrix when three CHEAP files (the + * `dist/` module-load closure guards: 448 ms, 53 ms, sub-second) were added to + * `SPAWN_CLI`. They cost nothing to run, but a count-split re-partitions on + * every insertion, and the reshuffle happened to land `cli-e2e` + `cli-limit-e2e` + * + `analyze-heap-oom-e2e` together on shard 1/3 — 32 files against 26 and 29 — + * which blew the 20-minute budget with four files still queued. Nothing about + * the added files caused it; they were simply the perturbation. + * + * So the split is done HERE, by weight, and only the chosen shard's files are + * handed to vitest. Two properties follow, and both are pinned in + * `test/unit/cross-platform-shard.test.ts`: + * + * - the heaviest suites are spread across shards by construction, so the + * busiest shard tracks the ideal rather than the luck of the sort order; + * - adding or removing a CHEAP file cannot move a heavy one, so registering a + * new platform-sensitive test is no longer a CI-stability gamble. That is the + * property whose absence caused this. + */ + +/** + * Measured wall-clock on the WINDOWS runner (the slowest platform, so it is the + * one that decides the budget), in seconds, from the last fully-green matrix run + * plus the timed files of the run that failed. + * + * Only files heavy enough to matter are listed; everything else is carried by + * {@link PER_FILE_OVERHEAD_SEC} alone. These are load-balancing hints, NOT + * assertions — no + * test asserts a runtime, and drift only makes the split slightly less even, so + * a stale entry is harmless and refreshing them is optional. Deliberately not + * auto-generated: a committed table is reviewable and works offline, and the + * alternative (timing files at CI runtime to decide the split) would make the + * partition depend on the very machine load it is trying to protect against. + */ +export const WINDOWS_WEIGHTS_SEC: Readonly> = { + 'test/integration/cli-e2e.test.ts': 361, + 'test/integration/worker-pool.test.ts': 222, + 'test/unit/incremental-vector-extension-ordering.test.ts': 87, + 'test/integration/cli-limit-e2e.test.ts': 75, + 'test/unit/hooks.test.ts': 26, + 'test/integration/analyze-heap-oom-e2e.test.ts': 23, + 'test/unit/git-utils.test.ts': 18, + 'test/integration/hooks-e2e.test.ts': 15, + 'test/integration/tree-sitter-languages.test.ts': 9, + 'test/unit/repo-manager.test.ts': 9, + 'test/unit/detect-changes-worktree.test.ts': 9, + 'test/integration/antigravity-hook-e2e.test.ts': 7, + 'test/unit/index-lock.test.ts': 5, + 'test/unit/setup.test.ts': 5, +}; + +/** + * Fixed cost every file pays regardless of what it asserts: a pool worker start, + * module graph evaluation, and (for most of this list) a native addon load. + * + * Added to EVERY file's weight, not just unmeasured ones, and that is the point. + * Calibrated against the last green Windows matrix: its busiest shard ran 736 s + * of wall clock over ~511 s of measured file time, so roughly 8 s per file is + * unattributed setup. Without this term the balancer treats a light file as + * nearly free and, having isolated the two monsters, piles every remaining file + * onto the other shards — trading a runtime imbalance for a file-count one that + * costs just as much. With it, the split balances runtime AND count together. + */ +const PER_FILE_OVERHEAD_SEC = 8; + +/** + * Scheduling weight for `file`: its measured runtime (0 if it was fast enough + * that vitest printed no duration) plus the per-file floor above. + */ +export function weightOf(file: string): number { + return (WINDOWS_WEIGHTS_SEC[file] ?? 0) + PER_FILE_OVERHEAD_SEC; +} + +/** + * Partition `files` into `total` shards and return the 1-based `index` one. + * + * Longest-processing-time first: sort by weight descending, then repeatedly give + * the next file to the lightest shard so far. LPT is the standard greedy for + * multiprocessor scheduling and is guaranteed within 4/3 of optimal — far more + * than enough here, where the goal is only "no shard gets two monsters". + * + * Ties break on the file path so the partition is DETERMINISTIC: every shard + * computes the same split independently, on a different machine, with no + * coordination — which is what lets each runner select its own slice. + * + * Returns files in the input list's original order, not weight order, so failure + * output and reruns stay readable. + */ +export function shardFiles( + files: readonly string[], + index: number, + total: number, +): readonly string[] { + if (!Number.isInteger(total) || total < 1) { + throw new Error(`shard total must be a positive integer, got ${total}`); + } + if (!Number.isInteger(index) || index < 1 || index > total) { + throw new Error(`shard index must be in 1..${total}, got ${index}`); + } + if (total === 1) return [...files]; + + const byWeightDesc = [...files].sort((a, b) => { + const diff = weightOf(b) - weightOf(a); + return diff !== 0 ? diff : a.localeCompare(b); + }); + + const loads = Array.from({ length: total }, () => 0); + const assigned = Array.from({ length: total }, () => new Set()); + for (const file of byWeightDesc) { + let lightest = 0; + for (let i = 1; i < total; i++) { + if (loads[i]! < loads[lightest]!) lightest = i; + } + assigned[lightest]!.add(file); + loads[lightest]! += weightOf(file); + } + + const mine = assigned[index - 1]!; + return files.filter((f) => mine.has(f)); +} + +/** Total weight of a file set — the shard cost this balancer is minimising. */ +export function shardWeight(files: readonly string[]): number { + return files.reduce((sum, f) => sum + weightOf(f), 0); +} diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 76bb3891e..325699953 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -186,6 +186,33 @@ const SPAWN_CLI = [ // exposed a file-backend double-admit race here (#2658 review); the reclaim is // now judgment-verified so a live holder is never displaced. 'test/integration/analyze-index-lock-concurrency.test.ts', + // The three `dist/` module-load closure guards, all built on the shared + // child-process probe in `test/helpers/module-load-probe.ts`. That probe IS + // the platform-varying part: it spawns `process.execPath` in array form, + // clears NODE_OPTIONS, addresses its target via `pathToFileURL` (Windows needs + // the `file:///C:/...` form — a bare absolute path is not a valid ESM + // specifier there), and renders every result through a `path.sep`→POSIX + // normalisation the anchors and offender regexes depend on. None of that is + // proven anywhere else. + // + // Cheap: measured on the Windows runner at 448 ms, 53 ms and sub-second. An + // earlier attempt to register them still turned the matrix red — not from + // their own cost, but because vitest sharded by file COUNT, so inserting any + // file re-partitioned the list and happened to cluster `cli-e2e` (361 s) with + // `cli-limit-e2e` (75 s) on one shard. The split is weight-aware now + // (`scripts/cross-platform-shard.ts`), so a cheap file can no longer move a + // heavy one. + // + // #2802: MCP startup must not eagerly load the analyze-only language + // provider registry or the group contract extractors. + 'test/integration/mcp/startup-language-closure.test.ts', + // PR #1383: `cli/mcp.js`'s static-import closure must stay leaf-only so no + // native binding initialises before the stdout sentinel installs. + 'test/integration/mcp/import-closure.test.ts', + // #2091/#2093/#2116: the scope-resolution registry must not load the optional + // tree-sitter grammars at import time. The offender regexes match grammar + // paths with either separator, which only the Windows runner proves. + 'test/integration/optional-grammars/registry-import-closure.test.ts', ]; // Worker threads tests — exercise real worker_threads which have diff --git a/gitnexus/scripts/run-cross-platform.ts b/gitnexus/scripts/run-cross-platform.ts index 3b829e3ca..5404a87c3 100644 --- a/gitnexus/scripts/run-cross-platform.ts +++ b/gitnexus/scripts/run-cross-platform.ts @@ -15,6 +15,7 @@ import path from 'path'; import { fileURLToPath } from 'url'; import { ALL_CROSS_PLATFORM } from './cross-platform-tests.js'; import { parseShardArg } from './shard-arg.js'; +import { shardFiles, shardWeight } from './cross-platform-shard.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '..'); @@ -29,8 +30,10 @@ if (missing.length > 0) { } // Optional sharding (CI): `--shard=/` splits the fixed file list across -// parallel matrix shards so each runner processes ~1/n of it. Passed straight -// through to vitest, which partitions the *given* files deterministically. The +// parallel matrix shards. The split is computed HERE, by measured weight, and +// only this shard's files are handed to vitest — it is NOT passed through, +// because vitest partitions by file COUNT and this suite's runtimes span three +// orders of magnitude (see cross-platform-shard.ts). The // Windows runner is ~5x slower than macOS/Linux on this spawn-heavy suite (~50 // CLI/worker process spawns), so a single shard was creeping past the watchdog // below; sharding keeps each runner well under it (see ci-tests.yml matrix). @@ -63,14 +66,22 @@ const timeoutMs = ? timeoutMinutes * 60 * 1000 : DEFAULT_TIMEOUT_MIN * 60 * 1000; +// Resolve the shard to an explicit file list. `--shard=i/n` is consumed here, +// never forwarded: forwarding it as well would re-partition this slice a second +// time and silently drop most of it. +const shardParts = shardArg?.replace('--shard=', '').split('/'); +const shardIndex = shardParts ? Number(shardParts[0]) : 1; +const shardTotal = shardParts ? Number(shardParts[1]) : 1; +const files = shardFiles(ALL_CROSS_PLATFORM, shardIndex, shardTotal); + console.log( - `Running ${ALL_CROSS_PLATFORM.length} platform-sensitive tests` + - `${shardArg ? ` (${shardArg.replace('--shard=', 'shard ')})` : ''}...\n`, + `Running ${files.length} of ${ALL_CROSS_PLATFORM.length} platform-sensitive tests` + + `${shardArg ? ` (shard ${shardIndex}/${shardTotal}, ~${shardWeight(files)}s measured weight)` : ''}...\n`, ); const startedAt = Date.now(); try { - execFileSync('npx', ['vitest', 'run', ...ALL_CROSS_PLATFORM, ...(shardArg ? [shardArg] : [])], { + execFileSync('npx', ['vitest', 'run', ...files], { cwd: ROOT, stdio: 'inherit', timeout: timeoutMs, diff --git a/gitnexus/skills/gitnexus-impact-analysis.md b/gitnexus/skills/gitnexus-impact-analysis.md index 45eb7ce87..0b81795de 100644 --- a/gitnexus/skills/gitnexus-impact-analysis.md +++ b/gitnexus/skills/gitnexus-impact-analysis.md @@ -17,22 +17,23 @@ description: "Use when the user wants to know what will break if they change som ## Workflow ``` -1. impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) or `node .gitnexus/run.cjs impact "X" --direction upstream --repo .` 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. detect_changes() → Map current git changes to affected flows +3. detect_changes({scope: "all"}) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` 4. Assess risk and report to user ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. +> If `.gitnexus/run.cjs` is missing, replace `node .gitnexus/run.cjs` with `npx gitnexus` in the fallback commands. ## Checklist ``` -- [ ] impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) or CLI fallback to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] detect_changes() for pre-commit check +- [ ] detect_changes({scope: "all"}) or CLI fallback for pre-commit check - [ ] Assess risk level and report to user ``` @@ -55,7 +56,7 @@ description: "Use when the user wants to know what will break if they change som ## Tools -**impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: ``` impact({ @@ -73,10 +74,10 @@ impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis. If MCP is unavailable, use `node .gitnexus/run.cjs detect-changes --scope all --repo .` instead: ``` -detect_changes({scope: "staged"}) +detect_changes({scope: "all"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -86,7 +87,7 @@ detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) or `node .gitnexus/run.cjs impact "validateUser" --direction upstream --repo .` → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/gitnexus/skills/gitnexus-review/SKILL.md b/gitnexus/skills/gitnexus-review/SKILL.md index 9eabc426c..7b47cc3f6 100644 --- a/gitnexus/skills/gitnexus-review/SKILL.md +++ b/gitnexus/skills/gitnexus-review/SKILL.md @@ -120,10 +120,17 @@ and do not claim a complete graph-backed review. review surface: when the diff changes what gets emitted or persisted, verify every schema/version constant gating caches, incremental writebacks, and fingerprint baselines was bumped or regenerated — in - GitNexus itself, for example: `INCREMENTAL_SCHEMA_VERSION` (the - incremental write set covers only changed files, so new cross-file edges - never reach an existing index without the bump), the parse-store - `SCHEMA_BUMP`, and both bench fingerprint sets. + GitNexus itself, for example: graph DDL needs no manual bump, because + `SCHEMA_FINGERPRINT` (`gitnexus/src/core/lbug/schema.ts`) is derived + from `NODE_SCHEMA_QUERIES` + `REL_SCHEMA_QUERIES` and moves on its own; + the check there is whether the diff changed any string in those arrays, + and, if it added a new DDL array, whether that array was folded into the + fingerprint. The hand-maintained ritual still applies where no + declarative artifact describes the invalidated set: the parse-store + `SCHEMA_BUMP` and both bench fingerprint sets still need an explicit + bump, re-checked against the base branch right before merge. Semantic + changes that leave the DDL untouched are outside the fingerprint; they + rely on the analyzer runner-identity receipt in the index metadata. ## Expert lenses diff --git a/gitnexus/src/cli/ai-context.ts b/gitnexus/src/cli/ai-context.ts index faeb13d76..bf0277b5f 100644 --- a/gitnexus/src/cli/ai-context.ts +++ b/gitnexus/src/cli/ai-context.ts @@ -191,18 +191,18 @@ ${tableBody}` return `${GITNEXUS_START_MARKER} # GitNexus — Code Intelligence -This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows)`}. Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows)`}. Use GitNexus graph tools to understand code, assess impact, and navigate safely. > Index stale? Run \`${runner} analyze\` from the project root — it auto-selects an available runner. ${bootstrapNote} ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run \`impact({target: "symbolName", direction: "upstream"})\` and report the blast radius (direct callers, affected processes, risk level) to the user.${ +- **MUST run impact analysis before editing.** Use \`impact({target: "symbolName", direction: "upstream"})\` (MCP) or \`${runner} impact "symbolName" --direction upstream --repo .\` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis.${ hasPdg - ? ` For unified PDG impact, add \`mode: "pdg"\` with optional \`line: \` — it returns statement-level \`affectedStatements\` over CDG + REACHING_DEF and inter-procedural symbols in \`interproceduralByDepth\`/\`byDepth\`; no-layer/degraded PDG results are UNKNOWN-risk notes (\`--pdg\` layer).` + ? ` For unified PDG impact, add \`mode: "pdg"\` with optional \`line: \` — it returns statement-level \`affectedStatements\` over CDG + REACHING_DEF and inter-procedural symbols in \`interproceduralByDepth\`/\`byDepth\`; no-layer/degraded PDG results are UNKNOWN-risk notes (\`--pdg\` layer). CLI equivalent: \`${runner} impact "symbolName" --direction upstream --mode pdg --line --repo .\`.` : '' } -- **MUST run \`detect_changes()\` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: \`detect_changes({scope: "compare", base_ref: ${JSON.stringify(markdownSafeBranch(defaultBranch))}})\`. +- **MUST analyze graph changes before committing.** Use \`detect_changes({scope: "all"})\` (MCP) or \`${runner} detect-changes --scope all --repo .\` (CLI fallback). For regression review: \`detect_changes({scope: "compare", base_ref: ${JSON.stringify(markdownSafeBranch(defaultBranch))}})\` or \`${runner} detect-changes --scope compare --base-ref ${JSON.stringify(markdownSafeBranch(defaultBranch))} --repo .\`. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. - When exploring unfamiliar code, use \`query({search_query: "concept"})\` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. - When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use \`context({name: "symbolName"})\`. @@ -214,10 +214,10 @@ This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${s ## Never Do -- NEVER edit a function, class, or method without first running \`impact\` on it. +- NEVER edit a function, class, or method before MCP/CLI impact analysis. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. - NEVER rename symbols with find-and-replace — use \`rename\` which understands the call graph. -- NEVER commit changes without running \`detect_changes()\` to check affected scope. +- NEVER commit before MCP/CLI graph change analysis. ## Resources diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 9475ee25f..6851c2478 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -15,6 +15,8 @@ import v8 from 'v8'; import cliProgress from 'cli-progress'; import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js'; import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js'; +import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js'; +import { causeChain } from '../lib/utils.js'; import { getOsPageSize, isLbugCheckpointIoError, @@ -101,15 +103,13 @@ const writeFatalToStderr = (label: string, err: unknown): void => { // #2068) is only reachable via `.cause`. Without this the user sees the // wrapper's main-thread stack and never the real frame. `cause.stack` already // begins with the cause's message, so we print the stack alone (not message + - // stack) to avoid repeating it. Depth-bounded so a cyclic `cause` can't loop - // (the phase runner wraps one level; the bound leaves headroom for future - // nesting); uses realStderrWrite so the redirected console.error's ANSI - // clear-line wrapping can't erase it (#1169). - const MAX_CAUSE_DEPTH = 5; - let cause: unknown = isErr ? (err as { cause?: unknown }).cause : undefined; - for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause instanceof Error; depth++) { + // stack) to avoid repeating it. `causeChain` owns the traversal and the depth + // bound that stops a cyclic `cause` looping — this used to be one of four + // hand-rolled copies that had already drifted apart on both. Uses + // realStderrWrite so the redirected console.error's ANSI clear-line wrapping + // can't erase it (#1169). The head is skipped: it was just printed above. + for (const cause of causeChain(isErr ? (err as { cause?: unknown }).cause : undefined)) { realStderrWrite(`\n Caused by: ${cause.stack ?? cause.message}\n`); - cause = (cause as { cause?: unknown }).cause; } }; @@ -1717,6 +1717,36 @@ const analyzeCommandImpl = async ( return; } + // An extracted edge whose FROM→TO label pair is missing from GitNexus's own + // relation DDL (#2789). `assertDeclaredPair` aborts the run rather than let + // the bulk COPY fail late and silently drop the edge, so the user sees a + // mid-run crash inside GitNexus internals with nothing to act on. Name the + // pair, the relationship and the file that produced it, and say plainly that + // a re-run cannot help — this is deterministic for the same input. + // Checked by TYPE (repo norm, #2385) BEFORE the message-text heuristics + // below, and through the `cause` chain because the ingestion phase runner + // rewraps every phase failure as `Phase 'X' failed: …`. + const undeclaredPair = findUndeclaredRelationPairError(err); + if (undeclaredPair !== undefined) { + // Render the error's OWN message indented — same idiom as the + // `LbugWipeError` and page-size branches below. `UndeclaredRelationPairError` + // builds a fully self-contained message (pair, relationship type, both node + // ids, source file, issue URL, `.gitnexusignore` workaround) precisely + // because `gitnexus serve` forwards only `err.message` over worker IPC. + // Re-rendering those fields here would be a second copy of one string, free + // to drift from the first — and the actionable half would reach CLI users + // only. `undeclaredPair.message`, not the outer `msg`: the real error may be + // several `cause` levels below the phase wrapper `msg` came from. + cliError(` ${undeclaredPair.message.replace(/\n/g, '\n ')}\n`, { + recoveryHint: 'undeclared-relation-pair', + labelPair: undeclaredPair.pairKey, + relationType: undeclaredPair.relationType, + sourceFile: undeclaredPair.sourceFile, + }); + process.exitCode = 1; + return; + } + // WAL corruption — the index file is unreadable. Give a clear recovery // path without a confusing stack trace (the native error message alone // is enough signal). diff --git a/gitnexus/src/cli/cli-message.ts b/gitnexus/src/cli/cli-message.ts index 61b65dd00..b9b509b52 100644 --- a/gitnexus/src/cli/cli-message.ts +++ b/gitnexus/src/cli/cli-message.ts @@ -60,7 +60,8 @@ export type RecoveryHint = | 'module-not-found' | 'gitnexusrc-invalid' | 'default-branch-invalid' - | 'index-lock-timeout'; + | 'index-lock-timeout' + | 'undeclared-relation-pair'; /** * Common shape for the optional structured-field bag passed to diff --git a/gitnexus/src/cli/group.ts b/gitnexus/src/cli/group.ts index 0053099b7..3111f69d0 100644 --- a/gitnexus/src/cli/group.ts +++ b/gitnexus/src/cli/group.ts @@ -250,10 +250,41 @@ export function registerGroupCommands(program: Command): void { } else { const summary = (raw as { summary?: Record })?.summary; const risk = (raw as { risk?: string })?.risk; - console.log(`Group impact for "${name}" (${String(opts.repo)}): risk=${risk ?? '?'}`); + // A truncated fan-out under-reports risk (mergeRisk only grows with + // traversed crossings), and the default human output used to print a + // bare `risk=` indistinguishable from a complete run — the JSON + // already carried `truncated`, but nobody reading the terminal saw it. + const riskFloor = + (raw as { riskEpistemic?: string })?.riskEpistemic === 'lower-bound' ? '+' : ''; + const boundaryOnly = + ( + raw as { + cross?: Array<{ fanout_status?: string }>; + } + )?.cross?.filter((entry) => entry.fanout_status === 'not_attempted').length ?? 0; + console.log( + `Group impact for "${name}" (${String(opts.repo)}): risk=${risk ?? '?'}${riskFloor}`, + ); if (summary) { + const boundaryNote = boundaryOnly > 0 ? ` (${boundaryOnly} boundary-only)` : ''; console.log( - ` direct=${summary.direct ?? 0} processes=${summary.processes_affected ?? 0} cross=${summary.cross_repo_hits ?? 0}`, + ` direct=${summary.direct ?? 0} processes=${summary.processes_affected ?? 0} cross=${summary.cross_repo_hits ?? 0}${boundaryNote}`, + ); + } + if (riskFloor) { + // `truncated` has two independent causes that point at different + // subsystems, so the note must name the one that actually fired: + // dropped crossings, or a local walk that never finished (most + // often the impact chunk cap, which any symbol with more than a + // thousand locally-impacted nodes hits on every run). `dropped` is + // deduped to distinct repos before it reaches here, so it counts + // repos — reporting it as crossings understates a fan-out cap the + // same way #2787's totals did. + const dropped = (raw as { truncatedRepos?: string[] })?.truncatedRepos ?? []; + console.log( + dropped.length > 0 + ? ` risk is a LOWER BOUND — fan-out stopped early; crossings to ${dropped.length} repo(s) not traversed: ${dropped.join(', ')}` + : ' risk is a LOWER BOUND — the local impact walk did not complete (every bridge crossing was traversed)', ); } } diff --git a/gitnexus/src/core/analyzer-identity.ts b/gitnexus/src/core/analyzer-identity.ts index 4289d1788..6bc80848f 100644 --- a/gitnexus/src/core/analyzer-identity.ts +++ b/gitnexus/src/core/analyzer-identity.ts @@ -82,6 +82,7 @@ type PackageManifest = { dependencies?: Record; optionalDependencies?: Record; peerDependencies?: Record; + devDependencies?: Record; }; type StatState = { @@ -100,6 +101,16 @@ type ReadableFileState = { symlinkTarget?: string; }; +/** + * State of a symbolic link recorded by its link text rather than by its + * target's payload. There is deliberately no `target` stat: the whole point of + * this shape is that the link was never resolved (see {@link RuntimeArtifact}). + */ +type SymlinkArtifactState = { + link: StatState; + symlinkTarget: string; +}; + type RuntimePackage = { root: string; locator: string; @@ -137,11 +148,28 @@ type RuntimeArtifactScanBudget = { edges: number; }; -type RuntimeArtifact = { - absolutePath: string; - canonicalPath: string; - kind: 'file' | 'symlink'; -}; +/** + * One runtime payload input. + * + * `file` and `symlink` contribute their target's CONTENT digest; `symlink` also + * carries its link text, so both a retarget and a byte change move the receipt. + * + * `unfollowed-symlink` is a symbolic link that does not resolve to a regular + * file — a linked directory, a dangling link, a device node. It contributes its + * `readlink` TEXT and nothing else. See {@link collectArtifacts} for why the + * scan records such links instead of following them. + */ +type RuntimeArtifact = + | { + absolutePath: string; + canonicalPath: string; + kind: 'file' | 'symlink'; + } + | { + absolutePath: string; + canonicalPath: string; + kind: 'unfollowed-symlink'; + }; type BuildEntry = { absolutePath: string; @@ -157,13 +185,21 @@ type CachedBuildEntry = { digest?: string; }; -type CachedArtifactEntry = { - absolutePath: string; - canonicalPath: string; - kind: RuntimeArtifact['kind']; - state: ReadableFileState; - digest: string; -}; +type CachedArtifactEntry = + | { + absolutePath: string; + canonicalPath: string; + kind: 'file' | 'symlink'; + state: ReadableFileState; + digest: string; + } + | { + absolutePath: string; + canonicalPath: string; + kind: 'unfollowed-symlink'; + state: SymlinkArtifactState; + digest: string; + }; type CachedBuildDirectoryGuard = { relativePath: string; @@ -410,6 +446,28 @@ function snapshotReadableFile(candidate: string): ReadableFileState { }; } +/** + * Snapshot a symbolic link without resolving it. Unlike + * {@link snapshotReadableFile} this never stats the target, so it is total over + * linked directories, dangling links, and links to device nodes — the inputs + * that make the readable-file snapshot throw. + */ +function snapshotSymlinkArtifact(candidate: string): SymlinkArtifactState { + const link = lstatSync(candidate, { bigint: true }); + if (!link.isSymbolicLink()) { + throw new Error(`Analyzer identity input is not a symbolic link: ${candidate}`); + } + return { link: statState(link), symlinkTarget: readlinkSync(candidate) }; +} + +function snapshotRuntimeArtifact( + artifact: RuntimeArtifact, +): ReadableFileState | SymlinkArtifactState { + return artifact.kind === 'unfollowed-symlink' + ? snapshotSymlinkArtifact(artifact.absolutePath) + : snapshotReadableFile(artifact.absolutePath); +} + function snapshotDirectory(candidate: string): StatState { const stat = lstatSync(candidate, { bigint: true }); if (!stat.isDirectory() || stat.isSymbolicLink()) { @@ -931,6 +989,100 @@ function runtimePackageLocator(packageRoot: string, runtimeRoot: string): string return `relative:${relative}`; } +/** Protocols that name a checkout-local package instead of a registry tarball. */ +const LOCAL_LINK_PROTOCOL_PATTERN = /^(?:file|link|workspace|portal):/; +/** npm's bare local-path shorthands: `./x`, `../x`, `/x`, `~/x`, `C:\x`. */ +const LOCAL_LINK_PATH_PATTERN = /^(?:\.\.?[/\\]|~[/\\]|[/\\]|[A-Za-z]:)/; + +function isLocallyLinkedSpecifier(specifier: unknown): boolean { + if (typeof specifier !== 'string') return false; + const value = specifier.trim(); + return LOCAL_LINK_PROTOCOL_PATTERN.test(value) || LOCAL_LINK_PATH_PATTERN.test(value); +} + +/** + * Whether a REALPATH'd package root lives inside some installed dependency + * tree. Used as the resolved-location half of "is this dependency a checkout + * this repository owns?" (see {@link undeclaredLocalDevDependencyNames}). + * + * The input must already be realpath'd: `resolveDependencyPackageRoot` returns + * `realpathSync.native`, so a package reached through a link out of + * `node_modules` reports its checkout location and a package that merely lives + * in `node_modules` reports a path that still carries the segment. + * + * `pathApi` is injectable so the Windows separator handling is unit-testable + * from a POSIX runner, exactly as {@link isInside} does. The separator sets + * differ deliberately: `\` is a legal filename character on POSIX, so only + * win32 may treat it as a boundary. + */ +function hasNodeModulesSegment(candidate: string, pathApi: typeof path = path): boolean { + const segments = pathApi.sep === '\\' ? candidate.split(/[\\/]+/) : candidate.split('/'); + return segments.includes('node_modules'); +} + +/** Test seam for {@link hasNodeModulesSegment} (see {@link _isInsideForTests}). */ +export const _hasNodeModulesSegmentForTests = hasNodeModulesSegment; + +/** + * How many dev dependencies may be admitted by RESOLVED LOCATION alone before + * the whole resolved-location channel is treated as untrustworthy and disabled. + * + * "Realpath carries no `node_modules` segment" is a proxy for "checkout-local", + * and a layout that materializes packages outside `node_modules` — pnpm with a + * relocated `virtual-store-dir`, a custom linker — makes every dev dependency + * pass it. Folding an entire dev tree into the receipt is not a graceful + * degradation: `runtimePackages`/`runtimeEntries`/`runtimeBytes` THROW, so a + * mis-fired proxy on a legitimate install would abort analyze outright. + * + * The bound is therefore on ADMISSIONS, and overflow admits NONE of them rather + * than an arbitrary prefix. A prefix would not bound the failure — the abort + * comes from the transitive payload of whichever trees get folded in — and it + * would make the receipt depend on an arbitrary slice of a sorted name list. + * Dropping the channel wholesale falls back to the specifier-only receipt, + * which is the behaviour that ships today and is known not to abort, and leaves + * the declared-intent half in {@link dependencyNames} untouched. + * + * Four is measured, not guessed. Monorepo and workspace links are declared + * (`file:`/`link:`/`workspace:`) and travel the uncapped declared half, so this + * channel only ever carries UNDECLARED `npm link ` — a manual, per-package + * developer action, in practice one or two packages. A mis-fire admits the + * entire dev-only set instead: 13 names in this repository's own install, tens + * in a typical application. The cap sits an order of magnitude below the + * mis-fire population and comfortably above realistic link counts. + */ +const MAX_UNDECLARED_LOCAL_DEV_DEPENDENCIES = 4; + +/** + * Dependency names whose resolved packages can contribute analyzer semantics. + * + * The three runtime sections are enumerated wholesale. `devDependencies` are + * deliberately not: a registry dev tool (vitest, eslint, typescript) is never + * loaded by the analyzer, and folding the dev tree into the receipt would churn + * `dependencyRuntime.digest` — and force a full re-analysis — on every unrelated + * devDependency bump. + * + * Locally linked dev dependencies are the exception. A `file:`/`link:`/ + * `workspace:` sibling is part of this checkout and ships code the analyzer + * imports at runtime: GitNexus links `gitnexus-shared`, whose schema constants + * feed `RELATION_SCHEMA`/`NODE_SCHEMA_QUERIES`. In `kind: 'source'` runs that + * sibling sits outside `buildRoot`, so leaving it out let a semantic change + * there alter analyzer behaviour while moving neither `build.digest` nor + * `dependencyRuntime.digest` — DDL-affecting edits were still caught by the + * schema fingerprint, semantics-only edits by nothing. + * + * An unresolvable link (a published install, where the sibling checkout does not + * exist) still contributes its `` edge, so the linked package appearing + * or disappearing remains a receipt change rather than a silent one. That is why + * the specifier check cannot be replaced by resolution: resolution returns + * `null` for an absent linked checkout exactly as it does for an uninstalled + * registry dev tool, and the two must not be conflated. + * + * This function is the DECLARED-INTENT half and is enumerated for every package + * in the dependency BFS, so it must stay a pure function of the manifest. The + * RESOLVED-LOCATION half — `npm link `, which leaves the specifier a + * registry range — lives in {@link undeclaredLocalDevDependencyNames} and is + * applied to the root package only. + */ function dependencyNames(manifest: PackageManifest): string[] { const names = new Set(); for (const section of [ @@ -941,6 +1093,12 @@ function dependencyNames(manifest: PackageManifest): string[] { if (!section || typeof section !== 'object') continue; for (const name of Object.keys(section)) names.add(name); } + const development = manifest.devDependencies; + if (development && typeof development === 'object') { + for (const [name, specifier] of Object.entries(development)) { + if (isLocallyLinkedSpecifier(specifier)) names.add(name); + } + } return [...names].sort(compareBytes); } @@ -980,6 +1138,53 @@ function resolveDependencyPackageRoot( } } +/** + * Dev dependencies that are locally linked by INSTALLED LOCATION rather than by + * declared specifier — the `npm link ` shape, where the manifest still + * carries a registry range while `node_modules/` is a symlink into a + * working checkout. {@link isLocallyLinkedSpecifier} is blind to those, yet the + * linked code is exactly as load-bearing for analyzer semantics as a declared + * `file:` sibling, so a semantic-only edit there would move neither digest. + * + * The resolver already knows: {@link resolveDependencyPackageRoot} returns a + * realpath, so a linked package reports a root outside every `node_modules` + * tree while an ordinary installed package cannot. + * + * Two properties are load-bearing and must not be relaxed: + * + * 1. ROOT ONLY. {@link dependencyNames} runs for every package in the BFS, and + * published tarballs keep their `devDependencies`, so probing dev-only names + * everywhere costs 1998 resolutions rather than the ~13 this manifest + * declares — measured on this install, with 0 true positives. Persisted + * `dependencyPathGuards` grow 2220 → 11049, and every guard is re-probed on + * each warm validation, so the cost is recurring and on the `status` path. + * Root-only costs 13 resolutions and ~29 guards. + * 2. An unresolvable name is NEVER admitted. `null` here means "uninstalled + * registry dev tool" far more often than "broken link", and admitting it + * would emit a `` edge for every dev tool absent from a published + * install. Declared links keep that edge through {@link dependencyNames}; + * undeclared ones have no declaration to honour. + * + * The admission count is bounded by {@link MAX_UNDECLARED_LOCAL_DEV_DEPENDENCIES}. + */ +function undeclaredLocalDevDependencyNames( + rootPackage: RuntimePackage, + pathGuards: Map, + limits: AnalyzerIdentityTraversalLimits, +): string[] { + const development = rootPackage.manifest.devDependencies; + if (!development || typeof development !== 'object') return []; + const admitted: string[] = []; + for (const [name, specifier] of Object.entries(development)) { + // Already carried by the declared half; resolving again would only add + // guards. Its `` edge is that half's responsibility. + if (isLocallyLinkedSpecifier(specifier)) continue; + const resolved = resolveDependencyPackageRoot(rootPackage.root, name, pathGuards, limits); + if (resolved !== null && !hasNodeModulesSegment(resolved)) admitted.push(name); + } + return admitted.length <= MAX_UNDECLARED_LOCAL_DEV_DEPENDENCIES ? admitted : []; +} + function collectRuntimePackages( packageRoot: string, directoryGuards: Map, @@ -1010,7 +1215,21 @@ function collectRuntimePackages( for (let index = 0; index < queue.length; index += 1) { const parent = queue[index]; - for (const dependencyName of dependencyNames(parent.manifest)) { + // The declared half is enumerated for every package; the resolved-location + // half is scoped to the root package, where the 1998-resolution / + // 8829-extra-guard blow-up documented on + // `undeclaredLocalDevDependencyNames` cannot occur. Dropping this scope is + // the expensive regression, so it is pinned by a guard-count test. + const dependencies = + parent.root === packageRoot + ? [ + ...new Set([ + ...dependencyNames(parent.manifest), + ...undeclaredLocalDevDependencyNames(parent, pathGuards, limits), + ]), + ].sort(compareBytes) + : dependencyNames(parent.manifest); + for (const dependencyName of dependencies) { budget.edges += 1; if (budget.edges > limits.runtimeEdges) { throw new Error( @@ -1108,16 +1327,24 @@ function collectArtifacts( ); } for (const entry of entries) { + const absolutePath = path.join(absoluteDir, entry.name); + const relativePath = path.relative(root, absolutePath).split(path.sep).join('/'); + const stat = lstatSync(absolutePath); // Nested dependencies are collected from their manifests as separate // packages. Only prune those separately traversed trees and VCS // metadata; generic cache/model directories can contain loadable code, // native addons, Wasm modules, or data consumed by the runtime. - if (entry.isDirectory() && PRUNED_RUNTIME_DIRECTORIES.has(entry.name)) { - continue; - } - const absolutePath = path.join(absoluteDir, entry.name); - const relativePath = path.relative(root, absolutePath).split(path.sep).join('/'); - const stat = lstatSync(absolutePath); + // + // Pruning is decided by NAME alone. These four names never carry analyzer + // payload in any form: `node_modules` is traversed separately through + // `resolveDependencyPackageRoot` (which follows links and guards each + // hop), and a `.git`/`.hg`/`.svn` entry is VCS metadata whether it is a + // directory, a symbolic link into a shared store, or — inside a submodule + // or linked worktree checkout — a regular file holding a gitdir pointer. + // Hashing that pointer would make analyzer identity depend on where the + // checkout happens to live, which is a false-stale source, not a + // semantic input. + if (PRUNED_RUNTIME_DIRECTORIES.has(entry.name)) continue; if (stat.isDirectory()) { if (depth >= limits.runtimeDepth) { throw new Error( @@ -1125,6 +1352,39 @@ function collectArtifacts( ); } pending.push({ absoluteDir: absolutePath, depth: depth + 1 }); + } else if (stat.isSymbolicLink() && !isFile(absolutePath)) { + // A symbolic link that does not resolve to a regular file must never + // reach the payload branch below: `snapshotReadableFile` stats the + // target, and a directory (or a dangling link) makes it throw, aborting + // the entire analyze. Workspace-linked checkouts made this reachable + // for every name, not just the pruned four — `dist -> build`, a + // vendored-grammar link, anything a sibling checkout ships. + // + // Such links are RECORDED by their link text rather than followed. + // Following them would (a) recurse without cycle protection — this + // traversal has none, so `self -> .` would ride the depth limit, which + // THROWS, trading one hard abort for another; (b) re-scan trees already + // reached by their real path, inflating the entry/byte budgets that + // also throw; and (c) need a whole containment/TOCTOU trust boundary + // for targets outside the package. Recording the text is cycle-free, + // costs one `readlink`, and still moves the receipt when the link is + // retargeted. The trade-off is that a link's target contributes no + // content of its own: when it points outside the package, only the + // link text is covered. Links that DO resolve to a regular file keep + // their content digest below, unchanged. + if (shouldHashRuntimePayload(relativePath)) { + budget.artifacts += 1; + if (budget.artifacts > limits.runtimePayloads) { + throw new Error( + `Analyzer runtime payload scan exceeded ${limits.runtimePayloads} payloads: ${root}`, + ); + } + artifacts.push({ + absolutePath, + canonicalPath: `${canonicalPrefix}/${relativePath}`, + kind: 'unfollowed-symlink', + }); + } } else if ( (stat.isFile() || stat.isSymbolicLink()) && shouldHashRuntimePayload(relativePath) @@ -1322,7 +1582,7 @@ function dependencySnapshot(inputs: DependencyInputs): unknown { absolutePath: artifact.absolutePath, canonicalPath: artifact.canonicalPath, kind: artifact.kind, - state: snapshotReadableFile(artifact.absolutePath), + state: snapshotRuntimeArtifact(artifact), })), directories: [...inputs.directoryGuards.entries()] .map(([absolutePath, guard]) => ({ absolutePath, ...guard })) @@ -1343,10 +1603,30 @@ function hashRuntimeArtifact( artifact: RuntimeArtifact, cache: CachedArtifactEntry | undefined, options: AnalyzerIdentityResolveOptions, -): { digest: string; state: ReadableFileState } { +): CachedArtifactEntry { + if (artifact.kind === 'unfollowed-symlink') { + // The link text is the entire payload, so there is no file read for a + // cached digest to amortize: recompute it and stay independent of the + // cache's freshness. The distinct frame label keeps a link recording from + // ever colliding with a content digest. + const state = snapshotSymlinkArtifact(artifact.absolutePath); + return { + ...artifact, + state, + digest: hashCanonicalFrames([ + ['runtime-payload-link-v1', artifact.kind, state.symlinkTarget], + ]), + }; + } + const before = snapshotReadableFile(artifact.absolutePath); - if (cache && SHA256_PATTERN.test(cache.digest) && isDeepStrictEqual(cache.state, before)) { - return { digest: cache.digest, state: before }; + if ( + cache && + cache.kind !== 'unfollowed-symlink' && + SHA256_PATTERN.test(cache.digest) && + isDeepStrictEqual(cache.state, before) + ) { + return { ...artifact, state: before, digest: cache.digest }; } const stable = hashStableFile(artifact.absolutePath); @@ -1363,7 +1643,7 @@ function hashRuntimeArtifact( path: artifact.absolutePath, bytes: stable.bytes, }); - return { digest, state: stable.state }; + return { ...artifact, state: stable.state, digest }; } function compareEdges(a: RuntimeDependencyEdge, b: RuntimeDependencyEdge): number { @@ -1445,7 +1725,7 @@ function hashDependencyRuntime( artifact.kind, digestBytes(hashed.digest), ]); - nextArtifacts.push({ ...artifact, state: hashed.state, digest: hashed.digest }); + nextArtifacts.push(hashed); } return { @@ -1479,6 +1759,19 @@ function isReadableFileState(value: unknown): value is ReadableFileState { ); } +function isSymlinkArtifactState(value: unknown): value is SymlinkArtifactState { + if (typeof value !== 'object' || value === null) return false; + const record = value as Record; + // `target === undefined` keeps a readable-file state from masquerading as an + // unresolved link recording, which would otherwise be validated against the + // wrong guard mode on the warm path. + return ( + isStatState(record.link) && + typeof record.symlinkTarget === 'string' && + record.target === undefined + ); +} + function isDependencyPathGuardResult(value: unknown): value is DependencyPathGuardResult { if (value === null) return true; if (typeof value !== 'object') return false; @@ -1649,15 +1942,18 @@ function isIdentityCachePayload( const artifactEntriesValid = record.artifactEntries.every((entry: unknown) => { if (typeof entry !== 'object' || entry === null) return false; const item = entry as Record; - return ( - typeof item.absolutePath === 'string' && - path.isAbsolute(item.absolutePath) && - typeof item.canonicalPath === 'string' && - (item.kind === 'file' || item.kind === 'symlink') && - isReadableFileState(item.state) && - typeof item.digest === 'string' && - SHA256_PATTERN.test(item.digest) - ); + if ( + typeof item.absolutePath !== 'string' || + !path.isAbsolute(item.absolutePath) || + typeof item.canonicalPath !== 'string' || + typeof item.digest !== 'string' || + !SHA256_PATTERN.test(item.digest) + ) { + return false; + } + return item.kind === 'unfollowed-symlink' + ? isSymlinkArtifactState(item.state) + : (item.kind === 'file' || item.kind === 'symlink') && isReadableFileState(item.state); }); const hasBuildRootGuard = record.buildDirectoryGuards.some( (entry: unknown) => @@ -2146,14 +2442,24 @@ function validateIdentityCache( } } for (const artifact of cache.artifactEntries) { - if ( - !add( - { absolutePath: artifact.absolutePath, mode: 'readable-file' }, - { type: 'readable-file', state: artifact.state }, - ) - ) { - return false; - } + // A recorded link is re-probed as a link, never as a readable file: the + // readable-file probe resolves the target and would report `null` for the + // very inputs this kind exists to describe, failing every warm validation. + const probe: { request: CacheGuardRequest; expected: CacheGuardResult } = + artifact.kind === 'unfollowed-symlink' + ? { + request: { absolutePath: artifact.absolutePath, mode: 'link' }, + expected: { + type: 'symlink', + state: artifact.state.link, + symlinkTarget: artifact.state.symlinkTarget, + }, + } + : { + request: { absolutePath: artifact.absolutePath, mode: 'readable-file' }, + expected: { type: 'readable-file', state: artifact.state }, + }; + if (!add(probe.request, probe.expected)) return false; } const entries = [...expected.entries()]; diff --git a/gitnexus/src/core/augmentation/engine.ts b/gitnexus/src/core/augmentation/engine.ts index 528b774cf..9e1189ad4 100644 --- a/gitnexus/src/core/augmentation/engine.ts +++ b/gitnexus/src/core/augmentation/engine.ts @@ -132,6 +132,7 @@ export async function augment(pattern: string, cwd?: string): Promise { MATCH (n) WHERE n.filePath = '${escaped}' AND n.name CONTAINS '${patternFirstWord}' RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, n.filePath AS filePath + ORDER BY id LIMIT 3 `, ); @@ -158,6 +159,7 @@ export async function augment(pattern: string, cwd?: string): Promise { MATCH (n) WHERE n.name CONTAINS '${patternFirstWord}' RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, n.filePath AS filePath + ORDER BY id LIMIT 5 `, ).catch(() => []); @@ -184,98 +186,119 @@ export async function augment(pattern: string, cwd?: string): Promise { const idList = uniqueSymbols.map((s) => `'${escapeCypherString(s.nodeId)}'`).join(', '); - // Batch fetch callers - const callersMap = new Map(); - try { + // Callers/callees are windowed PER SYMBOL, not out of one shared budget. + // A single `n.id IN [...] ... ORDER BY targetId LIMIT 15` sorts by the very + // column the rows are grouped by, which turns the cap into a single-bucket + // prefix: the alphabetically-first target takes every row (the hottest + // symbol in this repo's own index has 2163 callers) and the other four + // render with no callers at all. Raising the cap does not bound that, and + // ordering caller-major only moves the bucket. A per-target window is not + // expressible in one statement either — LadybugDB does not preserve a + // per-branch ORDER BY through UNION ALL — so fan out one bounded query per + // symbol (#2787). + // + // `id STARTS WITH 'File:'` sorts container nodes last: node ids are + // `Label:path:name`, so a bare `ORDER BY id` puts `File:` ahead of + // `Function:`/`Method:` and the "Called by" line renders bare filenames + // instead of the calling symbols the hint exists to name. + const NEIGHBOUR_CAP = 3; + const neighbourNames = async (nodeId: string, incoming: boolean): Promise => { + const edge = incoming + ? `(other)-[:CodeRelation {type: 'CALLS'}]->(n)` + : `(n)-[:CodeRelation {type: 'CALLS'}]->(other)`; const rows = await executeQuery( repoId, ` - MATCH (caller)-[:CodeRelation {type: 'CALLS'}]->(n) - WHERE n.id IN [${idList}] - RETURN n.id AS targetId, caller.name AS name - LIMIT 15 + MATCH ${edge} + WHERE n.id = '${escapeCypherString(nodeId)}' + RETURN other.name AS name + ORDER BY other.id STARTS WITH 'File:', other.id + LIMIT ${NEIGHBOUR_CAP} `, - ); + ).catch(() => []); + const names: string[] = []; for (const r of rows) { - const tid = r.targetId || r[0]; - const name = r.name || r[1]; - if (tid && name) { - if (!callersMap.has(tid)) callersMap.set(tid, []); - callersMap.get(tid)!.push(name); - } + const name = r.name || r[0]; + if (name) names.push(name); } - } catch { - /* skip */ - } + return names; + }; - // Batch fetch callees - const calleesMap = new Map(); - try { - const rows = await executeQuery( - repoId, - ` - MATCH (n)-[:CodeRelation {type: 'CALLS'}]->(callee) - WHERE n.id IN [${idList}] - RETURN n.id AS sourceId, callee.name AS name - LIMIT 15 - `, + // Keyed by nodeId, like the process/cohesion enrichments below — positional + // arrays would put two lookup disciplines in one object literal and make + // "stays index-aligned with uniqueSymbols" an unenforced invariant. + const neighbourMap = async (incoming: boolean): Promise> => + new Map( + await Promise.all( + uniqueSymbols.map( + async (s) => [s.nodeId, await neighbourNames(s.nodeId, incoming)] as const, + ), + ), ); - for (const r of rows) { - const sid = r.sourceId || r[0]; - const name = r.name || r[1]; - if (sid && name) { - if (!calleesMap.has(sid)) calleesMap.set(sid, []); - calleesMap.get(sid)!.push(name); - } - } - } catch { - /* skip */ - } // Batch fetch processes - const processesMap = new Map(); - try { - const rows = await executeQuery( - repoId, - ` + const fetchProcesses = async (): Promise> => { + const byNode = new Map(); + try { + const rows = await executeQuery( + repoId, + ` MATCH (n)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process) WHERE n.id IN [${idList}] RETURN n.id AS nodeId, p.heuristicLabel AS label, r.step AS step, p.stepCount AS stepCount `, - ); - for (const r of rows) { - const nid = r.nodeId || r[0]; - const label = r.label || r[1]; - const step = r.step || r[2]; - const stepCount = r.stepCount || r[3]; - if (nid && label) { - if (!processesMap.has(nid)) processesMap.set(nid, []); - processesMap.get(nid)!.push(`${label} (step ${step}/${stepCount})`); + ); + for (const r of rows) { + const nid = r.nodeId || r[0]; + const label = r.label || r[1]; + const step = r.step || r[2]; + const stepCount = r.stepCount || r[3]; + if (nid && label) { + if (!byNode.has(nid)) byNode.set(nid, []); + byNode.get(nid)!.push(`${label} (step ${step}/${stepCount})`); + } } + } catch { + /* skip */ } - } catch { - /* skip */ - } + return byNode; + }; // Batch fetch cohesion - const cohesionMap = new Map(); - try { - const rows = await executeQuery( - repoId, - ` + const fetchCohesion = async (): Promise> => { + const byNode = new Map(); + try { + const rows = await executeQuery( + repoId, + ` MATCH (n)-[:CodeRelation {type: 'MEMBER_OF'}]->(c:Community) WHERE n.id IN [${idList}] RETURN n.id AS nodeId, c.cohesion AS cohesion `, - ); - for (const r of rows) { - const nid = r.nodeId || r[0]; - const coh = r.cohesion ?? r[1] ?? 0; - if (nid) cohesionMap.set(nid, coh); + ); + for (const r of rows) { + const nid = r.nodeId || r[0]; + const coh = r.cohesion ?? r[1] ?? 0; + if (nid) byNode.set(nid, coh); + } + } catch { + /* skip */ } - } catch { - /* skip */ - } + return byNode; + }; + + // One wave, not three. `augment` runs from the Claude Code PreToolUse hook + // in a cold process against a <500ms budget, so nothing amortizes — and the + // process/cohesion queries depend only on `idList`, never on the neighbour + // results, so serializing them behind the fan-out bought nothing. Each query + // keeps its own try/catch, so one failing still degrades to an empty map + // instead of taking the others down. + const [callerMap, calleeMap, processesMap, cohesionMap] = await Promise.all([ + neighbourMap(true), + neighbourMap(false), + fetchProcesses(), + fetchCohesion(), + ]); // Assemble enriched results const enriched: Array<{ @@ -291,8 +314,8 @@ export async function augment(pattern: string, cwd?: string): Promise { enriched.push({ name: sym.name, filePath: sym.filePath, - callers: (callersMap.get(sym.nodeId) || []).slice(0, 3), - callees: (calleesMap.get(sym.nodeId) || []).slice(0, 3), + callers: callerMap.get(sym.nodeId) || [], + callees: calleeMap.get(sym.nodeId) || [], processes: processesMap.get(sym.nodeId) || [], cohesion: cohesionMap.get(sym.nodeId) || 0, }); diff --git a/gitnexus/src/core/embedding-checkpoint.ts b/gitnexus/src/core/embedding-checkpoint.ts new file mode 100644 index 000000000..042dd6273 --- /dev/null +++ b/gitnexus/src/core/embedding-checkpoint.ts @@ -0,0 +1,301 @@ +/** + * The single owner of `RepoMeta.embeddingCheckpoint` — how it is minted, and + * how a run decides what to do with one it finds. + * + * This module exists for the reason `embedding-count.ts` next door exists, and + * for a sharper one. A review of #2790 found that two hand-copied bodies of + * "measure the embedding count" had drifted inside a single change; the fix for + * that then created a SECOND pair of hand-copied publishers — of this record — + * and they had already drifted too: the CLI armed the attempt counter only + * after clearing its identity gate, the server derived it from the resumed + * marker alone. Worse, only one of the two READERS implemented `kind` at all, + * so a `'partial'` marker written by `gitnexus analyze` and resumed through + * `POST /api/embed` hit exactly the permanent wedge `kind` was introduced to + * remove. Prose promising two sites stay in step is not a mechanism. This is. + * + * Same tier as {@link ./embedding-count.ts} and {@link ./embedding-mode.ts}: + * type-only imports, no native dependencies, so `run-analyze.ts` can import it + * statically without dragging an embeddings module in (#2370) and `server/` + * can import it too. + */ + +import type { RepoMeta } from '../storage/repo-manager.js'; + +export type EmbeddingCheckpoint = NonNullable; +export type EmbeddingCheckpointKind = NonNullable; + +export interface EmbeddingRunIdentity { + model: string; + dimensions: number; + provider: string; +} + +export interface EmbeddingCheckpointProgress { + nodesProcessed: number; + totalNodes: number; + chunksProcessed: number; +} + +/** + * How many consecutive resume attempts may fail to clear a `'partial'` pending + * set before it is abandoned. + * + * Not a fresh guess: it is this repo's existing per-operation retry budget — + * the HTTP embedder spends `HTTP_MAX_RETRIES + 1 === 3` per request and the WAL + * driver `CHECKPOINT_RETRY_ATTEMPTS === 3` per flush. Each attempt here is a + * whole analyze invocation with those budgets nested underneath, so 3 is + * already generous while still converging within a day of ordinary committing. + */ +export const EMBEDDING_RESUME_MAX_ATTEMPTS = 3; + +/** + * The one home for the back-compat default. Markers written before #2790's + * follow-up carry no `kind`, and they must keep the stricter behavior — an + * implicit default scattered across read sites is precisely how the two + * readers came to disagree. + */ +export const checkpointKind = (checkpoint: EmbeddingCheckpoint): EmbeddingCheckpointKind => + checkpoint.kind ?? 'interrupted'; + +/** Pending-node count, tolerating the field's optionality. */ +export const pendingNodeCount = (checkpoint: EmbeddingCheckpoint): number => + checkpoint.pendingNodeIds?.length ?? 0; + +/** + * The attempt-chain rule, which is the subtlest thing here and was the part + * that had already diverged. + * + * The counter advances only when this run resumed a `'partial'` marker AND at + * least one node it was handed failed AGAIN. A resume that cleared its set but + * lost *different* nodes is a FRESH partial, so the budget resets — otherwise a + * steadily-flaky endpoint would exhaust the budget on nodes that were never the + * problem. `undefined` rather than `0` when the streak breaks, so a clean + * marker serializes without the field. + */ +export const nextAttemptCount = ( + resumedFrom: EmbeddingCheckpoint | undefined, + failedNodeIds: readonly string[], +): number | undefined => { + if (resumedFrom === undefined || checkpointKind(resumedFrom) !== 'partial') return undefined; + const resumedPending = new Set(resumedFrom.pendingNodeIds ?? []); + const failedAgain = failedNodeIds.some((id) => resumedPending.has(id)); + return failedAgain ? (resumedFrom.attempts ?? 0) + 1 : undefined; +}; + +/** + * Mint the marker written BEFORE a bounded write window opens. + * + * `'interrupted'`: if the process dies inside the window these nodes may hold a + * subset of their chunks, which is what forces resume to delete and regenerate + * them even when a persisted row carries the current content hash — and what + * makes an identity mismatch fail closed rather than mix vector spaces. No + * `attempts`: the retry bound belongs to `'partial'`, whose pending set + * provably holds zero rows and can therefore be abandoned safely. + */ +export const mintInterruptedCheckpoint = ( + identity: EmbeddingRunIdentity, + progress: EmbeddingCheckpointProgress, + pendingNodeIds: string[], + at: string = new Date().toISOString(), +): EmbeddingCheckpoint => ({ + at, + ...progress, + model: identity.model, + dimensions: identity.dimensions, + provider: identity.provider, + kind: 'interrupted', + pendingNodeIds, +}); + +/** + * Mint the marker written AFTER a run that completed while dropping nodes. + * + * `totalNodes` is reconstructed as complete + dropped, because `nodesProcessed` + * counts only the nodes that finished with a full row set. It feeds the resume + * log line and nothing else. + */ +export const mintPartialCheckpoint = ( + identity: EmbeddingRunIdentity, + result: { nodesProcessed: number; chunksProcessed: number; failedNodeIds: string[] }, + resumedFrom: EmbeddingCheckpoint | undefined, + at: string = new Date().toISOString(), +): EmbeddingCheckpoint => ({ + at, + nodesProcessed: result.nodesProcessed, + totalNodes: result.nodesProcessed + result.failedNodeIds.length, + chunksProcessed: result.chunksProcessed, + model: identity.model, + dimensions: identity.dimensions, + provider: identity.provider, + kind: 'partial', + attempts: nextAttemptCount(resumedFrom, result.failedNodeIds), + pendingNodeIds: result.failedNodeIds, +}); + +/** + * Mint the marker written when a run finished but its embedding count could not + * be verified. + * + * A third state, and it needs its own `kind` rather than borrowing `'partial'`: + * its pending set is EMPTY. Nothing was dropped and nothing needs re-embedding + * — the marker exists only to defeat the same-commit fast return so the next + * run re-derives a count, because clearing it while `stats.embeddings` still + * reads a stale zero is what arms a later `--force` to wipe live embeddings. + * Stamping it `'partial'` made `gitnexus status` tell the operator that N nodes + * had lost their embeddings, where N is zero. + */ +export const mintUnverifiedCountCheckpoint = ( + identity: EmbeddingRunIdentity, + progress: EmbeddingCheckpointProgress, + at: string = new Date().toISOString(), +): EmbeddingCheckpoint => ({ + at, + ...progress, + model: identity.model, + dimensions: identity.dimensions, + provider: identity.provider, + kind: 'unverified-count', + pendingNodeIds: [], +}); + +export type EmbeddingResumeDecision = + /** An explicit flag cleared it; nothing to resume. */ + | { readonly action: 'discard'; readonly log: string } + /** + * Keep running, but abandon the pending set — either the retry budget is + * spent, or the marker was written by a different embedding identity whose + * nodes provably hold zero rows so nothing is at risk. + */ + | { readonly action: 'abandon'; readonly log: string } + /** Resume: regenerate `pendingNodeIds` under the matching identity. */ + | { + readonly action: 'resume'; + readonly log: string; + readonly pendingNodeIds: ReadonlySet; + readonly resumedFrom: EmbeddingCheckpoint; + } + /** Fail closed: a half-written window under a foreign identity. */ + | { readonly action: 'abort'; readonly error: string }; + +/** + * The one implementation of "what should this run do with the checkpoint it + * found", shared by the CLI resume gate and `POST /api/embed`. + * + * Callers resolve the embedding identity lazily and pass it only when they need + * a verdict beyond the flag checks — resolving it is not free, and the flag + * paths short-circuit before it is needed. Pass `identity: undefined` to get + * just those; a decision that requires an identity returns `'resume'` only when + * one was supplied and matched. + */ +export const decideEmbeddingResume = ( + checkpoint: EmbeddingCheckpoint, + identity: EmbeddingRunIdentity | undefined, + options: { + force?: boolean; + dropEmbeddings?: boolean; + maxAttempts?: number; + } = {}, +): EmbeddingResumeDecision => { + const kind = checkpointKind(checkpoint); + const pending = pendingNodeCount(checkpoint); + const maxAttempts = options.maxAttempts ?? EMBEDDING_RESUME_MAX_ATTEMPTS; + + if (options.dropEmbeddings) { + return { action: 'discard', log: 'Discarding the embedding checkpoint (--drop-embeddings).' }; + } + if (options.force) { + return { + action: 'discard', + log: + 'Discarding the embedding checkpoint (--force)' + + `${pending > 0 ? ` and its ${pending} pending node(s)` : ''}; this run rebuilds from ` + + 'scratch. Pass --embeddings to regenerate them explicitly.', + }; + } + // Only the 'unverified-count' marker may skip the identity gate. Its pending + // set is empty BY DEFINITION and it exists purely to force a fresh count, so + // there is nothing a foreign embedding identity could corrupt. + // + // An empty pending set alone must NOT take this path. `onCheckpoint` mints an + // 'interrupted' marker with `pendingNodeIds: []` after every post-window + // save, and legacy markers may omit the field entirely — so keying on + // emptiness would silently clear an interrupted marker under a foreign + // provider instead of failing closed, which is exactly the vector-space + // mixing the 'interrupted' kind exists to prevent. + if (kind === 'unverified-count') { + return { + action: 'abandon', + log: 'Re-deriving the embedding count left unverified by the previous run.', + }; + } + if (kind === 'partial' && (checkpoint.attempts ?? 0) >= maxAttempts) { + return { + action: 'abandon', + log: + `Warning: ${pending} node(s) failed to embed on ${checkpoint.attempts} consecutive ` + + 'resume attempts and are being abandoned (#2790). The index is registered without ' + + 'them; `gitnexus status` stops reporting it as incomplete. Re-run ' + + '`gitnexus analyze --embeddings --force` once the embedding endpoint accepts them.', + }; + } + if (identity === undefined) { + return { + action: 'abort', + error: 'Cannot resume embedding checkpoint: no embedding identity was resolved.', + }; + } + + const providerDiffers = checkpoint.provider !== identity.provider; + const identityDiffers = + providerDiffers || + checkpoint.model !== identity.model || + checkpoint.dimensions !== identity.dimensions; + + if (identityDiffers && kind !== 'interrupted') { + // Those nodes hold zero rows, so there is no vector space to mix. Throwing + // here is what turned an exit-0 partial run into a permanent wedge: a hook + // or CI job without the endpoint's env vars resolves `provider: 'local'`, + // mismatches, and dies before any phase runs. + return { + action: 'abandon', + log: + `Warning: dropping ${pending} pending node(s) from a ${kind} embedding checkpoint: it ` + + `was written by a different embedding configuration (${checkpoint.model} at ` + + `${checkpoint.dimensions} dimensions) than this run resolves (${identity.model} at ` + + `${identity.dimensions}). Those nodes hold no embedding rows, so nothing is lost that ` + + 'a later `--embeddings` run cannot regenerate.', + }; + } + if (providerDiffers) { + return { + action: 'abort', + error: + 'Cannot resume embedding checkpoint: the embedding provider configuration differs. ' + + 'Restore the matching endpoint configuration or pass --drop-embeddings to rebuild without it.', + }; + } + if (identityDiffers) { + return { + action: 'abort', + error: + `Cannot resume embedding checkpoint: it uses ${checkpoint.model} at ` + + `${checkpoint.dimensions} dimensions, but this run resolves ${identity.model} at ` + + `${identity.dimensions}. Restore the matching embedding configuration or pass ` + + '--drop-embeddings to rebuild without it.', + }; + } + // Reached with an EMPTY pending set too — an 'interrupted' marker written by + // a post-window save names none. That still resumes rather than clearing: + // the marker's presence is what forces embedding generation on a run that + // would otherwise derive `shouldGenerateEmbeddings: false`, and it has now + // cleared the identity gate, so resuming is safe. + return { + action: 'resume', + log: + `Previous analyze ended at an embedding checkpoint (${checkpoint.nodesProcessed}/` + + `${checkpoint.totalNodes} nodes); resuming from persisted hashes` + + `${pending > 0 ? ` and regenerating ${pending} pending node(s)` : ''}.`, + pendingNodeIds: new Set(checkpoint.pendingNodeIds ?? []), + resumedFrom: checkpoint, + }; +}; diff --git a/gitnexus/src/core/embedding-count.ts b/gitnexus/src/core/embedding-count.ts new file mode 100644 index 000000000..c2058f15b --- /dev/null +++ b/gitnexus/src/core/embedding-count.ts @@ -0,0 +1,90 @@ +/** + * The single implementation of "how many embedding rows are actually persisted". + * + * Lives in its own module — beside {@link ./embedding-mode.ts} and with the same + * no-native-imports property — because the CLI (`run-analyze.ts`) and the server + * (`server/api.ts`) both publish `RepoMeta.stats.embeddings`, and a review of + * #2790 found the two hand-copied bodies had already drifted inside a single + * change: one used `?? Number.NaN`, the other `?? 0`, under a comment asserting + * they measured the field "the same way". Two publishers of one field must not + * be able to disagree, so there is now one function and three call sites. + * + * Keeping it out of `core/embeddings/` is deliberate: `run-analyze.ts` may only + * reach embedding modules through `await import(...)` (#2370 — no embeddings + * module loads unless a run actually needs one), and this counter runs on the + * ordinary finalization path of every analyze. + * + * TRI-STATE, and the asymmetry is the whole point. `unknown` means COULD NOT + * ASK — the query throws for reasons that have nothing to do with how many rows + * were written (table missing, connection closed, DB busy or read-only, the + * VECTOR-extension DML lock #2623) — and a fabricated count is never returned in + * its place. A wrong-LOW value is the dangerous direction: `stats.embeddings` is + * the sole input to the next run's `existingEmbeddingCount` → `deriveEmbeddingMode` + * → `shouldLoadCache`, so a false `0` is exactly what makes a later `--force` + * wipe live embeddings without loading the cache. + */ + +import { EMBEDDING_TABLE_NAME } from 'gitnexus-shared'; + +/** Callback shape both callers already satisfy with their `executeQuery`. */ +export type EmbeddingCountQuery = ( + cypher: string, +) => Promise> | undefined>; + +export type PersistedEmbeddingCount = + | { readonly kind: 'measured'; readonly count: number } + /** `reason` is operator-facing: each caller phrases its own warning around it. */ + | { readonly kind: 'unknown'; readonly reason: string }; + +export const EMBEDDING_COUNT_CYPHER = `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN count(e) AS cnt`; + +/** + * Count the persisted embedding rows, or report that the answer never arrived. + * + * A MISSING row or cell counts as unknown, NOT as zero: an empty table still + * answers with one row holding `0`, so no row / no cell means the query did not + * really answer. That distinction is what `?? Number.NaN` buys — with `?? 0`, + * `Number.isFinite(0)` is true and the unknown branch becomes unreachable for + * exactly the case it exists to catch. + */ +export const measurePersistedEmbeddingCount = async ( + runQuery: EmbeddingCountQuery, +): Promise => { + let rows: Array> | undefined; + try { + rows = await runQuery(EMBEDDING_COUNT_CYPHER); + } catch (err) { + return { kind: 'unknown', reason: err instanceof Error ? err.message : String(err) }; + } + const row = rows?.[0]; + const parsed = Number(row?.cnt ?? row?.[0] ?? Number.NaN); + if (Number.isFinite(parsed)) return { kind: 'measured', count: parsed }; + return { + kind: 'unknown', + reason: + row === undefined + ? 'the count query returned no row' + : 'the count query returned a non-numeric result', + }; +}; + +/** `undefined` ≡ unknown, for the call sites that only need the optional number. */ +export const persistedEmbeddingCountOrUndefined = ( + result: PersistedEmbeddingCount, +): number | undefined => (result.kind === 'measured' ? result.count : undefined); + +/** + * Fold a measurement into the `RepoMeta.stats.embeddings` a run publishes. + * + * The measurement was already shared; this fold was not, and it is the half + * that decides what actually lands on disk — the CLI's finalization and the + * server's `withMeasuredEmbeddingCount` each carried their own copy of the same + * two lines. NEVER a fabricated `0`: `unknown` carries `priorCount` forward + * (see this file's header for the chain a false zero arms), and `undefined` in + * means `undefined` out, so a repo that never had a count keeps the field + * absent rather than gaining an invented one. + */ +export const resolvePersistedEmbeddingCount = ( + measured: PersistedEmbeddingCount, + priorCount: number | undefined, +): number | undefined => (measured.kind === 'measured' ? measured.count : priorCount); diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts index e4976d71e..194f47313 100644 --- a/gitnexus/src/core/embeddings/ast-utils.ts +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -11,6 +11,7 @@ import { resolveLanguageKey, } from '../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../tree-sitter/safe-parse.js'; +import { getProvider } from '../ingestion/languages/index.js'; const parserCache = new Map(); @@ -30,7 +31,15 @@ export const ensureAndParse = async (content: string, filePath: string): Promise parserCache.set(parserKey, parserInstance); } - return parseSourceSafe(parserInstance, content); + // Same text the ingestion worker parses — otherwise a provider whose + // `preprocessSource` repairs a declaration (Swift conditional directives, + // C++ UE macros, Dart extension types) would leave embeddings looking at an + // error-recovered tree. Resolved from `language` so the transform and the + // parser always come from the same provider. Length-preserving, so node + // offsets still index `content`. + const parseContent = getProvider(language).preprocessSource?.(content, filePath) ?? content; + + return parseSourceSafe(parserInstance, parseContent); }; const FUNCTION_LIKE_TYPES = new Set([ diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index ffb5b1b13..f3cda5f23 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -39,6 +39,7 @@ import { resolveEmbeddingConfig, } from './config.js'; import { rankExactEmbeddingRows, type ExactEmbeddingRow } from './exact-search.js'; +import { EMBEDDING_COUNT_CYPHER } from '../embedding-count.js'; import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, STALE_HASH_SENTINEL } from '../lbug/schema.js'; import { loadVectorExtension, createVectorIndex } from '../lbug/lbug-adapter.js'; import { escapeCypherString } from '../lbug/cypher-escape.js'; @@ -175,7 +176,9 @@ const queryEmbeddableNodes = async ( } } catch (error) { if (isDev) { - logger.warn({ error }, `Query for ${label} nodes failed:`); + // `err` is pino's standard serializer key — an arbitrary key serializes an + // Error to `{}`, losing the message and stack (#2114). + logger.warn({ err: error }, `Query for ${label} nodes failed:`); } } } @@ -220,7 +223,8 @@ const queryFallbackFileNodes = async ( ); } catch (error) { if (isDev) { - logger.warn({ error }, 'Fallback File-node embedding query failed:'); + // `err`, not `error` — see the serializer note above (#2114). + logger.warn({ err: error }, 'Fallback File-node embedding query failed:'); } return []; } @@ -305,12 +309,156 @@ export const buildVectorIndex = async (): Promise => { }; export interface EmbeddingPipelineResult { + /** Nodes that finished the run with a COMPLETE set of embedding rows. */ nodesProcessed: number; chunksProcessed: number; vectorIndexReady: boolean; semanticMode: 'vector-index' | 'exact-scan'; + /** + * Nodes whose embeddings were dropped this run because at least one of their + * chunks lost its sub-batch to a failing endpoint (#2790). Their rows were + * deleted, so they now hold ZERO rows and the next incremental run re-embeds + * them from scratch. Empty on a clean run. + */ + failedNodeIds: string[]; } +/** + * How many sub-batches may fail back-to-back before the pipeline gives up. + * + * Surviving a transient endpoint hiccup is the whole point of tolerating a + * failed sub-batch (#2790) — but a genuinely dead endpoint would otherwise walk + * every remaining node deleting its rows on the way, which on an incremental run + * wipes surviving embeddings wholesale. Rethrowing once the ceiling is reached + * preserves the pre-#2790 behavior for a dead endpoint. Any successful sub-batch + * resets the counter, so only an unbroken run of failures trips it. + * + * ponytail: flat consecutive-failure ceiling; make it rate-based if flaky + * endpoints prove common. + */ +const MAX_CONSECUTIVE_SUB_BATCH_FAILURES = 5; + +/** + * Cumulative guard: the share of sub-batches allowed to fail across the whole + * run, and the minimum sample before that share means anything. + * + * The ceiling above only catches a total outage — any success resets it, so an + * endpoint load-shedding every other sub-batch walks the entire repo dropping + * half of it while `analyze` still exits 0 (#2790). Shaped after Resilience4j's + * `failureRateThreshold` + `minimumNumberOfCalls`: judge a failure RATE, but + * only once enough sub-batches have been attempted for a rate to mean anything + * — a 3-node repo losing its single sub-batch is 100% and must not abort. The + * rate sits below a live-traffic breaker's 50% (a batch indexer's job is to + * index the whole corpus, not to serve degraded traffic) and above Hadoop's + * single-digit `mapreduce.map.failures.maxpercent` (tolerating transient + * hiccups is the entire point of #2790). + * + * The sample floor SCALES with the run; it used to be a flat 20. A flat floor + * can exceed a run's ENTIRE sub-batch budget, and then the guard is not "not yet + * armed" — it is structurally off. Every resume run has exactly that shape by + * construction: its node set is only the pending ids, so it produces few + * sub-batches no matter how large the repo is, and the run whose whole purpose + * is retrying a suspect endpoint was the one run this guard could never fire in. + * Resilience4j can afford a constant `minimumNumberOfCalls` because a breaker + * sits on an unbounded call stream — the sample always arrives eventually; a + * batch indexer has a finite budget, so the minimum has to be expressed relative + * to it. Hadoop's `mapreduce.map.failures.maxpercent` takes the other extreme, a + * pure percentage with no minimum at all, which is why a 1-of-1 failure reads as + * "100% failed" there. The clamp below keeps both ends honest: + * + * floor = clamp(ceil(totalNodes / subBatchSize / 2), 5, 20) + * + * - ceil(totalNodes / subBatchSize / 2): the run's own sub-batch budget, halved + * — so the rate is judged over the back half of a short run: a real sample, + * still early enough to stop the run before it walks the rest of the corpus. + * Derived from the ACTUAL `subBatchSize` rather than assuming the default of + * 8, because `GITNEXUS_EMBEDDING_SUB_BATCH_SIZE` is exactly the knob an + * operator turns for a constrained or flaky endpoint — the population this + * guard protects — and a floor computed against the wrong divisor is the same + * structurally-off guard, just at a different repo size. Nodes that chunk + * into several chunks only produce MORE sub-batches, so the guard arms + * earlier still. + * - min 5: under five attempts a "rate" is one or two coin flips. A 3-node repo + * losing its single sub-batch is 100% and must be tolerated, not aborted — + * the exact case the old flat floor was written to protect, kept intact. + * - max 20 (the previous flat value, reached once a run has 40 sub-batches): + * large runs keep today's behavior byte for byte, and a purely proportional + * floor would perversely WEAKEN the guard at scale — half a 20k-node repo's + * sub-batches is 1250 of them (at subBatchSize 8) eaten before a rate could + * abort anything. + * + * Honest limit: this bounds the failure RATE, never the absolute loss. A run + * that steadily fails just under 25% of its sub-batches never aborts, so a + * single run may drop up to a quarter of the corpus and still exit 0 behind one + * warning line. That is by design — the dropped nodes are left holding ZERO rows + * so the next incremental run re-embeds them, which beats failing the whole + * analyze — but it is the real ceiling on what this guard promises. + * + * ponytail: lifetime ratio, evaluated only when a sub-batch fails. A sliding + * window would spot an endpoint that degrades late in a long run sooner — + * upgrade if partial-corpus reports keep arriving from runs that stayed under + * this bar. + */ +const MAX_SUB_BATCH_FAILURE_RATIO = 0.25; +const MIN_SUB_BATCHES_FLOOR = 5; +const MAX_SUB_BATCHES_FLOOR = 20; +/** Judge the rate over the back half of the run: half its sub-batch budget. */ +const RATIO_GUARD_SAMPLE_FRACTION_DIVISOR = 2; + +const minSubBatchesBeforeFailureRatioGuard = (totalNodes: number, subBatchSize: number): number => + Math.min( + MAX_SUB_BATCHES_FLOOR, + Math.max( + MIN_SUB_BATCHES_FLOOR, + Math.ceil(totalNodes / subBatchSize / RATIO_GUARD_SAMPLE_FRACTION_DIVISOR), + ), + ); + +/** + * The cumulative-ratio abort (#2790). Deliberately NOT the raw endpoint error + * the consecutive ceiling rethrows: the operator's next action differs — one bad + * batch is noise, a quarter of the corpus failing is an endpoint to fix before + * re-running. The retained streak error still names the real defect, carried in + * both the message and `cause`. + */ +const failureRatioAbortError = (failures: number, attempted: number, cause: unknown): Error => + new Error( + `[embed] Aborting: ${failures} of ${attempted} embed sub-batches failed ` + + `(${Math.round((failures / attempted) * 100)}%, limit ` + + `${Math.round(MAX_SUB_BATCH_FAILURE_RATIO * 100)}%) — too much of the corpus is failing to ` + + `embed for this run to produce a usable index. Fix the embedding endpoint and re-run. ` + + `Underlying failure: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + +/** + * Both failed: the run is aborting AND the cleanup DELETE for the dropped nodes + * threw (busy/read-only DB). The abort error stays primary — it is the one that + * names the real defect and the one the operator must act on — and is kept as + * `cause`; the cleanup failure is appended, because it also means those nodes may + * still hold partial rows carrying the CURRENT contentHash, which a later + * incremental run would read as fresh (the corruption the cleanup exists to + * prevent). Without this, a failing DELETE replaced the endpoint error outright. + */ +const abortWithFailedCleanupError = (abortErr: unknown, cleanupErr: unknown): Error => + new Error( + `${abortErr instanceof Error ? abortErr.message : String(abortErr)} — additionally, ` + + `cleanup of the dropped nodes' embedding rows FAILED, so some may still hold partial ` + + `rows that a later incremental run would read as fresh: ` + + `${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`, + { cause: abortErr }, + ); + +/** + * Cancellation is an instruction, not endpoint flakiness — it must abort the run + * instead of being absorbed as a tolerable sub-batch failure. Checked on the + * error shape as well as on our own signal, because an embed call handed the + * signal rejects with a `DOMException`/`Error` named `AbortError` rather than + * flipping anything the pipeline owns (e.g. a host-supplied transport signal). + */ +const isCancellationError = (err: unknown): boolean => + typeof err === 'object' && err !== null && (err as { name?: unknown }).name === 'AbortError'; + export interface EmbeddingPipelineCheckpoint { nodesProcessed: number; totalNodes: number; @@ -509,6 +657,7 @@ export const runEmbeddingPipeline = async ( chunksProcessed: 0, vectorIndexReady, semanticMode: vectorIndexReady ? 'vector-index' : 'exact-scan', + failedNodeIds: [], }; } @@ -520,7 +669,38 @@ export const runEmbeddingPipeline = async ( batchSize, Math.ceil(checkpointEveryNodes / batchSize) * batchSize, ); + // Nodes that finished with a COMPLETE row set. Distinct from the number of + // nodes traversed (see `traversedNodes` in the loop): a node whose chunks lost + // a sub-batch is walked past but NOT processed, and must not be counted as if + // its embeddings exist. let processedNodes = 0; + // Run-level: every node whose rows were dropped by the sub-batch failure path. + const failedNodeIds = new Set(); + // Run-level, not per-batch: a dead endpoint stays dead across batch boundaries. + let consecutiveSubBatchFailures = 0; + // Run-level tally behind MAX_SUB_BATCH_FAILURE_RATIO (#2790). Lifetime, never + // reset — a steadily-degraded endpoint is exactly the shape the consecutive + // counter's reset-on-success blinds it to. + let subBatchesAttempted = 0; + let subBatchFailures = 0; + // Sized from this run's own node set AND its configured sub-batch size, not + // a flat constant, so a resume run (whose node set is only the pending ids) + // and an operator-tuned `subBatchSize` both still arm the guard — see + // minSubBatchesBeforeFailureRatioGuard. + const minSubBatchesForRatioGuard = minSubBatchesBeforeFailureRatioGuard( + totalNodes, + finalConfig.subBatchSize, + ); + // #2790: the FIRST error of the current streak — the one the ceiling rethrows. + // Later failures in a streak degrade into generic noise: a misconfigured + // endpoint trips the HTTP client's circuit breaker after 3 rejections, so + // failures 4 and 5 are "circuit open … retry in 30s" (advice to wait for a + // condition that never changes) while the first still names the real defect + // ("unexpected response shape"). Boxed rather than held as a bare `unknown` + // so a thrown `undefined` can't be misread as "no error". Cleared with the + // counter below, so the rethrown error always belongs to the streak that + // actually tripped the ceiling. + let streakFirstError: { err: unknown } | undefined; onProgress({ phase: 'embedding', @@ -631,20 +811,69 @@ export const runEmbeddingPipeline = async ( // Embed chunk texts in sub-batches to control memory const EMBED_SUB_BATCH = finalConfig.subBatchSize; + // Nodes that lost at least one chunk in this batch. Collected as NODE ids, + // not sub-batch indices: `allTexts`/`allUpdates` are flat over the whole + // batch with no node alignment, so one node's chunks can straddle a + // sub-batch boundary and a failure never maps cleanly onto whole nodes. + const batchFailedNodeIds = new Set(); + // Set instead of thrown immediately so the cleanup DELETE below still runs. + // Carries EITHER abort reason — the consecutive-ceiling rethrow or the + // cumulative-ratio abort — hence the reason-neutral name. + let abortError: { err: unknown } | undefined; for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) { const subTexts = allTexts.slice(si, si + EMBED_SUB_BATCH); const subUpdates = allUpdates.slice(si, si + EMBED_SUB_BATCH); + subBatchesAttempted += 1; let embeddings: Float32Array[]; try { embeddings = await embedBatch(subTexts, { signal: pipelineOptions.signal }); } catch (embedErr) { - logger.error( - { embedErr }, - `❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`, + // Never swallow a cancel as a "tolerable" failure — the operator asked + // us to stop, and the run has no business continuing (or deleting rows). + if (pipelineOptions.signal?.aborted || isCancellationError(embedErr)) throw embedErr; + + // #2790: one transient hiccup used to discard an entire multi-hour run. + // Warn (not error — the run survives), drop this sub-batch's nodes, and + // keep going; the cleanup DELETE after this loop makes the drop safe. + const droppedNodeIds = new Set(subUpdates.map((u) => u.nodeId)); + for (const nodeId of droppedNodeIds) batchFailedNodeIds.add(nodeId); + consecutiveSubBatchFailures += 1; + subBatchFailures += 1; + const streakError = (streakFirstError ??= { err: embedErr }); + // Log under `err` so pino's standard serializer keeps the message and + // stack — the previous `{ embedErr }` key serialized to `{}` (#2114). + logger.warn( + { err: embedErr }, + `⚠️ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}...") — ` + + `dropping ${droppedNodeIds.size} node(s) from this run so the next run re-embeds them ` + + `(${consecutiveSubBatchFailures}/${MAX_CONSECUTIVE_SUB_BATCH_FAILURES} consecutive, ` + + `${subBatchFailures}/${subBatchesAttempted} sub-batches failed this run):`, ); - throw embedErr; + if (consecutiveSubBatchFailures >= MAX_CONSECUTIVE_SUB_BATCH_FAILURES) { + // Bail out — but via `break`, not `throw`, so the cleanup DELETE below + // still runs: sub-batches that succeeded before the endpoint went dark + // left partial rows carrying the CURRENT hash, which is exactly the + // read-as-fresh corruption this path exists to prevent. + abortError = streakError; + break; + } + if ( + subBatchesAttempted >= minSubBatchesForRatioGuard && + subBatchFailures / subBatchesAttempted >= MAX_SUB_BATCH_FAILURE_RATIO + ) { + // Same break-then-cleanup path as the ceiling: a run aborting because + // the endpoint is shedding a quarter of the corpus still must not leave + // this batch's touched nodes half embedded (#2790). + abortError = { + err: failureRatioAbortError(subBatchFailures, subBatchesAttempted, streakError.err), + }; + break; + } + continue; } + consecutiveSubBatchFailures = 0; + streakFirstError = undefined; const dbUpdates = subUpdates.map((u, i) => ({ ...u, @@ -655,10 +884,40 @@ export const runEmbeddingPipeline = async ( throwIfCancelled(); } - processedNodes += batch.length; - totalChunks += allUpdates.length; + // A failed sub-batch leaves the node HALF embedded, and half is worse than + // none: this batch's stale rows were already deleted above, a node's chunks + // can straddle a sub-batch boundary, and every surviving row carries the + // CURRENT contentHash. Both downstream hash-map builders collapse a node's + // rows to one entry (last row wins), so a half-embedded node would read as + // FRESH on every future run and its missing chunks would never come back. + // Deleting ALL of its rows leaves zero, which the incremental filter reads + // as "New node — needs embedding" — it self-heals next run. Kept inside the + // per-batch iteration so the inconsistency window stays one batch wide, + // matching the U6 / KTD7 delete-window rationale above. + if (batchFailedNodeIds.size > 0) { + try { + await deleteStaleEmbeddingRows(executeWithReusedStatement, [...batchFailedNodeIds]); + } catch (cleanupErr) { + // A failing cleanup DELETE must never swallow a pending abort: that + // error names the actual defect (the endpoint), while a busy or + // read-only DB is a second, downstream symptom. Without this the + // `throw` below is unreachable and the run reports only the DELETE. + if (!abortError) throw cleanupErr; + throw abortWithFailedCleanupError(abortError.err, cleanupErr); + } + for (const nodeId of batchFailedNodeIds) failedNodeIds.add(nodeId); + } + if (abortError) throw abortError.err; - const embeddingProgress = 20 + (processedNodes / totalNodes) * 70; + // Nodes walked past in this run so far. Drives progress percent and the + // checkpoint cadence, which must stay monotonic and hit `totalNodes` + // exactly — `processedNodes` can now lag behind and would silently stop the + // window-aligned and terminal checkpoints from ever firing. + const traversedNodes = batchIndex + batch.length; + processedNodes += batch.length - batchFailedNodeIds.size; + totalChunks += allUpdates.filter((u) => !batchFailedNodeIds.has(u.nodeId)).length; + + const embeddingProgress = 20 + (traversedNodes / totalNodes) * 70; onProgress({ phase: 'embedding', percent: Math.round(embeddingProgress), @@ -670,7 +929,7 @@ export const runEmbeddingPipeline = async ( if ( pipelineOptions.onCheckpoint && - (processedNodes % checkpointWindowNodeCount === 0 || processedNodes === totalNodes) + (traversedNodes % checkpointWindowNodeCount === 0 || traversedNodes === totalNodes) ) { await pipelineOptions.onCheckpoint({ nodesProcessed: processedNodes, @@ -683,10 +942,12 @@ export const runEmbeddingPipeline = async ( // Phase 4: Create vector index throwIfCancelled(); + // Report the nodes actually embedded, not `totalNodes`: with a flaky endpoint + // those differ, and a caller is about to read this number as truth. onProgress({ phase: 'indexing', percent: 90, - nodesProcessed: totalNodes, + nodesProcessed: processedNodes, totalNodes, }); @@ -699,26 +960,34 @@ export const runEmbeddingPipeline = async ( onProgress({ phase: 'ready', percent: 100, - nodesProcessed: totalNodes, + nodesProcessed: processedNodes, totalNodes, }); + if (failedNodeIds.size > 0) { + logger.warn( + `⚠️ Embedding pipeline finished with ${failedNodeIds.size} node(s) dropped after embed failures; ` + + 'their rows were removed so the next run re-embeds them.', + ); + } if (isDev) { logger.info( - `✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`, + `✅ Embedding pipeline complete! (${totalChunks} chunks from ${processedNodes}/${totalNodes} nodes)`, ); } return { - nodesProcessed: totalNodes, + nodesProcessed: processedNodes, chunksProcessed: totalChunks, vectorIndexReady, semanticMode: vectorIndexReady ? 'vector-index' : 'exact-scan', + failedNodeIds: [...failedNodeIds], }; } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; if (isDev) { - logger.error({ error }, '❌ Embedding pipeline error:'); + // `err`, not `error` — see the serializer note in queryEmbeddableNodes (#2114). + logger.error({ err: error }, '❌ Embedding pipeline error:'); } onProgress({ @@ -785,9 +1054,10 @@ export const semanticSearch = async ( } if (bestChunks.size === 0) { - const countRows = await executeQuery( - `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN count(e) AS cnt`, - ); + // The Cypher only. NOT `measurePersistedEmbeddingCount`: its tri-state + // exists so a publisher never writes a fabricated 0, whereas here `?? 0` + // is the right answer — an unknown count simply skips the exact scan. + const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER); const countRow = countRows[0]; const embeddingCount = Number(countRow?.cnt ?? countRow?.[0] ?? 0); const exactLimit = getExactScanLimit(); diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index e937cd364..82cce622a 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -11,7 +11,12 @@ * via `AbortSignal.timeout` on the underlying fetch. */ -import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; +import { + CircuitOpenError, + ResilientFetchExhaustedError, + isTerminalNetworkError, + resilientFetch, +} from 'gitnexus-shared'; const DEFAULT_HTTP_TIMEOUT_MS = 180_000; const MAX_HTTP_TIMEOUT_MS = 300_000; @@ -310,6 +315,46 @@ const isEmbeddingItem = (item: unknown): item is EmbeddingItem => item !== null && Array.isArray((item as { embedding?: unknown }).embedding); +/** + * Module-private signal that a 2xx response carried a body this client cannot + * use (unparseable, or parseable but wrong-shaped). Deliberately not exported: + * it never escapes {@link httpEmbedBatch}, which converts it into the + * user-facing {@link HttpEmbeddingError} it carries in `terminalMessage`. + * + * Thrown from inside the `fetchImpl` callback so `resilientFetch` classifies it + * as `retryable-network` — a truncated or HTML body is the same class of + * endpoint failure as a 503 and deserves the same backoff loop, and routing it + * through the retry loop also makes the circuit breaker see it as a failure + * instead of erasing the outage signal with `recordSuccess()`. See #2790. + */ +class RetryableEmbeddingBodyError extends Error { + constructor( + readonly terminalMessage: string, + options?: { cause?: unknown }, + ) { + super(terminalMessage, options?.cause !== undefined ? { cause: options.cause } : undefined); + this.name = 'RetryableEmbeddingBodyError'; + } +} + +/** + * Build the message for a 2xx body carrying the wrong number of vectors. + * + * Hoisted to module scope because two layers report this same fault — the + * in-loop check inside {@link httpEmbedBatch} and the defensive backstop in + * {@link httpEmbed} — and the operator must see one wording regardless of which + * one catches it. Names both counts: "0 vectors for 64 texts" is actionable, + * "unexpected response shape" is not. + */ +const countMismatchMessage = ( + received: number, + expected: number, + safeEndpoint: string, + batchIndex: number, +): string => + `Embedding endpoint returned ${received} vectors for ${expected} texts ` + + `(${safeEndpoint}, batch ${batchIndex})`; + /** * Send a single batch of texts to the embedding endpoint with retry. * @@ -347,7 +392,21 @@ const httpEmbedBatch = async ( requestBody.dimensions = dimensions; } + // Built on demand, not up front. Both describe faults, so in a healthy run — + // every call of it — they are garbage, and each `safeUrl` parses a URL. A + // 300k-chunk run at subBatchSize 8 makes ~37.5k `httpEmbedBatch` calls, so + // eager construction spent two `new URL()` per call describing failures that + // never happened. Same shape as `countMismatchMessage` above. + const unparseableMessage = (): string => + `Embedding endpoint returned an unparseable response (${safeUrl(url)}, batch ${batchIndex})`; + const unexpectedShapeMessage = (): string => + `Embedding endpoint returned an unexpected response shape (${safeUrl(url)}, batch ${batchIndex})`; + let resp: Response; + // Set by `fetchImpl` on the attempt that produced a usable body. Reads back + // as `undefined` only on a path that must already have thrown — see the + // defensive check after the retry loop. + let parsed: EmbeddingItem[] | undefined; try { throwIfAborted(requestOptions.signal); resp = await resilientFetch( @@ -368,7 +427,54 @@ const httpEmbedBatch = async ( const signal = requestOptions.signal ? AbortSignal.any([requestOptions.signal, timeoutSignal]) : timeoutSignal; - return globalThis.fetch(input, { ...init, signal }); + const attemptResp = await globalThis.fetch(input, { ...init, signal }); + // Non-OK bodies are none of our business: hand the response straight + // back so `resilientFetch` keeps classifying 4xx/5xx/429 unchanged. + if (!attemptResp.ok) return attemptResp; + + // The body is read *here*, inside the retried callback, rather than + // after `resilientFetch` returns. A reachable-but-wrong endpoint (a + // captive portal, a non-embeddings service, a truncated stream) can + // answer 200 with HTML or half a JSON document; parsing outside the + // loop made that a one-shot terminal failure while a 503 got three + // attempts. Throwing from in here gives a bad body the same backoff + // and the same breaker accounting as any other endpoint fault (#2790). + let payload: { data: EmbeddingItem[] }; + try { + payload = (await attemptResp.json()) as { data: EmbeddingItem[] }; + } catch (err) { + // Not every `.json()` rejection is a parse error: the per-attempt + // signal (`AbortSignal.any([caller, AbortSignal.timeout(...)])`) is + // wired to the body stream, so a stalled body rejects with the abort + // reason. Re-raise those untouched — `isTerminalNetworkError` is + // `resilientFetch`'s own predicate, so this test agrees with + // `classifyOutcome` by construction. Wrapping one would flip its + // verdict from `terminal-network` (returned without retry AND + // without touching the breaker, via `recordNeutral()`) to + // `retryable-network` (retried, then `breaker.recordFailure()`): the + // same timeout would take 3 attempts instead of 1, count toward the + // process-global `embeddings-http` breaker, and reach the operator as + // "unparseable response" so they never reach for the timeout knob. + if (isTerminalNetworkError(err)) throw err; + throw new RetryableEmbeddingBodyError(unparseableMessage(), { cause: err }); + } + if (!Array.isArray(payload?.data) || !payload.data.every(isEmbeddingItem)) { + throw new RetryableEmbeddingBodyError(unexpectedShapeMessage()); + } + // Cardinality belongs *inside* the retry loop. `every(isEmbeddingItem)` + // is vacuously true for `[]` and true for any array shorter than the + // request, so a 200 carrying `{"data": []}` — or half the vectors — + // used to be classified `success`, call `breaker.recordSuccess()` + // (erasing the outage signal), and only then fail terminally after a + // single attempt. A short body is a truncated body: same backoff, same + // breaker accounting as any other endpoint fault (#2790). + if (payload.data.length !== batch.length) { + throw new RetryableEmbeddingBodyError( + countMismatchMessage(payload.data.length, batch.length, safeUrl(url), batchIndex), + ); + } + parsed = payload.data; + return attemptResp; }, breakerKey: HTTP_BREAKER_KEY, retry: { @@ -390,6 +496,12 @@ const httpEmbedBatch = async ( { cause: err }, ); } + // Retries are exhausted on a bad 2xx body. Surface the message the sentinel + // carried, keeping the underlying parse error in `cause` only — the body + // text must never reach the `sanitizeReason` fallback and leak to stderr. + if (err instanceof RetryableEmbeddingBodyError) { + throw new HttpEmbeddingError(err.terminalMessage, { cause: err.cause }); + } if (err instanceof CircuitOpenError) { throw new HttpEmbeddingError( `Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`, @@ -425,25 +537,14 @@ const httpEmbedBatch = async ( ); } - // A reachable-but-wrong endpoint (e.g. a captive portal or a non-embeddings - // service) can answer 200 with an HTML/truncated body. Parse inside the - // typed-error boundary so that lands as an endpoint failure the CLI can - // classify, not a raw SyntaxError/TypeError on the generic stack-dump path. - let data: { data: EmbeddingItem[] }; - try { - data = (await resp.json()) as { data: EmbeddingItem[] }; - } catch (err) { - throw new HttpEmbeddingError( - `Embedding endpoint returned an unparseable response (${safeUrl(url)}, batch ${batchIndex})`, - { cause: err }, - ); + if (parsed === undefined) { + // Defensively unreachable: an OK response either sets `parsed` or throws + // out of `fetchImpl`. Kept so the narrowing holds without a non-null + // assertion, and so a future `resilientFetch` change can't return an + // unvalidated body silently. + throw new HttpEmbeddingError(unparseableMessage()); } - if (!Array.isArray(data?.data) || !data.data.every(isEmbeddingItem)) { - throw new HttpEmbeddingError( - `Embedding endpoint returned an unexpected response shape (${safeUrl(url)}, batch ${batchIndex})`, - ); - } - return data.data; + return parsed; }; /** @@ -482,10 +583,15 @@ export const httpEmbed = async ( config.timeoutMs, ); + // Defensive backstop, deliberately kept: `httpEmbedBatch` now rejects a + // short body from *inside* the retry loop (through the same + // `countMismatchMessage`), so in practice this branch is unreachable. It + // stays so a future change to the in-loop check can't silently hand a short + // vector list to the caller — that failure would land as a Kuzu/FLOAT[N] + // error far from its cause. if (items.length !== batch.length) { throw new HttpEmbeddingError( - `Embedding endpoint returned ${items.length} vectors for ${batch.length} texts ` + - `(${safeUrl(url)}, batch ${batchIndex})`, + countMismatchMessage(items.length, batch.length, safeUrl(url), batchIndex), ); } @@ -493,6 +599,18 @@ export const httpEmbed = async ( const vec = new Float32Array(item.embedding); // Fail fast on dimension mismatch rather than inserting bad vectors // into the FLOAT[N] column which would cause a cryptic Kuzu error. + // + // Unlike the cardinality check this one stays *outside* the retry loop, + // deliberately. The expected width is `config.dimensions ?? DEFAULT_DIMS` + // (GITNEXUS_EMBEDDING_DIMS), which is NOT the `dimensions` value + // `httpEmbedBatch` receives — that is `config.requestDimensions`, which + // `GITNEXUS_EMBEDDING_REQUEST_DIMS` can set to a different number or to + // `undefined` (`omit`). More importantly a width mismatch is an operator + // *configuration* error, not an endpoint fault: retrying it three times + // can never change the answer, and routing it through the retry loop + // would count a healthy endpoint's responses toward the shared circuit + // breaker. The message is an actionable config hint, so it is terminal + // on the first attempt by design (#2790). const expected = config.dimensions ?? DEFAULT_DIMS; if (vec.length !== expected) { const hint = config.dimensions @@ -539,6 +657,9 @@ export const httpEmbedQuery = async ( config.minIntervalMs, config.timeoutMs, ); + // Defensive backstop like the `httpEmbed` one above: an empty `data` array is + // now a cardinality mismatch (0 vectors for 1 text) rejected and retried + // inside `httpEmbedBatch`, so this branch is unreachable in practice. if (!items.length) { throw new HttpEmbeddingError(`Embedding endpoint returned empty response (${safeUrl(url)})`); } diff --git a/gitnexus/src/core/group/cross-impact.ts b/gitnexus/src/core/group/cross-impact.ts index 70c0af506..06c16d3fa 100644 --- a/gitnexus/src/core/group/cross-impact.ts +++ b/gitnexus/src/core/group/cross-impact.ts @@ -11,6 +11,7 @@ import type { CrossRepoImpact, GroupConfig, GroupImpactResult, + GroupImpactTruncationReason, MatchType, OutOfScopeLink, } from './types.js'; @@ -29,6 +30,7 @@ import { readBridgeMeta, } from './bridge-db.js'; import { BRIDGE_SCHEMA_VERSION } from './bridge-schema.js'; +import { compareCodeUnits } from '../../lib/utils.js'; // High limit for the local phase of group impact so collectImpactSymbolUids // sees (nearly) all symbols. Bypasses the MCP-facing default of 100. @@ -40,6 +42,26 @@ export const MAX_SUPPORTED_CROSS_DEPTH = 1; /** Default wall-clock budget for the Phase 1 `impact` leg when callers omit `timeoutMs`. */ export const DEFAULT_LOCAL_IMPACT_TIMEOUT_MS = 30_000; +/** + * Cap on neighbour fan-outs attempted per group-impact request. + * + * The bound used to be the wall clock alone, which made the cutoff a function + * of machine load: an idle host traversed more crossings and `mergeRisk` + * escalated to CRITICAL at three, while a loaded host stopped at two and + * reported HIGH or lower — same graph, same arguments, different verdict + * (#2787). A count is deterministic, and the neighbour list carries a total + * order over the full crossing identity (confidence DESC, then repo, uid, + * contract), so the cap keeps the strongest crossings rather than an arbitrary + * prefix. + * + * 50 is borrowed from `MAX_CROSSINGS_TO_TRY` (cross-trace.ts) on cost, not on + * scope — that one caps ContractLinks per repo pair inside a trace, this one + * caps the total across all neighbour repos in one impact call, and group impact + * had no numeric cap at all before #2787. The wall clock stays as a hang + * backstop below; this is the bound that normally binds. + */ +export const MAX_NEIGHBOR_FANOUT = 50; + const CY_NEIGHBORS_UPSTREAM = ` MATCH (consumer:Contract)-[l:ContractLink]->(provider:Contract) WHERE provider.repo = $localRepo @@ -349,14 +371,51 @@ function extractProcessNames(impact: unknown): string[] { // No behavior change — `'UNKNOWN'` was already handled correctly at the // `(localRisk === 'LOW' || localRisk === 'UNKNOWN')` branch below. export function mergeRisk(localRisk: string, cross: CrossRepoImpact[]): string { - const highConf = cross.some((c) => c.contract.confidence >= 0.85); + const traversed = cross.filter((c) => c.fanout_status !== 'not_attempted'); + const highConf = traversed.some((c) => c.contract.confidence >= 0.85); if (localRisk === 'CRITICAL') return 'CRITICAL'; - if (cross.length >= 3) return 'CRITICAL'; + if (traversed.length >= 3) return 'CRITICAL'; if (highConf) return 'HIGH'; - if (cross.length > 0 && (localRisk === 'LOW' || localRisk === 'UNKNOWN')) return 'MEDIUM'; + if (traversed.length > 0 && (localRisk === 'LOW' || localRisk === 'UNKNOWN')) return 'MEDIUM'; return localRisk; } +/** + * Build the truncation fields every `runGroupImpact` return path shares. + * + * `riskEpistemic` must follow `truncated` mechanically: it is the marker that + * tells a caller the `risk` value is a floor rather than a verdict, and + * `mergeRisk` can only under-report once a crossing is dropped. Attaching it at + * each return let two of the four paths set `truncated` without it, so a + * truncated result read as complete — deriving it in one place is what keeps + * the invariant from drifting again (#2787). + */ +function truncationFields( + truncated: boolean, + // Only read on the truncated branch, so the not-truncated call sites omit it + // rather than passing a reason that is thrown away. + reasonIfTruncated: GroupImpactTruncationReason = 'partial', +): Pick { + if (!truncated) return { truncated: false }; + return { truncated: true, truncationReason: reasonIfTruncated, riskEpistemic: 'lower-bound' }; +} + +function addCrossImpact(cross: CrossRepoImpact[], candidate: CrossRepoImpact): void { + const sameBoundary = (entry: CrossRepoImpact): boolean => + entry.repo_path === candidate.repo_path && entry.contract.id === candidate.contract.id; + + if (candidate.fanout_status === 'not_attempted') { + if (cross.some(sameBoundary)) return; + } else { + const boundaryOnlyIndex = cross.findIndex( + (entry) => sameBoundary(entry) && entry.fanout_status === 'not_attempted', + ); + if (boundaryOnlyIndex >= 0) cross.splice(boundaryOnlyIndex, 1); + } + + cross.push(candidate); +} + export async function ensureBridgeReady( groupDir: string, ): Promise<{ handle: BridgeHandle } | { error: string }> { @@ -431,7 +490,18 @@ export async function resolveBridgeNeighbors( const n = rowToNeighbor(raw); if (n) neighbors.push(n); } - neighbors.sort((a, b) => b.confidence - a.confidence); + // Sort on the FULL crossing identity — the same triple the fan-out dedups on + // below (`repo\0uid\0contractId`). Two links that share (confidence, repo, + // uid) but differ in contract both survive that dedup, so leaving contractId + // out of the comparator makes them compare 0 and fall back to raw bridge row + // order, which is what decides who lands past MAX_NEIGHBOR_FANOUT (#2787). + neighbors.sort( + (a, b) => + b.confidence - a.confidence || + compareCodeUnits(a.neighborRepo, b.neighborRepo) || + compareCodeUnits(a.neighborUid, b.neighborUid) || + compareCodeUnits(a.contractId, b.contractId), + ); return neighbors; } @@ -497,7 +567,7 @@ export async function runGroupImpact( group: name, cross: [], outOfScope: [], - truncated: true, + ...truncationFields(true, 'timeout'), truncatedRepos: [], summary: { direct: 0, @@ -507,7 +577,6 @@ export async function runGroupImpact( }, risk: 'UNKNOWN', timeoutMs, - truncationReason: 'timeout', crossDepthWarning, }; } @@ -531,7 +600,7 @@ export async function runGroupImpact( group: name, cross: [], outOfScope: [], - truncated: false, + ...truncationFields(false), truncatedRepos: [], summary: { direct: 0, @@ -554,7 +623,7 @@ export async function runGroupImpact( group: name, cross: [], outOfScope: [], - truncated: Boolean((local as { partial?: boolean }).partial), + ...truncationFields(Boolean((local as { partial?: boolean }).partial), 'partial'), truncatedRepos: [], summary: { direct: s.direct ?? 0, @@ -564,7 +633,6 @@ export async function runGroupImpact( }, risk: String((local as { risk?: string }).risk ?? 'LOW'), timeoutMs, - truncationReason: (local as { partial?: boolean }).partial ? 'partial' : undefined, crossDepthWarning, }; } @@ -576,6 +644,10 @@ export async function runGroupImpact( const cross: CrossRepoImpact[] = []; const outOfScope: OutOfScopeLink[] = []; const truncatedRepos: string[] = []; + /** Real `impactByUid` fan-outs issued — what MAX_NEIGHBOR_FANOUT bounds. */ + let attemptedFanouts = 0; + /** True when the wall-clock backstop, not the count cap, cut the fan-out. */ + let fanoutTimedOut = false; try { const neighbors = await resolveBridgeNeighbors(handle, { @@ -587,7 +659,12 @@ export async function runGroupImpact( const seen = new Set(); for (const n of neighbors) { - if (servicePrefix && !fileMatchesServicePrefix(n.neighborFilePath, servicePrefix)) { + const manifestOnly = n.neighborUid.startsWith('manifest::'); + if ( + servicePrefix && + !manifestOnly && + !fileMatchesServicePrefix(n.neighborFilePath, servicePrefix) + ) { continue; } if (!repoInSubgroup(n.neighborRepo, subgroup)) { @@ -604,8 +681,18 @@ export async function runGroupImpact( if (seen.has(key)) continue; seen.add(key); - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) { + // Deterministic bound first: the count decides the cutoff on every host. + // Manifest-only crossings are exempt because they cost no `impactByUid` + // and dropping them regresses #2784. + if (!manifestOnly && attemptedFanouts >= MAX_NEIGHBOR_FANOUT) { + truncatedRepos.push(n.neighborRepo); + continue; + } + // Wall clock second, as a hang backstop only. It is load-dependent by + // construction, so when it is what fired the response must say `timeout`, + // not the generic `partial` it used to report. + if (!manifestOnly && deadline - Date.now() <= 0) { + fanoutTimedOut = true; truncatedRepos.push(n.neighborRepo); continue; } @@ -621,11 +708,41 @@ export async function runGroupImpact( continue; } + // A manifest link can prove the repository boundary even when its far + // endpoint has no graph symbol of its own (for example, a string-based + // RPC dispatch). Those endpoints deliberately use a deterministic + // `manifest::...` UID. Calling impactByUid with that synthetic value can + // never succeed because it is not a node in the verified neighbor + // repository; dropping the crossing here made `group sync` report the + // manifest link while `group impact` silently returned cross=0 (#2722). + // + // Preserve the proven crossing with an empty local fan-out. Real UIDs + // still take the normal impactByUid path below, where failures remain + // truncations rather than false successful traversals. + if (manifestOnly) { + addCrossImpact(cross, { + repo: regName, + repo_path: n.neighborRepo, + contract: { + id: n.contractId, + type: n.contractType as ContractType, + match_type: 'manifest', + confidence: n.confidence, + }, + by_depth: {}, + affected_processes: [], + fanout_status: 'not_attempted', + }); + continue; + } + + const remainingMs = deadline - Date.now(); // Phase-2 hardening: race each impactByUid against a per-call // timeout derived from the remaining budget. Without this wrap a // single hung neighbor would pin the request past the clamped // timeout, which Codex's adversarial review on PR #1331 flagged // as the still-open half of CodeQL #184 / js/resource-exhaustion. + attemptedFanouts += 1; const { value: fan, timedOut: neighborTimedOut } = await safeNeighborImpact( deps.port, neighborHandle.id, @@ -640,11 +757,12 @@ export async function runGroupImpact( remainingMs, ); if (neighborTimedOut || fan == null) { + if (neighborTimedOut) fanoutTimedOut = true; truncatedRepos.push(n.neighborRepo); continue; } - cross.push({ + addCrossImpact(cross, { repo: regName, repo_path: n.neighborRepo, contract: { @@ -671,7 +789,12 @@ export async function runGroupImpact( group: name, cross, outOfScope, - truncated, + // The risk VALUE is never clamped down. `mergeRisk` is monotone increasing + // in the traversed-crossing count, so truncation can only under-report — + // and under-reporting a blast radius is the unsafe direction (an agent told + // LOW proceeds; told CRITICAL it stops). Marking the floor keeps the + // warning intact while making the incompleteness legible. + ...truncationFields(truncated, fanoutTimedOut ? 'timeout' : 'partial'), truncatedRepos: [...new Set(truncatedRepos)], summary: { direct: localSum.direct ?? 0, @@ -681,10 +804,12 @@ export async function runGroupImpact( }, risk: mergeRisk(localRisk, cross), timeoutMs, - truncationReason: truncated ? 'partial' : undefined, crossDepthWarning, }; return result; } export { normalizeServicePrefix, fileMatchesServicePrefix } from './group-path-utils.js'; +// Re-exported, not redefined: the single definition lives in lib/utils.ts, but +// this module's own surface is what the #2787 regression test imports it from. +export { compareCodeUnits }; diff --git a/gitnexus/src/core/group/cross-trace.ts b/gitnexus/src/core/group/cross-trace.ts index 16293b179..7c115de01 100644 --- a/gitnexus/src/core/group/cross-trace.ts +++ b/gitnexus/src/core/group/cross-trace.ts @@ -26,6 +26,7 @@ import { GroupNotFoundError, loadGroupConfig } from './config-parser.js'; import { getGroupDir } from './storage.js'; import { ensureBridgeReady, MAX_SUPPORTED_CROSS_DEPTH } from './cross-impact.js'; +import { compareCodeUnits } from '../../lib/utils.js'; import { closeBridgeDb, queryBridge } from './bridge-db.js'; import type { GroupPdgFlowHop, @@ -257,7 +258,7 @@ RETURN consumer.symbolUid AS consumerUid, l.confidence AS confidence, l.contractId AS contractId, consumer.type AS contractType -ORDER BY l.confidence DESC +ORDER BY l.confidence DESC, contractId, consumerUid, providerUid LIMIT ${MAX_CROSSINGS_TO_TRY + 1} `; @@ -279,7 +280,7 @@ RETURN consumer.symbolUid AS consumerUid, l.confidence AS confidence, l.contractId AS contractId, consumer.type AS contractType -ORDER BY l.confidence DESC +ORDER BY l.confidence DESC, contractId, consumerUid, providerUid LIMIT ${MAX_CROSSINGS_TO_TRY + 1} `; @@ -321,6 +322,32 @@ function rowToCrossing(r: Record): CrossingRow | null { }; } +/** + * Sort a crossing list on the full crossing identity, then apply + * `MAX_CROSSINGS_TO_TRY`. + * + * Both bridge queries already order by confidence DESC; this re-sorts + * defensively so the cap keeps the strongest candidates even if a tuple-mode + * driver reorders rows. The comparator mirrors BOTH queries' `ORDER BY` + * key-for-key — a shorter comparator reproduces less than the order it is + * defending, leaving ties on raw row order (#2787) — so it lives in one place + * rather than being hand-synced across the two callers and two Cypher clauses. + */ +function capCrossings(all: CrossingRow[]): { crossings: CrossingRow[]; truncated: boolean } { + all.sort( + (a, b) => + b.confidence - a.confidence || + compareCodeUnits(a.contractId, b.contractId) || + compareCodeUnits(a.consumerUid, b.consumerUid) || + compareCodeUnits(a.providerUid, b.providerUid), + ); + const truncated = all.length > MAX_CROSSINGS_TO_TRY; + return { + crossings: truncated ? all.slice(0, MAX_CROSSINGS_TO_TRY) : all, + truncated, + }; +} + async function listCrossingsBetween( handle: BridgeHandle, fromRepo: string, @@ -335,14 +362,7 @@ async function listCrossingsBetween( const c = rowToCrossing(raw); if (c) all.push(c); } - // The query already orders by confidence DESC; re-sort defensively so the cap - // keeps the strongest candidates even if a tuple-mode driver reorders rows. - all.sort((a, b) => b.confidence - a.confidence); - const truncated = all.length > MAX_CROSSINGS_TO_TRY; - return { - crossings: truncated ? all.slice(0, MAX_CROSSINGS_TO_TRY) : all, - truncated, - }; + return capCrossings(all); } function destRowToCrossing(r: Record): CrossingRow | null { @@ -375,12 +395,7 @@ async function listCrossingsFrom( const c = destRowToCrossing(raw); if (c) all.push(c); } - all.sort((a, b) => b.confidence - a.confidence); - const truncated = all.length > MAX_CROSSINGS_TO_TRY; - return { - crossings: truncated ? all.slice(0, MAX_CROSSINGS_TO_TRY) : all, - truncated, - }; + return capCrossings(all); } // ── Cross-member symbol resolution ─────────────────────────────────────── diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index f28f3fac4..a6e0b046f 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -102,6 +102,10 @@ RETURN sym.id AS uid, sym.name AS name, sym.filePath AS filePath, // degenerate edge-less node NOR inflates the uniqueness count and masks the real // handler. `LIMIT 2` bounds materialization: distinguishing unique (1) from // ambiguous (>=2) never needs more than two rows (the count guard stays exact). +// +// determinism: probe — uniqueness discriminator, not a window. `toResolvedSymbol` +// reads row 0 only when `rows.length === 1`; a 2-row result is discarded whole, +// so WHICH two rows came back can never reach a caller. export const RESOLVE_BY_NAME_QUERY = ` MATCH (n) WHERE labels(n) IN ['Function','Method','CodeElement'] AND n.name = $name AND n.filePath <> '' @@ -114,6 +118,10 @@ LIMIT 2`; // the precise rung — it survives aliases and local same-name collisions that a // repo-wide name lookup cannot, and only resolves on a unique match within that // module. `LIMIT 2` keeps the uniqueness count exact (see RESOLVE_BY_NAME_QUERY). +// +// determinism: probe — uniqueness discriminator, not a window. Same consumer as +// RESOLVE_BY_NAME_QUERY: `toResolvedSymbol` reads row 0 only when exactly one row +// came back, and discards a 2-row result whole. export const RESOLVE_IN_MODULE_QUERY = ` MATCH (n) WHERE labels(n) IN ['Function','Method','CodeElement'] AND n.name = $name AND (n.filePath STARTS WITH $fileDot OR n.filePath STARTS WITH $fileSlash) diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 4462f51f9..272c27ce6 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -23,7 +23,7 @@ export const CUSTOM_CONTRACT_RESOLVE_QUERY = `MATCH (n) WHERE labels(n) IN ['Function','Method','Class','Interface','Struct','Enum','Trait','Constructor','TypeAlias','Impl','Macro','Union','Typedef','Property','Record','Delegate','Annotation','Template','Const','Static','CodeElement'] AND n.name = $symbolName RETURN n.id AS uid, n.name AS name, n.filePath AS filePath - ORDER BY n.filePath ASC + ORDER BY n.filePath ASC, n.id ASC LIMIT 1`; /** @@ -242,7 +242,7 @@ export class ManifestExtractor { `MATCH (handler)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route) WHERE route.name = $normalized RETURN handler.id AS uid, handler.name AS name, handler.filePath AS filePath - ORDER BY handler.filePath ASC + ORDER BY handler.filePath ASC, handler.id ASC LIMIT 1`, { normalized }, ); @@ -255,7 +255,7 @@ export class ManifestExtractor { rows = await executor( `MATCH (n) WHERE labels(n) IN ['Function','Method','Class','Interface'] AND n.name = $contract RETURN n.id AS uid, n.name AS name, n.filePath AS filePath - ORDER BY n.filePath ASC + ORDER BY n.filePath ASC, n.id ASC LIMIT 1`, { contract: link.contract }, ); @@ -279,7 +279,7 @@ export class ManifestExtractor { rows = await executor( `MATCH (n) WHERE labels(n) IN ['Function','Method'] AND n.name = $methodName RETURN n.id AS uid, n.name AS name, n.filePath AS filePath - ORDER BY n.filePath ASC + ORDER BY n.filePath ASC, n.id ASC LIMIT 1`, { methodName }, ); @@ -287,7 +287,7 @@ export class ManifestExtractor { rows = await executor( `MATCH (n) WHERE labels(n) IN ['Class','Interface'] AND n.name = $serviceName RETURN n.id AS uid, n.name AS name, n.filePath AS filePath - ORDER BY n.filePath ASC + ORDER BY n.filePath ASC, n.id ASC LIMIT 1`, { serviceName }, ); @@ -304,7 +304,7 @@ export class ManifestExtractor { rows = await executor( `MATCH (n) WHERE labels(n) IN ['Module'] AND n.name = $contract RETURN n.id AS uid, n.name AS name, n.filePath AS filePath - ORDER BY n.filePath ASC + ORDER BY n.filePath ASC, n.id ASC LIMIT 1`, { contract: link.contract }, ); @@ -312,7 +312,7 @@ export class ManifestExtractor { rows = await executor( `MATCH (f:File) WHERE f.filePath = $contract RETURN f.id AS uid, f.name AS name, f.filePath AS filePath - ORDER BY f.filePath ASC + ORDER BY f.filePath ASC, f.id ASC LIMIT 1`, { contract: link.contract }, ); diff --git a/gitnexus/src/core/group/service.ts b/gitnexus/src/core/group/service.ts index c18587ce2..f1356dcec 100644 --- a/gitnexus/src/core/group/service.ts +++ b/gitnexus/src/core/group/service.ts @@ -14,7 +14,10 @@ import { repoInSubgroup, } from './group-path-utils.js'; import { getDefaultGitnexusDir, getGroupDir, listGroups, readContractRegistry } from './storage.js'; -import { syncGroup } from './sync.js'; +// `./sync.js` is imported LAZILY in `groupSync` — see the comment at its call +// site. It statically pulls the six contract extractors and, through them, the +// native tree-sitter binding; a static import here puts all of that on MCP +// server startup, which never syncs. import { logger } from '../logger.js'; import type { ContractRegistry, @@ -338,6 +341,12 @@ export class GroupService { return { error: `Group "${name}" not found. Run group_list to see configured groups.` }; throw err; } + // Lazy: `sync.js` reaches the six contract extractors and the native + // tree-sitter binding. `groupSync` is the ONLY consumer — the other seven + // group tools never need it — so deferring it here keeps that closure off + // MCP server startup entirely and off every non-sync group call. The CLI + // already does exactly this at `cli/group.ts`'s sync command. + const { syncGroup } = await import('./sync.js'); const result = await syncGroup(config, { groupDir, exactOnly: Boolean(params.exactOnly), diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index 42f6df368..db9d1989f 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -134,6 +134,18 @@ export interface GroupImpactResult { cross_repo_hits: number; }; risk: string; + /** + * `'lower-bound'` when the fan-out was cut short, so `risk` is a FLOOR, not a + * verdict. Same vocabulary as single-repo `impact`'s `epistemic` field. + * + * `mergeRisk` is monotone increasing in the number of traversed crossings, so + * dropping a crossing can only move the reported risk DOWN — a fully + * truncated fan-out returns bare `localRisk`, making a symbol whose blast + * radius crosses a repo boundary indistinguishable from one with no + * cross-repo consumers. Absent this marker a truncated run emits a + * confident-looking `risk` byte-identical to a complete one. + */ + riskEpistemic?: 'lower-bound'; /** * Milliseconds budget applied to the **Phase 1 local impact** leg (`safeLocalImpact`). * If the walk hits this wall first, expect `truncationReason: 'timeout'` and a partial `local` payload. @@ -177,6 +189,12 @@ export interface CrossRepoImpact { }; by_depth: Record; affected_processes: string[]; + /** + * Present when the bridge proves a repository boundary but the far endpoint + * has no graph symbol, so local fan-out cannot be attempted. Omitted for + * completed fan-out to preserve the existing serialized result shape. + */ + fanout_status?: 'not_attempted'; } export interface OutOfScopeLink { diff --git a/gitnexus/src/core/incremental/subgraph-extract.ts b/gitnexus/src/core/incremental/subgraph-extract.ts index 9f2e49a56..e0f0e41eb 100644 --- a/gitnexus/src/core/incremental/subgraph-extract.ts +++ b/gitnexus/src/core/incremental/subgraph-extract.ts @@ -55,10 +55,12 @@ import { isSpringAutoConfigurationDeclaration, isSpringAutoConfigurationSyntheticClass, } from '../ingestion/frameworks/spring/auto-configuration.js'; +import { isSpringAopEvidenceNode } from '../ingestion/frameworks/spring/aop.js'; const isGraphWideNode = (node: GraphNode): boolean => node.label === 'Community' || node.label === 'Process' || + isSpringAopEvidenceNode(node) || isSpringAutoConfigurationSyntheticClass(node); /** @@ -98,6 +100,10 @@ const isGraphWideRelationship = (relationship: GraphRelationship): boolean => relationship.type === 'TAINT_PATH' || relationship.type === 'CALL_SUMMARY' || relationship.type === 'INJECTS' || + // Spring pointcut matching (#2416) is repository-wide. A third-file change + // can alter annotation-name visibility or the set matched by a wildcard, + // even when neither endpoint file changed. + relationship.type === 'ADVISED_BY' || isSpringAutoConfigurationDeclaration(relationship); /** @@ -144,10 +150,13 @@ export const extractChangedSubgraph = ( /** * Public — derive the EFFECTIVE write-set: `toWriteSet` expanded by one - * hop along every edge in the new graph that crosses the writable - * boundary (one endpoint in a writable file, the other in an unchanged - * file). The unchanged-side file is pulled in so its stale rows are - * deleted + rewritten in lockstep with the changed side. + * hop along every file-owned edge in the new graph that crosses the + * writable boundary (one endpoint in a writable file, the other in an + * unchanged file). Graph-wide relationships are excluded: their owner + * phase delete-alls and re-extracts them independently, so following them + * here would turn high-fan-out metadata into a near-full-repository write. + * For ordinary edges, the unchanged-side file is pulled in so its stale + * rows are deleted + rewritten in lockstep with the changed side. * * Single pass over the edge list. Does NOT mutate `toWriteSet`. The * orchestrator MUST feed the returned set to both `deleteNodesForFiles` @@ -161,6 +170,7 @@ export const computeEffectiveWriteSet = ( const nodeFilePaths = indexNodeFilePaths(fullGraph); const expanded = new Set(toWriteSet); fullGraph.forEachRelationship((r: GraphRelationship) => { + if (isGraphWideRelationship(r)) return; const sourcePath = nodeFilePaths.get(r.sourceId); const targetPath = nodeFilePaths.get(r.targetId); if (!sourcePath || !targetPath) return; // skip edges to graph-wide nodes diff --git a/gitnexus/src/core/index-freshness.ts b/gitnexus/src/core/index-freshness.ts index 4ff800bfa..26c107f38 100644 --- a/gitnexus/src/core/index-freshness.ts +++ b/gitnexus/src/core/index-freshness.ts @@ -1,8 +1,10 @@ +import { checkpointKind } from './embedding-checkpoint.js'; import type { RepoMeta } from '../storage/repo-manager.js'; export const INDEX_INCOMPLETE_REASONS = [ 'incremental-in-progress', 'embedding-checkpoint-pending', + 'embedding-count-unverified', ] as const; export type IndexIncompleteReason = (typeof INDEX_INCOMPLETE_REASONS)[number]; @@ -13,6 +15,18 @@ export function getIndexIncompleteReasons( ): IndexIncompleteReason[] { const reasons: IndexIncompleteReason[] = []; if (meta?.incrementalInProgress) reasons.push('incremental-in-progress'); - if (meta?.embeddingCheckpoint) reasons.push('embedding-checkpoint-pending'); + if (meta?.embeddingCheckpoint) { + // The three checkpoint kinds are not one operator-facing state. GUARDRAILS + // and the runbook document `embedding-checkpoint-pending` as "N node(s) + // lost their embeddings to endpoint failures" — true for 'interrupted' and + // 'partial', a lie for 'unverified-count', whose pending set is empty and + // whose only defect is a count nobody could measure (see the `kind` doc in + // repo-manager.ts). + reasons.push( + checkpointKind(meta.embeddingCheckpoint) === 'unverified-count' + ? 'embedding-count-unverified' + : 'embedding-checkpoint-pending', + ); + } return reasons; } diff --git a/gitnexus/src/core/ingestion/cfg/callee-cell-format.ts b/gitnexus/src/core/ingestion/cfg/callee-cell-format.ts new file mode 100644 index 000000000..337fb7173 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/callee-cell-format.ts @@ -0,0 +1,44 @@ +/** + * Wire format of the `BasicBlock.callees` / `BasicBlock.calleeIds` cells. + * + * A LEAF module on purpose: it declares two string constants and imports + * nothing. `cfg/emit.ts` produces those cells and `mcp/local/pdg-impact.ts` + * parses them, but `emit.ts` is analyze-only and drags the whole CFG closure + * (reaching-defs, control-dependence, post-dominators, synthetic-escape, + * call-site-harvest) behind it — 8 modules evaluated at every MCP server start + * just to read two strings, since ESM evaluates a module to import any binding + * from it (#2802 review). Splitting the format constants out deletes that cost + * rather than deferring it, which is the same bar #2802 held its own proposals + * to. + * + * `emit.ts` RE-EXPORTS both names, so every existing importer keeps working and + * the producer/consumer pair still resolves to one definition — the drift this + * shared constant exists to prevent stays impossible. + */ + +/** + * Reserved token placed in `BasicBlock.callees` when a statement's call sites + * were truncated at the per-statement site cap: the recorded callee list is then + * INCOMPLETE, so over-cap callees are absent. `*` is not a valid identifier + * leaf, so it cannot collide with a real callee name. The impact bridge treats a + * slice containing this sentinel as "callees unknown" and keeps reach + * callgraph-equal (proven), rather than falsely labeling an absent-but-real + * callee `unproven-bridge`. + */ +export const CALLEES_TRUNCATED_SENTINEL = '*'; + +/** + * Inner separator for the `BasicBlock.calleeIds` cell (resolved callee symbol + * ids). A TAB is used — NOT a space — because resolved ids embed `filePath` and + * C++ overload shape tags with multi-word primitive types (e.g. `unsigned char`, + * `long double`), so an id can legitimately contain a space; a space-joined cell + * then fragments on read and silently drops inter-procedural reach to that + * callee (#2227 tri-review). A tab cannot appear in a tree-sitter-derived id + * token (paths/identifiers/type tokens are tab-free) and round-trips intact + * through `escapeCSVField` (tab is in its preserved set) and the RFC-4180 COPY + * reader (every cell is quoted). Producer (`calleeIdsOfBlock`) and consumer + * (`splitCalleeIds`) both resolve to this single constant so they cannot drift. + * The sibling `callees` (leaf-name) cell stays space-joined — leaf names are + * bare identifiers and never contain a space. + */ +export const CALLEE_ID_SEP = '\t'; diff --git a/gitnexus/src/core/ingestion/cfg/emit.ts b/gitnexus/src/core/ingestion/cfg/emit.ts index 2bc0f8f77..456470ec7 100644 --- a/gitnexus/src/core/ingestion/cfg/emit.ts +++ b/gitnexus/src/core/ingestion/cfg/emit.ts @@ -31,34 +31,19 @@ import { augmentForPostDom } from './synthetic-escape.js'; import { DEFAULT_PDG_MAX_SITES_PER_STATEMENT } from './visitors/call-site-harvest.js'; import { calleeIdPosKey } from '../scope-resolution/graph-bridge/callee-id-sink.js'; import { encodeReachingDefReasonPairs } from './reaching-def-reason-codec.js'; +import { CALLEES_TRUNCATED_SENTINEL, CALLEE_ID_SEP } from './callee-cell-format.js'; import type { BasicBlockData, BindingEntry, FunctionCfg } from './types.js'; /** - * Reserved token placed in `BasicBlock.callees` when a statement's call sites - * were truncated at {@link DEFAULT_PDG_MAX_SITES_PER_STATEMENT}: the recorded - * callee list is then INCOMPLETE, so over-cap callees are absent. `*` is not a - * valid identifier leaf, so it cannot collide with a real callee name. The - * impact bridge treats a slice containing this sentinel as "callees unknown" and - * keeps reach callgraph-equal (proven), rather than falsely labeling an - * absent-but-real callee `unproven-bridge`. + * Cell-format constants live in the LEAF module `callee-cell-format.ts` and are + * re-exported here so every existing importer keeps working. The consumer side + * (`mcp/local/pdg-impact.ts`) imports them from the leaf directly: importing any + * binding from THIS module evaluates it, and with it the whole analyze-only CFG + * closure — 8 modules on every MCP server start to read two strings (#2802 + * review). Producer and consumer still resolve to one definition, so the drift + * these shared constants exist to prevent stays impossible. */ -export const CALLEES_TRUNCATED_SENTINEL = '*'; - -/** - * Inner separator for the `BasicBlock.calleeIds` cell (resolved callee symbol - * ids). A TAB is used — NOT a space — because resolved ids embed `filePath` and - * C++ overload shape tags with multi-word primitive types (e.g. `unsigned char`, - * `long double`), so an id can legitimately contain a space; a space-joined cell - * then fragments on read and silently drops inter-procedural reach to that - * callee (#2227 tri-review). A tab cannot appear in a tree-sitter-derived id - * token (paths/identifiers/type tokens are tab-free) and round-trips intact - * through `escapeCSVField` (tab is in its preserved set) and the RFC-4180 COPY - * reader (every cell is quoted). Producer ({@link calleeIdsOfBlock}) and - * consumer (`splitCalleeIds`) import this single constant so they cannot drift. - * The sibling `callees` (leaf-name) cell stays space-joined — leaf names are - * bare identifiers and never contain a space. - */ -export const CALLEE_ID_SEP = '\t'; +export { CALLEES_TRUNCATED_SENTINEL, CALLEE_ID_SEP } from './callee-cell-format.js'; /** * Default per-function CFG edge cap. A pathological generated function could diff --git a/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts b/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts index 58edeca7d..37b5f2b14 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/analysis-features.ts @@ -27,3 +27,19 @@ export const SPRING_CONDITIONALS_FEATURE: AnalysisFeatureDescriptor = { version: 1, appliesTo: (filePaths) => filePaths.some(isSpringConditionOrAutoConfigurationFile), }; + +/** + * Candidate-language approximation, not a claim that the file contains AOP. + * Kotlin scripts are included because `.kts` is a supported Kotlin input. + */ +function isJvmSourceFile(filePath: string): boolean { + const normalized = filePath.replaceAll('\\', '/').toLowerCase(); + return normalized.endsWith('.java') || normalized.endsWith('.kt') || normalized.endsWith('.kts'); +} + +/** Durable completeness contract for Spring proxy/advice evidence (#2416). */ +export const SPRING_AOP_FEATURE: AnalysisFeatureDescriptor = { + id: 'spring.aop-advice', + version: 1, + appliesTo: (filePaths) => filePaths.some(isJvmSourceFile), +}; diff --git a/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts b/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts index eb05e44f2..928f3e805 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts @@ -6,7 +6,7 @@ export interface SpringAnnotationArgument { function splitTopLevel(value: string, separator: string): string[] | null { const parts: string[] = []; const stack: string[] = []; - let quote: '"' | "'" | null = null; + let quote: '"' | "'" | '"""' | null = null; let escaped = false; let start = 0; @@ -22,12 +22,24 @@ function splitTopLevel(value: string, separator: string): string[] | null { for (let index = 0; index < value.length; index++) { const char = value[index]; + if (quote === '"""') { + if (value.startsWith('"""', index)) { + quote = null; + index += 2; + } + continue; + } if (quote !== null) { if (escaped) escaped = false; else if (char === '\\') escaped = true; else if (char === quote) quote = null; continue; } + if (value.startsWith('"""', index)) { + quote = '"""'; + index += 2; + continue; + } if (char === '"' || char === "'") { quote = char; continue; @@ -53,7 +65,7 @@ function splitTopLevel(value: string, separator: string): string[] | null { function topLevelAssignment(argument: string): number { const stack: string[] = []; - let quote: '"' | "'" | null = null; + let quote: '"' | "'" | '"""' | null = null; let escaped = false; // Keep the same deliberate generic-delimiter policy as splitTopLevel. const closing = new Map([ @@ -65,12 +77,24 @@ function topLevelAssignment(argument: string): number { for (let index = 0; index < argument.length; index++) { const char = argument[index]; + if (quote === '"""') { + if (argument.startsWith('"""', index)) { + quote = null; + index += 2; + } + continue; + } if (quote !== null) { if (escaped) escaped = false; else if (char === '\\') escaped = true; else if (char === quote) quote = null; continue; } + if (argument.startsWith('"""', index)) { + quote = '"""'; + index += 2; + continue; + } if (char === '"' || char === "'") { quote = char; continue; @@ -119,6 +143,12 @@ export function parseSpringAnnotationArguments( export function parseStaticStringLiteral(value: string): string | null { const trimmed = value.trim(); + if (trimmed.startsWith('"""')) { + if (trimmed.length < 6 || !trimmed.endsWith('"""')) return null; + const raw = trimmed.slice(3, -3); + if (raw.includes('"""') || /\$(?:\{|[A-Za-z_])/.test(raw)) return null; + return raw; + } const match = /^"((?:\\.|[^"\\])*)"$/s.exec(trimmed); if (match === null) return null; if (/(^|[^\\])\$(?:\{|[A-Za-z_])/.test(match[1])) return null; diff --git a/gitnexus/src/core/ingestion/frameworks/spring/aop-candidates.ts b/gitnexus/src/core/ingestion/frameworks/spring/aop-candidates.ts new file mode 100644 index 000000000..3e1e8b2c4 --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/aop-candidates.ts @@ -0,0 +1,118 @@ +import type { GraphNode } from 'gitnexus-shared'; +import type { SpringAopStaticPointcut } from './aop.js'; + +export interface SpringAopOwnedMethod { + readonly method: GraphNode; + readonly owner: GraphNode; +} + +export interface SpringAopCandidateIndex { + readonly totalCandidates: number; + candidatesFor(pointcut: SpringAopStaticPointcut): readonly SpringAopOwnedMethod[]; +} + +interface OwnerBucket { + readonly id: string; + readonly qualifiedName: string; + readonly candidates: SpringAopOwnedMethod[]; +} + +function ownerPatternLiteralPrefix(pattern: string): string { + const wildcardIndex = pattern.indexOf('*'); + const descendantIndex = pattern.indexOf('..'); + const firstDynamicIndex = [wildcardIndex, descendantIndex] + .filter((index) => index >= 0) + .reduce((first, index) => Math.min(first, index), pattern.length); + return pattern.slice(0, firstDynamicIndex); +} + +function lowerBoundByQualifiedName(buckets: readonly OwnerBucket[], prefix: string): number { + let low = 0; + let high = buckets.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + if ((buckets[middle]?.qualifiedName ?? '') < prefix) low = middle + 1; + else high = middle; + } + return low; +} + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +/** + * Build immutable candidate lists once per pipeline run. The pointcut matcher + * remains the final authority; this index only returns safe supersets. + */ +export function createSpringAopCandidateIndex( + candidates: readonly SpringAopOwnedMethod[], + methodAnnotations: ReadonlyMap>, +): SpringAopCandidateIndex { + const allCandidates = [...candidates]; + const candidatesByAnnotation = new Map(); + const candidatesByExactOwner = new Map(); + const ownerBucketsById = new Map(); + + for (const candidate of allCandidates) { + for (const annotation of methodAnnotations.get(candidate.method.id) ?? []) { + const annotated = candidatesByAnnotation.get(annotation) ?? []; + annotated.push(candidate); + candidatesByAnnotation.set(annotation, annotated); + } + + const qualifiedName = candidate.owner.properties.qualifiedName; + if (typeof qualifiedName !== 'string' || qualifiedName.length === 0) continue; + const exactOwnerCandidates = candidatesByExactOwner.get(qualifiedName) ?? []; + exactOwnerCandidates.push(candidate); + candidatesByExactOwner.set(qualifiedName, exactOwnerCandidates); + + let bucket = ownerBucketsById.get(candidate.owner.id); + if (bucket === undefined) { + bucket = { id: candidate.owner.id, qualifiedName, candidates: [] }; + ownerBucketsById.set(candidate.owner.id, bucket); + } + bucket.candidates.push(candidate); + } + + const ownerBuckets = [...ownerBucketsById.values()].sort( + (left, right) => + compareStrings(left.qualifiedName, right.qualifiedName) || compareStrings(left.id, right.id), + ); + const candidatesByOwnerPattern = new Map(); + + return { + totalCandidates: allCandidates.length, + candidatesFor(pointcut) { + if (pointcut.kind === 'annotation') { + return candidatesByAnnotation.get(pointcut.annotation) ?? []; + } + if (!pointcut.ownerPattern.includes('*') && !pointcut.ownerPattern.includes('..')) { + return candidatesByExactOwner.get(pointcut.ownerPattern) ?? []; + } + + // Unqualified wildcard patterns (for example `*Service` or `Order*`) + // match the owner simple name. The qualified-name prefix index cannot + // safely narrow those, so preserve the full candidate superset. + if (!pointcut.ownerPattern.includes('.')) return allCandidates; + + const prefix = ownerPatternLiteralPrefix(pointcut.ownerPattern); + if (prefix.length === 0) return allCandidates; + const cached = candidatesByOwnerPattern.get(pointcut.ownerPattern); + if (cached !== undefined) return cached; + + const selected: SpringAopOwnedMethod[] = []; + for ( + let index = lowerBoundByQualifiedName(ownerBuckets, prefix); + index < ownerBuckets.length; + index += 1 + ) { + const bucket = ownerBuckets[index]; + if (bucket === undefined || !bucket.qualifiedName.startsWith(prefix)) break; + selected.push(...bucket.candidates); + } + candidatesByOwnerPattern.set(pointcut.ownerPattern, selected); + return selected; + }, + }; +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/aop.ts b/gitnexus/src/core/ingestion/frameworks/spring/aop.ts new file mode 100644 index 000000000..fe82eaed6 --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/aop.ts @@ -0,0 +1,707 @@ +import type { GraphNode, ParsedFile, Range, ScopeId } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { + resolveCallerGraphId, + resolveDefGraphId, +} from '../../scope-resolution/graph-bridge/ids.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import { stripBidiAndZeroWidth } from '../../utils/ast-helpers.js'; +import { + parseSpringAnnotationArguments, + parseStaticStringLiteral, +} from './annotation-arguments.js'; +import { createSpringAnnotationNameResolver } from './bean-candidates.js'; + +export const SPRING_AOP_REASON_PREFIX = 'spring-aop:v1:'; +export const SPRING_AOP_EVIDENCE_DESCRIPTION_PREFIX = 'Spring AOP: '; +export const SPRING_AOP_EVIDENCE_ID_PREFIX = 'CodeElement:spring-aop:'; +const MAX_POINTCUT_LENGTH = 1_000; + +export type SpringAopBehavior = + | 'transactional' + | 'caching' + | 'cacheable' + | 'cache-evict' + | 'cache-put' + | 'authorization'; + +export type SpringAopAdviceKind = + | 'around' + | 'before' + | 'after' + | 'after-returning' + | 'after-throwing' + | 'pointcut'; + +export interface SpringAopAnnotationFact { + readonly name: string; + readonly text: string; + readonly line: number; + /** Kotlin use-site targets do not describe a callable annotation here. */ + readonly useSiteTarget?: string; +} + +export interface SpringAopOwnerFact< + Annotation extends SpringAopAnnotationFact = SpringAopAnnotationFact, +> { + readonly ownerScopeId: ScopeId; + readonly ownerKind: 'class' | 'callable'; + readonly ownerFilePath?: string; + /** Exact syntax range used only as a fail-closed bridge for collapsed language scopes. */ + readonly ownerRange?: Range; + /** The language models this owner/member as static, but it belongs to a singleton instance. */ + readonly singletonInstance?: true; + readonly annotations: readonly Annotation[]; +} + +export interface SpringAopMetadataAdapter { + getFacts(filePath: string): readonly SpringAopOwnerFact[]; + isPackageVisibilityIncomplete(filePath: string): boolean; +} + +export interface SpringAopBehaviorReason { + readonly kind: 'behavior'; + readonly annotation: string; + readonly behavior: SpringAopBehavior; + readonly declaredOn: 'class' | 'method'; + readonly activation: 'unknown'; + readonly proxy: 'possible'; +} + +export interface SpringAopAdviceReason { + readonly kind: 'advice'; + readonly annotation: string; + readonly advice: Exclude; + readonly pointcut: string; + readonly match: 'static'; + readonly activation: 'unknown'; + readonly proxy: 'possible'; +} + +export interface SpringAopPointcutReason { + readonly kind: 'pointcut'; + readonly annotation: string; + readonly pointcut: string | null; + readonly match: 'static' | 'unresolved'; + readonly resolution: 'resolved' | 'unknown'; +} + +export interface SpringAopAspectReason { + readonly kind: 'aspect'; + readonly annotation: string; + readonly activation: 'unknown'; + readonly registration: 'unknown'; +} + +export type SpringAopReason = + | SpringAopBehaviorReason + | SpringAopAdviceReason + | SpringAopPointcutReason + | SpringAopAspectReason; + +export interface SpringAopAspectRecord { + readonly ownerId: string; + readonly annotation: string; + readonly line: number; +} + +export interface SpringAopBehaviorRecord { + readonly ownerId: string; + readonly ownerKind: 'class' | 'callable'; + readonly annotation: string; + readonly behavior: SpringAopBehavior; + readonly line: number; +} + +export interface SpringAopAdviceRecord { + readonly ownerId: string; + readonly annotation: string; + readonly advice: SpringAopAdviceKind; + readonly pointcut: string | null; + readonly line: number; +} + +export interface SpringAopGraphMetadata { + readonly candidateFilePaths: ReadonlySet; + readonly aspectClassIds: ReadonlySet; + readonly singletonInstanceClassIds: ReadonlySet; + readonly aspects: readonly SpringAopAspectRecord[]; + readonly behaviors: readonly SpringAopBehaviorRecord[]; + readonly advices: readonly SpringAopAdviceRecord[]; +} + +interface MutableSpringAopGraphMetadata { + readonly candidateFilePaths: Set; + readonly aspectClassIds: Set; + readonly singletonInstanceClassIds: Set; + readonly aspects: SpringAopAspectRecord[]; + readonly behaviors: SpringAopBehaviorRecord[]; + readonly advices: SpringAopAdviceRecord[]; +} + +const metadataByGraph = new WeakMap(); + +function graphMetadata(graph: KnowledgeGraph): MutableSpringAopGraphMetadata { + let metadata = metadataByGraph.get(graph); + if (metadata === undefined) { + metadata = { + candidateFilePaths: new Set(), + aspectClassIds: new Set(), + singletonInstanceClassIds: new Set(), + aspects: [], + behaviors: [], + advices: [], + }; + metadataByGraph.set(graph, metadata); + } + return metadata; +} + +export function getSpringAopGraphMetadata(graph: KnowledgeGraph): SpringAopGraphMetadata { + return graphMetadata(graph); +} + +const ASPECT_ANNOTATION = 'org.aspectj.lang.annotation.Aspect'; + +const BEHAVIOR_ANNOTATIONS = new Map([ + ['org.springframework.transaction.annotation.Transactional', 'transactional'], + ['jakarta.transaction.Transactional', 'transactional'], + ['javax.transaction.Transactional', 'transactional'], + ['org.springframework.cache.annotation.Cacheable', 'cacheable'], + ['org.springframework.cache.annotation.CacheEvict', 'cache-evict'], + ['org.springframework.cache.annotation.CachePut', 'cache-put'], + ['org.springframework.cache.annotation.Caching', 'caching'], + ['org.springframework.security.access.prepost.PreAuthorize', 'authorization'], + ['org.springframework.security.access.prepost.PostAuthorize', 'authorization'], + ['org.springframework.security.access.prepost.PreFilter', 'authorization'], + ['org.springframework.security.access.prepost.PostFilter', 'authorization'], + ['org.springframework.security.access.annotation.Secured', 'authorization'], + ['jakarta.annotation.security.RolesAllowed', 'authorization'], + ['javax.annotation.security.RolesAllowed', 'authorization'], +]); + +const ADVICE_ANNOTATIONS = new Map([ + ['org.aspectj.lang.annotation.Around', 'around'], + ['org.aspectj.lang.annotation.Before', 'before'], + ['org.aspectj.lang.annotation.After', 'after'], + ['org.aspectj.lang.annotation.AfterReturning', 'after-returning'], + ['org.aspectj.lang.annotation.AfterThrowing', 'after-throwing'], + ['org.aspectj.lang.annotation.Pointcut', 'pointcut'], +]); + +const RECOGNIZED_AOP_ANNOTATIONS = new Set([ + ASPECT_ANNOTATION, + ...BEHAVIOR_ANNOTATIONS.keys(), + ...ADVICE_ANNOTATIONS.keys(), +]); + +const CAPTURE_RELEVANT_SIMPLE_NAMES = new Set( + [...RECOGNIZED_AOP_ANNOTATIONS].map((name) => simpleName(name)), +); + +function simpleName(name: string): string { + const separator = name.lastIndexOf('.'); + return separator === -1 ? name : name.slice(separator + 1); +} + +export function hasSpringAopRelevantAnnotation( + annotations: readonly Pick[], +): boolean { + return annotations.some((annotation) => + CAPTURE_RELEVANT_SIMPLE_NAMES.has(simpleName(annotation.name)), + ); +} + +function ownerGraphNode( + fact: SpringAopOwnerFact, + indexes: ScopeResolutionIndexes, + nodeLookup: GraphNodeLookup, + graph: KnowledgeGraph, + exactOwnerByRange: ReadonlyMap, +): GraphNode | undefined { + const ownerScope = indexes.scopeTree.getScope(fact.ownerScopeId); + + let ownerId: string | undefined; + if (fact.ownerKind === 'class' && ownerScope !== undefined) { + const classDef = ownerScope.ownedDefs.find( + (definition) => definition.type === 'Class' || definition.type === 'Interface', + ); + if (classDef !== undefined) + ownerId = resolveDefGraphId(classDef.filePath, classDef, nodeLookup); + } else if (ownerScope !== undefined) { + ownerId = resolveCallerGraphId(fact.ownerScopeId, indexes, nodeLookup); + } + + if (ownerId === undefined && fact.ownerFilePath !== undefined && fact.ownerRange !== undefined) { + const kind = fact.ownerKind === 'class' ? 'class' : 'callable'; + const fallback = exactOwnerByRange.get( + `${kind}\0${fact.ownerFilePath}\0${fact.ownerRange.startLine - 1}\0${fact.ownerRange.endLine - 1}`, + ); + if (fallback !== null && fallback !== undefined) ownerId = fallback.id; + } + if (ownerId === undefined) return undefined; + const owner = graph.getNode(ownerId); + return owner === undefined || owner.label === 'File' ? undefined : owner; +} + +function staticPointcutExpression(annotationText: string): string | null { + const args = parseSpringAnnotationArguments(annotationText); + if (args === null) return null; + const pointcutArguments = args.filter( + (argument) => + argument.name === undefined || argument.name === 'value' || argument.name === 'pointcut', + ); + if (pointcutArguments.length !== 1) return null; + const allowedCompanionArguments = new Set(['returning', 'throwing', 'argNames']); + if ( + args.some( + (argument) => + argument !== pointcutArguments[0] && + (argument.name === undefined || !allowedCompanionArguments.has(argument.name)), + ) + ) { + return null; + } + const argument = pointcutArguments[0]; + if (argument === undefined) return null; + const parsed = parseStaticStringLiteral(argument.value); + return parsed === null ? null : sanitizePointcut(parsed); +} + +function sanitizePointcut(value: string): string | null { + const normalized = stripBidiAndZeroWidth(value).replace(/\s+/g, ' ').trim(); + return normalized.length > 0 && normalized.length <= MAX_POINTCUT_LENGTH ? normalized : null; +} + +/** + * Resolve syntax facts after imports and package visibility are complete. + * Language adapters own AST shape only; this shared layer owns framework FQNs. + */ +export function createSpringAopMetadataAttacher( + adapter: SpringAopMetadataAdapter, +) { + return ( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, + indexes: ScopeResolutionIndexes, + ): void => { + const resolveAnnotation = createSpringAnnotationNameResolver(indexes); + const metadata = graphMetadata(graph); + const exactOwnerByRange = new Map(); + for (const node of graph.iterNodes()) { + const kind = + node.label === 'Method' + ? 'callable' + : node.label === 'Class' || node.label === 'Interface' + ? 'class' + : undefined; + if (kind === undefined || typeof node.properties.filePath !== 'string') continue; + const key = `${kind}\0${node.properties.filePath}\0${node.properties.startLine}\0${node.properties.endLine}`; + exactOwnerByRange.set(key, exactOwnerByRange.has(key) ? null : node); + } + let classIdByMethod: ReadonlyMap | undefined; + + const singletonOwnerId = (owner: GraphNode): string | undefined => { + if (owner.label === 'Class' || owner.label === 'Interface') return owner.id; + if (owner.label !== 'Method') return undefined; + if (classIdByMethod === undefined) { + const owners = new Map(); + for (const relationship of graph.iterRelationshipsByType('HAS_METHOD')) { + owners.set(relationship.targetId, relationship.sourceId); + } + classIdByMethod = owners; + } + return classIdByMethod.get(owner.id); + }; + + for (const parsed of parsedFiles) { + // The set is populated only by registered language adapters. The shared + // matcher can therefore reject same-qualified-name symbols from other + // languages without naming Java/Kotlin in framework-generic code. + metadata.candidateFilePaths.add(parsed.filePath); + const incomplete = adapter.isPackageVisibilityIncomplete(parsed.filePath); + const resolvedAnnotations = new Map(); + for (const fact of adapter.getFacts(parsed.filePath)) { + const owner = ownerGraphNode(fact, indexes, nodeLookup, graph, exactOwnerByRange); + const ownerScope = indexes.scopeTree.getScope(fact.ownerScopeId); + if (owner === undefined) continue; + if (fact.singletonInstance === true) { + const singletonId = singletonOwnerId(owner); + if (singletonId !== undefined) metadata.singletonInstanceClassIds.add(singletonId); + } + + for (const annotation of fact.annotations) { + // `@get:`, `@field:`, etc. target generated/property elements rather + // than the callable represented by this fact. Guessing would overstate + // proxy behavior, so Kotlin use-site targets fail closed. + if (annotation.useSiteTarget !== undefined) continue; + const enclosingScope = ownerScope?.parent ?? null; + const cacheKey = `${enclosingScope ?? ''}\0${annotation.name}`; + let resolved = resolvedAnnotations.get(cacheKey); + if (!resolvedAnnotations.has(cacheKey)) { + resolved = resolveAnnotation( + annotation.name, + parsed, + enclosingScope, + RECOGNIZED_AOP_ANNOTATIONS, + incomplete, + ); + resolvedAnnotations.set(cacheKey, resolved); + } + if (resolved === undefined) continue; + + if (resolved === ASPECT_ANNOTATION && fact.ownerKind === 'class') { + metadata.aspectClassIds.add(owner.id); + metadata.aspects.push({ + ownerId: owner.id, + annotation: resolved, + line: annotation.line, + }); + continue; + } + + const behavior = BEHAVIOR_ANNOTATIONS.get(resolved); + if (behavior !== undefined) { + metadata.behaviors.push({ + ownerId: owner.id, + ownerKind: fact.ownerKind, + annotation: resolved, + behavior, + line: annotation.line, + }); + continue; + } + + const advice = ADVICE_ANNOTATIONS.get(resolved); + if (advice !== undefined && fact.ownerKind === 'callable' && owner.label === 'Method') { + metadata.advices.push({ + ownerId: owner.id, + annotation: resolved, + advice, + pointcut: staticPointcutExpression(annotation.text), + line: annotation.line, + }); + } + } + } + } + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isBoundedString(value: unknown, maxLength = MAX_POINTCUT_LENGTH): value is string { + return typeof value === 'string' && value.length > 0 && value.length <= maxLength; +} + +const BEHAVIORS = new Set([ + 'transactional', + 'caching', + 'cacheable', + 'cache-evict', + 'cache-put', + 'authorization', +]); +const ADVICES = new Set>([ + 'around', + 'before', + 'after', + 'after-returning', + 'after-throwing', +]); + +export function encodeSpringAopReason(reason: SpringAopReason): string { + return `${SPRING_AOP_REASON_PREFIX}${JSON.stringify(reason)}`; +} + +/** Decode only the current, validated reason contract; malformed/foreign rows fail closed. */ +export function decodeSpringAopReason(value: unknown): SpringAopReason | undefined { + if (typeof value !== 'string' || !value.startsWith(SPRING_AOP_REASON_PREFIX)) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(value.slice(SPRING_AOP_REASON_PREFIX.length)); + } catch { + return undefined; + } + if (!isRecord(parsed) || !isBoundedString(parsed.annotation)) return undefined; + + if ( + parsed.kind === 'behavior' && + typeof parsed.behavior === 'string' && + BEHAVIORS.has(parsed.behavior as SpringAopBehavior) && + BEHAVIOR_ANNOTATIONS.get(parsed.annotation) === parsed.behavior && + (parsed.declaredOn === 'class' || parsed.declaredOn === 'method') && + parsed.activation === 'unknown' && + parsed.proxy === 'possible' + ) { + return parsed as unknown as SpringAopBehaviorReason; + } + if ( + parsed.kind === 'advice' && + typeof parsed.advice === 'string' && + ADVICES.has(parsed.advice as Exclude) && + ADVICE_ANNOTATIONS.get(parsed.annotation) === parsed.advice && + isBoundedString(parsed.pointcut) && + parsed.match === 'static' && + parsed.activation === 'unknown' && + parsed.proxy === 'possible' + ) { + return parsed as unknown as SpringAopAdviceReason; + } + if ( + parsed.kind === 'pointcut' && + ADVICE_ANNOTATIONS.has(parsed.annotation) && + (parsed.pointcut === null || isBoundedString(parsed.pointcut)) && + ((parsed.match === 'static' && + parsed.resolution === 'resolved' && + typeof parsed.pointcut === 'string') || + (parsed.match === 'unresolved' && parsed.resolution === 'unknown')) + ) { + return parsed as unknown as SpringAopPointcutReason; + } + if ( + parsed.kind === 'aspect' && + parsed.annotation === ASPECT_ANNOTATION && + parsed.activation === 'unknown' && + parsed.registration === 'unknown' + ) { + return parsed as unknown as SpringAopAspectReason; + } + return undefined; +} + +export function isSpringAopEvidenceNode(node: GraphNode): boolean { + return node.label === 'CodeElement' && node.id.startsWith(SPRING_AOP_EVIDENCE_ID_PREFIX); +} + +export interface SpringAopExecutionPointcut { + readonly kind: 'execution'; + readonly ownerPattern: string; + readonly methodPattern: string; + readonly visibility?: 'public'; + readonly parameterCount?: number; +} + +export interface SpringAopWithinPointcut { + readonly kind: 'within'; + readonly ownerPattern: string; +} + +export interface SpringAopAnnotationPointcut { + readonly kind: 'annotation'; + readonly annotation: string; +} + +export type SpringAopStaticPointcut = + | SpringAopExecutionPointcut + | SpringAopWithinPointcut + | SpringAopAnnotationPointcut; + +const TYPE_PATTERN = /^[A-Za-z_$*][A-Za-z0-9_$.*]*$/; +const METHOD_PATTERN = /^[A-Za-z_$*][A-Za-z0-9_$*]*$/; + +/** Parse the deliberately narrow, fully static pointcut subset supported in v1. */ +export function parseSpringAopPointcut(expression: string): SpringAopStaticPointcut | null { + const normalized = sanitizePointcut(expression); + if (normalized === null) return null; + const annotation = /^@annotation\s*\(\s*([A-Za-z_$][A-Za-z0-9_$.]*)\s*\)$/.exec(normalized); + if (annotation !== null) { + const annotationName = annotation[1]; + return annotationName !== undefined && BEHAVIOR_ANNOTATIONS.has(annotationName) + ? { kind: 'annotation', annotation: annotationName } + : null; + } + const within = /^within\s*\(\s*([^()]+?)\s*\)$/.exec(normalized); + if (within !== null) { + const ownerPattern = within[1]?.trim(); + return ownerPattern !== undefined && validTypePattern(ownerPattern) + ? { kind: 'within', ownerPattern } + : null; + } + + const execution = /^execution\s*\(\s*([^()]*)\(([^()]*)\)\s*\)$/.exec(normalized); + if (execution === null) return null; + const head = execution[1]?.trim(); + const parameters = execution[2]?.trim(); + if (head === undefined || parameters === undefined) return null; + const tokens = head.split(/\s+/); + let visibility: 'public' | undefined; + let returnPattern: string; + let qualifiedMethod: string; + if (tokens.length === 2) { + [returnPattern, qualifiedMethod] = tokens as [string, string]; + } else if (tokens.length === 3 && tokens[0] === 'public') { + const publicTokens = tokens as [string, string, string]; + visibility = 'public'; + returnPattern = publicTokens[1]; + qualifiedMethod = publicTokens[2]; + } else { + return null; + } + if (returnPattern !== '*') return null; + + const separator = qualifiedMethod.lastIndexOf('.'); + const ownerPattern = separator === -1 ? '*' : qualifiedMethod.slice(0, separator); + const methodPattern = separator === -1 ? qualifiedMethod : qualifiedMethod.slice(separator + 1); + if (!validTypePattern(ownerPattern) || !METHOD_PATTERN.test(methodPattern)) return null; + + let parameterCount: number | undefined; + if (parameters === '..') parameterCount = undefined; + else if (parameters === '') parameterCount = 0; + else { + const parameterPatterns = parameters.split(',').map((part) => part.trim()); + if (parameterPatterns.some((part) => part !== '*')) return null; + parameterCount = parameterPatterns.length; + } + + return { + kind: 'execution', + ownerPattern, + methodPattern, + ...(visibility === undefined ? {} : { visibility }), + ...(parameterCount === undefined ? {} : { parameterCount }), + }; +} + +function validTypePattern(pattern: string): boolean { + return ( + TYPE_PATTERN.test(pattern) && + // A simple exact type name needs the aspect's package/import scope to + // resolve correctly. That context is not retained in the v1 record, so + // fail closed instead of claiming a static match. Unqualified wildcard + // patterns are self-contained and match the declaring type's simple name. + (pattern.includes('.') || pattern.includes('*')) && + !pattern.includes('...') && + pattern.split('..').length <= 2 && + !pattern.endsWith('.') + ); +} + +function findLiteral( + value: string, + literal: string, + startIndex: number, + endExclusive: number, +): number { + const prefixLengths = new Array(literal.length).fill(0); + for (let index = 1, prefixLength = 0; index < literal.length; index += 1) { + while (prefixLength > 0 && literal[index] !== literal[prefixLength]) { + prefixLength = prefixLengths[prefixLength - 1] ?? 0; + } + if (literal[index] === literal[prefixLength]) prefixLength += 1; + prefixLengths[index] = prefixLength; + } + + for (let index = startIndex, prefixLength = 0; index < endExclusive; index += 1) { + while (prefixLength > 0 && value[index] !== literal[prefixLength]) { + prefixLength = prefixLengths[prefixLength - 1] ?? 0; + } + if (value[index] === literal[prefixLength]) prefixLength += 1; + if (prefixLength === literal.length) return index - literal.length + 1; + } + return -1; +} + +/** Match a single-segment glob in O(pattern + value) without regex backtracking. */ +function segmentGlobMatches(pattern: string, value: string): boolean { + if (!pattern.includes('*')) return pattern === value; + const literalChunks = pattern.split('*').filter((chunk) => chunk.length > 0); + if (literalChunks.length === 0) return true; + + let firstMiddleChunk = 0; + let lastMiddleChunkExclusive = literalChunks.length; + let cursor = 0; + let middleEndExclusive = value.length; + + if (!pattern.startsWith('*')) { + const prefix = literalChunks[0] ?? ''; + if (!value.startsWith(prefix)) return false; + cursor = prefix.length; + firstMiddleChunk = 1; + } + + if (!pattern.endsWith('*')) { + const suffix = literalChunks[literalChunks.length - 1] ?? ''; + const suffixStart = value.length - suffix.length; + if (suffixStart < cursor || !value.endsWith(suffix)) return false; + middleEndExclusive = suffixStart; + lastMiddleChunkExclusive -= 1; + } + + for (let index = firstMiddleChunk; index < lastMiddleChunkExclusive; index += 1) { + const chunk = literalChunks[index] ?? ''; + const matchIndex = findLiteral(value, chunk, cursor, middleEndExclusive); + if (matchIndex === -1) return false; + cursor = matchIndex + chunk.length; + } + return cursor <= middleEndExclusive; +} + +function patternSegmentsMatch( + patternSegments: readonly string[], + valueSegments: readonly string[], +) { + return ( + patternSegments.length === valueSegments.length && + patternSegments.every((segment, index) => + segmentGlobMatches(segment, valueSegments[index] ?? ''), + ) + ); +} + +/** Match Spring's narrow type-pattern subset without compiling repository input as regex. */ +function typePatternMatches(pattern: string, value: string): boolean { + if (!validTypePattern(pattern) || value.length === 0) return false; + if (pattern === '*') return true; + + const valueSegments = value.split('.'); + if (valueSegments.some((segment) => segment.length === 0)) return false; + if (!pattern.includes('.')) { + return segmentGlobMatches(pattern, valueSegments[valueSegments.length - 1] ?? ''); + } + const pieces = pattern.split('..'); + if (pieces.length === 1) return patternSegmentsMatch(pattern.split('.'), valueSegments); + + const leftSegments = (pieces[0] ?? '').split('.'); + const rightSegments = (pieces[1] ?? '').split('.'); + if (valueSegments.length < leftSegments.length + rightSegments.length) return false; + return ( + patternSegmentsMatch(leftSegments, valueSegments.slice(0, leftSegments.length)) && + patternSegmentsMatch(rightSegments, valueSegments.slice(-rightSegments.length)) + ); +} + +export function springAopPointcutMatches( + pointcut: SpringAopStaticPointcut, + owner: GraphNode, + method: GraphNode, + methodAnnotations: ReadonlySet = new Set(), +): boolean { + if (method.label !== 'Method') return false; + if (pointcut.kind === 'annotation') return methodAnnotations.has(pointcut.annotation); + const qualifiedName = owner.properties.qualifiedName; + if (typeof qualifiedName !== 'string') return false; + if (!typePatternMatches(pointcut.ownerPattern, qualifiedName)) return false; + if (pointcut.kind === 'within') return true; + if ( + pointcut.visibility === 'public' && + method.properties.visibility !== 'public' && + // Java and Kotlin interface methods are public when no visibility modifier + // is present. Their extractors retain that absence as `package`; explicit + // private members remain private and therefore fail this exception. + !(owner.label === 'Interface' && method.properties.visibility === 'package') + ) { + return false; + } + if ( + pointcut.parameterCount !== undefined && + method.properties.parameterCount !== pointcut.parameterCount + ) { + return false; + } + return segmentGlobMatches(pointcut.methodPattern, method.properties.name); +} diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index c4ba324ba..329faf42f 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -114,9 +114,17 @@ interface LanguageProviderConfig { * The current C++ UE-macro preprocessor relies on the practical fact that * UE reflection macros and module-export tokens are ASCII-only. * - * Must be a pure function — same input always yields the same output. Called - * once per file, on every code path that re-parses (parsing-processor, import - * processor, heritage processor, call processor, parse worker). + * Must be a pure function — same input always yields the same output, and + * re-applying it to its own output changes nothing. + * + * Applied by the parse worker (`parse-worker.ts`), by `extractParsedFile` + * (`scope-extractor-bridge.ts`) on the parse-cache-miss path, and by the + * embedding parse (`embeddings/ast-utils.ts`, which does not go through the + * bridge). Any *new* path that re-parses a file must apply it too, or the two + * halves of the pipeline analyze different programs — and note the set is not + * closed today: language-owned re-parse helpers reached through other + * provider hooks (e.g. `populateRangeBindings`) still see raw text. + * `test/unit/preprocess-source-parity.test.ts` pins the bridge equivalence. * * Default: undefined (no preprocessing — `file.content` is parsed verbatim). */ diff --git a/gitnexus/src/core/ingestion/languages/cpp/adl.ts b/gitnexus/src/core/ingestion/languages/cpp/adl.ts index 7a1bd9f66..4c9efb744 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/adl.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/adl.ts @@ -58,8 +58,8 @@ * - `noAdlSites` — call sites with parenthesized function (capture-time) * - `classToNamespaceQualifiedName` — class def → its enclosing namespace * qualified name (`populateCppAssociatedNamespaces` time) - * - `adlIndex` / `adlIndexSource` — the lazily-built candidate index and the - * `parsedFiles` reference it was built from (first-`pickCppAdlCandidates` + * - `adlIndexByPass` — the lazily-built candidate index, memoized weakly on + * the `parsedFiles` array it was built from (first-`pickCppAdlCandidates` * time; see `ensureAdlIndex`) * * The class→namespace map uses qualified names (not scope IDs) because @@ -176,8 +176,21 @@ export interface AdlCandidateIndex { readonly seqByNodeId: Map; } -let adlIndex: AdlCandidateIndex | undefined; -let adlIndexSource: readonly ParsedFile[] | undefined; +/** + * Per-pass memo: `parsedFiles` array identity → the built index. `WeakMap`-keyed + * so the entry lives only as long as the caller's `parsedFiles` array does, and + * is reclaimed with the pass — mirrors `qualifiedNsIndexByPass` in + * `inline-namespaces.ts` and `moduleScopeIndexByPass` in `file-local-linkage.ts`. + * + * Weak keying is load-bearing, not stylistic: the index holds + * `SymbolDefinition` references reaching into every ParsedFile's scopes, so the + * previous module-level strong `let` pair (index + source array) kept the whole + * C/C++ ParsedFile set alive past the point `scope-resolution/pipeline/phase.ts` + * evicts `files`/`contents`/`preExtractedByPath` and calls `forceGc()` to + * reclaim it — C++ is 7th of 16 `SCOPE_RESOLVERS` entries, so the retention + * survived nine later language passes plus emit. + */ +let adlIndexByPass = new WeakMap(); function siteKey(filePath: string, line: number, col: number): string { return `${filePath}:${line}:${col}`; @@ -373,25 +386,39 @@ export function validateAdlSeqCoverage(idx: AdlCandidateIndex): string[] { } /** Build the ADL index on first use of a given `parsedFiles` set; reuse it for - * all subsequent call sites in the same pipeline run. Reset by + * every subsequent call site in the same pipeline run. Reset by * `clearCppAdlState`. * + * Lifetime: the memoized entry is reachable only while the caller still holds + * the `parsedFiles` array. Once the pipeline drops it, the entry — and the + * `SymbolDefinition` references it holds into those files' scopes — becomes + * collectable; nothing here pins it (see {@link adlIndexByPass}). + * * Staleness is keyed on `parsedFiles` reference identity ONLY, but the index * is a function of THREE inputs: `parsedFiles` (namespace/friend candidates), * `scopes` (`classDefsBySimple`, read from `scopes.defs.byId`), and the - * module-level `classToNamespaceQualifiedName` (friend-candidate keys). This - * is sound for the current pipeline because all three are built together once - * per `runScopeResolution` pass and `clearCppAdlState` runs in - * `loadResolutionConfig` at the start of every pass. Callers MUST call - * `clearCppAdlState` between any two passes that change `scopes` or - * `classToNamespaceQualifiedName` while reusing the same `parsedFiles` array - * reference — otherwise a stale index would be served. (No such caller exists - * today; widening the guard to also key on `scopes` is deferred until one - * does.) */ -function ensureAdlIndex(scopes: ScopeResolutionIndexes, parsedFiles: readonly ParsedFile[]): void { - if (adlIndex !== undefined && adlIndexSource === parsedFiles) return; - adlIndex = buildAdlIndex(scopes, parsedFiles); - adlIndexSource = parsedFiles; + * module-level `classToNamespaceQualifiedName` (friend-candidate keys). The + * WeakMap key covers only the first, so the explicit clear is still REQUIRED + * for the other two: a later pass may hand back the same `parsedFiles` + * reference with different `scopes` / class→namespace state, and identity + * alone would serve the stale index. This is sound for the current pipeline + * because all three are built together once per `runScopeResolution` pass and + * `clearCppAdlState` runs in `loadResolutionConfig` at the start of every + * pass. Callers MUST call `clearCppAdlState` between any two passes that + * change `scopes` or `classToNamespaceQualifiedName` while reusing the same + * `parsedFiles` array reference — otherwise a stale index would be served. + * (No such caller exists today; widening the key to also cover `scopes` is + * deferred until one does.) */ +function ensureAdlIndex( + scopes: ScopeResolutionIndexes, + parsedFiles: readonly ParsedFile[], +): AdlCandidateIndex { + let index = adlIndexByPass.get(parsedFiles); + if (index === undefined) { + index = buildAdlIndex(scopes, parsedFiles); + adlIndexByPass.set(parsedFiles, index); + } + return index; } /** Record per-call-site argument info. Called once per call site from @@ -504,8 +531,14 @@ export function clearCppAdlState(): void { argInfoSiteKeysByFile.clear(); noAdlSiteKeysByFile.clear(); classToNamespaceQualifiedName.clear(); - adlIndex = undefined; - adlIndexSource = undefined; + // The candidate index is dropped by REASSIGNING a fresh `WeakMap`: `WeakMap` + // has no `.clear()`, and swapping the instance discards every memoized entry + // at once — the exact invalidation the previous `adlIndex = undefined` pair + // provided. Required even though the map is keyed on `parsedFiles` identity, + // because the index also depends on `scopes` and + // `classToNamespaceQualifiedName`, which the key cannot observe (see + // {@link ensureAdlIndex}). + adlIndexByPass = new WeakMap(); } /** @@ -572,15 +605,20 @@ export function pickCppAdlCandidates( if (args === undefined || args.length === 0) return undefined; // Build the workspace-wide ADL candidate index once; reuse for every site. - ensureAdlIndex(scopes, parsedFiles); + const idx = ensureAdlIndex(scopes, parsedFiles); // Collect associated namespace QNames from every participating class-typed arg // and from function-reference args. const associatedNamespaces = new Set(); for (const arg of args) { - collectAssociatedNamespacesForAdlArg(arg, scopes, associatedNamespaces); + collectAssociatedNamespacesForAdlArg(arg, scopes, idx, associatedNamespaces); if (arg.functionRefText !== undefined) { - collectFunctionTypeAssociatedNamespaces(arg.functionRefText, scopes, associatedNamespaces); + collectFunctionTypeAssociatedNamespaces( + arg.functionRefText, + scopes, + idx, + associatedNamespaces, + ); } } if (associatedNamespaces.size === 0) return undefined; @@ -593,8 +631,6 @@ export function pickCppAdlCandidates( // (`friendCandidates`, ISO C++ `[basic.lookup.argdep]` §2). // Dedup by nodeId and sort by visitation sequence so the candidate list is // byte-for-byte identical to the legacy file-major scan order. - const idx = adlIndex; - if (idx === undefined) return undefined; const bySeq = new Map(); const seenKey = new Set(); const collectFrom = (buckets: Map>): void => { @@ -621,12 +657,13 @@ export function pickCppAdlCandidates( function collectAssociatedNamespacesForAdlArg( arg: CppAdlArgInfo, scopes: ScopeResolutionIndexes, + idx: AdlCandidateIndex, associatedNamespaces: Set, ): void { // For template args this may be the template name itself (e.g. `vector`); // simple-name lookup can match project classes with the same name (known // V1/V2 simplification). - addAssociatedNamespaceForClassName(arg.simpleClassName, scopes, associatedNamespaces); + addAssociatedNamespaceForClassName(arg.simpleClassName, scopes, idx, associatedNamespaces); // Includes template-owner namespaces (e.g. `std` in std::vector). If // that surfaces extra candidates, merged-candidate overload narrowing in @@ -637,17 +674,18 @@ function collectAssociatedNamespacesForAdlArg( if (ns.length > 0) associatedNamespaces.add(ns); } for (const className of arg.templateArgClassNames) { - addAssociatedNamespaceForClassName(className, scopes, associatedNamespaces); + addAssociatedNamespaceForClassName(className, scopes, idx, associatedNamespaces); } } function addAssociatedNamespaceForClassName( simpleClassName: string, scopes: ScopeResolutionIndexes, + idx: AdlCandidateIndex, associatedNamespaces: Set, ): void { if (simpleClassName.length === 0) return; - const classLookup = findCppClassDefBySimpleName(simpleClassName); + const classLookup = findCppClassDefBySimpleName(idx, simpleClassName); if (classLookup === undefined) return; const { classDef, ambiguous } = classLookup; const nsQName = classToNamespaceQualifiedName.get(classDef.nodeId); @@ -762,11 +800,12 @@ function findNamespaceDefInScope(scope: { * ISO C++ `[basic.lookup.argdep]` §2: enumerations contribute their * enclosing namespace to the associated set, just like class types. */ function findCppClassDefBySimpleName( + idx: AdlCandidateIndex, simpleName: string, ): { classDef: SymbolDefinition; ambiguous: boolean } | undefined { // `classDefsBySimple` preserves `scopes.defs.byId` order, so `[0]` is the // legacy first-match and `length > 1` is the legacy `ambiguous` flag. - const matches = adlIndex?.classDefsBySimple.get(simpleName); + const matches = idx.classDefsBySimple.get(simpleName); if (matches === undefined) return undefined; const first = matches[0]; if (first === undefined) return undefined; @@ -780,10 +819,9 @@ function findCppClassDefBySimpleName( function collectFunctionTypeAssociatedNamespaces( refText: string, scopes: ScopeResolutionIndexes, + idx: AdlCandidateIndex, out: Set, ): void { - const idx = adlIndex; - if (idx === undefined) return; const colonIdx = refText.lastIndexOf('::'); if (colonIdx !== -1) { // Qualified ref: extract namespace prefix and normalise :: → dot notation. @@ -796,7 +834,7 @@ function collectFunctionTypeAssociatedNamespaces( // contributing `a` to the associated set (false-positive CALLS edge). const matches = idx.nsFunctionsByQName.get(nsText)?.get(simpleName); if (matches !== undefined) { - for (const def of matches) collectAssociatedNamespacesForFunctionDef(def, scopes, out); + for (const def of matches) collectAssociatedNamespacesForFunctionDef(def, scopes, idx, out); } return; } @@ -807,32 +845,34 @@ function collectFunctionTypeAssociatedNamespaces( // the function's own enclosing namespace. const matches = idx.nsFunctionsBySimple.get(refText); if (matches !== undefined) { - for (const def of matches) collectAssociatedNamespacesForFunctionDef(def, scopes, out); + for (const def of matches) collectAssociatedNamespacesForFunctionDef(def, scopes, idx, out); } } function collectAssociatedNamespacesForFunctionDef( def: SymbolDefinition, scopes: ScopeResolutionIndexes, + idx: AdlCandidateIndex, out: Set, ): void { const parameterTypes = def.parameterTypeClasses?.map((typeClass) => typeClass.base); for (const paramType of parameterTypes ?? def.parameterTypes ?? []) { - collectAssociatedNamespacesForFunctionTypeText(paramType, scopes, out); + collectAssociatedNamespacesForFunctionTypeText(paramType, scopes, idx, out); } if (def.returnType !== undefined) { - collectAssociatedNamespacesForFunctionTypeText(def.returnType, scopes, out); + collectAssociatedNamespacesForFunctionTypeText(def.returnType, scopes, idx, out); } } function collectAssociatedNamespacesForFunctionTypeText( typeText: string, scopes: ScopeResolutionIndexes, + idx: AdlCandidateIndex, out: Set, ): void { for (const token of extractCppTypeNameTokens(typeText)) { if (isIgnoredCppAdlNamespace(token.namespaceName)) continue; - addAssociatedNamespaceForClassName(token.simpleName, scopes, out); + addAssociatedNamespaceForClassName(token.simpleName, scopes, idx, out); if (token.namespaceName !== '') out.add(token.namespaceName); } } diff --git a/gitnexus/src/core/ingestion/languages/cpp/captures.ts b/gitnexus/src/core/ingestion/languages/cpp/captures.ts index a13cba09d..ce1ed009c 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/captures.ts @@ -8,6 +8,7 @@ import { import { getCppParser, getCppScopeQuery } from './query.js'; import { getTreeSitterBufferSize } from '../../constants.js'; import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; +import { stripUeMacros } from '../../cpp-ue-preprocessor.js'; import { normalizeQualifiedName } from '../../utils/qualified-name.js'; import { splitCppInclude, splitCppUsingDecl } from './import-decomposer.js'; import { @@ -159,8 +160,12 @@ export function emitCppScopeCaptures( ): readonly CaptureMatch[] { let tree = cachedTree as ReturnType['parse']> | undefined; if (tree === undefined) { - tree = parseSourceSafe(getCppParser(), sourceText, undefined, { - bufferSize: getTreeSitterBufferSize(sourceText), + // Idempotent re-application: `extractParsedFile` already preprocesses, but + // direct emitter callers (benchmarks, capture goldens) must see the same + // program the pipeline does. + const parseText = stripUeMacros(sourceText); + tree = parseSourceSafe(getCppParser(), parseText, undefined, { + bufferSize: getTreeSitterBufferSize(parseText), }); } diff --git a/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts b/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts index 134738dfd..47f76d845 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts @@ -14,12 +14,14 @@ * 2. **Transitive qualified visibility.** `outer::foo()` resolves to * `outer::v1::foo()` when `v1` is inline. The qualified-namespace * receiver resolver (`resolveCppQualifiedNamespaceMember`) walks - * inline-namespace children transitively when collecting candidates. + * inline-namespace children transitively when collecting candidates — + * once per pipeline run, into {@link QualifiedNsMemberIndex} (#2788). * * State lifecycle: capture-time `markCppInlineNamespaceRange` records each * inline namespace's source range; `populateCppInlineNamespaceScopes` * resolves ranges to `ScopeId`s during `populateOwners`. Cleared via - * `clearCppInlineNamespaces`, called from `clearFileLocalNames`. + * `clearCppInlineNamespaces`, called from + * `cppScopeResolver.loadResolutionConfig` at the start of every pass. * * STL idiom this enables: `std::__1::vector` (libc++) and `std::__cxx11` * (libstdc++) are inline namespaces of `std`. With this support, @@ -32,7 +34,10 @@ import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexe import { isOverloadAmbiguousAfterNormalization, narrowOverloadCandidates, + type OverloadNarrowingHookCtx, } from '../../scope-resolution/passes/overload-narrowing.js'; +import { isOverloadableCallable } from '../../utils/callable-labels.js'; +import { isSemanticModelValidatorEnabled } from '../../utils/env.js'; import { CPP_CONVERSION_ONLY_ARG_TYPE_PREFIXES, cppConversionRank } from './conversion-rank.js'; interface RangeKey { @@ -44,6 +49,11 @@ interface RangeKey { const inlineNamespaceRangesByFile = new Map>(); const inlineNamespaceScopeIds = new Set(); +/** Bumped by every writer of {@link inlineNamespaceScopeIds}. The qualified-ns + * index is a function of both `parsedFiles` and that Set; its WeakMap key sees + * only the first, so the memo stamps this epoch and a missed + * {@link clearCppInlineNamespaces} degrades to a rebuild, not a stale answer. */ +let inlineNamespaceEpoch = 0; function rangeKey(r: RangeKey): string { return `${r.startLine}:${r.startCol}:${r.endLine}:${r.endCol}`; @@ -85,10 +95,19 @@ export function applyCppInlineNamespaceSideChannel( for (const r of ranges) set.add(r); } -/** Clear all inline-namespace state. Called from `clearFileLocalNames`. */ +/** Clear all inline-namespace state. Called from + * `cppScopeResolver.loadResolutionConfig` at the start of every pass. + * + * The qualified-namespace index is dropped by REASSIGNING a fresh `WeakMap` + * (`WeakMap` has no `.clear()`) so its entries — and the `SymbolDefinition` + * references they hold — are released here rather than waiting on the key. + * Correctness does not rest on that: {@link inlineNamespaceEpoch} invalidates + * any surviving entry. */ export function clearCppInlineNamespaces(): void { inlineNamespaceRangesByFile.clear(); inlineNamespaceScopeIds.clear(); + inlineNamespaceEpoch++; + qualifiedNsIndexByPass = new WeakMap(); } /** Resolve captured ranges to actual ScopeIds by matching scope ranges @@ -98,6 +117,7 @@ export function clearCppInlineNamespaces(): void { export function populateCppInlineNamespaceScopes(parsed: ParsedFile): void { const ranges = inlineNamespaceRangesByFile.get(parsed.filePath); if (ranges === undefined || ranges.size === 0) return; + inlineNamespaceEpoch++; for (const scope of parsed.scopes) { if (scope.kind !== 'Namespace') continue; if (ranges.has(rangeKey(scope.range))) { @@ -115,17 +135,316 @@ export function isCppInlineNamespaceScope(scopeId: ScopeId): boolean { } /** - * Walk every parsed file looking for a Namespace scope whose qualified - * name matches `receiverName`, collect its callable ownedDefs matching - * `memberName`, transitively descending into any inline-namespace - * children (since they're members of the enclosing namespace under ISO - * C++). + * Qualified-namespace member index — built **once** per pipeline run from + * `parsedFiles` and reused by every qualified call site. + * + * The legacy lookup re-scanned every parsed file (rebuilding a per-file + * `scopesById` map each time) once **per qualified call site**, making the + * scope-resolution emit phase O(callsites × scopes): 25.3 min of a 33-min + * analyze on a 1,473-file C++ repo, 75% of total self-time in this one + * function (#2788). Mirrors the same fix #1990 applied to ADL + * (`pickCppAdlCandidates` → {@link AdlCandidateIndex}); per-site cost drops + * to a Map lookup once a `(receiver, member)` pair has been resolved. + * + * **Why the index is a graph and not a flattened member table.** Every + * `Namespace` scope is a legal qualified receiver on its own — in + * `outer { inline v1 { inline v2 { … } } }`, `v2::foo()` is valid C++ — so an + * eager table has to record, for every namespace, every member of every inline + * descendant. On a chain of depth D that is D + (D−1) + … + 1 entries: quadratic + * in **both** time and memory, and materializing it recursively also recursed D + * deep. Neither bound was theoretical — a generated chain 6,000 deep with one + * function per level exhausted a 4 GB heap, and a member-less one 8,000 deep + * threw `RangeError: Maximum call stack size exceeded`. Nothing on the analyze + * path catches either (`run.ts` only wraps the CFG/PDG emit block; `phase.ts`'s + * `try` has a `finally` and no `catch`), so one pathological generated file + * aborted the whole `analyze` — the failure class of #2769. + * + * Storing the *shape* instead — each namespace's own members plus links to its + * inline children — makes the build linear in scopes+defs, and a call site pays + * one iterative pre-order walk of the receiver's inline subtree, memoized per + * `(receiver, member)`. Real inline nesting is 1–2 deep (`std::__1`, + * `std::__cxx11`), so that walk is 2–3 nodes. + */ +interface QualifiedNsMemberIndex { + /** + * Namespace simple name → the scopes declaring it, in the order the legacy + * linear scan visited them (file-major, then `parsed.scopes` declaration + * order). Nothing downstream observes that order — every multi-candidate + * path in {@link resolveCppQualifiedNamespaceMember} either narrows to a + * unique survivor or returns `'ambiguous'` — but it is free to preserve, and + * a future tie-break should inherit the legacy order rather than Map + * insertion happenstance. + */ + readonly rootsByReceiver: ReadonlyMap; + /** + * Memo of resolved candidate lists, receiver → member → defs. Filled on + * demand by {@link qualifiedNsMembers}; only pairs a call site actually asks + * for are ever materialized, which is what keeps the deep-chain case linear. + */ + readonly membersByReceiver: Map>; +} + +/** One `Namespace` scope as the qualified-lookup walk sees it. */ +interface QualifiedNsNode { + /** This scope's OWN callable defs bucketed by member simple name, in + * `ownedDefs` order. Computed once at build time, so a namespace with many + * members costs one pass no matter how many distinct members are queried. */ + readonly ownMembers: ReadonlyMap; + /** Inline-namespace children in `parsed.scopes` declaration order. Direct + * object links rather than `ScopeId` lookups, so the walk needs no scope + * table and ids that repeat across files can never splice one file's + * children under another file's namespace. */ + readonly inlineChildren: readonly QualifiedNsNode[]; +} + +/** Build-time view of {@link QualifiedNsNode}: `inlineChildren` is appended to + * as the build's second pass links each child, then only ever read. */ +interface MutableNsNode extends QualifiedNsNode { + readonly inlineChildren: QualifiedNsNode[]; +} + +type NsScope = ParsedFile['scopes'][number]; + +/** + * Dev/test-only mutation tripwire on the arrays this module shares across call + * sites ({@link NO_DEFS} and every `membersByReceiver` memo entry): an in-place + * `.sort()`/`.splice()` ever added to `overload-narrowing.ts` would corrupt + * every LATER resolution of the same pair, not just its own call, and freezing + * makes that a loud `TypeError` instead. DEV-ONLY because the types already + * reject it — the memo and `narrowOverloadCandidates`' parameter are both + * `readonly SymbolDefinition[]`, so only a cast gets past — while the freeze is + * not free: V8 moves a frozen array to `PACKED_FROZEN_ELEMENTS`, off the + * builtin fast path for the `.filter`/`.map`/`.some` narrowing runs over it at + * every multi-candidate call site (measured 4.6×; large bench arm 100.3 → 70.6ms + * with it off). Gated on `isSemanticModelValidatorEnabled()` — the OPT-IN form, + * not `adl.ts`'s opt-out `NODE_ENV !== 'production'`: `NODE_ENV` is unset in a + * CLI `analyze`, so the opt-out form would keep paying the freeze exactly where + * the cost lands. Read once at load because this one is per-call-site. + */ +const FREEZE_SHARED_CANDIDATES = isSemanticModelValidatorEnabled(); + +/** Shared empties: most namespace scopes declare no callables, and most + * qualified receivers name no namespace at all. `NO_DEFS` is process-wide, so + * it gets the same dev-only freeze the memoized arrays do. */ +const NO_MEMBERS: ReadonlyMap = new Map(); +const NO_DEFS: readonly SymbolDefinition[] = FREEZE_SHARED_CANDIDATES ? Object.freeze([]) : []; + +/** Simple (last-segment) name of a def, matching the legacy scan exactly — + * including the empty-string fallback for a def with no `qualifiedName`, and + * the empty last segment of a trailing-dot name. `lastIndexOf` + `slice` + * rather than `split('.').pop()`: same result, no intermediate array, and this + * runs per callable def and per namespace scope at build. */ +function simpleNameOf(def: SymbolDefinition): string { + const qualified = def.qualifiedName; + if (qualified == null) return ''; + const lastDot = qualified.lastIndexOf('.'); + return lastDot === -1 ? qualified : qualified.slice(lastDot + 1); +} + +/** + * Per-pass memo: `parsedFiles` array identity → the built index. `WeakMap`-keyed + * so the entry lives only as long as the caller's `parsedFiles` array does, and + * is reclaimed with the pass — mirrors `moduleScopeIndexByPass` in + * `file-local-linkage.ts`. + * + * Weak keying is load-bearing, not stylistic: the index holds + * `SymbolDefinition` references reaching into every ParsedFile's scopes, so a + * module-level strong `let` pair (index + source array) kept the whole C/C++ + * ParsedFile set alive past the point `scope-resolution/pipeline/phase.ts` + * evicts `files`/`contents`/`preExtractedByPath` and calls `forceGc()` to + * reclaim it — C++ is 7th of 16 `SCOPE_RESOLVERS` entries, so the retention + * survived nine later language passes plus emit (104.2 MB measured). + */ +let qualifiedNsIndexByPass = new WeakMap(); + +/** Memo cell: the index plus the {@link inlineNamespaceEpoch} it was built under. */ +interface MemoizedIndex { + readonly epoch: number; + readonly index: QualifiedNsMemberIndex; +} + +/** Build the index in two linear passes per file: one to make a node per + * `Namespace` scope, one to link inline children and register receivers. + * Linking needs both endpoints to exist and `parsed.scopes` does not promise + * parents precede children, hence two passes rather than one — but only the + * first pass filters `parsed.scopes`; it hands the second the `[scope, node]` + * pairs it already found. */ +function buildQualifiedNsMemberIndex(parsedFiles: readonly ParsedFile[]): QualifiedNsMemberIndex { + const rootsByReceiver = new Map(); + + for (const parsed of parsedFiles) { + const nodesByScope = new Map(); + const namespaces: [NsScope, MutableNsNode][] = []; + for (const sc of parsed.scopes) { + if (sc.kind !== 'Namespace') continue; + const node: MutableNsNode = { ownMembers: bucketOwnMembers(sc), inlineChildren: [] }; + nodesByScope.set(sc.id, node); + namespaces.push([sc, node]); + } + + for (const [sc, node] of namespaces) { + // A non-`Namespace` parent (a Module scope, say) has no node, so the + // inline child links to nothing — hence the `?.` below. + if (sc.parent !== null && inlineNamespaceScopeIds.has(sc.id)) { + nodesByScope.get(sc.parent)?.inlineChildren.push(node); + } + const nsDef = findNamespaceDefInScope(sc); + if (nsDef === undefined) continue; + const nsName = simpleNameOf(nsDef); + let roots = rootsByReceiver.get(nsName); + if (roots === undefined) { + roots = []; + rootsByReceiver.set(nsName, roots); + } + roots.push(node); + } + } + return { rootsByReceiver, membersByReceiver: new Map() }; +} + +/** Bucket a namespace scope's own callable `ownedDefs` by member simple name. + * No descent: inline children are separate nodes reached by the walk. */ +function bucketOwnMembers(scope: NsScope): ReadonlyMap { + let members: Map | undefined; + for (const def of scope.ownedDefs) { + if (!isOverloadableCallable(def.type)) continue; + const simple = simpleNameOf(def); + members ??= new Map(); + let arr = members.get(simple); + if (arr === undefined) { + arr = []; + members.set(simple, arr); + } + arr.push(def); + } + return members ?? NO_MEMBERS; +} + +/** Candidates for `receiver::member`, memoized on the index per pair so a + * repeated call site costs two Map lookups. */ +function qualifiedNsMembers( + index: QualifiedNsMemberIndex, + receiverName: string, + memberName: string, +): readonly SymbolDefinition[] { + const roots = index.rootsByReceiver.get(receiverName); + // No namespace by that name — the most common outcome in real source. Left + // unmemoized deliberately: it is already O(1), and memoizing would grow the + // index by an entry per unresolved receiver name in the workspace. + if (roots === undefined) return NO_DEFS; + let byMember = index.membersByReceiver.get(receiverName); + if (byMember === undefined) { + byMember = new Map(); + index.membersByReceiver.set(receiverName, byMember); + } + let hits = byMember.get(memberName); + if (hits === undefined) { + // Memoization makes this array shared by every call site in the pass rather + // than rebuilt per call, so it carries the dev-only mutation tripwire (see + // {@link FREEZE_SHARED_CANDIDATES}). + hits = gatherQualifiedNsMember(roots, memberName); + if (FREEZE_SHARED_CANDIDATES) hits = Object.freeze(hits); + byMember.set(memberName, hits); + } + return hits; +} + +/** Pre-order depth-first walk of every scope named `receiverName` and its + * transitive inline-namespace children, collecting callables named + * `memberName` — the index twin of the legacy + * `findMemberInNamespaceTransitive`. + * + * Iterative, not recursive: nesting depth is whatever the input file says, and + * a recursive descent threw `RangeError` past ~8k levels (see + * {@link QualifiedNsMemberIndex}). */ +function gatherQualifiedNsMember( + roots: readonly QualifiedNsNode[], + memberName: string, +): readonly SymbolDefinition[] { + const hits: SymbolDefinition[] = []; + const stack: QualifiedNsNode[] = []; + // `visited` spans ALL roots, reproducing the legacy per-receiver `seenNodeId` + // dedup: `namespace ns { inline namespace ns { … } }` registers both scopes + // under receiver `ns`, and the outer walk already collected the inner's defs. + // Node identity is equivalent to the legacy nodeId key because a node always + // contributes the same defs. It doubles as the guard that a malformed parent + // cycle terminates instead of spinning — the one regression an explicit stack + // could otherwise introduce over recursion's `RangeError`. + const visited = new Set(); + for (const root of roots) { + stack.push(root); + while (stack.length > 0) { + const node = stack.pop(); + if (visited.has(node)) continue; + visited.add(node); + const own = node.ownMembers.get(memberName); + if (own !== undefined) for (const def of own) hits.push(def); + // Reverse push so children pop in declaration order — the legacy walk's + // pre-order depth-first candidate order (see + // {@link QualifiedNsMemberIndex.rootsByReceiver}). + const kids = node.inlineChildren; + for (let i = kids.length - 1; i >= 0; i--) stack.push(kids[i]); + } + } + return hits.length === 0 ? NO_DEFS : hits; +} + +/** Build the index on first use of a given `parsedFiles` set; reuse it for + * every subsequent call site that passes the same array reference AND the same + * {@link inlineNamespaceEpoch}. + * + * Both keys are needed because the index is a function of TWO inputs: + * `parsedFiles` and the module-level `inlineNamespaceScopeIds` (which inline + * children get descended into). A later pass may hand back the same + * `parsedFiles` reference with different inline-namespace state, so identity + * alone would serve a stale index — the epoch is what makes that a checked + * rebuild instead of a documented obligation on the clear site. + * + * Lifetime: the entry is reachable only while the caller still holds the + * `parsedFiles` array; nothing here pins it (see {@link qualifiedNsIndexByPass}). + * + * Same STALENESS contract as `ensureAdlIndex` in `adl.ts`, minus that + * sibling's dev-gated `validateAdlSeqCoverage` guard: this index has no `?? 0` + * defaulting read to silently collide on, so there is nothing for such a guard + * to catch. */ +function qualifiedNsMemberIndex(parsedFiles: readonly ParsedFile[]): QualifiedNsMemberIndex { + const memo = qualifiedNsIndexByPass.get(parsedFiles); + if (memo !== undefined && memo.epoch === inlineNamespaceEpoch) return memo.index; + const index = buildQualifiedNsMemberIndex(parsedFiles); + qualifiedNsIndexByPass.set(parsedFiles, { epoch: inlineNamespaceEpoch, index }); + return index; +} + +/** Constant narrowing hooks for the C++ qualified-receiver path — hoisted so + * the literal is not reallocated at every multi-candidate call site. */ +const CPP_NARROWING_HOOKS: OverloadNarrowingHookCtx = { + conversionRankFn: cppConversionRank, + conversionOnlyArgTypePrefixes: CPP_CONVERSION_ONLY_ARG_TYPE_PREFIXES, +}; + +/** + * Find the Namespace scopes whose simple name matches `receiverName` and + * return their callable members matching `memberName`, transitively + * including inline-namespace children (since they're members of the + * enclosing namespace under ISO C++). Served from a per-pipeline index + * ({@link QualifiedNsMemberIndex}), not a per-call-site workspace scan. * * Returns the most specific (innermost) match — for `outer::foo()` * where `inline namespace v1` declares `foo`, returns `v1::foo`. When * multiple inline-namespace children declare the same name, ISO C++ * leaves the call ambiguous; returns `'ambiguous'` so the caller * suppresses edge emission rather than picking arbitrarily (#1564). + * + * Two production call sites, both in `scope-resolver.ts`: + * - `resolveQualifiedReceiverMember` — the `outer::foo()` qualified-receiver + * hook. Passes `callsite`, so a multi-candidate set can be narrowed by + * arity and argument types. + * - `resolveAdlCandidates` — resolving a `using ns::name;` named import back + * to its namespace member, for template-class method bodies where the + * lexical walk misses that visibility. Passes NO `callsite`, which makes + * the narrowing filters below pass-throughs: on that path any receiver + * whose member has more than one candidate returns `'ambiguous'` (and the + * caller then skips it) rather than being disambiguated. */ export function resolveCppQualifiedNamespaceMember( receiverName: string, @@ -134,27 +453,7 @@ export function resolveCppQualifiedNamespaceMember( _scopes: ScopeResolutionIndexes, callsite?: Callsite, ): SymbolDefinition | 'ambiguous' | undefined { - const allHits: SymbolDefinition[] = []; - const seenNodeId = new Set(); - for (const parsed of parsedFiles) { - const scopesById = new Map(); - for (const sc of parsed.scopes) scopesById.set(sc.id, sc); - for (const scope of parsed.scopes) { - if (scope.kind !== 'Namespace') continue; - const nsDef = findNamespaceDefInScope(scope); - if (nsDef === undefined) continue; - const nsName = nsDef.qualifiedName?.split('.').pop() ?? nsDef.qualifiedName ?? ''; - if (nsName !== receiverName) continue; - // Found a matching namespace scope in this file. Collect ALL - // members transitively through any inline-namespace children. - const hits = findMemberInNamespaceTransitive(scope, scopesById, memberName); - for (const hit of hits) { - if (seenNodeId.has(hit.nodeId)) continue; - seenNodeId.add(hit.nodeId); - allHits.push(hit); - } - } - } + const allHits = qualifiedNsMembers(qualifiedNsMemberIndex(parsedFiles), receiverName, memberName); if (allHits.length === 0) return undefined; if (allHits.length === 1) return allHits[0]; @@ -167,12 +466,7 @@ export function resolveCppQualifiedNamespaceMember( allHits, callsite?.arity, callsite?.argumentTypes, - callsite !== undefined - ? { - conversionRankFn: cppConversionRank, - conversionOnlyArgTypePrefixes: CPP_CONVERSION_ONLY_ARG_TYPE_PREFIXES, - } - : undefined, + callsite !== undefined ? CPP_NARROWING_HOOKS : undefined, ); if (narrowed.length === 1) return narrowed[0]; if (narrowed.length === 0) return undefined; @@ -182,46 +476,6 @@ export function resolveCppQualifiedNamespaceMember( return 'ambiguous'; } -/** Recursively search a namespace scope and any inline-namespace - * descendants for callable defs with the given simple name. Non-inline - * nested namespaces are NOT traversed — they require explicit - * qualification (`outer::nested::foo`). Returns ALL matches so the - * caller can detect same-name ambiguity across inline children (#1564). */ -function findMemberInNamespaceTransitive( - scope: { - readonly id: ScopeId; - readonly ownedDefs: readonly SymbolDefinition[]; - readonly parent: ScopeId | null; - }, - scopesById: ReadonlyMap< - ScopeId, - { - readonly id: ScopeId; - readonly kind: string; - readonly parent: ScopeId | null; - readonly ownedDefs: readonly SymbolDefinition[]; - } - >, - memberName: string, -): SymbolDefinition[] { - const results: SymbolDefinition[] = []; - // Check this scope's own ownedDefs first. - for (const def of scope.ownedDefs) { - if (def.type !== 'Function' && def.type !== 'Method' && def.type !== 'Constructor') continue; - const simple = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; - if (simple === memberName) results.push(def); - } - // Descend into inline-namespace children. - for (const childScope of scopesById.values()) { - if (childScope.parent !== scope.id) continue; - if (childScope.kind !== 'Namespace') continue; - if (!inlineNamespaceScopeIds.has(childScope.id)) continue; - const childHits = findMemberInNamespaceTransitive(childScope, scopesById, memberName); - for (const hit of childHits) results.push(hit); - } - return results; -} - function findNamespaceDefInScope(scope: { readonly ownedDefs: readonly SymbolDefinition[]; }): SymbolDefinition | undefined { diff --git a/gitnexus/src/core/ingestion/languages/cpp/interpret.ts b/gitnexus/src/core/ingestion/languages/cpp/interpret.ts index b330a3fc0..1627232c5 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/interpret.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/interpret.ts @@ -77,7 +77,76 @@ export function interpretCppTypeBinding(captures: CaptureMatch): ParsedTypeBindi source = 'annotation'; } - return { boundName: name, rawTypeName: normalizeCppTypeName(type), source }; + const declaredSpelling = cppPointerSpelling(captures, type, name); + return declaredSpelling === undefined + ? { boundName: name, rawTypeName: normalizeCppTypeName(type), source } + : { boundName: name, rawTypeName: normalizeCppTypeName(type), declaredSpelling, source }; +} + +/** Anchors whose capture spans a whole declaration, so the declarator — and + * with it the pointer — is inside the captured text. */ +const CPP_WHOLE_DECLARATION_ANCHORS = [ + '@type-binding.parameter', + '@type-binding.annotation', + '@type-binding.assignment', +] as const; + +const squeezeWhitespace = (text: string): string => text.replace(/\s+/g, ''); + +/** + * `User* repos` — the written spelling, when tree-sitter-cpp hangs the pointer + * off the DECLARATOR rather than the type. + * + * `@type-binding.type` is a bare `User` there, so the binding records `User` and + * nothing downstream can tell `repos[0]` (pointer subscript, element `User`) + * from `grid[0]` on a class with `operator[]` (element: whatever the operator + * returns). The index step needs that distinction and declines without it, so + * the pointer is reconstructed here — at the capture layer, from the anchor, + * which spans the whole declaration. + * + * EXACT SHAPE ONLY: the declaration must be precisely ` * ` once + * whitespace is removed. `const T*`, `T**`, an array declarator, a reference, + * a function pointer — none match, and none get a spelling, so the index step + * declines for them. A loose match would hand back container evidence that is + * not there and re-mint the confidently wrong edge this exists to prevent. + */ +function cppPointerSpelling( + captures: CaptureMatch, + typeText: string, + nameText: string, +): string | undefined { + // The target below always contains a literal `*`, so an anchor whose text has + // none can never equal it. Testing that FIRST is the whole optimisation: this + // runs on every C++ type binding and the overwhelming majority declare no + // pointer, so the squeezes (three `replace` passes over the declaration, the + // type and the name) are skipped entirely for them. `target` is built lazily + // on the first star-bearing anchor for the same reason. + let target: string | undefined; + for (const anchor of CPP_WHOLE_DECLARATION_ANCHORS) { + const text = captures[anchor]?.text; + if (text === undefined || !text.includes('*')) continue; + if (target === undefined) { + const type = squeezeWhitespace(typeText); + const name = squeezeWhitespace(nameText); + if (type.length === 0 || name.length === 0) return undefined; + target = `${type}*${name}`; + } + if (squeezeWhitespace(text) === target) return `${typeText.trim()}*`; + } + return undefined; +} + +/** Declaration specifiers that carry no type identity. Held in ONE place so + * every consumer agrees on the keyword list — the resolver's element-type + * hook strips exactly the same set before deciding whether a spelling is a + * pointer, and a silently diverging copy there would make the two disagree + * about what `const T*` is. */ +const CPP_SPECIFIER_RE = + /\b(const|volatile|restrict|static|extern|inline|mutable|constexpr|consteval)\b/g; + +/** Remove C++ declaration specifiers from `text` and trim the result. */ +export function stripCppSpecifiers(text: string): string { + return text.replace(CPP_SPECIFIER_RE, '').trim(); } /** @@ -90,11 +159,7 @@ export function interpretCppTypeBinding(captures: CaptureMatch): ParsedTypeBindi * by base-name fallback in resolveClassBindingForName. */ export function normalizeCppTypeName(text: string): string { - let t = text.trim(); - // Strip const, volatile, restrict, static, extern, inline, mutable, constexpr - t = t - .replace(/\b(const|volatile|restrict|static|extern|inline|mutable|constexpr|consteval)\b/g, '') - .trim(); + let t = stripCppSpecifiers(text); // Strip pointer stars while (t.endsWith('*')) t = t.slice(0, -1).trim(); while (t.startsWith('*')) t = t.slice(1).trim(); diff --git a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts index a1b891b32..13f21183e 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts @@ -9,6 +9,7 @@ import { tagNamespacePrefixes, } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import { extractElementTypeFromString } from '../../type-extractors/shared.js'; import { cppProvider } from '../c-cpp.js'; import { cppArityCompatibility } from './arity.js'; import { CPP_CONVERSION_ONLY_ARG_TYPE_PREFIXES, cppConversionRank } from './conversion-rank.js'; @@ -46,6 +47,12 @@ import { clearCppMemberLookupState, resolveCppReceiverMember, } from './member-lookup.js'; +import { stripCppSpecifiers } from './interpret.js'; + +/** A pointee worth binding: a bare identifier, not `T**`, `T[]`, `A::B` or a + * template spelling. Hoisted — a literal here would mint a fresh RegExp on + * every subscript the resolver folds. */ +const CPP_SIMPLE_POINTEE_RE = /^[A-Za-z_]\w*$/; /** * Per-pass memo of the augmented `#include`-resolution file set @@ -247,6 +254,29 @@ export const cppScopeResolver: ScopeResolver = { return mro.includes(lhsDef.nodeId); }, + // Subscript route only — C++ collection views are method calls. + // + // Two container spellings reach a subscript. A POINTER is one of them: `p[i]` + // on a `User*` is pointer arithmetic yielding a `User`, so the trailing `*` is + // peeled here rather than by `stripTypePreservingDecoration` (C++ declares + // none) — and peeling it at a bare class lookup instead would be wrong, since + // that is the route by which `p->m()` must keep seeing `User`'s own members. + // The other is a template container (`std::vector`), left to the shared + // extractor. + // + // Fed the annotation AS WRITTEN, since `normalizeCppTypeName` strips both the + // star and the array brackets at capture. `undefined` = "not a container", so + // an `operator[]`-bearing class does not fold `grid[0]` onto `Grid`. + elementTypeOf: (containerType, via) => { + if (via.kind !== 'index') return undefined; + const t = stripCppSpecifiers(containerType); + if (t.endsWith('*')) { + const pointee = t.slice(0, -1).trim(); + return CPP_SIMPLE_POINTEE_RE.test(pointee) ? pointee : undefined; + } + return extractElementTypeFromString(t); + }, + // C++ is statically typed — disable field fallback heuristic fieldFallbackOnMethodLookup: false, // C++ needs return type propagation across #include boundaries diff --git a/gitnexus/src/core/ingestion/languages/csharp/accessor-unwrap.ts b/gitnexus/src/core/ingestion/languages/csharp/accessor-unwrap.ts index d0ea89789..bca732185 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/accessor-unwrap.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/accessor-unwrap.ts @@ -1,16 +1,18 @@ +import type { ElementAccessRoute } from '../../scope-resolution/contract/scope-resolver.js'; +import { extractElementTypeFromString } from '../../type-extractors/shared.js'; + /** - * C# collection-accessor unwrapping. + * C# container element-type unwrapping. * * When the compound-receiver resolver encounters a trailing * `.Values` / `.Keys` on a dotted member-access chain, it calls the - * provider's `unwrapCollectionAccessor` hook to find the element + * provider's `elementTypeOf` hook to find the element * type. This module supplies the C# implementation — recognizing * Dictionary-family generics and returning the value or key type. * - * Other languages (Python, Java, TypeScript) use method-call syntax - * for the same access (`.values()` / `.keys()`), which the compound- - * receiver's call-expression branch already handles; they leave this - * hook undefined. + * Other languages (Python, Java, TypeScript) use method-call syntax for the + * same access (`.values()` / `.keys()`), which the compound-receiver's + * call-expression branch already handles; they answer only the `index` route. */ /** Extract (K, V) from `Dictionary` / `IDictionary` / @@ -46,12 +48,19 @@ function extractDictionaryArgs(rawName: string): { key: string; value: string } * receiver / accessor combination we don't recognize, letting the * compound-receiver pass fall through to the regular field walk. */ -export function unwrapCsharpCollectionAccessor( - receiverType: string, - accessor: string, +export function unwrapCsharpElementType( + containerType: string, + via: ElementAccessRoute, ): string | undefined { - if (accessor !== 'Values' && accessor !== 'Keys') return undefined; - const args = extractDictionaryArgs(receiverType); + const args = extractDictionaryArgs(containerType); + // Subscript on a dictionary yields the VALUE type — `dict["k"]` is a V, never + // a K. Previously this route returned nothing at all for C#, because the + // dictionary parse below was reachable only from the accessor hook. + if (via.kind === 'index') { + if (args !== undefined) return args.value; + return extractElementTypeFromString(containerType); + } + if (via.name !== 'Values' && via.name !== 'Keys') return undefined; if (args === undefined) return undefined; - return accessor === 'Values' ? args.value : args.key; + return via.name === 'Values' ? args.value : args.key; } diff --git a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts index 469d52363..eb1d3b10d 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts @@ -21,7 +21,7 @@ import { } from './index.js'; import { populateCsharpNamespaceSiblings } from './namespace-siblings.js'; import { loadCsharpResolutionConfig, type CsharpResolutionConfig } from './resolution-config.js'; -import { unwrapCsharpCollectionAccessor } from './accessor-unwrap.js'; +import { unwrapCsharpElementType } from './accessor-unwrap.js'; const csharpScopeResolver: ScopeResolver = { // Construction is keyword-prefixed: `new Service(db).doWork()` (#2708). @@ -89,7 +89,7 @@ const csharpScopeResolver: ScopeResolver = { // `data.Values` / `data.Keys` on Dictionary-like receivers unwrap // to the value / key element type. Other languages use method-call // syntax for the same access and leave this hook undefined. - unwrapCollectionAccessor: unwrapCsharpCollectionAccessor, + elementTypeOf: unwrapCsharpElementType, // C# matches legacy DAG by collapsing member-call CALLS edges to // `(caller, target)` — multiple `g.Greet(...)` sites from Main diff --git a/gitnexus/src/core/ingestion/languages/dart/captures.ts b/gitnexus/src/core/ingestion/languages/dart/captures.ts index 3157c0841..c3a09b91b 100644 --- a/gitnexus/src/core/ingestion/languages/dart/captures.ts +++ b/gitnexus/src/core/ingestion/languages/dart/captures.ts @@ -39,13 +39,13 @@ import { computeDartArityMetadata } from './arity-metadata.js'; import { synthesizeDartReceiverBinding } from './receiver-binding.js'; import { synthesizeDartSignatureBindings } from './signature-bindings.js'; import { getDartParser, getDartScopeQuery } from './query.js'; +import { preprocessDartExtensionTypes } from './extension-type-preprocess.js'; import { recordCacheHit, recordCacheMiss } from './cache-stats.js'; import { getTreeSitterBufferSize } from '../../constants.js'; import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; import { encodeMarker } from '../../utils/heritage-marker.js'; import { DART_BUILT_INS } from './built-ins.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; -import { preprocessDartExtensionTypes } from './extension-type-preprocess.js'; import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js'; const FUNCTION_DECL_TAGS = [ @@ -113,6 +113,9 @@ export function emitDartScopeCaptures( _filePath: string, cachedTree?: unknown, ): readonly CaptureMatch[] { + // Idempotent re-application: `extractParsedFile` already preprocesses, but + // direct emitter callers (benchmarks, capture goldens) must see the same + // program the pipeline does. const parseText = preprocessDartExtensionTypes(sourceText); let tree: Parser.Tree; if (cachedTree !== undefined && cachedTree !== null) { diff --git a/gitnexus/src/core/ingestion/languages/go/captures.ts b/gitnexus/src/core/ingestion/languages/go/captures.ts index ffc624c87..baff677af 100644 --- a/gitnexus/src/core/ingestion/languages/go/captures.ts +++ b/gitnexus/src/core/ingestion/languages/go/captures.ts @@ -16,6 +16,31 @@ import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js'; +/** Range for a capture that marks no source span. */ +const ZERO_RANGE = Object.freeze({ startLine: 0, startCol: 0, endLine: 0, endCol: 0 }); + +/** + * The callee-position marker, allocated once for the whole process. + * + * This capture is a PRESENCE FLAG, not a payload: `scope-extractor.ts` reads it + * only as `match['@reference.callee-position'] !== undefined`, and the tag is + * listed in `KNOWN_SUB_TAGS`, so `anchorCaptureFor` never considers its range + * when picking a reference anchor. Nothing else in the pipeline touches it. + * (Its range used to duplicate `@reference.read`'s exactly, and that capture is + * still emitted, so the one place that scans every capture's range — the + * synthetic Module-scope span in `scope-extractor.ts` — sees the same maximum.) + * + * Minting it per site from the read node was therefore pure cost, and not a + * cheap one: `node.text` is an `input.substring` behind N-API index marshals, + * and `startPosition` / `endPosition` are two more round-trips each. A frozen + * singleton carries exactly the same information — its presence — for free. + */ +const CALLEE_POSITION_MARKER: Capture = Object.freeze({ + name: '@reference.callee-position', + range: ZERO_RANGE, + text: '', +}); + const GO_CALLABLE_CAPTURE_OPTIONS = { functionNodeTypes: new Set(['function_declaration', 'method_declaration', 'func_literal']), callNodeTypes: new Set(['call_expression']), @@ -45,6 +70,25 @@ const GO_CALLABLE_CAPTURE_OPTIONS = { }, } as const; +/** + * Is this `selector_expression` in CALLEE position — the `function` child of an + * enclosing `call_expression` — rather than free-standing? + * + * `h.dep.Work` in `h.dep.Work()` is in callee position; `f := h.dep.Work` (a + * method value, or a func-typed field value) is not. + * + * Callee position alone does NOT make the selector a non-read. Go dispatches + * `h.dep.Work()` through a func-typed struct field just as readily as through a + * method, and this predicate cannot tell the two apart — `call_expression` has + * the same shape either way, and the tail's declaration may live in another + * package. See {@link emitGoScopeCaptures} for what is done with the answer. + */ +function isCalleeOfMemberCall(node: SyntaxNode): boolean { + const parent = node.parent; + if (parent === null || parent.type !== 'call_expression') return false; + return parent.childForFieldName('function')?.id === node.id; +} + export function emitGoScopeCaptures( sourceText: string, _filePath: string, @@ -183,6 +227,34 @@ export function emitGoScopeCaptures( } } + // Mark — do NOT drop — the read site on a member call's callee. + // + // The `@reference.read` pattern matches every `selector_expression`, so + // `h.dep.Work()` yields THREE sites: the call on `Work`, a read on the inner + // `h.dep` (the genuine field read), and a read on the OUTER `h.dep.Work`. + // + // That third site is a phantom ONLY when `Work` is a method: it then + // resolves through `findOwnedMember` (which prefers methods over fields) and + // emits an ACCESSES edge to the METHOD duplicating the CALLS edge at the + // same position — `v.impl.DoWork()` emitting both `CALLS -> DoWork` and + // `ACCESSES -> DoWork`. + // + // When `Work` is a FUNC-TYPED STRUCT FIELD (`Work func() error` — callback + // structs, hook structs, hand-rolled mocks) the very same syntax is a + // genuine field read followed by an indirect call through the value it + // holds, and that read is the field's only ACCESSES evidence. Dropping the + // site here deleted it (#2782 review). + // + // Method-vs-field is not knowable at capture: `call_expression` has the same + // shape either way and the tail's declaration may live in another package. + // So the capture layer records the POSITION and edge emission — which knows + // the resolved target's kind — decides. See `ReferenceSite.inCalleePosition` + // and `tryEmitEdge`. + const readNode = nodeMap['@reference.read']; + if (readNode !== undefined && isCalleeOfMemberCall(readNode)) { + grouped['@reference.callee-position'] = CALLEE_POSITION_MARKER; + } + // Structural receiver chain for a call whose receiver is itself an // expression, so resolution can type it by folding over structure // instead of re-parsing the receiver's source text. Self-gating: a diff --git a/gitnexus/src/core/ingestion/languages/go/interpret.ts b/gitnexus/src/core/ingestion/languages/go/interpret.ts index c2711a27a..d3a9d6113 100644 --- a/gitnexus/src/core/ingestion/languages/go/interpret.ts +++ b/gitnexus/src/core/ingestion/languages/go/interpret.ts @@ -79,11 +79,23 @@ export function interpretGoTypeBinding(captures: CaptureMatch): ParsedTypeBindin return { boundName: name, rawTypeName: normalizedType, source }; } +/** Shallow `map[K]V` spelling match — the key is anything up to the first `]`, + * so a nested map key (`map[map[a]b]V`) is deliberately NOT recognised. Held + * in one place so the capture-side normalizers and the resolver's element-type + * hook cannot drift on what counts as a map spelling. */ +const GO_MAP_VALUE_RE = /^map\[[^\]]+\]\s*(.+)$/; + +/** `map[K]V` → `V`, trimmed. `undefined` when `text` is not a map spelling. */ +export function goMapValueType(text: string): string | undefined { + const match = GO_MAP_VALUE_RE.exec(text); + return match === null ? undefined : match[1].trim(); +} + export function normalizeGoTypeName(text: string): string { let t = text.trim(); t = stripGoOuterTypePrefixes(t); - const mapMatch = t.match(/^map\[[^\]]+\]\s*(.+)$/); - if (mapMatch) t = mapMatch[1].trim(); + const mapValue = goMapValueType(t); + if (mapValue !== undefined) t = mapValue; t = stripGoOuterTypePrefixes(t.replace(/^(?:<-)?chan(?:<-)?\s+/, '')); if (t.startsWith('func(')) { const retMatch = t.match(/^func\([^)]*\)\s*(.*)$/); @@ -111,8 +123,8 @@ export function normalizeGoReturnType(text: string): string { t = t.slice(1, closeIdx).trim(); } t = stripGoOuterTypePrefixes(t); - const mapMatch = t.match(/^map\[[^\]]+\]\s*(.+)$/); - if (mapMatch) t = mapMatch[1].trim(); + const mapValue = goMapValueType(t); + if (mapValue !== undefined) t = mapValue; t = stripGoOuterTypePrefixes(t.replace(/^(?:<-)?chan(?:<-)?\s+/, '')); if (t.startsWith('func(')) { const retMatch = t.match(/^func\([^)]*\)\s*(.*)$/); diff --git a/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts index e55595a63..ff891d079 100644 --- a/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts @@ -15,6 +15,11 @@ import { import { detectGoInterfaceImplementations } from './interface-impls.js'; import { populateGoRangeBindings } from './range-binding.js'; import { expandGoWildcardNames } from './expand-wildcards.js'; +import { goMapValueType } from './interpret.js'; + +/** Slice `[]T` and array `[N]T` / `[...]T` → the element spelling. Hoisted — + * a literal inside the hook would mint a fresh RegExp per folded subscript. */ +const GO_SLICE_ELEMENT_RE = /^\[[^\]]*\]\s*(.+)$/; export const goScopeResolver: ScopeResolver = { language: SupportedLanguages.Go, @@ -33,6 +38,41 @@ export const goScopeResolver: ScopeResolver = { arityCompatibility: (callsite, def) => goArityCompatibility(def, callsite), + // Only `*` — a pointer leaves the method set reachable by selector unchanged, + // so `*Host` and `Host` name the same class for receiver typing. `[]` and + // `map[…]` are deliberately NOT stripped here: they are containers whose + // member set differs from the element's, and unwrapping them belongs to the + // index step that consumed a subscript. (Field bindings never reach this + // anyway — `normalizeGoTypeName` already strips them at capture. The one + // binding that arrives decorated is the receiver self-binding, kept raw on + // purpose for `method-owners.ts`.) + stripTypePreservingDecoration: (typeName) => + typeName.startsWith('*') ? typeName.slice(1).trim() : undefined, + + // The subscript counterpart of the stripper above, and the reason the two are + // separate hooks: `*` is safe to strip at any class lookup, `[]` / `map[K]` + // only where the source actually indexed. Answers the `index` route only — + // Go has no property-style collection view. + // + // Reads the AS-WRITTEN spelling (`[]*User`), not the capture-normalized name: + // `normalizeGoTypeName` already collapsed that to `User`, which is exactly the + // ambiguity that made an unanswered index step fold onto the container. The + // element is returned STILL DECORATED (`*User`) — the index step looks it up + // through `stripTypePreservingDecoration` above, so the pointer resolves. + elementTypeOf: (containerType, via) => { + if (via.kind !== 'index') return undefined; + const t = containerType.trim(); + // `map[K]V` — a subscript yields V. Literally the same matcher + // `normalizeGoTypeName` uses, so the two cannot disagree on what a map + // spelling is. + const mapValue = goMapValueType(t); + if (mapValue !== undefined) return mapValue; + // Slice `[]T` and array `[N]T` / `[...]T`. + const sliceMatch = GO_SLICE_ELEMENT_RE.exec(t); + if (sliceMatch !== null) return sliceMatch[1]!.trim(); + return undefined; + }, + buildMro: (graph, parsedFiles, nodeLookup) => buildMro(graph, parsedFiles, nodeLookup, defaultLinearize), diff --git a/gitnexus/src/core/ingestion/languages/java.ts b/gitnexus/src/core/ingestion/languages/java.ts index d55e4a63b..047a87791 100644 --- a/gitnexus/src/core/ingestion/languages/java.ts +++ b/gitnexus/src/core/ingestion/languages/java.ts @@ -43,6 +43,76 @@ import { resolveJavaImportTarget, } from './java/index.js'; +/** + * Java names the platform owns, matched against a BARE IDENTIFIER — a dropped + * receiver's chain base (`System` in `System.out.println(...)`) or the bare + * spelling of that base's declared type (`String raw` ⇒ `raw.trim()`). This set + * is the ONLY positive evidence `classifyReceiverOrigin` has that a lost edge + * pointed OUTSIDE the analyzed program; without it every Java drop hedges + * `impact()` down to `epistemic: 'lower-bound'` (#2744). + * + * TYPE names only — deliberately no method names, unlike `csharp.ts`. The same + * hook also gates `type-env.ts`'s return-type inference and the #2545 free-call + * shadow guard, both keyed on the CALLEE name; Java method names are camelCase + * and collide constantly with user code (`format`, `add`, `get`, `run`, `apply`), + * so listing them would silently suppress real in-program resolutions. Receiver + * bases are what this pass actually asks about, and those are types. + * + * Inclusion rule, applied to every entry below: a name earns a place only when + * (a) it plausibly appears as a receiver base or bare declared type, and (b) an + * application defining its OWN type by that name is implausible. Rule (b) is the + * hard gate. A name listed here can never be reported as in-program from the + * fallthrough arm, so a wrong entry silently erases a real uncertainty signal, + * whereas a missing one only costs a hedge — the failure is asymmetric, so this + * set under-includes on purpose. + * + * Deliberately ABSENT, each for rule (b) — all are ordinary domain nouns an + * application really does declare, and `Map`/`Set`/`Collection` are the worst + * case because a same-package Java type needs no import to shadow them: + * `Map`, `Set`, `Collection`, `Stream`, `Number`, `Record`, `Error`. + * (`Record` and `Error` also fail rule (a): `java.lang.Record` has no callable + * static surface and application code never receives a bare `Error`.) `Void` + * is absent on rule (a) alone — no `Void` instance exists to be a receiver. + * `List` IS included: unlike `Map`, a hand-rolled `List` would fight the + * near-universal `java.util.List` import, and it is the highest-value declared + * receiver type in the language. + */ +const BUILT_INS: ReadonlySet = new Set([ + // java.lang — implicitly imported, so these appear unqualified everywhere. + 'System', + 'String', + 'Integer', + 'Long', + 'Double', + 'Boolean', + 'Character', + 'Byte', + 'Short', + 'Float', + 'Object', + 'Math', + 'Thread', + 'Runtime', + 'Class', + 'StringBuilder', + 'StringBuffer', + 'Exception', + 'RuntimeException', + 'Throwable', + 'Iterable', + 'Comparable', + 'Runnable', + 'Enum', + // java.util — an explicit import, but an unresolvable one: the import target + // is outside the workspace, so it produces no in-program binding and the base + // still reaches this set (verified against real scope extraction, not assumed). + 'Optional', + 'List', + 'Arrays', + 'Collections', + 'Objects', +]); + const orderJavaSameNameTypeCandidates = ({ callSiteFilePath, candidates, @@ -122,6 +192,7 @@ export const javaProvider = defineLanguage({ // ── Javadoc → description (issue #2270) ── descriptionExtractor: createLeadingDocDescriptionExtractor(), + builtInNames: BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks ── emitScopeCaptures: emitJavaScopeCaptures, diff --git a/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts index 3fd9db774..7a7f82685 100644 --- a/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts +++ b/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts @@ -10,6 +10,7 @@ import { } from '../jvm/package-facts.js'; import { getJavaPackageFact, setJavaPackageFact } from './package-facts.js'; import type { JavaSpringConfigConsumerFact } from './spring-config-bindings.js'; +import type { JavaSpringAopFact } from './spring-aop.js'; import type { JavaSpringConditionalFact } from './spring-conditionals.js'; import type { JavaSpringDiClassFact } from './spring-di.js'; @@ -19,12 +20,14 @@ export interface JavaCaptureSideChannel { readonly kind: 'java'; readonly packageFact: JvmPackageFact; readonly classAnnotations: readonly JavaClassAnnotationFact[]; + readonly springAopFacts?: readonly JavaSpringAopFact[]; readonly springConfigConsumers?: readonly JavaSpringConfigConsumerFact[]; readonly springConditionalFacts?: readonly JavaSpringConditionalFact[]; readonly springDiFacts?: readonly JavaSpringDiClassFact[]; } const classAnnotations = createClassAnnotationFactStore(); +const springAopFacts = new Map(); const springConfigConsumers = new Map(); const springConditionalFacts = new Map(); const springDiFacts = new Map(); @@ -32,11 +35,21 @@ const springDiFacts = new Map(); /** Clear facts retained by a prior workspace pass in a long-lived process. */ export function clearJavaClassAnnotationFacts(): void { classAnnotations.clear(); + springAopFacts.clear(); springConfigConsumers.clear(); springConditionalFacts.clear(); springDiFacts.clear(); } +export function setJavaSpringAopFacts(filePath: string, facts: readonly JavaSpringAopFact[]): void { + if (facts.length === 0) springAopFacts.delete(filePath); + else springAopFacts.set(filePath, facts); +} + +export function getJavaSpringAopFacts(filePath: string): readonly JavaSpringAopFact[] { + return springAopFacts.get(filePath) ?? []; +} + /** Store the annotation syntax collected by Java's existing scope-query traversal. */ export function setJavaClassAnnotationFacts( filePath: string, @@ -90,12 +103,14 @@ export function collectJavaCaptureSideChannel( filePath: string, ): JavaCaptureSideChannel | undefined { const facts = classAnnotations.get(filePath); + const aopFacts = springAopFacts.get(filePath) ?? []; const configConsumers = springConfigConsumers.get(filePath) ?? []; const conditionFacts = springConditionalFacts.get(filePath) ?? []; const diFacts = springDiFacts.get(filePath) ?? []; const packageFact = getJavaPackageFact(filePath); if ( facts.length === 0 && + aopFacts.length === 0 && configConsumers.length === 0 && conditionFacts.length === 0 && diFacts.length === 0 && @@ -107,6 +122,7 @@ export function collectJavaCaptureSideChannel( kind: 'java', packageFact: packageFact ?? UNKNOWN_JVM_PACKAGE_FACT, classAnnotations: facts, + ...(aopFacts.length > 0 ? { springAopFacts: aopFacts } : {}), ...(configConsumers.length > 0 ? { springConfigConsumers: configConsumers } : {}), ...(conditionFacts.length > 0 ? { springConditionalFacts: conditionFacts } : {}), ...(diFacts.length > 0 ? { springDiFacts: diFacts } : {}), @@ -128,6 +144,7 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void { !Array.isArray(data.classAnnotations) ) { setJavaClassAnnotationFacts(parsed.filePath, []); + setJavaSpringAopFacts(parsed.filePath, []); setJavaSpringConfigConsumerFacts(parsed.filePath, []); setJavaSpringConditionalFacts(parsed.filePath, []); setJavaSpringDiFacts(parsed.filePath, []); @@ -135,6 +152,10 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void { return; } setJavaClassAnnotationFacts(parsed.filePath, data.classAnnotations); + setJavaSpringAopFacts( + parsed.filePath, + Array.isArray(data.springAopFacts) ? data.springAopFacts : [], + ); setJavaSpringConfigConsumerFacts( parsed.filePath, Array.isArray(data.springConfigConsumers) ? data.springConfigConsumers : [], diff --git a/gitnexus/src/core/ingestion/languages/java/captures.ts b/gitnexus/src/core/ingestion/languages/java/captures.ts index dd463c83e..60d315df5 100644 --- a/gitnexus/src/core/ingestion/languages/java/captures.ts +++ b/gitnexus/src/core/ingestion/languages/java/captures.ts @@ -35,6 +35,7 @@ import { getTreeSitterBufferSize } from '../../constants.js'; import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; import { setJavaClassAnnotationFacts, + setJavaSpringAopFacts, setJavaSpringConfigConsumerFacts, setJavaSpringConditionalFacts, setJavaSpringDiFacts, @@ -44,6 +45,7 @@ import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captur import { captureJavaSpringConfigConsumerFacts } from './spring-config-bindings.js'; import { captureJavaSpringDiClassFact, type JavaSpringDiClassFact } from './spring-di.js'; import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js'; +import { captureJavaSpringAopFacts, type JavaSpringAopFact } from './spring-aop.js'; import { captureJavaSpringConditionalFacts, type JavaSpringConditionalFact, @@ -132,6 +134,8 @@ export function emitJavaScopeCaptures( const rawMatches = getJavaScopeQuery().matches(tree.rootNode); const out: CaptureMatch[] = []; const classAnnotations = new Map>(); + const springAopFacts: JavaSpringAopFact[] = []; + const springAopTypeNodeIds = new Set(); const springConditionalFacts: JavaSpringConditionalFact[] = []; const springDiFacts: JavaSpringDiClassFact[] = []; const springDiClassNodeIds = new Set(); @@ -154,6 +158,15 @@ export function emitJavaScopeCaptures( } if (Object.keys(grouped).length === 0) continue; + const springAopTypeNode = [ + nodeIfType(nodeMap['@scope.class'], 'class_declaration'), + nodeIfType(nodeMap['@scope.class'], 'interface_declaration'), + ].find((node): node is SyntaxNode => node !== null); + if (springAopTypeNode !== undefined && !springAopTypeNodeIds.has(springAopTypeNode.id)) { + springAopTypeNodeIds.add(springAopTypeNode.id); + springAopFacts.push(...captureJavaSpringAopFacts(springAopTypeNode, filePath)); + } + const springDiClassNode = nodeIfType(nodeMap['@scope.class'], 'class_declaration'); if (springDiClassNode !== null && !springDiClassNodeIds.has(springDiClassNode.id)) { springDiClassNodeIds.add(springDiClassNode.id); @@ -375,6 +388,7 @@ export function emitJavaScopeCaptures( filePath, captureJavaSpringConfigConsumerFacts(tree.rootNode, filePath), ); + setJavaSpringAopFacts(filePath, springAopFacts); setJavaSpringConditionalFacts(filePath, springConditionalFacts); setJavaSpringDiFacts(filePath, springDiFacts); diff --git a/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts index 3b27c5d41..86c94bccb 100644 --- a/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts @@ -30,6 +30,7 @@ import { } from './index.js'; import { populateJavaPackageSiblings } from './package-siblings.js'; import { attachSpringBeanCandidateMetadata } from './spring-bean-metadata.js'; +import { attachJavaSpringAopMetadata } from './spring-aop.js'; import { attachJavaSpringConfigBindings } from './spring-config-bindings.js'; import { attachJavaSpringConditionalMetadata } from './spring-conditionals.js'; import { attachJavaSpringDiMetadata } from './spring-di.js'; @@ -88,6 +89,7 @@ const javaScopeResolver: ScopeResolver = { populateRangeBindings: populateJavaCrossFileReturnTypes, emitPostResolutionEdges: (graph, parsedFiles, nodeLookup, indexes, ctx) => { attachSpringBeanCandidateMetadata(graph, parsedFiles, nodeLookup, indexes); + attachJavaSpringAopMetadata(graph, parsedFiles, nodeLookup, indexes); attachJavaSpringConditionalMetadata(graph, parsedFiles, nodeLookup, indexes); attachJavaSpringDiMetadata(graph, parsedFiles, nodeLookup, indexes); attachJavaSpringConfigBindings(graph, parsedFiles, nodeLookup, indexes, ctx); diff --git a/gitnexus/src/core/ingestion/languages/java/spring-aop.ts b/gitnexus/src/core/ingestion/languages/java/spring-aop.ts new file mode 100644 index 000000000..724e30691 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/java/spring-aop.ts @@ -0,0 +1,61 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { + createSpringAopMetadataAttacher, + hasSpringAopRelevantAnnotation, + type SpringAopOwnerFact, +} from '../../frameworks/spring/aop.js'; +import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { getJavaSpringAopFacts } from './capture-side-channel.js'; +import { isJavaPackageSiblingVisibilityIncomplete } from './package-siblings.js'; +import { javaSpringAnnotationFacts, type JavaAnnotationSyntaxFact } from './spring-di.js'; + +export type JavaSpringAopAnnotationFact = JavaAnnotationSyntaxFact; + +export type JavaSpringAopFact = SpringAopOwnerFact; + +function scopeId(filePath: string, node: SyntaxNode, kind: 'Class' | 'Function') { + return makeScopeId({ + filePath, + range: nodeToCapture('@spring-aop.owner', node).range, + kind, + }); +} + +/** + * Capture Spring AOP syntax while Java's existing class traversal already has + * the AST node in hand. Import/FQN resolution and pointcut matching remain in + * the shared post-resolution layer. + */ +export function captureJavaSpringAopFacts( + classNode: SyntaxNode, + filePath: string, +): JavaSpringAopFact[] { + const facts: JavaSpringAopFact[] = []; + const classAnnotations = javaSpringAnnotationFacts(classNode); + if (hasSpringAopRelevantAnnotation(classAnnotations)) { + facts.push({ + ownerScopeId: scopeId(filePath, classNode, 'Class'), + ownerKind: 'class', + annotations: classAnnotations, + }); + } + + const body = classNode.childForFieldName('body'); + if (body === null) return facts; + for (const member of body.namedChildren) { + if (member.type !== 'method_declaration') continue; + const annotations = javaSpringAnnotationFacts(member); + if (!hasSpringAopRelevantAnnotation(annotations)) continue; + facts.push({ + ownerScopeId: scopeId(filePath, member, 'Function'), + ownerKind: 'callable', + annotations, + }); + } + return facts; +} + +export const attachJavaSpringAopMetadata = createSpringAopMetadataAttacher({ + getFacts: getJavaSpringAopFacts, + isPackageVisibilityIncomplete: isJavaPackageSiblingVisibilityIncomplete, +}); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts index c1e8ce4f8..3c211bcc0 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts @@ -49,10 +49,12 @@ import { } from '../jvm/package-facts.js'; import { getCompanionScopesForFile, markCompanionScope } from './companion-scopes.js'; import { getKotlinPackageFact, setKotlinPackageFact } from './package-facts.js'; +import type { KotlinSpringAopFact } from './spring-aop.js'; import type { KotlinSpringConditionalFact } from './spring-conditionals.js'; import type { KotlinSpringDiClassFact } from './spring-di.js'; const classAnnotations = createClassAnnotationFactStore(); +const springAopFacts = new Map(); const springConditionalFacts = new Map(); const springDiFacts = new Map(); @@ -70,6 +72,8 @@ export interface KotlinCaptureSideChannel { readonly packageFact: JvmPackageFact; /** Class annotation syntax collected by the existing scope traversal. */ readonly classAnnotations: readonly ClassAnnotationFact[]; + /** Spring proxy/advice syntax captured per class or callable owner. */ + readonly springAopFacts?: readonly KotlinSpringAopFact[]; /** Profile, conditional, and auto-configuration syntax captured per owner. */ readonly springConditionalFacts?: readonly KotlinSpringConditionalFact[]; /** Constructor, property, and method injection syntax captured per class. */ @@ -78,10 +82,23 @@ export interface KotlinCaptureSideChannel { export function clearKotlinClassAnnotationFacts(): void { classAnnotations.clear(); + springAopFacts.clear(); springConditionalFacts.clear(); springDiFacts.clear(); } +export function setKotlinSpringAopFacts( + filePath: string, + facts: readonly KotlinSpringAopFact[], +): void { + if (facts.length === 0) springAopFacts.delete(filePath); + else springAopFacts.set(filePath, facts); +} + +export function getKotlinSpringAopFacts(filePath: string): readonly KotlinSpringAopFact[] { + return springAopFacts.get(filePath) ?? []; +} + export function setKotlinClassAnnotationFacts( filePath: string, facts: readonly ClassAnnotationFact[], @@ -129,12 +146,14 @@ export function collectKotlinCaptureSideChannel( ): KotlinCaptureSideChannel | undefined { const companionScopes = getCompanionScopesForFile(filePath); const annotationFacts = classAnnotations.get(filePath); + const aopFacts = springAopFacts.get(filePath) ?? []; const conditionFacts = springConditionalFacts.get(filePath) ?? []; const diFacts = springDiFacts.get(filePath) ?? []; const packageFact = getKotlinPackageFact(filePath); if ( companionScopes.length === 0 && annotationFacts.length === 0 && + aopFacts.length === 0 && conditionFacts.length === 0 && diFacts.length === 0 && packageFact === undefined @@ -146,6 +165,7 @@ export function collectKotlinCaptureSideChannel( companionScopes, packageFact: packageFact ?? UNKNOWN_JVM_PACKAGE_FACT, classAnnotations: annotationFacts, + ...(aopFacts.length > 0 ? { springAopFacts: aopFacts } : {}), ...(conditionFacts.length > 0 ? { springConditionalFacts: conditionFacts } : {}), ...(diFacts.length > 0 ? { springDiFacts: diFacts } : {}), }; @@ -170,6 +190,7 @@ export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { !Array.isArray(data.classAnnotations) ) { classAnnotations.set(parsed.filePath, []); + setKotlinSpringAopFacts(parsed.filePath, []); setKotlinSpringConditionalFacts(parsed.filePath, []); setKotlinSpringDiFacts(parsed.filePath, []); setKotlinPackageFact(parsed.filePath, UNKNOWN_JVM_PACKAGE_FACT); @@ -179,6 +200,10 @@ export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { markCompanionScope(parsed.filePath, scopeId); } classAnnotations.set(parsed.filePath, data.classAnnotations); + setKotlinSpringAopFacts( + parsed.filePath, + Array.isArray(data.springAopFacts) ? data.springAopFacts : [], + ); setKotlinSpringConditionalFacts( parsed.filePath, Array.isArray(data.springConditionalFacts) ? data.springConditionalFacts : [], diff --git a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts index 00868c174..4e063c3cf 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts @@ -20,6 +20,7 @@ import { getKotlinParser, getKotlinScopeQuery } from './query.js'; import { markCompanionScope } from './companion-scopes.js'; import { setKotlinClassAnnotationFacts, + setKotlinSpringAopFacts, setKotlinSpringConditionalFacts, setKotlinSpringDiFacts, } from './capture-side-channel.js'; @@ -27,6 +28,7 @@ import { captureKotlinPackageFact } from './package-facts.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; import { captureKotlinSpringDiClassFact, type KotlinSpringDiClassFact } from './spring-di.js'; import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js'; +import { captureKotlinSpringAopFacts, type KotlinSpringAopFact } from './spring-aop.js'; import { captureKotlinSpringConditionalFacts, type KotlinSpringConditionalFact, @@ -93,6 +95,8 @@ export function emitKotlinScopeCaptures( const out: CaptureMatch[] = []; const classAnnotations = new Map>(); + const springAopFacts: KotlinSpringAopFact[] = []; + const springAopTypeNodeIds = new Set(); const springConditionalFacts: KotlinSpringConditionalFact[] = []; const springDiFacts: KotlinSpringDiClassFact[] = []; const springDiClassNodeIds = new Set(); @@ -119,6 +123,18 @@ export function emitKotlinScopeCaptures( } if (Object.keys(grouped).length === 0) continue; + // tree-sitter-kotlin represents both classes and interfaces with + // `class_declaration`; `object_declaration` is the separate object form. + const springAopTypeNode = [ + nodeIfType(groupedNodes['@scope.class'], 'class_declaration'), + nodeIfType(groupedNodes['@scope.class'], 'object_declaration'), + nodeIfType(groupedNodes['@scope.class'], 'companion_object'), + ].find((node): node is SyntaxNode => node !== null); + if (springAopTypeNode !== undefined && !springAopTypeNodeIds.has(springAopTypeNode.id)) { + springAopTypeNodeIds.add(springAopTypeNode.id); + springAopFacts.push(...captureKotlinSpringAopFacts(springAopTypeNode, filePath)); + } + const springDiClassNode = nodeIfType(groupedNodes['@scope.class'], 'class_declaration'); if (springDiClassNode !== null && !springDiClassNodeIds.has(springDiClassNode.id)) { springDiClassNodeIds.add(springDiClassNode.id); @@ -323,6 +339,7 @@ export function emitKotlinScopeCaptures( } setKotlinClassAnnotationFacts(filePath, materializeClassAnnotationFacts(classAnnotations)); + setKotlinSpringAopFacts(filePath, springAopFacts); setKotlinSpringConditionalFacts(filePath, springConditionalFacts); setKotlinSpringDiFacts(filePath, springDiFacts); out.push(...synthesizeCallableFlowCaptures(tree.rootNode, KOTLIN_CALLABLE_CAPTURE_OPTIONS)); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts index bad2b85d0..af101af3c 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts @@ -5,6 +5,7 @@ import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolv import { resolveDefGraphId } from '../../scope-resolution/graph-bridge/ids.js'; import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; import { isClassLike } from '../../scope-resolution/scope/walkers.js'; +import { indexOnlyElementType } from '../../type-extractors/shared.js'; import { kotlinProvider } from '../kotlin.js'; import { kotlinArityCompatibility, @@ -21,6 +22,7 @@ import { import { isKotlinStaticOnly } from './owners.js'; import { populateKotlinPackageSiblings } from './package-siblings.js'; import { attachKotlinSpringBeanCandidateMetadata } from './spring-bean-metadata.js'; +import { attachKotlinSpringAopMetadata } from './spring-aop.js'; import { clearKotlinPackageFacts } from './package-facts.js'; import { attachKotlinSpringDiMetadata } from './spring-di.js'; import { attachKotlinSpringConditionalMetadata } from './spring-conditionals.js'; @@ -116,6 +118,16 @@ export const kotlinScopeResolver: ScopeResolver = { isSuperReceiver: (text) => text.trim() === 'super', + // Subscript route only — Kotlin's collection views (`.values`, `.keys`) are + // properties on the stdlib types, resolved by the ordinary member walk rather + // than by unwrapping a generic here. + // + // Fed the annotation AS WRITTEN (`List`, `Map`), which + // `normalizeKotlinType` had already reduced to `User`. `undefined` means "not + // a container", so an `operator fun get` class does not fold `cache[k]` onto + // `Cache` itself. + elementTypeOf: indexOnlyElementType, + isStaticOnly: isKotlinStaticOnly, fieldFallbackOnMethodLookup: false, @@ -127,6 +139,7 @@ export const kotlinScopeResolver: ScopeResolver = { populateNamespaceSiblings: populateKotlinPackageSiblings, emitPostResolutionEdges: (graph, parsedFiles, nodeLookup, indexes) => { attachKotlinSpringBeanCandidateMetadata(graph, parsedFiles, nodeLookup, indexes); + attachKotlinSpringAopMetadata(graph, parsedFiles, nodeLookup, indexes); attachKotlinSpringConditionalMetadata(graph, parsedFiles, nodeLookup, indexes); attachKotlinSpringDiMetadata(graph, parsedFiles, nodeLookup, indexes); }, diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-aop.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-aop.ts new file mode 100644 index 000000000..797651619 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-aop.ts @@ -0,0 +1,77 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { + createSpringAopMetadataAttacher, + type SpringAopOwnerFact, +} from '../../frameworks/spring/aop.js'; +import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { getKotlinSpringAopFacts } from './capture-side-channel.js'; +import { isKotlinPackageSiblingVisibilityIncomplete } from './package-siblings.js'; +import { kotlinSpringAnnotationFacts, type KotlinAnnotationSyntaxFact } from './spring-di.js'; + +export type KotlinSpringAopAnnotationFact = KotlinAnnotationSyntaxFact; + +export type KotlinSpringAopFact = SpringAopOwnerFact; + +function scopeId(filePath: string, node: SyntaxNode, kind: 'Class' | 'Function') { + return makeScopeId({ + filePath, + range: nodeToCapture('@spring-aop.owner', node).range, + kind, + }); +} + +function ownerRange(node: SyntaxNode) { + return nodeToCapture('@spring-aop.owner', node).range; +} + +/** + * Capture Spring AOP syntax from the class node already surfaced by Kotlin's + * scope query. The shared layer resolves annotations and rejects non-default + * use-site targets after imports and package visibility have finalized. + */ +export function captureKotlinSpringAopFacts( + classNode: SyntaxNode, + filePath: string, +): KotlinSpringAopFact[] { + const facts: KotlinSpringAopFact[] = []; + const classAnnotations = kotlinSpringAnnotationFacts(classNode); + const singletonInstance = + classNode.type === 'object_declaration' || classNode.type === 'companion_object'; + // Kotlin import aliases can give a Spring annotation any local simple name. + // Capture annotated owners conservatively, then let the post-import shared + // resolver keep only recognized Spring AOP annotations. Objects also retain + // an empty owner fact so the shared phase can distinguish their singleton + // instance members from true static methods without naming Kotlin. + if (classAnnotations.length > 0 || singletonInstance) { + facts.push({ + ownerScopeId: scopeId(filePath, classNode, 'Class'), + ownerKind: 'class', + ownerFilePath: filePath, + ownerRange: ownerRange(classNode), + ...(singletonInstance ? { singletonInstance: true } : {}), + annotations: classAnnotations, + }); + } + + const body = classNode.namedChildren.find((child) => child.type === 'class_body'); + if (body === undefined) return facts; + for (const member of body.namedChildren) { + if (member.type !== 'function_declaration') continue; + const annotations = kotlinSpringAnnotationFacts(member); + if (annotations.length === 0) continue; + facts.push({ + ownerScopeId: scopeId(filePath, member, 'Function'), + ownerKind: 'callable', + ownerFilePath: filePath, + ownerRange: ownerRange(member), + ...(singletonInstance ? { singletonInstance: true } : {}), + annotations, + }); + } + return facts; +} + +export const attachKotlinSpringAopMetadata = createSpringAopMetadataAttacher({ + getFacts: getKotlinSpringAopFacts, + isPackageVisibilityIncomplete: isKotlinPackageSiblingVisibilityIncomplete, +}); diff --git a/gitnexus/src/core/ingestion/languages/python/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/python/scope-resolver.ts index 4aecc28b4..12b20ded8 100644 --- a/gitnexus/src/core/ingestion/languages/python/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/python/scope-resolver.ts @@ -17,6 +17,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import { indexOnlyElementType } from '../../type-extractors/shared.js'; import { pythonProvider } from '../python.js'; import { isPythonImportedModule, @@ -75,6 +76,17 @@ const pythonScopeResolver: ScopeResolver = { isSuperReceiver: (text) => /^super\s*\(/.test(text), + // Subscript route only — Python spells collection views as method calls + // (`.values()`), which the compound resolver's call branch already handles. + // + // The hook receives the annotation AS WRITTEN (`List[User]`, `Dict[str, User]`), + // not the name `interpret.ts`'s `stripGeneric` reduced it to, so `undefined` + // here means "this spelling is not a container" — which is what stops + // `cfg['k'].run()` on a `__getitem__`-bearing class from folding onto the + // class itself. Answering the route at all is what keeps `repos[0].save()` + // resolving. + elementTypeOf: indexOnlyElementType, + // Python is dynamically typed — field-fallback heuristic on, return- // type propagation across imports on. Both default to true; listed // explicitly here for documentation. diff --git a/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts index da9221420..5fd0f1570 100644 --- a/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts @@ -2,6 +2,7 @@ import type { ParsedFile } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import { indexOnlyElementType } from '../../type-extractors/shared.js'; import { rustProvider } from '../rust.js'; import { rustArityCompatibility, rustMergeBindings, resolveRustImportTarget } from './index.js'; import { populateRustOwners } from './method-owners.js'; @@ -165,6 +166,16 @@ export const rustScopeResolver: ScopeResolver = { isSuperReceiver: () => false, + // Subscript route only — Rust has no property-style collection view; `.iter()` + // / `.values()` are method calls the compound resolver's call branch handles. + // + // Fed the annotation AS WRITTEN (`&Vec`, `HashMap`), which + // `normalizeRustTypeName` had already collapsed to `User` by the time the fold + // saw it. Returning `undefined` is the answer "not a container" and makes the + // index step decline rather than fold onto the receiver's own class — an + // `Index`-impl type would otherwise take its own members as the element's. + elementTypeOf: indexOnlyElementType, + populateRangeBindings: populateRustRangeBindings, fieldFallbackOnMethodLookup: false, diff --git a/gitnexus/src/core/ingestion/languages/swift.ts b/gitnexus/src/core/ingestion/languages/swift.ts index d19a8fb52..1ae31679d 100644 --- a/gitnexus/src/core/ingestion/languages/swift.ts +++ b/gitnexus/src/core/ingestion/languages/swift.ts @@ -37,6 +37,7 @@ import { swiftMergeBindings, swiftArityCompatibility, } from './swift/index.js'; +import { preprocessSwiftConditionalDirectives } from './swift/conditional-directive-preprocess.js'; /** Swift init/deinit declarations have special names and Constructor label. */ const swiftExtractFunctionName = ( @@ -178,6 +179,7 @@ const BUILT_INS: ReadonlySet = new Set([ export const swiftProvider = defineLanguage({ id: SupportedLanguages.Swift, extensions: ['.swift'], + preprocessSource: preprocessSwiftConditionalDirectives, entryPointPatterns: [ /^viewDidLoad$/, /^viewWillAppear$/, diff --git a/gitnexus/src/core/ingestion/languages/swift/captures.ts b/gitnexus/src/core/ingestion/languages/swift/captures.ts index b9b111454..e3d398695 100644 --- a/gitnexus/src/core/ingestion/languages/swift/captures.ts +++ b/gitnexus/src/core/ingestion/languages/swift/captures.ts @@ -46,6 +46,7 @@ import { computeSwiftArityMetadata } from './arity-metadata.js'; import { synthesizeSwiftReceiverBinding } from './receiver-binding.js'; import { synthesizeSwiftSignatureBindings } from './signature-bindings.js'; import { getSwiftParser, getSwiftScopeQuery } from './query.js'; +import { preprocessSwiftConditionalDirectives } from './conditional-directive-preprocess.js'; import { recordCacheHit, recordCacheMiss } from './cache-stats.js'; import { getTreeSitterBufferSize } from '../../constants.js'; import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; @@ -89,10 +90,16 @@ export function emitSwiftScopeCaptures( cachedTree?: unknown, ): readonly CaptureMatch[] { // Reuse the parse phase's cached Tree when available; otherwise parse. + // `extractParsedFile` already applies `preprocessSource` on this path, but + // this emitter is also called directly (benchmarks, capture goldens, the + // scope-capture tripwire), and those callers must see the same program the + // pipeline does. The transform is idempotent, so applying it twice is a + // no-op; it is length-preserving, so offsets still index `sourceText`. let tree = cachedTree as ReturnType['parse']> | undefined; if (tree === undefined) { - tree = parseSourceSafe(getSwiftParser(), sourceText, undefined, { - bufferSize: getTreeSitterBufferSize(sourceText), + const parseText = preprocessSwiftConditionalDirectives(sourceText); + tree = parseSourceSafe(getSwiftParser(), parseText, undefined, { + bufferSize: getTreeSitterBufferSize(parseText), }); recordCacheMiss(); } else { diff --git a/gitnexus/src/core/ingestion/languages/swift/conditional-directive-preprocess.ts b/gitnexus/src/core/ingestion/languages/swift/conditional-directive-preprocess.ts new file mode 100644 index 000000000..78ea5aa14 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/swift/conditional-directive-preprocess.ts @@ -0,0 +1,283 @@ +/** + * A conditional-compilation directive occupying a whole line. + * + * `[^\S\r\n]` (horizontal whitespace) rather than `[ \t]` so NBSP / + * ideographic-space indentation and a leading BOM are recognized too. Leading + * whitespace is *not* required: nesting is decided from the scanner's brace + * depth, not from indentation (a column-0 `#if` inside a class body is exactly + * the shape that loses its enclosing declaration). + */ +const SWIFT_CONDITIONAL_DIRECTIVE_RE = /^[^\S\r\n]*#(if|elseif|else|endif)\b[^\r\n]*$/; + +/** + * Cheap whole-file precondition, mirroring `stripUeMacros`'s `HAS_UE_HINT`. + * Derived from the line pattern so the two can never drift apart. + */ +const HAS_CONDITIONAL_DIRECTIVE_HINT = new RegExp(SWIFT_CONDITIONAL_DIRECTIVE_RE.source, 'm'); + +interface SwiftPreprocessScanState { + blockCommentDepth: number; + multilineStringPounds: number | null; + /** Net `{` minus `}` seen in code position. May go negative on broken source. */ + braceDepth: number; +} + +/** + * An `#if` … `#endif` run. Blanking is decided per group, so a group is either + * fully blanked or fully preserved — never half-erased. + */ +interface DirectiveGroup { + braceDepthAtStart: number; + /** Indices into the alternating `parts` array of this group's directive lines. */ + directiveLines: number[]; + /** Net brace delta of every branch so far — what the parser sees once all survive. */ + totalDelta: number; + /** False as soon as one branch is not brace-balanced on its own. */ + allBranchesBalanced: boolean; + currentBranchDelta: number; +} + +function hasTripleQuoteAt(line: string, index: number): boolean { + return line.startsWith('"""', index); +} + +/** + * Length of the multiline terminator at `index`, or 0. + * + * A plain `"""` string always closes at `"""`, whatever follows it — an + * adjacent `#` is the next token, not part of the delimiter. A raw `#"""` + * string closes at `"""` followed by *at least* its own pound count. + */ +function matchingMultilineCloseLength(line: string, index: number, poundCount: number): number { + if (!hasTripleQuoteAt(line, index)) return 0; + if (poundCount === 0) return 3; + return line.startsWith('#'.repeat(poundCount), index + 3) ? 3 + poundCount : 0; +} + +function skipRegularString(line: string, startIndex: number, rawPoundCount: number): number { + const endPounds = '#'.repeat(rawPoundCount); + let index = startIndex + (rawPoundCount > 0 ? rawPoundCount + 1 : 1); + + while (index < line.length) { + if (line[index] === '"') { + if (rawPoundCount === 0) return index + 1; + if (line.startsWith(endPounds, index + 1)) return index + 1 + rawPoundCount; + } + if (rawPoundCount === 0 && line[index] === '\\') index++; + index++; + } + + return line.length; +} + +/** + * Skip an extended regex literal (`#/…/#`), whose body may legally contain + * `/*` — which would otherwise open a block comment that never closes. + * `slashIndex` points at the `/` that follows the opening pound run. + */ +function skipExtendedRegexLiteral(line: string, slashIndex: number, poundCount: number): number { + const closePounds = '#'.repeat(poundCount); + let index = slashIndex + 1; + + while (index < line.length) { + if (line[index] === '\\') { + index += 2; + continue; + } + if (line[index] === '/' && line.startsWith(closePounds, index + 1)) { + return index + 1 + poundCount; + } + index++; + } + + return line.length; +} + +function scanSwiftLine(line: string, state: SwiftPreprocessScanState): void { + let index = 0; + + while (index < line.length) { + if (state.blockCommentDepth > 0) { + if (line.startsWith('/*', index)) { + state.blockCommentDepth++; + index += 2; + } else if (line.startsWith('*/', index)) { + state.blockCommentDepth--; + index += 2; + } else { + index++; + } + continue; + } + + if (state.multilineStringPounds !== null) { + const closeLength = matchingMultilineCloseLength(line, index, state.multilineStringPounds); + if (closeLength > 0) { + state.multilineStringPounds = null; + index += closeLength; + continue; + } + // Non-raw multiline strings honour backslash escapes, so `\"""` is string + // data and not a terminator. Raw strings escape with `\#`, so a bare + // backslash there is literal. + index += state.multilineStringPounds === 0 && line[index] === '\\' ? 2 : 1; + continue; + } + + if (line.startsWith('//', index)) return; + if (line.startsWith('/*', index)) { + state.blockCommentDepth = 1; + index += 2; + continue; + } + + if (line[index] === '#') { + // Count the pound run exactly once and always advance past it, so a long + // run of bare `#` stays linear instead of being re-walked per character. + let poundCount = 1; + while (line[index + poundCount] === '#') poundCount++; + const afterPounds = index + poundCount; + + if (hasTripleQuoteAt(line, afterPounds)) { + state.multilineStringPounds = poundCount; + index = afterPounds + 3; + } else if (line[afterPounds] === '"') { + index = skipRegularString(line, index, poundCount); + } else if (line[afterPounds] === '/') { + index = skipExtendedRegexLiteral(line, afterPounds, poundCount); + } else { + index = afterPounds; + } + continue; + } + + if (hasTripleQuoteAt(line, index)) { + state.multilineStringPounds = 0; + index += 3; + continue; + } + if (line[index] === '"') { + index = skipRegularString(line, index, 0); + continue; + } + + if (line[index] === '{') state.braceDepth++; + else if (line[index] === '}') state.braceDepth--; + index++; + } +} + +/** + * Blank nested Swift conditional-compilation directives before parsing. + * + * tree-sitter-swift 0.7.1 does not admit `directive` as a `class_body` child + * (`vendor/tree-sitter-swift/src/node-types.json`), so error recovery can + * discard the enclosing declaration. Replacing only the directive text with + * spaces preserves `.length`, line endings, and every declaration offset. + * + * ponytail: delete this file (and the `preprocessSource` wiring in `swift.ts`) + * once a vendored tree-sitter-swift release includes upstream PR #583 ("Allow + * #if/#elseif/#else/#endif directives inside type bodies"); see also upstream + * issue #298 and PR #599. `.github/vendored-grammars.json` has no `"hold"` on + * swift, so the bump bot will land that release on its own. + * + * A directive group is blanked only when **all** of these hold: + * + * - it opens at `braceDepth > 0` — nesting, not indentation, is what the + * grammar rejects; top-level directives are valid source-file members and + * are left intact + * - it is not inside a multiline string literal or a block comment — blanking + * a line that carries a block-comment terminator would un-terminate the + * comment and swallow the rest of the file, and blanking inside a literal + * would rewrite program data + * - every branch is brace-balanced. A group that splits a declaration header + * (`#if` … `func f() async {` … `#else` … `func f() {` … `#endif`) leaves + * one unmatched `{` once both branches survive, which re-parents every + * later top-level declaration. Such a group degrades to the pre-fix + * behavior instead. + * + * Known residuals, all of which degrade to "directive left in place" rather + * than to corrupted output: string interpolation is not parsed with full Swift + * expression fidelity, so a nested multiline string inside an interpolation can + * confuse the scanner; a bare `/…/` regex literal containing `/*` opens a + * phantom block comment; and a multiline extended regex literal is treated as + * ending at its first line. + * + * Byte length is *not* preserved when a blanked directive line carries + * non-ASCII trailing text (` #if os(iOS) // 日本語 🔥` is 23 UTF-16 code units + * either way, but shrinks from 31 UTF-8 bytes to 23). C++ sidesteps this + * because UE macro tokens are ASCII-only; a Swift directive line can carry any + * trailing comment. Per the `LanguageProvider.preprocessSource` contract this is + * safe only while no consumer slices the original UTF-8 bytes by `startIndex`: + * node-tree-sitter reports UTF-16 code-unit indices, and every in-process + * consumer slices the JS string, whose `.length` *is* preserved. + * + * Must stay pure and idempotent — see `LanguageProvider.preprocessSource`. + */ +export function preprocessSwiftConditionalDirectives(sourceText: string): string { + if (!HAS_CONDITIONAL_DIRECTIVE_HINT.test(sourceText)) return sourceText; + + // Alternating [line, terminator, line, terminator, …, line]; joining it back + // reproduces the input byte for byte, including bare-`\r` endings. + const parts = sourceText.split(/(\r\n|\n|\r)/); + const state: SwiftPreprocessScanState = { + blockCommentDepth: 0, + multilineStringPounds: null, + braceDepth: 0, + }; + const openGroups: DirectiveGroup[] = []; + let blankedAny = false; + + for (let index = 0; index < parts.length; index += 2) { + const text = parts[index]!; + const insideLiteralOrComment = + state.multilineStringPounds !== null || state.blockCommentDepth > 0; + const match = insideLiteralOrComment ? null : SWIFT_CONDITIONAL_DIRECTIVE_RE.exec(text); + + const braceDepthAtStart = state.braceDepth; + scanSwiftLine(text, state); + const braceDelta = state.braceDepth - braceDepthAtStart; + + const openGroup = openGroups[openGroups.length - 1]; + if (match === null) { + if (openGroup !== undefined) openGroup.currentBranchDelta += braceDelta; + continue; + } + + // Directive lines are blanked as a unit, so their own braces (if any) never + // reach the parser and must not count toward a branch's balance. + if (match[1] === 'if') { + openGroups.push({ + braceDepthAtStart, + directiveLines: [index], + totalDelta: 0, + allBranchesBalanced: true, + currentBranchDelta: 0, + }); + continue; + } + // A `#else`/`#endif` with no open `#if` is malformed source — leave it be. + if (openGroup === undefined) continue; + + openGroup.directiveLines.push(index); + openGroup.totalDelta += openGroup.currentBranchDelta; + openGroup.allBranchesBalanced &&= openGroup.currentBranchDelta === 0; + openGroup.currentBranchDelta = 0; + if (match[1] !== 'endif') continue; + + openGroups.pop(); + const parentGroup = openGroups[openGroups.length - 1]; + // Every branch survives preprocessing, so the enclosing branch sees the sum + // — not one branch's delta. + if (parentGroup !== undefined) parentGroup.currentBranchDelta += openGroup.totalDelta; + + if (openGroup.braceDepthAtStart > 0 && openGroup.allBranchesBalanced) { + // Safe to mutate in place: every line of this group has already been + // scanned, and the decision never reopens. + for (const line of openGroup.directiveLines) parts[line] = ' '.repeat(parts[line]!.length); + blankedAny = true; + } + } + + return blankedAny ? parts.join('') : sourceText; +} diff --git a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts index fa8050bfa..fe7a30da4 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts @@ -23,6 +23,7 @@ import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexe import { typescriptProvider } from '../typescript.js'; import { loadTsconfigPaths, type TsconfigPaths } from '../../language-config.js'; import { buildSuffixIndex, type SuffixIndex } from '../../import-resolvers/utils.js'; +import { indexOnlyElementType } from '../../type-extractors/shared.js'; import { typescriptArityCompatibility, typescriptMergeBindings, @@ -106,6 +107,12 @@ function makeTsResolveImportTarget(): ScopeResolver['resolveImportTarget'] { } const typescriptScopeResolver: ScopeResolver = { + // One hook, both routes. TypeScript exposes collection views as METHOD calls + // (`.values()`), which the call-expression branch already handles, so the + // accessor route yields nothing here — but it is now the same hook rather + // than a second one left undefined. + elementTypeOf: indexOnlyElementType, + // Construction is keyword-prefixed: `new Service(db).doWork()` (#2708). constructionSyntax: { keyword: 'new' }, language: SupportedLanguages.TypeScript, @@ -155,10 +162,10 @@ const typescriptScopeResolver: ScopeResolver = { fieldFallbackOnMethodLookup: false, propagatesReturnTypesAcrossImports: true, - // TypeScript uses `.values()` / `.keys()` method-call syntax for - // collection views -- no property-style accessors like C#'s - // `Dictionary.Values`. Leave `unwrapCollectionAccessor` - // undefined and let the regular member-call branch handle them. + // TypeScript uses `.values()` / `.keys()` method-call syntax for collection + // views -- no property-style accessors like C#'s `Dictionary.Values` -- + // so `elementTypeOf` answers only the `index` route and lets the regular + // member-call branch handle the rest. // // `collapseMemberCallsByCallerTarget` left undefined (= false) -- // TypeScript legacy DAG emits one edge per call site, so diff --git a/gitnexus/src/core/ingestion/pipeline-phases/index.ts b/gitnexus/src/core/ingestion/pipeline-phases/index.ts index 0095c6bef..359d80a65 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/index.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/index.ts @@ -25,6 +25,12 @@ export { springAutoConfigurationPhase, type SpringAutoConfigurationOutput, } from './spring-auto-configuration.js'; +export { + springAopPhase, + springAopInheritancePhase, + type SpringAopOutput, + type SpringAopInheritanceOutput, +} from './spring-aop.js'; export { pruneLocalSymbolsPhase, type PruneLocalSymbolsOutput } from './prune-local-symbols.js'; export { taintSummariesPhase, type TaintSummariesOutput } from './taint-summaries.js'; export { callSummariesPhase, type CallSummariesOutput } from './call-summaries.js'; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/spring-aop.ts b/gitnexus/src/core/ingestion/pipeline-phases/spring-aop.ts new file mode 100644 index 000000000..8e4ac3cc5 --- /dev/null +++ b/gitnexus/src/core/ingestion/pipeline-phases/spring-aop.ts @@ -0,0 +1,618 @@ +/** + * Phase: springAop + * + * Materializes statically visible Spring proxy/advice behavior after every + * language resolver has attached normalized metadata to the shared graph. + * The post-MRO export in this file propagates declarative behavior through + * METHOD_OVERRIDES/METHOD_IMPLEMENTS without changing @annotation semantics. + * + * @deps scopeResolution + * @reads Class/Method nodes, HAS_METHOD edges, shared Spring AOP metadata + * @writes synthetic CodeElement nodes, DEFINES/DECLARES/ADVISED_BY edges + */ + +import type { GraphNode } from 'gitnexus-shared'; +import { generateId } from '../../../lib/utils.js'; +import { + decodeSpringAopReason, + encodeSpringAopReason, + getSpringAopGraphMetadata, + parseSpringAopPointcut, + SPRING_AOP_EVIDENCE_DESCRIPTION_PREFIX, + springAopPointcutMatches, + type SpringAopAdviceRecord, + type SpringAopAspectRecord, + type SpringAopBehaviorRecord, + type SpringAopPointcutReason, +} from '../frameworks/spring/aop.js'; +import { + createSpringAopCandidateIndex, + type SpringAopCandidateIndex, + type SpringAopOwnedMethod, +} from '../frameworks/spring/aop-candidates.js'; +import { toZeroBasedLine } from '../utils/line-base.js'; +import type { PipelineContext, PipelinePhase } from './types.js'; +import { logger } from '../../logger.js'; + +export const DEFAULT_SPRING_AOP_MAX_CANDIDATE_INSPECTIONS_PER_ADVICE = 100_000; +export const DEFAULT_SPRING_AOP_MAX_CANDIDATE_INSPECTIONS = 2_000_000; +export const DEFAULT_SPRING_AOP_MAX_ADVISED_EDGES_PER_ADVICE = 25_000; +export const DEFAULT_SPRING_AOP_MAX_ADVISED_EDGES = 100_000; + +export interface SpringAopOutput { + readonly advisedByEdges: number; + readonly evidenceNodes: number; + readonly unresolvedPointcuts: number; + readonly candidateInspections: number; + readonly truncatedAdvices: number; +} + +export interface SpringAopInheritanceOutput { + readonly inheritedBehaviorEdges: number; +} + +function simpleName(name: string): string { + const separator = name.lastIndexOf('.'); + return separator === -1 ? name : name.slice(separator + 1); +} + +function eligibleMethod( + node: GraphNode, + owner: GraphNode | undefined, + singletonInstanceClassIds: ReadonlySet, +): boolean { + return ( + node.label === 'Method' && + (node.properties.isStatic !== true || + (owner !== undefined && singletonInstanceClassIds.has(owner.id))) && + node.properties.visibility !== 'private' + ); +} + +function eligibleOwner(node: GraphNode): boolean { + return node.label === 'Class' || node.label === 'Interface'; +} + +function addEvidenceNode( + ctx: PipelineContext, + owner: GraphNode, + annotation: string, + line: number, + discriminator: string, + description: string, +): GraphNode { + const evidenceId = generateId( + 'CodeElement', + `spring-aop:${owner.id}:${line}:${annotation}:${discriminator}`, + ); + const evidence: GraphNode = { + id: evidenceId, + label: 'CodeElement', + properties: { + name: `@${simpleName(annotation)}`, + filePath: owner.properties.filePath, + startLine: toZeroBasedLine(line), + endLine: toZeroBasedLine(line), + isExported: false, + description: `${SPRING_AOP_EVIDENCE_DESCRIPTION_PREFIX}${description}`, + }, + }; + ctx.graph.addNode(evidence); + + const fileId = generateId('File', owner.properties.filePath); + if (ctx.graph.getNode(fileId) !== undefined) { + ctx.graph.addRelationship({ + id: generateId('DEFINES', `${fileId}->${evidenceId}`), + sourceId: fileId, + targetId: evidenceId, + type: 'DEFINES', + confidence: 1, + reason: 'spring-aop:evidence', + }); + } + return evidence; +} + +function emitBehavior( + ctx: PipelineContext, + record: SpringAopBehaviorRecord, + classMethods: ReadonlyMap, + ownerByMethod: ReadonlyMap, + singletonInstanceClassIds: ReadonlySet, +): { edges: number; evidence: number } { + const owner = ctx.graph.getNode(record.ownerId); + if (owner === undefined || (!eligibleOwner(owner) && owner.label !== 'Method')) { + return { edges: 0, evidence: 0 }; + } + if ( + owner.label === 'Method' && + !eligibleMethod(owner, ownerByMethod.get(owner.id), singletonInstanceClassIds) + ) { + return { edges: 0, evidence: 0 }; + } + const evidence = addEvidenceNode( + ctx, + owner, + record.annotation, + record.line, + `behavior:${record.behavior}`, + `${record.behavior} interceptor; activation unknown; proxy possible`, + ); + const reason = encodeSpringAopReason({ + kind: 'behavior', + annotation: record.annotation, + behavior: record.behavior, + declaredOn: record.ownerKind === 'class' ? 'class' : 'method', + activation: 'unknown', + proxy: 'possible', + }); + + const sources = + record.ownerKind === 'class' ? [owner, ...(classMethods.get(owner.id) ?? [])] : [owner]; + let edges = 0; + for (const source of sources) { + if ( + !eligibleOwner(source) && + !eligibleMethod(source, ownerByMethod.get(source.id), singletonInstanceClassIds) + ) { + continue; + } + ctx.graph.addRelationship({ + id: generateId( + 'ADVISED_BY', + `${source.id}->${evidence.id}:${record.line}:${record.behavior}`, + ), + sourceId: source.id, + targetId: evidence.id, + type: 'ADVISED_BY', + confidence: 1, + reason, + }); + edges++; + } + return { edges, evidence: 1 }; +} + +function emitAspect(ctx: PipelineContext, record: SpringAopAspectRecord): number { + const owner = ctx.graph.getNode(record.ownerId); + if (owner === undefined || !eligibleOwner(owner)) return 0; + const evidence = addEvidenceNode( + ctx, + owner, + record.annotation, + record.line, + 'aspect', + 'AspectJ aspect marker; registration unknown; activation unknown', + ); + ctx.graph.addRelationship({ + id: generateId('DECLARES', `${owner.id}->${evidence.id}`), + sourceId: owner.id, + targetId: evidence.id, + type: 'DECLARES', + confidence: 1, + reason: encodeSpringAopReason({ + kind: 'aspect', + annotation: record.annotation, + activation: 'unknown', + registration: 'unknown', + }), + }); + return 1; +} + +function pointcutReason(record: SpringAopAdviceRecord, resolved: boolean): SpringAopPointcutReason { + return { + kind: 'pointcut', + annotation: record.annotation, + pointcut: record.pointcut, + match: resolved ? 'static' : 'unresolved', + resolution: resolved ? 'resolved' : 'unknown', + }; +} + +interface SpringAopAdviceBudget { + readonly maxInspectionsPerAdvice: number; + readonly maxInspections: number; + readonly maxEdgesPerAdvice: number; + readonly maxEdges: number; + inspections: number; + edges: number; +} + +function resolveBudget(value: number | undefined, fallback: number): number { + return Number.isSafeInteger(value) && value !== undefined && value >= 0 ? value : fallback; +} + +function budgetReached(limit: number, used: number): boolean { + return limit !== 0 && used >= limit; +} + +function emitAdvice( + ctx: PipelineContext, + record: SpringAopAdviceRecord, + aspectClassIds: ReadonlySet, + ownerByMethod: ReadonlyMap, + candidateIndex: SpringAopCandidateIndex, + methodAnnotations: ReadonlyMap>, + budget: SpringAopAdviceBudget, +): { + edges: number; + evidence: number; + unresolved: number; + inspections: number; + truncated: boolean; +} { + const adviceNode = ctx.graph.getNode(record.ownerId); + if (adviceNode === undefined || adviceNode.label !== 'Method') { + return { edges: 0, evidence: 0, unresolved: 0, inspections: 0, truncated: false }; + } + const staticPointcut = record.pointcut; + const parsed = staticPointcut === null ? null : parseSpringAopPointcut(staticPointcut); + const isAdviceMethod = record.advice !== 'pointcut'; + const adviceOwner = ownerByMethod.get(adviceNode.id); + const activeAspect = adviceOwner !== undefined && aspectClassIds.has(adviceOwner.id); + const resolved = parsed !== null && (!isAdviceMethod || activeAspect); + const evidence = addEvidenceNode( + ctx, + adviceNode, + record.annotation, + record.line, + `pointcut:${record.advice}:${record.pointcut ?? ''}`, + `${record.advice} pointcut ${record.pointcut ?? ''}; resolution ${ + resolved ? 'resolved' : 'unknown' + }`, + ); + ctx.graph.addRelationship({ + id: generateId('DECLARES', `${adviceNode.id}->${evidence.id}`), + sourceId: adviceNode.id, + targetId: evidence.id, + type: 'DECLARES', + confidence: 1, + reason: encodeSpringAopReason(pointcutReason(record, resolved)), + }); + + if (!isAdviceMethod || !resolved || parsed === null || staticPointcut === null) { + return { + edges: 0, + evidence: 1, + unresolved: resolved ? 0 : 1, + inspections: 0, + truncated: false, + }; + } + + let edges = 0; + let inspections = 0; + let truncated = false; + for (const candidate of candidateIndex.candidatesFor(parsed)) { + if ( + budgetReached(budget.maxInspectionsPerAdvice, inspections) || + budgetReached(budget.maxInspections, budget.inspections) + ) { + truncated = true; + break; + } + inspections += 1; + budget.inspections += 1; + if (candidate.method.id === adviceNode.id) continue; + if (aspectClassIds.has(candidate.owner.id)) continue; + if ( + !springAopPointcutMatches( + parsed, + candidate.owner, + candidate.method, + methodAnnotations.get(candidate.method.id), + ) + ) { + continue; + } + if ( + budgetReached(budget.maxEdgesPerAdvice, edges) || + budgetReached(budget.maxEdges, budget.edges) + ) { + truncated = true; + break; + } + ctx.graph.addRelationship({ + id: generateId( + 'ADVISED_BY', + `${candidate.method.id}->${adviceNode.id}:${record.line}:${record.advice}`, + ), + sourceId: candidate.method.id, + targetId: adviceNode.id, + type: 'ADVISED_BY', + confidence: 0.95, + reason: encodeSpringAopReason({ + kind: 'advice', + annotation: record.annotation, + advice: record.advice, + pointcut: staticPointcut, + match: 'static', + activation: 'unknown', + proxy: 'possible', + }), + }); + edges++; + budget.edges += 1; + } + return { edges, evidence: 1, unresolved: 0, inspections, truncated }; +} + +export const springAopPhase: PipelinePhase = { + name: 'springAop', + deps: ['scopeResolution'], + + async execute(ctx: PipelineContext): Promise { + const metadata = getSpringAopGraphMetadata(ctx.graph); + if ( + metadata.aspects.length === 0 && + metadata.behaviors.length === 0 && + metadata.advices.length === 0 + ) { + return { + advisedByEdges: 0, + evidenceNodes: 0, + unresolvedPointcuts: 0, + candidateInspections: 0, + truncatedAdvices: 0, + }; + } + + ctx.onProgress({ + phase: 'enriching', + percent: 97, + message: 'Resolving Spring proxy and advice edges...', + stats: { filesProcessed: 0, totalFiles: 0, nodesCreated: ctx.graph.nodeCount }, + }); + + const classMethods = new Map(); + const ownerByMethod = new Map(); + const methodAnnotations = new Map>(); + const candidates: SpringAopOwnedMethod[] = []; + for (const relationship of ctx.graph.iterRelationshipsByType('HAS_METHOD')) { + const owner = ctx.graph.getNode(relationship.sourceId); + const method = ctx.graph.getNode(relationship.targetId); + if (owner === undefined || !eligibleOwner(owner) || method?.label !== 'Method') continue; + const ownerFilePath = owner.properties.filePath; + if (typeof ownerFilePath !== 'string' || !metadata.candidateFilePaths.has(ownerFilePath)) { + continue; + } + ownerByMethod.set(method.id, owner); + if (!eligibleMethod(method, owner, metadata.singletonInstanceClassIds)) continue; + const methods = classMethods.get(owner.id) ?? []; + methods.push(method); + classMethods.set(owner.id, methods); + candidates.push({ method, owner }); + } + + let advisedByEdges = 0; + let evidenceNodes = 0; + let unresolvedPointcuts = 0; + let candidateInspections = 0; + let truncatedAdvices = 0; + const budget: SpringAopAdviceBudget = { + maxInspectionsPerAdvice: resolveBudget( + ctx.options?.springAopMaxCandidateInspectionsPerAdvice, + DEFAULT_SPRING_AOP_MAX_CANDIDATE_INSPECTIONS_PER_ADVICE, + ), + maxInspections: resolveBudget( + ctx.options?.springAopMaxCandidateInspections, + DEFAULT_SPRING_AOP_MAX_CANDIDATE_INSPECTIONS, + ), + maxEdgesPerAdvice: resolveBudget( + ctx.options?.springAopMaxAdvisedEdgesPerAdvice, + DEFAULT_SPRING_AOP_MAX_ADVISED_EDGES_PER_ADVICE, + ), + maxEdges: resolveBudget( + ctx.options?.springAopMaxAdvisedEdges, + DEFAULT_SPRING_AOP_MAX_ADVISED_EDGES, + ), + inspections: 0, + edges: 0, + }; + for (const aspect of metadata.aspects) evidenceNodes += emitAspect(ctx, aspect); + for (const behavior of metadata.behaviors) { + // `@annotation` matches annotations declared directly on the method. + // Class-level behavior is fanned out by emitBehavior, but must not be + // copied here (that would model @within/@target semantics instead). + if (behavior.ownerKind === 'callable') { + const annotations = methodAnnotations.get(behavior.ownerId) ?? new Set(); + annotations.add(behavior.annotation); + methodAnnotations.set(behavior.ownerId, annotations); + } + const emitted = emitBehavior( + ctx, + behavior, + classMethods, + ownerByMethod, + metadata.singletonInstanceClassIds, + ); + advisedByEdges += emitted.edges; + evidenceNodes += emitted.evidence; + } + const candidateIndex = createSpringAopCandidateIndex(candidates, methodAnnotations); + for (const advice of metadata.advices) { + const emitted = emitAdvice( + ctx, + advice, + metadata.aspectClassIds, + ownerByMethod, + candidateIndex, + methodAnnotations, + budget, + ); + advisedByEdges += emitted.edges; + evidenceNodes += emitted.evidence; + unresolvedPointcuts += emitted.unresolved; + candidateInspections += emitted.inspections; + if (emitted.truncated) { + truncatedAdvices += 1; + logger.warn( + `[spring-aop] truncated advice ${advice.ownerId}: ` + + `${emitted.inspections} candidate inspections, ${emitted.edges} edges emitted; ` + + `run totals ${budget.inspections} inspections/${budget.edges} edges`, + ); + } + } + + if (truncatedAdvices > 0) { + ctx.onProgress({ + phase: 'enriching', + percent: 97, + message: 'Spring AOP advice resolution truncated by configured budgets', + detail: `${truncatedAdvices} advice method(s); ${candidateInspections} inspections; ${budget.edges} advice edges`, + stats: { filesProcessed: 0, totalFiles: 0, nodesCreated: ctx.graph.nodeCount }, + }); + } + + return { + advisedByEdges, + evidenceNodes, + unresolvedPointcuts, + candidateInspections, + truncatedAdvices, + }; + }, +}; + +interface InheritedBehaviorWorkItem { + readonly sourceId: string; + readonly evidenceId: string; + readonly reason: string; +} + +function sameMethodSignature(left: GraphNode, right: GraphNode): boolean { + if (left.properties.name !== right.properties.name) return false; + const leftCount = left.properties.parameterCount; + const rightCount = right.properties.parameterCount; + if (typeof leftCount === 'number' && typeof rightCount === 'number' && leftCount !== rightCount) { + return false; + } + const leftTypes = left.properties.parameterTypes; + const rightTypes = right.properties.parameterTypes; + return !( + Array.isArray(leftTypes) && + Array.isArray(rightTypes) && + leftTypes.length > 0 && + rightTypes.length > 0 && + (leftTypes.length !== rightTypes.length || + leftTypes.some((type, index) => type !== rightTypes[index])) + ); +} + +/** + * Propagate behavior evidence across the inheritance decisions materialized by + * MRO. This is deliberately separate from pointcut matching: `@annotation` + * continues to mean an annotation declared directly on the callable. + */ +export const springAopInheritancePhase: PipelinePhase = { + name: 'springAopInheritance', + deps: ['springAop', 'mro'], + + async execute(ctx: PipelineContext): Promise { + const metadata = getSpringAopGraphMetadata(ctx.graph); + const ownerByMethod = new Map(); + const classMethods = new Map(); + for (const relationship of ctx.graph.iterRelationshipsByType('HAS_METHOD')) { + const owner = ctx.graph.getNode(relationship.sourceId); + const method = ctx.graph.getNode(relationship.targetId); + if (owner === undefined || !eligibleOwner(owner) || method?.label !== 'Method') continue; + ownerByMethod.set(method.id, owner); + const methods = classMethods.get(owner.id) ?? []; + methods.push(method); + classMethods.set(owner.id, methods); + } + + const childrenByParent = new Map>(); + for (const type of ['EXTENDS', 'IMPLEMENTS'] as const) { + for (const relationship of ctx.graph.iterRelationshipsByType(type)) { + const children = childrenByParent.get(relationship.targetId) ?? new Set(); + children.add(relationship.sourceId); + childrenByParent.set(relationship.targetId, children); + } + } + + const implementingMethods = new Map>(); + for (const relationship of ctx.graph.iterRelationshipsByType('METHOD_IMPLEMENTS')) { + const methods = implementingMethods.get(relationship.targetId) ?? new Set(); + methods.add(relationship.sourceId); + implementingMethods.set(relationship.targetId, methods); + } + + const overridingClasses = new Map>(); + for (const relationship of ctx.graph.iterRelationshipsByType('METHOD_OVERRIDES')) { + const classes = overridingClasses.get(relationship.targetId) ?? new Set(); + classes.add(relationship.sourceId); + overridingClasses.set(relationship.targetId, classes); + } + + const queue: InheritedBehaviorWorkItem[] = []; + const emittedKeys = new Set(); + for (const relationship of ctx.graph.iterRelationshipsByType('ADVISED_BY')) { + if (decodeSpringAopReason(relationship.reason)?.kind !== 'behavior') continue; + const key = `${relationship.sourceId}\0${relationship.targetId}\0${relationship.reason}`; + emittedKeys.add(key); + queue.push({ + sourceId: relationship.sourceId, + evidenceId: relationship.targetId, + reason: relationship.reason, + }); + } + + let inheritedBehaviorEdges = 0; + const enqueue = (source: GraphNode, item: InheritedBehaviorWorkItem): void => { + const owner = ownerByMethod.get(source.id); + if ( + !eligibleOwner(source) && + !eligibleMethod(source, owner, metadata.singletonInstanceClassIds) + ) { + return; + } + const key = `${source.id}\0${item.evidenceId}\0${item.reason}`; + if (emittedKeys.has(key)) return; + emittedKeys.add(key); + ctx.graph.addRelationship({ + id: generateId('ADVISED_BY', `${source.id}->${item.evidenceId}:inherited:${item.reason}`), + sourceId: source.id, + targetId: item.evidenceId, + type: 'ADVISED_BY', + confidence: 1, + reason: item.reason, + }); + inheritedBehaviorEdges += 1; + queue.push({ sourceId: source.id, evidenceId: item.evidenceId, reason: item.reason }); + }; + + for (let cursor = 0; cursor < queue.length; cursor += 1) { + const item = queue[cursor]; + if (item === undefined) continue; + const source = ctx.graph.getNode(item.sourceId); + if (source === undefined) continue; + + if (eligibleOwner(source)) { + for (const childId of childrenByParent.get(source.id) ?? []) { + const child = ctx.graph.getNode(childId); + if (child === undefined || !eligibleOwner(child)) continue; + enqueue(child, item); + for (const method of classMethods.get(child.id) ?? []) enqueue(method, item); + } + continue; + } + if (source.label !== 'Method') continue; + + for (const methodId of implementingMethods.get(source.id) ?? []) { + const method = ctx.graph.getNode(methodId); + if (method !== undefined) enqueue(method, item); + } + for (const classId of overridingClasses.get(source.id) ?? []) { + const matches = (classMethods.get(classId) ?? []).filter((method) => + sameMethodSignature(method, source), + ); + const [match] = matches; + if (matches.length === 1 && match !== undefined) enqueue(match, item); + } + } + + return { inheritedBehaviorEdges }; + }, +}; diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index e99515468..4e858bd3f 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -34,6 +34,8 @@ import { scopeResolutionPhase, springConfigPhase, springAutoConfigurationPhase, + springAopPhase, + springAopInheritancePhase, pruneLocalSymbolsPhase, taintSummariesPhase, callSummariesPhase, @@ -56,6 +58,14 @@ export interface PipelineOptions { * to retain those nodes under `skipGraphPhases`. */ skipGraphPhases?: boolean; + /** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */ + springAopMaxCandidateInspectionsPerAdvice?: number; + /** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */ + springAopMaxCandidateInspections?: number; + /** Per-advice Spring AOP `ADVISED_BY` edge cap. `0` disables this cap. */ + springAopMaxAdvisedEdgesPerAdvice?: number; + /** Aggregate Spring AOP advice-edge cap for one analysis. `0` disables this cap. */ + springAopMaxAdvisedEdges?: number; /** * Build the control-flow-graph / PDG substrate (#2081 M1, opt-in via `--pdg`). * Off by default: workers skip all CFG work and emit no `cfgSideChannel`, and @@ -262,8 +272,8 @@ export interface PipelineOptions { * Phase dependency graph: * * scan → structure → [springConfig, markdown, cobol] → parse → [routes, tools, orm] - * → crossFile → scopeResolution → springAutoConfiguration → pruneLocalSymbols - * → mro → di → communities → processes + * → crossFile → scopeResolution → [springAutoConfiguration, springAop] → pruneLocalSymbols + * → mro → springAopInheritance → di → communities → processes * * To add a new phase: create a file in pipeline-phases/, export the phase * object, and `.register()` it at the appropriate position below. Opt-in @@ -290,6 +300,7 @@ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { .register(crossFilePhase) .register(scopeResolutionPhase) .register(springAutoConfigurationPhase) + .register(springAopPhase) .register(pruneLocalSymbolsPhase) // M4 (#2084): interprocedural taint fixpoint — the first real opt-in // pdg-gated phase. Off ⇒ absent ⇒ byte-identical graph. No always-on @@ -297,6 +308,7 @@ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { .register(taintSummariesPhase, { enabledWhen: (o) => o.pdg === true }) .register(callSummariesPhase, { enabledWhen: (o) => o.pdg === true }) .register(mroPhase, { enabledWhen: (o) => !o.skipGraphPhases }) + .register(springAopInheritancePhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(diPhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(communitiesPhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(processesPhase, { enabledWhen: (o) => !o.skipGraphPhases }) diff --git a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts index 626f83728..b87fa3b19 100644 --- a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts +++ b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts @@ -49,7 +49,16 @@ export function extractParsedFile( if (provider.emitScopeCaptures === undefined) return undefined; if (sourceText.trim().length === 0) return undefined; try { - const captures = provider.emitScopeCaptures(sourceText, filePath, cachedTree, { sourceKind }); + // A provider that rewrites source before parsing must see the same text + // here that the parse worker fed tree-sitter, or the two halves of the + // pipeline analyze different programs. Only the cache-miss path re-parses; + // with a cached tree the emitter ignores the text. The transform is + // length-preserving, so every offset still indexes the original. + const parseText = + cachedTree === undefined + ? (provider.preprocessSource?.(sourceText, filePath) ?? sourceText) + : sourceText; + const captures = provider.emitScopeCaptures(parseText, filePath, cachedTree, { sourceKind }); return extractScope(captures, filePath, provider); } catch (err) { const message = `scope extraction failed for ${filePath}: ${ diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index a8f55edc3..231d37c97 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -911,6 +911,13 @@ function pass3CollectImports( // ─── Pass 4: collect type bindings ───────────────────────────────────────── +/** Cap on the retained as-written annotation. Real container spellings are a + * handful of characters; a multi-line mapped/conditional type is neither a + * container any `elementTypeOf` parses nor worth keeping one copy of per + * binding on a kernel-scale repo. Over the cap the spelling is dropped, which + * makes an index step decline — the safe direction. */ +const MAX_DECLARED_SPELLING_LENGTH = 256; + function pass4CollectTypeBindings( matches: readonly CaptureMatch[], drafts: readonly ScopeDraft[], @@ -953,11 +960,41 @@ function pass4CollectTypeBindings( provider.bindingScopeFor?.(match, draftToScope(innermost), scopeTree) ?? autoHostedId; const host = draftById.get(hostId) ?? innermost; - const typeRef: TypeRef = { - rawName: parsed.rawTypeName, - declaredAtScope: host.id, - source: parsed.source, - }; + // The annotation as the source wrote it, kept only when the provider's + // interpretation is not already it. `interpretTypeBinding` normalizes + // container spellings away (`User[]` → `User`, `List[User]` → `User`, + // `[]*User` → `User`), which makes a reduced container indistinguishable + // from a class of the same name — and an index step folding on that + // ambiguity typed `grid[0]` as `Grid`. Read at the one place the + // distinction matters; see `TypeRef.declaredSpelling`. + // + // Read from the capture rather than from `ParsedTypeBinding` deliberately: + // `@type-binding.type` is the shared anchor EVERY provider already reads to + // build `rawTypeName`, so nothing has to be threaded through fourteen + // interpreters (and none can forget to). + // A provider may override when its grammar keeps part of the written type + // outside `@type-binding.type` (C++ hangs `*` on the declarator). + const writtenType = (parsed.declaredSpelling ?? match['@type-binding.type']?.text)?.trim(); + const declaredSpelling = + writtenType !== undefined && + writtenType.length > 0 && + writtenType.length <= MAX_DECLARED_SPELLING_LENGTH && + writtenType !== parsed.rawTypeName + ? writtenType + : undefined; + const typeRef: TypeRef = + declaredSpelling === undefined + ? { + rawName: parsed.rawTypeName, + declaredAtScope: host.id, + source: parsed.source, + } + : { + rawName: parsed.rawTypeName, + declaredSpelling, + declaredAtScope: host.id, + source: parsed.source, + }; // Prefer stronger sources when multiple matches fire for the same // bound name in the same scope. Example: `u: User = find()` matches // both the annotation and constructor-inferred patterns; the explicit @@ -1105,6 +1142,14 @@ function pass5CollectReferences( // that logs nothing. const receiverChain = extractReceiverChain(match); + // Callee-position marker: a member-read capture that is actually the callee + // of an enclosing call (`obj.f` in `obj.f()`). Recorded, not acted on — + // whether the read is a phantom or a genuine func-typed-field read depends + // on the resolved tail's kind, which only edge emission knows. Emitted by + // languages whose read pattern has no call-position exclusion; absent + // everywhere else, so the site stays byte-identical for them. + const inCalleePosition = match['@reference.callee-position'] !== undefined; + const site: ReferenceSite = { name: nameCap.text, atRange: anchor.range, @@ -1122,6 +1167,7 @@ function pass5CollectReferences( ...(argumentTypes !== undefined ? { argumentTypes } : {}), ...(argumentTypeClasses !== undefined ? { argumentTypeClasses } : {}), ...(receiverChain !== undefined ? { receiverChain } : {}), + ...(inCalleePosition ? { inCalleePosition: true } : {}), }; referenceSites.push(site); } @@ -1522,6 +1568,7 @@ const KNOWN_SUB_TAGS: ReadonlySet = new Set([ '@reference.name', '@reference.qualified-name', '@reference.property-key', + '@reference.callee-position', '@reference.receiver', '@reference.operator', '@reference.arity', diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index 174190464..aa0641367 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -15,7 +15,7 @@ * - propagatesReturnTypesAcrossImports (default true) * - fieldFallbackOnMethodLookup (default true — turn OFF for * statically-typed languages; the heuristic over-connects) - * - unwrapCollectionAccessor — property-style collection views + * - elementTypeOf — container element type, by subscript or accessor * - collapseMemberCallsByCallerTarget — one edge per caller/target * - populateNamespaceSiblings — cross-file implicit visibility * - hoistTypeBindingsToModule — enable ONLY when method return @@ -268,6 +268,7 @@ * `docs/plans/2026-04-20-001-refactor-emit-pipeline-generalization-plan.md`. */ +import type { DecorationStripper } from '../scope/walkers.js'; import type { BindingRef, Callsite, @@ -313,6 +314,11 @@ export interface ImportResolutionContext { * `RegistryProviders.constraintCompatibility`'s third parameter. */ export type { ConstraintContext } from 'gitnexus-shared'; +/** How a container's element was reached in the source. */ +export type ElementAccessRoute = + | { readonly kind: 'index' } + | { readonly kind: 'accessor'; readonly name: string }; + export interface ScopeResolver { /** Identity for telemetry + per-language flag check. */ readonly language: SupportedLanguages; @@ -708,22 +714,56 @@ export interface ScopeResolver { readonly fieldFallbackOnMethodLookup?: boolean; /** - * Unwrap a property-style collection accessor on a typed receiver - * to its element type. Called by `resolveCompoundReceiverClass` - * when walking dotted member-access chains of the form - * `receiver.Accessor`. The provider returns the element type's - * simple name, or `undefined` when the accessor doesn't unwrap — - * in which case the regular field-walk resumes. + * Element type of a container, reached either by a subscript (`repos[0]`) or + * by a property-style collection view (`dict.Values`). Returns the element + * type's simple name, or `undefined` when the container does not unwrap by + * that route — in which case the caller resumes its normal walk. * - * Use this only for languages that expose collection views as - * properties rather than method calls; languages whose collection - * views are `.values()` / `.keys()` method calls leave this - * undefined and let the normal call-expression branch handle them. + * ONE hook for both routes, deliberately. They were previously two + * (`unwrapCollectionAccessor` for the property route, `unwrapCollectionElement` + * for the subscript route), which meant a language implementing one silently + * got nothing for the other: C# parsed `Dictionary` for `.Values` but + * returned nothing for `list[0]`, and TypeScript did the reverse. Two entries + * answering one question, each accreting an implementation per language. + * + * `via` carries the route so a provider can distinguish them where it matters + * (a `Dictionary` yields its VALUE type by subscript but either type by + * accessor name); a provider that does not care can ignore it. + * + * Consulted ONLY where the source actually performed the access. It is NOT a + * general type-name normalizer: unwrapping a container at a bare class lookup + * would let `repos.find(x)` fold to `Repo.find`, because a container's member + * set is not its element's. For the same reason it is deliberately separate + * from `stripTypePreservingDecoration`: a pointer or a nullable leaves the + * member set unchanged and is safe to strip at the lookup; a container is not. + * + * ## The `index` route is REQUIRED, and `undefined` means "not a container" + * + * A language that leaves the `index` route unanswered gets NO index folding — + * the structural fold declines the step rather than passing the position + * through. That is not a default worth softening. `undefined` used to mean + * "fall back to identity", on the theory that a capture layer which already + * reduced the container (Go's `normalizeGoTypeName`, C#/TypeScript's + * `stripGeneric`) leaves nothing to unwrap. The theory holds for a container; + * it is false for an ordinary class the source happened to subscript, and + * `rawName` cannot tell those apart — `repos: User[]` and `grid: Grid` both + * arrive as a bare resolvable class name. Identity there typed `grid[0].run()` + * as `Grid.run` and `t[0].Render()` as `Table.Render`: a wrong owner, which + * this pipeline ranks strictly below a missing edge. + * + * The hook is therefore handed `TypeRef.declaredSpelling` — the annotation AS + * WRITTEN, retained by the scope extractor precisely because capture-time + * normalization destroys it — falling back to `rawName` only when nothing was + * normalized away. So a provider sees `User[]`, `List[User]`, `[]*User` or + * `Dictionary`, never the post-reduction `User`, and answering + * `undefined` for a spelling it does not recognize as a container is an + * ANSWER, not an absence. + * + * Implementations may return a still-DECORATED element name (Go's `[]*User` + * yields `*User`): the index step looks the element up through + * `stripTypePreservingDecoration`, so the pointer resolves. */ - readonly unwrapCollectionAccessor?: ( - receiverType: string, - accessor: string, - ) => string | undefined; + readonly elementTypeOf?: (containerType: string, via: ElementAccessRoute) => string | undefined; /** * Collapse member-call CALLS edges by `(caller, target)` rather @@ -1114,6 +1154,39 @@ export interface ScopeResolver { */ readonly hoistTypeBindingsToModule?: boolean; + /** + * Strip ONE layer of type-preserving decoration off a declared type name, + * or return `undefined` when there is nothing left to strip. + * + * Exists because a declared type is stored as written. Go's + * `synthesizeGoReceiverBinding` keeps `typeNode.text`, so a pointer-receiver + * method binds its receiver to the literal `*Host` — which matches no class + * binding, so receiver-chain resolution declines at the base and every + * `h.field.method()` in the dominant Go idiom loses its `CALLS` edge (#2766). + * The stored binding is deliberately left decorated (`method-owners.ts` + * consumes `*T` vs `T` to model Go's value and pointer method sets), so the + * normalization belongs at LOOKUP, never as a rewrite of the binding. + * + * TYPE-PRESERVING ONLY. Pointer, reference, `const`, nullable, borrow, + * deref-transparent smart pointer and sigil all leave the member set + * unchanged. A CONTAINER — array, slice, map, `Option` — does not: stripping + * one here would type `repos: Repo[]` as `Repo` and let `repos.find(x)` fold + * to `Repo.find`, a confident wrong edge the ambiguity gate cannot catch + * because `Repo` binds uniquely. Containers are unwrapped only by an index + * step that consumed a subscript. + * + * Consulted ONLY after every undecorated lookup has failed, and only by + * receiver-chain base and step resolution — the shared class lookup keeps + * exact-name behaviour for its other ~two dozen callers, several of which are + * shaped `findClassBindingInScope(...) ?? otherResolver(...)` and would have + * their fallback suppressed by a global widening. + * + * Leave undefined for languages whose declared types carry no type-preserving + * decoration. Measured: only Go needs it for a receiver base; Rust, C#, Swift, + * TypeScript and C++ need it for field types. + */ + readonly stripTypePreservingDecoration?: DecorationStripper; + /** * Whether the compound-receiver resolver should strip C-style cast * expressions from receiver-position text before resolving it — @@ -1140,7 +1213,7 @@ export interface ScopeResolver { * * A second opting language must extend the classifier grammar or * convert this toggle into a per-language classifier hook (the - * `unwrapCollectionAccessor` pattern) — do not flip this flag for + * `elementTypeOf` pattern) — do not flip this flag for * another language as-is. * * Known non-goal: the compound-receiver options built from this diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/edges.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/edges.ts index f7f246f26..1bf72b6ef 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/edges.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/edges.ts @@ -15,9 +15,10 @@ * language-agnostic — no language needs to change it. */ -import type { Reference, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import type { NodeLabel, Reference, ScopeId, SymbolDefinition } from 'gitnexus-shared'; import type { KnowledgeGraph } from '../../../graph/types.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { CALL_TARGET_TYPES } from '../../model/symbol-table.js'; import type { GraphNodeLookup } from '../graph-bridge/node-lookup.js'; import { resolveCallerGraphId, resolveDefGraphId } from '../graph-bridge/ids.js'; import type { CalleeIdSink } from './callee-id-sink.js'; @@ -70,6 +71,41 @@ export function mapReferenceKindToEdgeType( } } +/** + * Is this read site a PHANTOM — a duplicate of the call happening beside it? + * + * Some languages' member-read capture also matches the callee of a member call, + * so `obj.f()` produces a `call` site on `f` AND a `read` site on `obj.f` at the + * same position (Go's `@reference.read` matches every `selector_expression`). + * The capture layer marks that second site `inCalleePosition` rather than + * dropping it, because the two cases it covers are indistinguishable there: + * + * - tail resolves to a METHOD → phantom. The ACCESSES edge duplicates the + * CALLS edge emitted for the same source position; suppress it. + * - tail resolves to a FIELD → genuine. `h.dep.Work()` where + * `Work func() error` reads a func-typed struct field and calls the value + * it holds. That read is the field's only ACCESSES evidence — keep it. + * + * Only the resolved target's kind separates them, which is why this decision + * lives at emission and not in any language's capture code. Sites without the + * marker are untouched, so a genuine method VALUE (`f := obj.method`) — never in + * callee position — keeps its read. + * + * "Invoked rather than read" is `CALL_TARGET_TYPES` — the canonical callable + * target set (`FREE_CALLABLE_TYPES` ∪ Method/Constructor), not a local copy. A + * hand-rolled `{Function, Method, Constructor}` silently omits `Macro` (C/C++) + * and `Delegate` (C#), the two labels that set exists to add, and drops the + * `satisfies` guard in `symbol-table.ts` that compile-enforces every free + * callable label against `LABEL_BEHAVIOR`. + */ +function isPhantomCalleeRead( + site: { readonly kind: string; readonly inCalleePosition?: boolean }, + targetDef: { readonly type: NodeLabel }, +): boolean { + if (site.kind !== 'read' || site.inCalleePosition !== true) return false; + return CALL_TARGET_TYPES.has(targetDef.type); +} + /** * Resolve caller + target to graph ids and emit the edge. Returns true * if the edge was emitted (not deduped, not skipped). @@ -87,6 +123,9 @@ export function tryEmitEdge( readonly inScope: ScopeId; readonly atRange: { startLine: number; startCol: number }; readonly kind: string; + /** See {@link isPhantomCalleeRead}. Set by the extractor from the + * language's `@reference.callee-position` marker; absent otherwise. */ + readonly inCalleePosition?: boolean; }, targetDef: SymbolDefinition, reason: string, @@ -95,6 +134,10 @@ export function tryEmitEdge( collapseByCallerTarget = false, calleeCapture?: CalleeIdCaptureCtx, ): boolean { + // A read that only exists because it is the callee of the call beside it, and + // whose tail resolved to a callable, is that call restated as an access — + // checked first because it needs no id resolution. + if (isPhantomCalleeRead(site, targetDef)) return false; // Inheritance edges are emitted directly by `preEmitInheritanceEdges` (which // owns the enclosing-class caller and the EXTENDS-vs-IMPLEMENTS type), so this // generic bridge derives caller + edge type purely from the site. @@ -152,6 +195,15 @@ export function tryEmitEdge( * * All other invariants of `tryEmitEdge` apply: dedup key shape, collapse * flag honoring, edge-type mapping, caller-id resolution. + * + * ONE deliberate exception: the {@link isPhantomCalleeRead} suppression is not + * applied here, because it keys on the target's node label and this entry point + * is handed an id rather than a def. Reachable only for a `read` site marked + * `inCalleePosition` whose receiver typed as an object-literal VALUE — a + * JS/TS-shaped registration. No language that sets the marker resolves through + * this bridge today (verified for Go, whose func-valued struct-literal fields + * resolve through the owned-member path instead). A language adding the marker + * must re-check this path. */ export function tryEmitEdgeWithExplicitTargetId( graph: KnowledgeGraph, diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts index abe8b48e4..6f942d88b 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts @@ -54,17 +54,19 @@ import { definitionIdPosition } from '../utils/definition-id.js'; * restricted to function/class-likes, those calls correctly fall * through to the File-node fallback at the bottom of the walk. */ +export const CALLER_ANCHOR_LABELS: ReadonlySet = new Set([ + 'Function', + 'Method', + 'Constructor', + 'Module', + 'Class', + 'Interface', + 'Struct', + 'Enum', +]); + function isCallerAnchorLabel(label: NodeLabel): boolean { - return ( - label === 'Function' || - label === 'Method' || - label === 'Constructor' || - label === 'Module' || - label === 'Class' || - label === 'Interface' || - label === 'Struct' || - label === 'Enum' - ); + return CALLER_ANCHOR_LABELS.has(label); } function rangeContainsPoint( diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts index 1c6b94410..a8f90393a 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts @@ -244,38 +244,53 @@ export function buildGraphNodeLookup(graph: KnowledgeGraph): GraphNodeLookup { return lookup; } +/** + * Every label {@link buildGraphNodeLookup} registers — and therefore the ONLY + * labels `resolveDefGraphId` can ever return an id for. Both endpoints of every + * scope-resolution edge come from that lookup (the one exception is the File + * fallback in `resolveCallerGraphId`), so this set defines the whole FROM/TO + * surface those edges can produce. + * + * That makes it load-bearing for the LadybugDB relation DDL: a label added here + * without the matching `FROM x TO y` pairs in `RELATION_SCHEMA` crashes + * `analyze` at `assertDeclaredPair` on whichever codebase first emits the pair + * (#2792). `test/unit/schema-pair-coverage.test.ts` derives the required pairs + * from this set and fails in CI instead. + */ +export const LINKABLE_LABELS: ReadonlySet = new Set([ + 'Function', + 'Method', + 'Constructor', + // Program-like module declarations are provider-gated callable-value + // targets and need the same def→graph bridge. + 'Module', + 'Class', + 'Interface', + 'Struct', + 'Enum', + // Trait nodes are linkable so MRO builders can bridge PHP/Rust trait + // defs between scope-resolution DefIds and the graph's node ids. + // IMPLEMENTS edges from classes to traits are otherwise invisible to + // the scope-resolution MRO pass. + 'Trait', + // Variable / Property are linkable too — receiver-bound write/read + // ACCESSES edges target field nodes (e.g. `user.name = "x"` → + // ACCESSES edge to User's `name` Variable/Property node). + 'Variable', + 'Property', + // Const is linkable so the value-receiver-owner bridge in + // `receiver-bound-calls.ts` Case 5 can translate the scope-resolution + // `Variable` def for `export const fooService = {...}` to the canonical + // `Const:filePath:name` graph node id, against which object-literal + // method symbols register their `ownerId` (PR #1718 / issue #1358). + 'Const', + // Macro nodes are linkable so a macro invocation (`log!(…)`) resolved + // via `MacroRegistry` can bridge its scope-resolution `Macro` def to + // the legacy `@definition.macro` graph node and emit the `USES` edge + // (Rust #1934 F72; also covers C/C++ `#define` macro defs). + 'Macro', +]); + export function isLinkableLabel(label: NodeLabel): boolean { - return ( - label === 'Function' || - label === 'Method' || - label === 'Constructor' || - // Program-like module declarations are provider-gated callable-value - // targets and need the same def→graph bridge. - label === 'Module' || - label === 'Class' || - label === 'Interface' || - label === 'Struct' || - label === 'Enum' || - // Trait nodes are linkable so MRO builders can bridge PHP/Rust trait - // defs between scope-resolution DefIds and the graph's node ids. - // IMPLEMENTS edges from classes to traits are otherwise invisible to - // the scope-resolution MRO pass. - label === 'Trait' || - // Variable / Property are linkable too — receiver-bound write/read - // ACCESSES edges target field nodes (e.g. `user.name = "x"` → - // ACCESSES edge to User's `name` Variable/Property node). - label === 'Variable' || - label === 'Property' || - // Const is linkable so the value-receiver-owner bridge in - // `receiver-bound-calls.ts` Case 5 can translate the scope-resolution - // `Variable` def for `export const fooService = {...}` to the canonical - // `Const:filePath:name` graph node id, against which object-literal - // method symbols register their `ownerId` (PR #1718 / issue #1358). - label === 'Const' || - // Macro nodes are linkable so a macro invocation (`log!(…)`) resolved - // via `MacroRegistry` can bridge its scope-resolution `Macro` def to - // the legacy `@definition.macro` graph node and emit the `USES` edge - // (Rust #1934 F72; also covers C/C++ `#define` macro defs). - label === 'Macro' - ); + return LINKABLE_LABELS.has(label); } diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/compound-receiver.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/compound-receiver.ts index 6b77da5dd..39f98b467 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/compound-receiver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/compound-receiver.ts @@ -21,12 +21,13 @@ */ import type { ScopeId, SymbolDefinition, TypeRef } from 'gitnexus-shared'; -import type { ScopeResolver } from '../contract/scope-resolver.js'; +import type { ElementAccessRoute, ScopeResolver } from '../contract/scope-resolver.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; import type { WorkspaceResolutionIndex } from '../workspace-index.js'; import { stripTemplateArguments } from '../../utils/template-arguments.js'; import type { DecodedReceiverChain } from '../../utils/receiver-chain-codec.js'; import { decodeReceiverChain } from '../../utils/receiver-chain-codec.js'; +import type { DecorationStripper } from '../scope/walkers.js'; import { findClassBindingInScope, findEnclosingClassDef, @@ -77,14 +78,13 @@ interface ResolveCompoundReceiverOptions { * class, walk its fields and try the lookup on each field's class. * Phase-9C "unified fixpoint" — Python-shaped heuristic. */ readonly fieldFallback?: boolean; - /** Language-specific accessor unwrap — `data.Values` on a - * Dictionary-typed receiver yields V (C#), etc. Returns the - * element type's simple name, or `undefined` to let the regular - * field-walk handle the access. */ - readonly unwrapCollectionAccessor?: ( - receiverType: string, - accessor: string, - ) => string | undefined; + /** Container -> element, by subscript (`repos[0]`) or accessor (`data.Values` + * on a `Dictionary` yields V). Returns the element type's simple name, + * or `undefined`. See the `ScopeResolver` field of the same name for why the + * two routes share one hook, and for why `undefined` on the `index` route + * means "not a container" — a step that gets it DECLINES, so a language must + * answer that route to get index folding at all. */ + readonly elementTypeOf?: (containerType: string, via: ElementAccessRoute) => string | undefined; /** Walk up from the class scope to ancestor (Module) scopes when * looking up a method's return-type typeBinding. Only enable for * languages that hoist return-type bindings to Module scope (C#); @@ -121,6 +121,13 @@ interface ResolveCompoundReceiverOptions { * (`const Config = make(1); Config.db.query()` emitted `entry → Database.query`), * the exact wrong-edge failure this work exists to avoid. */ readonly strictBaseBinding?: boolean; + /** Per-language type-preserving decoration stripper, from the `ScopeResolver` + * contract. Passed to the class lookup at the base and step sites so a + * decorated declared type (`*Host`) resolves to its class. Absent for + * languages whose declared types carry no such decoration, and never applied + * by the shared lookup's other callers — see the contract's own note on why + * this is opt-in rather than global. */ + readonly stripTypePreservingDecoration?: DecorationStripper; } /** Is this hop the language's construction selector applied to the class @@ -276,35 +283,77 @@ function resolveConstructionExpressionClass( } /** - * One step of the fold: the type of `memberName` on `owner`, or `undefined`. + * One position in a fold: the class the chain has reached, PLUS the declared + * type text that produced it. * - * A method's return type and a field's declared type both live in the owning - * class scope's `typeBindings`, keyed by name, so a `call` step and a `field` - * step are the same lookup — the step's `kind` carries no resolution - * difference, only intent. Walks the MRO so an inherited member resolves. + * The declared type is carried for two reasons, and the index step is its only + * reader. First, it is the CONTAINER evidence that step demands: capture + * normalizes `repos: User[]` down to `User`, so the resolved class alone cannot + * tell a reduced container from a class the source merely subscripted, and + * folding on that ambiguity typed `grid[0].run()` as `Grid.run`. Second, a + * declared type that named NO class is still a usable position — + * `Promise` and `[]Repo` match nothing in the workspace — because a later + * step may unwrap it; keeping only the class would strand those shapes at the + * step that produced them. * - * Deliberately does NOT consult the field fallback that - * `resolveCompoundReceiverClass` offers. That fallback iterates every field of - * the owner and re-resolves each one's class looking for a same-named member — - * O(fields x depth x names) per step, the shape behind the 128 GB blowup in - * #1871 — and it answers a DIFFERENT question ("does any field's type have a - * member of this name?"), which is a guess. Structure exists precisely so this - * does not have to guess. + * `undefined` when the position was reached by a route that has no declared type + * to report (a static class receiver, say). A step needing one declines rather + * than guessing. */ +interface FoldState { + /** + * Absent when the position's declared type named no class — `Promise` + * and `[]Repo` name nothing in the workspace. ONE signal, not two: an earlier + * version carried a separate `unresolvedDeclaredType` flag alongside a `def` + * holding the PREVIOUS position, which no path ever read. Two sources of + * truth for one fact, and the dead `def` read as intentional. + * + * Only an unwrapping step (await, index) can advance from an absent `def`; + * every other step declines, because folding on against the previous class + * would look the next member up on the wrong owner. + */ + readonly def: SymbolDefinition | undefined; + /** + * The declared type AS WRITTEN — the spelling when capture normalized one + * away (`TypeRef.declaredSpelling`), else `rawName` (they are the same string + * when nothing was normalized). Only the index step reads it, and only the + * as-written form separates `repos: User[]` from `grid: Grid`: both reduce to + * a bare, resolvable class name. + */ + readonly declaredType?: string; + readonly declaredAtScope?: ScopeId; +} + function typeOfMemberOnClass( owner: SymbolDefinition, memberName: string, scopes: ScopeResolutionIndexes, index: WorkspaceResolutionIndex, options: ResolveCompoundReceiverOptions, -): SymbolDefinition | undefined { +): FoldState | undefined { const classScopeByDefId = index.classScopeByDefId; const ownerChain = [owner.nodeId, ...scopes.methodDispatch.mroFor(owner.nodeId)]; for (const ownerId of ownerChain) { const classScope = classScopeByDefId.get(ownerId); const memberType = classScope?.typeBindings.get(memberName); if (memberType !== undefined) { - return findClassBindingInScope(memberType.declaredAtScope, memberType.rawName, scopes); + const def = findClassBindingInScope( + memberType.declaredAtScope, + memberType.rawName, + scopes, + options.stripTypePreservingDecoration, + ); + // The declared type is reported even when it resolved to no class: + // `Promise` and `[]Repo` name nothing in the workspace, and an + // await or index step unwrapping them is exactly how they become + // resolvable. Returning `undefined` here would strand those shapes. A + // `def` that stayed absent is reported as absent, NOT as the previous + // owner, so nothing may fold an ordinary member off this position. + return { + def, + declaredType: memberType.declaredSpelling ?? memberType.rawName, + declaredAtScope: memberType.declaredAtScope, + }; } // Languages whose binding-scope hook hoists a method's return-type binding // out of the class body and onto an ancestor (Module) scope keep NOTHING in @@ -320,7 +369,26 @@ function typeOfMemberOnClass( if (curScope === undefined) break; const hoisted = curScope.typeBindings.get(memberName); if (hoisted !== undefined) { - return findClassBindingInScope(hoisted.declaredAtScope, hoisted.rawName, scopes); + const def = findClassBindingInScope( + hoisted.declaredAtScope, + hoisted.rawName, + scopes, + // Same stripper the primary branch above passes. Omitting it here + // meant a decorated declared type (`*Host`) resolved on one branch + // and not the other, for the same member of the same class. + options.stripTypePreservingDecoration, + ); + // Identical to the primary branch: a declared type that named no + // class is still a usable position when the next step unwraps it. + // Returning `undefined` here made `svc.getMap()['k'].run()` decline + // while byte-identical `byId['k'].run()` resolved, purely because ten + // languages route return-type bindings through this hoisted branch + // and the other through the class scope. + return { + def, + declaredType: hoisted.declaredSpelling ?? hoisted.rawName, + declaredAtScope: hoisted.declaredAtScope, + }; } curId = curScope.parent; } @@ -362,15 +430,92 @@ export function foldReceiverChain( // `receiverChain` is dropped before resolving the base: it describes the // whole receiver, and handing it back to the resolver would re-enter this // fold on the base and never terminate. - let current = resolveCompoundReceiverClass(chain.baseReceiverName, inScope, scopes, index, { + const baseDef = resolveCompoundReceiverClass(chain.baseReceiverName, inScope, scopes, index, { ...options, fieldFallback: false, receiverChain: undefined, strictBaseBinding: true, }); - if (current === undefined) return undefined; + // The base's own declared type is carried too, so a chain whose FIRST step is + // an unwrap (`repos[0].save()` — index applied directly to the base) has the + // container spelling available. Without it that shape declines at step 1. + // Looked up ONLY when something will read it: the base failed to resolve (so + // an unwrap step is the last chance), or an index step will unwrap the + // container. `resolveCompoundReceiverClass` already walked the scope chain for + // this same name above, so doing it unconditionally duplicated that walk on + // every fold — and the U10 census says 86% of chains are pure field/call and + // never read it. + const needsBaseDeclaredType = + baseDef === undefined || chain.steps.some((step) => step.kind === 'index'); + const baseBinding = needsBaseDeclaredType + ? findReceiverTypeBinding(inScope, chain.baseReceiverName, scopes) + : undefined; + + // A base whose declared type names no class is NOT automatically a dead end: + // `repos: User[]` binds to the literal `User[]`, which matches no class + // because a container is not one. That position is still usable IF the next + // step unwraps it — which is exactly what an index step does. Carrying it + // forward with the marker lets that step recover; every other step kind + // declines on the marker, so nothing folds against a phantom owner. + if (baseDef === undefined && baseBinding === undefined) return undefined; + let current: FoldState = { + def: baseDef, + declaredType: baseBinding?.declaredSpelling ?? baseBinding?.rawName, + declaredAtScope: baseBinding?.declaredAtScope, + }; for (const step of chain.steps) { + // A position whose declared type named no class can only be advanced by an + // unwrapping step. Folding an ordinary member off it would look the member + // up on the PREVIOUS class — a wrong owner, silently. + // + // `await` IS identity, and soundly so: every language whose capture layer + // reduces the wrapper has ALREADY done it by the time a binding reaches the + // fold (TypeScript strips `Promise`, C# strips `Task`), and awaiting a + // value that is NOT a thenable yields that same value — so both regimes land + // on the identical answer and there is nothing to distinguish. + if (step.kind === 'await') continue; + if (step.kind === 'index') { + // An index step is NOT identity, and that asymmetry with `await` above is + // the whole point. `await` on a non-promise is a no-op; a subscript on a + // non-container is not — it yields the element of an indexer whose type + // is a different class entirely. + // + // The NORMALIZED type name cannot tell the two regimes apart: capture + // reduces `repos: User[]` to `User`, so a reduced container and a class + // the source merely subscripted (`grid: Grid` where `Grid` declares an + // index signature) both arrive as a bare, resolvable class name. Falling + // back to identity there kept `current` on the CONTAINER and looked the + // next member up on it — `grid[0].run()` → `Grid.run`, `t[0].Render()` → + // `Table.Render`. A confidently wrong owner, which is strictly worse than + // no edge and invisible to a bench that scores edge PRESENCE. + // + // So the step demands positive evidence instead: `declaredType`, the + // AS-WRITTEN spelling (`TypeRef.declaredSpelling`, preserved precisely + // because `rawName` threw it away), handed to the language's + // `elementTypeOf`. A provider that does not recognize the spelling as a + // container is answering "not a container", and the only sound move is to + // decline — a language must therefore answer the `index` route to get + // index folding at all. + const declared = current.declaredType; + const element = + declared === undefined ? undefined : options.elementTypeOf?.(declared, { kind: 'index' }); + if (element === undefined) return undefined; + const scopeForLookup = current.declaredAtScope ?? inScope; + const elementClass = findClassBindingInScope( + scopeForLookup, + element, + scopes, + options.stripTypePreservingDecoration, + ); + if (elementClass === undefined) return undefined; + current = { + def: elementClass, + declaredType: element, + declaredAtScope: scopeForLookup, + }; + continue; + } // Construction is NOT an ordinary member lookup. `Factory.new` on a class // constant denotes an instance of Factory, and the cascade already encodes // that (`isConstructionSelectorHop`) along with the class-constant test that @@ -381,13 +526,23 @@ export function foldReceiverChain( // first. That turned a correct edge into a WRONG one (Ruby // `Factory.new.run` → `Product.run`), which is the failure mode this whole // line of work exists to avoid. Decline and let the cascade answer. + // + // Placed AFTER the name-free continues above, deliberately. When it sat + // first, `options.constructionSyntax?.selector === step.name` compared + // `undefined === undefined` for every await/index step in a language with no + // construction selector, vetoing the entire fold before it ran — which is + // why those receivers minted a chain, fired the gate, and produced no edge. + // Position, not a guard, is what makes that unreachable: only named steps + // get here. if (options.constructionSyntax?.selector === step.name) return undefined; - - const next = typeOfMemberOnClass(current, step.name, scopes, index, options); + if (current.def === undefined) return undefined; + const next = typeOfMemberOnClass(current.def, step.name, scopes, index, options); if (next === undefined) return undefined; current = next; } - return current; + // A chain that ended without a class returns undefined naturally — no + // separate guard, because `def` IS the signal. + return current.def; } export function resolveCompoundReceiverClass( @@ -480,7 +635,12 @@ export function resolveCompoundReceiverClass( return findClassBindingInScope(rhsTb.declaredAtScope, arg, scopes); } - const viaTb = findClassBindingInScope(tb.declaredAtScope, tb.rawName, scopes); + const viaTb = findClassBindingInScope( + tb.declaredAtScope, + tb.rawName, + scopes, + options.stripTypePreservingDecoration, + ); if (viaTb !== undefined) return viaTb; // Member-alias / call-result shapes store the RHS path on rawName @@ -714,7 +874,7 @@ export function resolveCompoundReceiverClass( // the final segment and unwraps the receiver's generic, return // the element class directly. Resolved before the field-walk // because Dictionary-family types aren't local class defs. - if (options.unwrapCollectionAccessor !== undefined && parts.length >= 2) { + if (options.elementTypeOf !== undefined && parts.length >= 2) { const last = parts[parts.length - 1]; const headInner = parts[0]; if (last === undefined || headInner === undefined) return undefined; @@ -742,7 +902,12 @@ export function resolveCompoundReceiverClass( prefixType = cur; } if (prefixType !== undefined) { - const elemName = options.unwrapCollectionAccessor(prefixType.rawName, last); + // `rawName`, not `declaredSpelling`, and deliberately: the accessor route + // only fires on a multi-arg container (`Dictionary`), which every + // provider's capture-time normalization leaves ALONE — so the two are the + // same string here, and reading the spelling would change nothing except + // to widen an unmeasured surface. + const elemName = options.elementTypeOf(prefixType.rawName, { kind: 'accessor', name: last }); if (elemName !== undefined) { return findClassBindingInScope(prefixType.declaredAtScope, elemName, scopes); } diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts index 038ad2437..e75aa0045 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts @@ -61,6 +61,7 @@ import { findReceiverTypeBinding, findValueBindingInScope, isClassLike, + type DecorationStripper, } from '../scope/walkers.js'; import { tryEmitEdge, @@ -82,6 +83,10 @@ import type { ResolutionOutcomeRecorder, ResolutionSuppressionReason, } from '../resolution-outcome.js'; +import { classifyReceiverShape } from '../resolution-outcome.js'; +import type { ReceiverOrigin } from '../resolution-outcome.js'; +import { decodeReceiverChain } from '../../utils/receiver-chain-codec.js'; +import type { DecodedReceiverChain } from '../../utils/receiver-chain-codec.js'; /** Subset of `ScopeResolver` consumed by this pass. Accepting the * subset rather than the full provider keeps tests and partial @@ -92,10 +97,11 @@ type ReceiverBoundProviderSubset = Pick< | 'isSuperReceiverInContext' | 'fieldFallbackOnMethodLookup' | 'collapseMemberCallsByCallerTarget' - | 'unwrapCollectionAccessor' + | 'elementTypeOf' | 'hoistTypeBindingsToModule' | 'stripReceiverCastExpressions' | 'constructionSyntax' + | 'stripTypePreservingDecoration' | 'resolveQualifiedReceiverMember' | 'resolveReceiverMember' | 'resolveThisViaEnclosingClass' @@ -113,8 +119,17 @@ function resolveClassBindingForName( scopeId: string, rawClassName: string, scopes: ScopeResolutionIndexes, + /** + * OPT-IN, and deliberately not passed by the emitting cases. `findClass + * BindingInScope`'s own docstring explains why the stripper is opt-in: a name + * that previously bound nothing starts binding, which SUPPRESSES the + * `?? otherResolver(...)` fallbacks several callers rely on. Case 4 therefore + * keeps exact-name behaviour and only `classifyReceiverOrigin` — which emits + * no edge and can only change a diagnostic label — passes it. + */ + stripDecoration?: DecorationStripper, ): SymbolDefinition | undefined { - const direct = findClassBindingInScope(scopeId, rawClassName, scopes); + const direct = findClassBindingInScope(scopeId, rawClassName, scopes, stripDecoration); if (direct !== undefined) return direct; if (!rawClassName.includes('<')) return undefined; @@ -128,7 +143,7 @@ function resolveClassBindingForName( // default to [] before checking `.length`. const qnameIds = scopes.qualifiedNames.get(baseName) ?? []; if (qnameIds.length === 0) { - return findClassBindingInScope(scopeId, baseName, scopes); + return findClassBindingInScope(scopeId, baseName, scopes, stripDecoration); } const matches: SymbolDefinition[] = []; for (const id of qnameIds) { @@ -149,7 +164,123 @@ function resolveClassBindingForName( // safety in non-ODR or mixed-language edge cases. } - return findClassBindingInScope(scopeId, baseName, scopes); + return findClassBindingInScope(scopeId, baseName, scopes, stripDecoration); +} + +/** A bare, undecorated identifier and nothing else — see {@link isBareTypeName}. */ +const BARE_TYPE_NAME_RE = /^[A-Za-z_$][\w$]*$/; + +/** + * A type name a built-in test may be asked about: a bare, undecorated + * identifier and nothing else. + * + * `Promise`, `[]Repo` and `Option` all name a built-in CONTAINER + * whose ELEMENT is very often in-program, and an await/index/unwrap step is + * exactly how a receiver chain reaches that element. Answering "external" + * because the outer spelling matched a built-in would relabel a real in-program + * drop, which is the failure this whole function exists to stop. A decorated or + * dotted spelling is likewise not a built-in name, it merely contains one. + */ +function isBareTypeName(rawName: string): boolean { + return BARE_TYPE_NAME_RE.test(rawName); +} + +/** + * Is this dropped receiver rooted inside the analyzed program? + * + * Asks of the receiver's BASE — the leftmost name the chain hangs off — what + * this index can DEMONSTRATE. Three answers, and the asymmetry between them is + * the whole point: + * + * - `in-program` — the base's declared type resolves here, or the base itself is + * a class, a qualified name, or a value this program declares. A real edge was + * lost; the hedge must fire. + * - `external` — POSITIVE evidence that the target is outside: the language + * itself names the base (or its bare declared type) a built-in. `console.log`, + * `fetch(...)`, `JSON.stringify` reach code no index contains, so there is no + * node an edge could have pointed at and nothing was lost. + * - `unknown` — everything else. An absence of evidence is NOT evidence of + * externality: an unannotated parameter (`function f(svc) { svc.a().b(); }`) + * is recorded nowhere in the scope model at all, and calling that "external" + * published `epistemic: 'exact'` over a genuinely missing in-program caller — + * strictly worse than hedging, because it is a confident wrong answer rather + * than an admitted gap. `unknown` counts WITH `in-program` in + * `summarizeUnresolvedReceivers`, which is the safe direction. + * + * Uses the AST-derived chain base when one was minted, and falls back to the + * head of the receiver text otherwise — never a regex over the source line. + * + * Exported for the unit tests that pin the three-way split; the pass is its only + * production caller. + */ +export function classifyReceiverOrigin( + decoded: DecodedReceiverChain | undefined, + inScope: string, + receiverName: string, + scopes: ScopeResolutionIndexes, + options: { + /** The language's type-preserving decoration stripper. Without it a Go + * pointer receiver — `func (h *Host)` binds `h` to the literal `*Host` — + * resolves to no class and the whole method body's drops were reported as + * external. Same hook the three receiver-chain lookups in + * `compound-receiver.ts` already receive. */ + readonly stripTypePreservingDecoration?: DecorationStripper; + /** `LanguageProvider.isBuiltInName`, threaded through the pass options the + * same way `emitFreeCallFallback` receives it. THE only source of positive + * external evidence available here; languages that declare no built-in set + * simply never produce an `external` verdict, which is the safe default. */ + readonly isBuiltInName?: (name: string) => boolean; + } = {}, +): ReceiverOrigin { + // The chain's base is authoritative. Without one, take the head of the + // receiver text up to the first member/call punctuation. + const base = decoded?.baseReceiverName ?? /^[A-Za-z_$][\w$]*/.exec(receiverName)?.[0]; + if (base === undefined || base.length === 0) return 'unknown'; + const strip = options.stripTypePreservingDecoration; + const isBuiltIn = options.isBuiltInName; + + // The base's declared TYPE, when it has one, is the strongest signal about + // where the member lives: `inputs.stream()` has an in-program base bound to + // `List`, whose `stream` is in the JDK. + const binding = findReceiverTypeBinding(inScope, base, scopes); + if (binding !== undefined) { + // `resolveClassBindingForName`, not a bare lookup: it also strips template + // arguments, so an in-program generic base (`Box b; b.open()`) + // resolves instead of being mislabelled and dropped from the hedge. + if ( + resolveClassBindingForName(binding.declaredAtScope, binding.rawName, scopes, strip) !== + undefined + ) { + return 'in-program'; + } + // The declared type is not one this index contains. That is only proof of + // externality when the language itself names it — otherwise the type merely + // failed to resolve (an alias, an inferred callable, a generic parameter), + // and we fall through to ask what the index knows about the base itself. + if (isBareTypeName(binding.rawName) && isBuiltIn?.(binding.rawName) === true) { + return 'external'; + } + } + // Anything else this index knows by that name (namespace, module, free fn). + // O(1), so it goes ahead of the scope-chain walks below: all three checks are + // arms of the same `in-program` disjunction and none has a side effect, so + // answering from the index first is free and changes no verdict. + if (scopes.qualifiedNames.has(base)) return 'in-program'; + // A type the program declares, used as a static receiver. + if (findClassBindingInScope(inScope, base, scopes, strip) !== undefined) return 'in-program'; + // A VALUE the program declares — an object-literal service, a local whose + // initializer we could not type (`const loc = makeIt(); loc.getUser().save()`), + // a field. The type channel had nothing usable to say about these, but the + // program demonstrably declares the name, so the lost edge is in-program and + // failing to type it is a resolver defect. This is the channel Case 5 already + // dispatches on; consulting it here keeps the diagnostic honest about the + // same population. + if (findValueBindingInScope(inScope, base, scopes) !== undefined) return 'in-program'; + + // Positive external evidence, and the only kind reachable from this pass. + if (isBuiltIn?.(base) === true) return 'external'; + + return 'unknown'; } export function emitReceiverBoundCalls( @@ -168,6 +299,14 @@ export function emitReceiverBoundCalls( * `undefined` ⇒ zero overhead, byte-identity (R4). Per-file capture * contexts are built from this + `parsed.filePath` in the loop. */ readonly calleeIdSink?: CalleeIdSink; + /** `LanguageProvider.isBuiltInName`. Passed through the options bag rather + * than widened into `ReceiverBoundProviderSubset`, mirroring how + * `emitFreeCallFallback` receives the same hook — the subset exists to keep + * test providers small, and this pass reads nothing else off the language + * provider. Consumed ONLY by `classifyReceiverOrigin`, so leaving it unset + * degrades a drop's label to `unknown` (the safe direction) and changes no + * edge. */ + readonly isBuiltInName?: (name: string) => boolean; } = {}, ): number { let emitted = 0; @@ -180,10 +319,17 @@ export function emitReceiverBoundCalls( const hoistTypeBindingsToModule = provider.hoistTypeBindingsToModule === true; const compoundOpts = { fieldFallback, - unwrapCollectionAccessor: provider.unwrapCollectionAccessor, + elementTypeOf: provider.elementTypeOf, hoistTypeBindingsToModule, stripReceiverCastExpressions: provider.stripReceiverCastExpressions === true, constructionSyntax: provider.constructionSyntax, + stripTypePreservingDecoration: provider.stripTypePreservingDecoration, + }; + // Loop-invariant: both hooks come off the pass arguments, so the options bag + // for `classifyReceiverOrigin` is built once here rather than per dropped site. + const receiverOriginOpts = { + stripTypePreservingDecoration: provider.stripTypePreservingDecoration, + isBuiltInName: options.isBuiltInName, }; // Build an interface → implementors map from IMPLEMENTS edges. @@ -402,7 +548,23 @@ export function emitReceiverBoundCalls( // the end of the site loop, not here — a later case may still resolve // the site, and only a site that survives every case is a real drop. let compoundReceiverUnresolved = false; - if (receiverName.includes('.') || receiverName.includes('(')) { + // The punctuation test is a C-family heuristic and it is the reason + // `repos[0].save()` is INVISIBLE in all 14 languages: a subscript receiver + // contains neither `.` nor `(`, so this case never fired, the fold was + // never consulted, and no drop was recorded either — the call vanished + // with the instrument blind to it. PHP `->` and `::` receivers are lost + // the same way. + // + // A minted receiver chain is the STRUCTURAL answer to the same question: + // the capture layer walked the real AST and found the receiver is an + // expression, whatever punctuation it happens to be spelled with. Trusting + // that instead of the text is the substitution this whole line of work + // exists to make. + if ( + receiverName.includes('.') || + receiverName.includes('(') || + site.receiverChain !== undefined + ) { const currentClass = resolveCompoundReceiverClass( receiverName, site.inScope, @@ -1341,6 +1503,10 @@ export function emitReceiverBoundCalls( // dropped site (its callee is unknown by definition, so the drop cannot // be attributed to any target symbol). if (compoundReceiverUnresolved && !handledSites.has(siteKey)) { + // Decoded once: both the shape census and the origin classifier read the + // same chain, and this is inside the drop guard so a resolved site pays + // nothing. + const decodedChain = decodeReceiverChain(site.receiverChain); options.recordResolutionOutcome?.({ kind: 'suppressed', reason: 'receiver-unresolved', @@ -1354,6 +1520,22 @@ export function emitReceiverBoundCalls( // recorded here too. Carry the kind so a consumer can separate a // dropped CALL from a dropped property access. siteKind: site.kind, + // Structural, from the AST-derived chain the emitter minted — never + // re-derived from the source line. + // `decodeReceiverChain` opens with a non-string guard, so the + // undefined case needs no ternary here. + receiverShape: classifyReceiverShape(decodedChain), + // Whether anything was actually lost. An external target has no node + // to point at, so its absence is completeness, not uncertainty — but + // ONLY a positive built-in match may say so. Everything the index + // cannot demonstrate stays `unknown` and keeps hedging. + receiverOrigin: classifyReceiverOrigin( + decodedChain, + site.inScope, + receiverName, + scopes, + receiverOriginOpts, + ), }); } } diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index aa6268d52..e11ae7c9a 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -810,6 +810,10 @@ export function runScopeResolution( { recordResolutionOutcome, calleeIdSink: calleeIdAccumulator, + // The pass's only source of positive EXTERNAL evidence for a dropped + // receiver (`console.log`, `fetch(...)`). Same hook, same spelling as + // the `emitFreeCallFallback` wiring below. + isBuiltInName: provider.languageProvider.isBuiltInName, }, ); const unresolvedReceiverExtras = diff --git a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts index 07b9d3083..a0e5e0919 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts @@ -1,4 +1,5 @@ import type { Range, ReferenceKind } from 'gitnexus-shared'; +import type { DecodedReceiverChain } from '../utils/receiver-chain-codec.js'; export type ResolutionSuppressionReason = | 'adl-ordinary-lookup-blocked' @@ -61,6 +62,120 @@ export type ResolutionOutcome = * persisted `RepoMeta.unresolvedReceiverMembers` artifact is unchanged. */ readonly siteKind?: ReferenceKind; + /** + * Structural shape of the receiver whose type could not be established. + * + * Derived from the site's ENCODED RECEIVER CHAIN — the compact string the + * capture emitters mint by walking the real AST — never from the source + * line. Re-deriving a shape textually would mean regex-classifying the + * number that gates this work, which is exactly the textual-shape dispatch + * the structural-receiver line of work exists to remove. + * + * Lets a consumer ask "which KIND of receiver are we losing?" instead of + * only "how many". Without it, `callDropsByExtension` is the finest + * available split and a language's bucket says nothing about whether the + * cause is one defect or five. + * + * NOTE ON COVERAGE: only drops that REACH the recorder carry a shape, and + * Case 0's gate fires on receiver punctuation, so shapes that mint no + * reference site at all (`?.`, explicit type args, subscript) are absent + * from this breakdown entirely — they are the INVISIBLE-GAP population the + * bench shape arm exists to see. A shape census here is a census of the + * VISIBLE drops, not of all lost calls. + * + * Diagnostic only. `summarizeUnresolvedReceivers` ignores it, so the + * persisted `RepoMeta.unresolvedReceiverMembers` artifact is unchanged. + */ + readonly receiverShape?: ReceiverShape; + /** + * Whether the receiver is rooted INSIDE the analyzed program. + * + * The single most important distinction a static-analysis tool can make + * about a call it did not resolve, and the one this codebase previously + * collapsed: + * + * - `in-program` — the receiver's base is a local, parameter, field or a + * type this index knows. Failing to type it is a RESOLVER DEFECT: a real + * caller exists in the graph and was lost. + * - `external` — the base is rooted in code this index does not contain + * (`System.out.println`, `fetch(...)`, `os.environ.setdefault`). There is + * NO node to point an edge at, so nothing was lost. A compiler resolves + * these against the JDK / BCL / lib.d.ts; without those, the honest + * answer is "outside the program", not "unknown". + * + * Only `in-program` makes an `impact` count a lower bound. Reporting an + * external target as uncertainty is what made the hedge fire on nearly + * every real codebase and taught readers to ignore it. + */ + readonly receiverOrigin?: ReceiverOrigin; }; +/** + * How a dropped receiver was spelled, structurally. + * + * - `chain-call` every recorded step is a call — `svc.getUser().save()` + * - `chain-field` every recorded step is a field — `h.repo.save()` + * - `chain-mixed` the chain interleaves both — `svc.getUser().addr.save()` + * - `chain-unwrap` the chain contains an `await` or `index` step — the shapes + * this work exists to expose. They were previously counted as + * FIELDS, because the classifier's parameter widened `kind` to + * `string` and they fell into its `else`. + * - `no-chain` the site carried no chain, so the receiver was a compound + * expression the capture walk could not reduce to a nameable + * base (it stopped early, or the base was unencodable) + */ + +export type ReceiverShape = + | 'chain-call' + | 'chain-field' + | 'chain-mixed' + | 'chain-unwrap' + | 'no-chain'; + +/** + * Is the receiver rooted inside the analyzed program, or outside it? + * + * `unknown` is for a site carrying no usable base at all — neither a chain nor + * an explicit receiver — where the question cannot be asked. It is treated as + * `in-program` for hedging purposes, because assuming completeness we cannot + * demonstrate is the unsafe direction. + */ +export type ReceiverOrigin = 'in-program' | 'external' | 'unknown'; + +/** Classify a dropped receiver from its encoded chain. `undefined` chain ⇒ + * `no-chain`; an undecodable one is also `no-chain`, since what we know about + * it is exactly that no usable structure survived. + * + * Takes `DecodedReceiverChain` rather than a structural duck-type: widening + * `kind` to `string` let `await` and `index` fall into an `else` branch and be + * counted as FIELDS, so the two shapes this work exists to expose were + * censused as `chain-field`. The discriminated union makes a new step kind a + * compile error instead of a silent bucket. */ +export function classifyReceiverShape(decoded: DecodedReceiverChain | undefined): ReceiverShape { + if (decoded === undefined || decoded.steps.length === 0) return 'no-chain'; + let calls = 0; + let fields = 0; + let unwraps = 0; + for (const step of decoded.steps) { + switch (step.kind) { + case 'call': + calls++; + break; + case 'field': + fields++; + break; + case 'await': + case 'index': + unwraps++; + break; + } + } + // An unwrap step dominates: a chain containing one fails for reasons a pure + // field or call chain does not, so folding it into either bucket would + // misattribute the population a fix has to target. + if (unwraps > 0) return 'chain-unwrap'; + if (calls > 0 && fields > 0) return 'chain-mixed'; + return calls > 0 ? 'chain-call' : 'chain-field'; +} + export type ResolutionOutcomeRecorder = (outcome: ResolutionOutcome) => void; diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts index e250f88a5..b4b8aeab9 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts @@ -321,10 +321,67 @@ export function moduleScopeIdOf( * * Without (2) we'd miss every cross-file class-receiver call. */ +/** + * Every class-like definition visible for `name`, from the scope chain AND the + * qualified-name index, deduped by `nodeId`. + * + * Exists because `walkScopeChain` returns the FIRST match and cannot report a + * collision, so a caller that widens what a name can match (the decoration + * normalizer below) has no way to tell "one answer" from "picked the nearest of + * several". Mirrors `findAllCallableBindingsInScope`, which solved the same + * problem for callables. + */ +export function findAllClassBindingsInScope( + startScope: ScopeId, + name: string, + scopes: ScopeResolutionIndexes, +): readonly SymbolDefinition[] { + const inScope = findAllBindingsInScope(startScope, name, scopes, (def) => isClassLike(def.type)); + // The scope chain wins outright when it binds the name: an inner binding + // shadows anything the qualified-name index would contribute. + if (inScope.length > 0) return inScope; + + const byNodeId = new Map(); + for (const id of scopes.qualifiedNames.get(name)) { + const def = scopes.defs.get(id); + if (def !== undefined && isClassLike(def.type)) byNodeId.set(def.nodeId, def); + } + return [...byNodeId.values()]; +} + +/** + * Strip one layer of type-preserving decoration off a declared type name, or + * `undefined` when there is nothing left to strip. Supplied per language through + * the `ScopeResolver` contract; the core never names a language (AGENTS.md R6). + * + * TYPE-PRESERVING only — pointer, reference, `const`, nullable, borrow, + * deref-transparent smart pointer, sigil. A CONTAINER (array, slice, map, + * `Option`) changes the member set, so stripping one here would type + * `repos: Repo[]` as `Repo` and let `repos.find(x)` fold to `Repo.find`. Those + * are unwrapped only by an index step that consumed a subscript. + */ +export type DecorationStripper = (typeName: string) => string | undefined; + +/** Bounded so a pathological stripper cannot spin. Real decoration nests + * shallowly (`*[]T`, `const T&`); three layers is generous. */ +const MAX_DECORATION_LAYERS = 3; + export function findClassBindingInScope( startScope: ScopeId, receiverName: string, scopes: ScopeResolutionIndexes, + /** + * OPT-IN. When supplied, a name that binds nothing is retried with decoration + * stripped one layer at a time, and each retry must resolve to exactly ONE + * class-like definition or it declines. + * + * Opt-in rather than global because roughly two dozen call sites use the shape + * `findClassBindingInScope(...) ?? otherResolver(...)`: turning a former + * `undefined` into a hit SUPPRESSES the fallback that used to answer, which + * would retarget inheritance edges and bypass generic-specialization + * selection. Only receiver-chain base and step resolution passes this. + */ + stripDecoration?: DecorationStripper, ): SymbolDefinition | undefined { const local = walkScopeChain(startScope, receiverName, scopes, (def) => isClassLike(def.type)); if (local !== undefined) return local; @@ -350,6 +407,25 @@ export function findClassBindingInScope( } } } + + // Decoration fallback (opt-in). Every branch above works on the name exactly + // as written; only when none of them bound anything do we consider that the + // name may be a decorated spelling of one that would. + if (stripDecoration !== undefined) { + let current = receiverName; + for (let layer = 0; layer < MAX_DECORATION_LAYERS; layer++) { + const stripped = stripDecoration(current); + if (stripped === undefined || stripped === current || stripped.length === 0) break; + current = stripped; + const candidates = findAllClassBindingsInScope(startScope, current, scopes); + // Exactly one, or decline. Two same-named classes reachable from here mean + // the decoration was carrying the only disambiguating information, and + // picking the nearest would mint a confident wrong edge — the failure this + // whole line of work exists to avoid. A missing edge is recoverable. + if (candidates.length === 1) return candidates[0]; + if (candidates.length > 1) return undefined; + } + } return undefined; } @@ -787,10 +863,28 @@ export function findAllCallableBindingCandidatesInScope( * `findCallableBindingInScope`: once any callable binding is found in a * scope, outer scopes are not consulted. */ -export function findAllCallableBindingsInScope( +/** + * Every definition visible for `name` at the NEAREST scope that binds it, + * filtered by `predicate` and deduped by `nodeId`. + * + * THE shared "collect all at the nearest binding scope" walk. `walkScopeChain` + * answers the first-match question; this answers the how-many question, which is + * what a caller needs before it can decline on ambiguity. + * + * Stops at the first scope that binds the name at all: an inner binding SHADOWS + * an outer one, so continuing would report a shadowed outer definition as a + * competing candidate and decline a name that is unambiguous at this point. + * + * Returns `[]` on a cycle or a missing scope. That is deliberate and matters: + * an earlier copy of this walk `break`-ed instead and fell through to a + * qualified-name fallback, so the same malformed input produced a different + * answer depending on which copy the caller happened to reach. + */ +function findAllBindingsInScope( startScope: ScopeId, - callableName: string, + name: string, scopes: ScopeResolutionIndexes, + predicate: (def: SymbolDefinition) => boolean, ): readonly SymbolDefinition[] { let currentId: ScopeId | null = startScope; const visited = new Set(); @@ -805,24 +899,16 @@ export function findAllCallableBindingsInScope( if (scope.kind !== 'Object') { const out: SymbolDefinition[] = []; const seen = new Set(); - const pushCallable = (def: SymbolDefinition): void => { - if (def.type !== 'Function' && def.type !== 'Method' && def.type !== 'Constructor') return; + const push = (def: SymbolDefinition): void => { + if (!predicate(def)) return; if (seen.has(def.nodeId)) return; seen.add(def.nodeId); out.push(def); }; - const localBindings = scope.bindings.get(callableName); - if (localBindings !== undefined) { - for (const b of localBindings) { - pushCallable(b.def); - } - } - - const importedBindings = lookupBindingsAt(currentId, callableName, scopes); - for (const b of importedBindings) { - pushCallable(b.def); - } + // Local first: a binding in this scope shadows an imported one. + for (const b of scope.bindings.get(name) ?? []) push(b.def); + for (const b of lookupBindingsAt(currentId, name, scopes)) push(b.def); if (out.length > 0) return out; } @@ -831,6 +917,19 @@ export function findAllCallableBindingsInScope( return []; } +export function findAllCallableBindingsInScope( + startScope: ScopeId, + callableName: string, + scopes: ScopeResolutionIndexes, +): readonly SymbolDefinition[] { + return findAllBindingsInScope( + startScope, + callableName, + scopes, + (def) => def.type === 'Function' || def.type === 'Method' || def.type === 'Constructor', + ); +} + /** * ISO C++ `[basic.lookup.unqual]` §7: ADL is suppressed when ordinary * unqualified lookup finds: diff --git a/gitnexus/src/core/ingestion/scope-resolution/unresolved-receivers.ts b/gitnexus/src/core/ingestion/scope-resolution/unresolved-receivers.ts index db6cfb03b..11e7d7f61 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/unresolved-receivers.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/unresolved-receivers.ts @@ -31,6 +31,58 @@ export interface UnresolvedReceiverSummary { /** Distinct member names beyond the cap, omitted from `counts`. Absent when * nothing was dropped from the map. */ readonly omittedNames?: number; + /** + * Call sites dropped whose receiver was rooted OUTSIDE the indexed program, + * by member name — `System.out.println`, `fetch(...)`, `os.environ.*`. + * + * Kept SEPARATE rather than filtered away. These do not make a count a lower + * bound (there is no in-graph node an edge could have reached), but erasing + * them at summary time would leave the persisted artifact unable to + * distinguish "clean index" from "76 drops we judged external" — with no + * audit path and no way back without a re-index. That is the same collapse + * `EpistemicCauses` exists to undo, and the judgement being recorded here is a + * heuristic, so it must stay reversible. + */ + readonly externalCounts?: Readonly>; + /** Total external-rooted call sites, including any beyond the cap. */ + readonly externalSites?: number; + /** + * Distinct member names beyond the cap, omitted from `externalCounts`. Absent + * when nothing was dropped from that map. + * + * The exact twin of `omittedNames`, and it exists for the same reason. Past + * the cap `lookupExternalCallCount` returns `undefined` for a truncated name, + * which is indistinguishable from "this member had no external drops" — so a + * symbol with real boundary evidence reads as having none. One map carrying a + * truncation marker and the other silently losing entries also made the + * persisted artifact self-contradictory: `externalSites` would exceed the sum + * of `externalCounts` with nothing to explain the difference. + */ + readonly externalOmittedNames?: number; +} + +/** + * Rank a name→count map and cap it at {@link MAX_UNRESOLVED_RECEIVER_MEMBERS}. + * + * Highest count first, name as a tiebreak so the persisted map is stable across + * runs — an unstable ordering would churn the metadata file (and its diff) on + * every analyze for no behavioural reason. ONE comparator, shared by the + * in-program and external maps: two hand-copied comparators that must stay + * identical or the artifact churns on one map and not the other is exactly the + * drift this contract cannot tolerate. + * + * `omitted` is the number of distinct names past the cap, so the caller can + * report truncation rather than silently losing entries. + */ +function rankAndCap(counts: Map): { + kept: [string, number][]; + omitted: number; +} { + const ranked = [...counts.entries()].sort( + ([aName, aCount], [bName, bCount]) => bCount - aCount || aName.localeCompare(bName), + ); + const kept = ranked.slice(0, MAX_UNRESOLVED_RECEIVER_MEMBERS); + return { kept, omitted: ranked.length - kept.length }; } /** @@ -43,7 +95,9 @@ export function summarizeUnresolvedReceivers( outcomes: readonly ResolutionOutcome[], ): UnresolvedReceiverSummary | undefined { const counts = new Map(); + const externalCounts = new Map(); let totalSites = 0; + let externalSites = 0; for (const outcome of outcomes) { if (outcome.kind !== 'suppressed' || outcome.reason !== 'receiver-unresolved') continue; if (outcome.name.length === 0) continue; @@ -56,24 +110,43 @@ export function summarizeUnresolvedReceivers( // A missing `siteKind` counts as a call: the only emitter always sets it, and // erring toward `lower-bound` is the safe direction for an epistemic signal. if (outcome.siteKind !== undefined && outcome.siteKind !== 'call') continue; + // Routed, not discarded. External-rooted drops (`console.log(...)`, + // `fetch(...)`) reach code this index does not contain, so there is no node + // an edge could have pointed at and nothing was lost — they must not hedge. + // But they stay in the artifact under their own key so the split is + // auditable and reversible. + // + // `external` is a POSITIVE determination made by `classifyReceiverOrigin` + // from a language built-in match, never a fallthrough: a receiver the + // classifier could not place lands in `unknown`, which counts here WITH + // `in-program`, because assuming a completeness we cannot demonstrate is + // the unsafe direction. + if (outcome.receiverOrigin === 'external') { + externalSites++; + externalCounts.set(outcome.name, (externalCounts.get(outcome.name) ?? 0) + 1); + continue; + } totalSites++; counts.set(outcome.name, (counts.get(outcome.name) ?? 0) + 1); } - if (totalSites === 0) return undefined; + // An index whose only drops were external-rooted still reports the split, so + // "nothing was lost" is distinguishable from "nothing was measured". + if (totalSites === 0 && externalSites === 0) return undefined; - // Highest count first, name as a tiebreak so the persisted map is stable - // across runs — an unstable ordering would churn the metadata file (and its - // diff) on every analyze for no behavioural reason. - const ranked = [...counts.entries()].sort( - ([aName, aCount], [bName, bCount]) => bCount - aCount || aName.localeCompare(bName), - ); - const kept = ranked.slice(0, MAX_UNRESOLVED_RECEIVER_MEMBERS); - const omittedNames = ranked.length - kept.length; + const { kept, omitted: omittedNames } = rankAndCap(counts); + const { kept: externalKept, omitted: externalOmittedNames } = rankAndCap(externalCounts); return { counts: Object.fromEntries(kept), totalSites, ...(omittedNames > 0 ? { omittedNames } : {}), + ...(externalSites > 0 + ? { + externalCounts: Object.fromEntries(externalKept), + externalSites, + ...(externalOmittedNames > 0 ? { externalOmittedNames } : {}), + } + : {}), }; } @@ -102,3 +175,22 @@ export function lookupUnresolvedCallCount( if (typeof sites !== 'number' || !Number.isFinite(sites) || sites <= 0) return undefined; return sites; } + +/** + * Look up the EXTERNAL-rooted dropped-call count for a member name. + * + * Companion to `lookupUnresolvedCallCount`, and prototype-safe for the same + * reason: the map is revived from JSON, so `constructor` / `toString` and + * friends would otherwise return a function. + */ +export function lookupExternalCallCount( + summary: UnresolvedReceiverSummary | undefined, + symName: string, +): number | undefined { + const counts = summary?.externalCounts; + if (counts === undefined || symName.length === 0) return undefined; + if (!Object.hasOwn(counts, symName)) return undefined; + const sites = counts[symName]; + if (typeof sites !== 'number' || !Number.isFinite(sites) || sites <= 0) return undefined; + return sites; +} diff --git a/gitnexus/src/core/ingestion/type-extractors/shared.ts b/gitnexus/src/core/ingestion/type-extractors/shared.ts index 17cb962cd..1da3ff12e 100644 --- a/gitnexus/src/core/ingestion/type-extractors/shared.ts +++ b/gitnexus/src/core/ingestion/type-extractors/shared.ts @@ -1,3 +1,4 @@ +import type { ElementAccessRoute } from '../scope-resolution/contract/scope-resolver.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; /** Which type argument to extract from a multi-arg generic container. @@ -698,6 +699,27 @@ export function extractElementTypeFromString( return undefined; } +/** + * `elementTypeOf` for a language with no property-style collection view: + * answer the subscript route from the written spelling, decline the accessor + * route. + * + * Languages whose collection views are spelled as method calls (`.values()`, + * `.iter()`) never reach the accessor route — the compound resolver's + * call-expression branch handles those — so the whole hook is this one + * decision. Declining (returning `undefined`) is the answer "this spelling is + * not a container", which is what stops an index-overloading class from + * folding `x[k].m()` onto its own members. + * + * Languages that DO expose a collection view as a property (C#'s `.Values`) + * need a bespoke body instead. + */ +export const indexOnlyElementType = ( + containerType: string, + via: ElementAccessRoute, +): string | undefined => + via.kind === 'index' ? extractElementTypeFromString(containerType) : undefined; + // ── Return type text helpers ───────────────────────────────────────────── // extractReturnTypeName works on raw return-type text already stored in // SymbolDefinition (e.g. "User", "Promise", "User | null", "*User"). diff --git a/gitnexus/src/core/ingestion/utils/call-analysis.ts b/gitnexus/src/core/ingestion/utils/call-analysis.ts index ea43911d9..86be8b8cd 100644 --- a/gitnexus/src/core/ingestion/utils/call-analysis.ts +++ b/gitnexus/src/core/ingestion/utils/call-analysis.ts @@ -2,6 +2,7 @@ import type { MixedChainStep } from 'gitnexus-shared'; import type { SyntaxNode } from './ast-helpers.js'; import { CALL_ARGUMENT_LIST_TYPES } from './ast-helpers.js'; +import { subscriptBase } from './callable-flow-captures.js'; /** Node types representing call expressions across supported languages. */ export const CALL_EXPRESSION_TYPES = new Set([ @@ -16,6 +17,24 @@ export const CALL_EXPRESSION_TYPES = new Set([ /** * Hard limit on chain depth to prevent runaway recursion. * For `a.b().c().d()`, the chain has depth 2 (b and c before d). + * + * A chain deeper than this is DISCARDED WHOLE, not truncated: + * `extractMixedChain` returns an undefined base and the encoder refuses to mint + * a partial chain, because a base-side prefix decodes cleanly as a shorter, + * complete-looking chain and would type the receiver against the wrong member. + * Correct, but it means a builder chain one hop too long contributes nothing at + * all rather than degrading. + * + * DELIBERATELY NOT RAISED. Measured (see bench/receiver-resolution/BASELINE.md, + * `fourHopChain`): a 4-step chain mints NOTHING at this cap — confirmed by + * probing the emitter directly — and the site still RESOLVES, because the text + * cascade that owns the fallback path runs to `COMPOUND_RECEIVER_MAX_DEPTH` (8) + * and answers where the structural fold declined. + * + * So the cap bounds which chains are typed STRUCTURALLY, not which calls + * resolve. Raising it moves work from the cascade to the fold without changing + * any edge, and the fixture that proves it is committed so the next person to + * reach for this number has the measurement rather than the intuition. */ export const MAX_CHAIN_DEPTH = 3; @@ -353,6 +372,51 @@ export const extractReceiverNode = (nameNode: SyntaxNode): SyntaxNode | undefine // ── Chained-call extraction ─────────────────────────────────────────────── /** Node types representing member/field access across languages. */ +/** + * Await expressions, per grammar. The walk previously stopped here — an await + * node is neither a call nor a field access — so `(await svc.getUserAsync()).save()` + * minted NO chain at all and the receiver fell to the text cascade. + */ +const AWAIT_EXPRESSION_NODE_TYPES = new Set([ + 'await_expression', // TS/JS/C#/Rust + 'await', // Python +]); + +/** + * Subscript / index expressions, per grammar. Same story as await: the walk + * stopped, so `repos[0].save()` minted no chain — which is why `indexElement` + * is an INVISIBLE-GAP (no edge AND no recorded drop) in all 14 languages, the + * most uniform cell in the matrix. + */ +const SUBSCRIPT_NODE_TYPES = new Set([ + 'subscript_expression', // TS/JS/PHP/C/C++ + 'subscript', // Python + 'index_expression', // Go/Rust + 'element_access_expression', // C# + 'array_access', // Java + 'indexing_expression', // Kotlin +]); + +/** + * Can the chain walk descend into this node, or is it the base? + * + * ONE predicate for all four branches. The call and field branches previously + * tested only the call/field sets, so a receiver like `x[0].f().g()` stopped at + * the subscript and returned the literal text `x[0]` as the base — which + * `isEncodableSegment` accepts, minting a chain whose base binds to nothing. And + * `(await f()).g.h()` returned `await f()`, rejected on whitespace, minting no + * chain at all. Adding a fifth step kind must not require remembering four + * separate call sites. + */ +function isChainableReceiverNode(node: SyntaxNode): boolean { + return ( + CALL_EXPRESSION_TYPES.has(node.type) || + FIELD_ACCESS_NODE_TYPES.has(node.type) || + AWAIT_EXPRESSION_NODE_TYPES.has(node.type) || + SUBSCRIPT_NODE_TYPES.has(node.type) + ); +} + const FIELD_ACCESS_NODE_TYPES = new Set([ 'member_expression', // TS/JS 'member_access_expression', // C# @@ -497,11 +561,25 @@ const TRANSPARENT_RECEIVER_WRAPPERS = new Set([ 'parenthesized_expression', // `(svc)` ]); +/** + * Iteration bound for the wrapper peel. Its OWN constant, not `MAX_CHAIN_DEPTH`. + * + * The two answer unrelated questions — "how many chain hops do we type?" versus + * "how many redundant parens might someone write?" — and sharing one number + * meant raising the chain cap silently widened this loop as a side effect. That + * coupling is easy to miss precisely because the shared name reads as + * intentional. `((x))` nests twice; nothing real nests deeply. + */ +const MAX_TRANSPARENT_WRAPPER_DEPTH = 3; + /** Peel transparent wrappers off a base receiver node. */ function unwrapTransparentReceiver(node: SyntaxNode): SyntaxNode { let current = node; - // Bounded: `((x))` nests twice; nothing real nests deeply. - for (let i = 0; i < MAX_CHAIN_DEPTH && TRANSPARENT_RECEIVER_WRAPPERS.has(current.type); i++) { + for ( + let i = 0; + i < MAX_TRANSPARENT_WRAPPER_DEPTH && TRANSPARENT_RECEIVER_WRAPPERS.has(current.type); + i++ + ) { const inner = current.namedChildren?.find((c) => c !== null); if (inner === undefined || inner === null) break; current = inner; @@ -516,6 +594,12 @@ export function extractMixedChain( let current: SyntaxNode = receiverNode; while (chain.length < MAX_CHAIN_DEPTH) { + // Peel transparent wrappers at LOOP ENTRY, not only where a base is + // returned. `(await svc.getUserAsync()).save()` hands this walk a + // `parenthesized_expression`, which matches no branch below, so the walk + // fell straight through to the base case with an empty chain and minted + // nothing — the await step could never be reached. + current = unwrapTransparentReceiver(current); if (CALL_EXPRESSION_TYPES.has(current.type)) { // ── Call expression: extract method name + inner receiver ──────────── const funcNode = @@ -572,10 +656,7 @@ export function extractMixedChain( } if (!innerReceiver) break; - if ( - CALL_EXPRESSION_TYPES.has(innerReceiver.type) || - FIELD_ACCESS_NODE_TYPES.has(innerReceiver.type) - ) { + if (isChainableReceiverNode(innerReceiver)) { current = innerReceiver; } else { return { @@ -624,10 +705,7 @@ export function extractMixedChain( if (!innerObject) break; - if ( - CALL_EXPRESSION_TYPES.has(innerObject.type) || - FIELD_ACCESS_NODE_TYPES.has(innerObject.type) - ) { + if (isChainableReceiverNode(innerObject)) { current = innerObject; } else { return { @@ -635,6 +713,50 @@ export function extractMixedChain( baseReceiverName: unwrapTransparentReceiver(innerObject).text || undefined, }; } + } else if (AWAIT_EXPRESSION_NODE_TYPES.has(current.type)) { + // Name-free: the awaited call's method name already lives on its own + // `call` step, so this records only that an await happened. + chain.unshift({ kind: 'await' }); + const inner = + current.childForFieldName?.('argument') ?? + current.childForFieldName?.('expression') ?? + current.namedChildren?.find((c: SyntaxNode) => c !== null) ?? + null; + if (!inner) break; + if (isChainableReceiverNode(inner)) { + current = inner; + } else { + return { + chain, + baseReceiverName: unwrapTransparentReceiver(inner).text || undefined, + }; + } + } else if (SUBSCRIPT_NODE_TYPES.has(current.type)) { + // Name-free: a subscript key is a VALUE, not an identifier the resolver + // could look up, so there is no member name to record. + chain.unshift({ kind: 'index' }); + // Shared per-grammar table — it knows Python's `value` and Java's `array`, + // which a locally-written ladder omitted (they worked only because the + // container happened to be the first named child, an ordering coincidence + // rather than a contract). Falls back for grammars whose subscript node + // carries no `index` field. + const obj = + subscriptBase(current) ?? + current.childForFieldName?.('object') ?? + current.childForFieldName?.('argument') ?? + current.childForFieldName?.('operand') ?? + current.childForFieldName?.('expression') ?? + current.namedChildren?.find((c: SyntaxNode) => c !== null) ?? + null; + if (!obj) break; + if (isChainableReceiverNode(obj)) { + current = obj; + } else { + return { + chain, + baseReceiverName: unwrapTransparentReceiver(obj).text || undefined, + }; + } } else if (current.type === 'selector') { // ── Dart: flat selector siblings (user.address.save() uses selector nodes) ── // Extract field name from unconditional_assignable_selector child diff --git a/gitnexus/src/core/ingestion/utils/callable-flow-captures.ts b/gitnexus/src/core/ingestion/utils/callable-flow-captures.ts index eb18a0f8b..4ca872ec4 100644 --- a/gitnexus/src/core/ingestion/utils/callable-flow-captures.ts +++ b/gitnexus/src/core/ingestion/utils/callable-flow-captures.ts @@ -1092,7 +1092,10 @@ function unaryOperator(node: SyntaxNode): string | undefined { * `tbl[i]()` join (#2522 review). Field names cover the grammars that field * their subscript nodes; others keep the generic traversal. */ -function subscriptBase(node: SyntaxNode): SyntaxNode | null { +/** The container operand of a subscript node, per grammar. Exported because the + * receiver-chain walk needs the same per-grammar answer — two divergent field + * tables for one question is how a new grammar gets half-supported. */ +export function subscriptBase(node: SyntaxNode): SyntaxNode | null { if (node.childForFieldName('index') === null) return null; return ( node.childForFieldName('argument') ?? // C/C++ subscript_expression diff --git a/gitnexus/src/core/ingestion/utils/receiver-chain-codec.ts b/gitnexus/src/core/ingestion/utils/receiver-chain-codec.ts index 4bb68eace..6d4b8678f 100644 --- a/gitnexus/src/core/ingestion/utils/receiver-chain-codec.ts +++ b/gitnexus/src/core/ingestion/utils/receiver-chain-codec.ts @@ -16,10 +16,10 @@ * and every chain array as a distinct allocation on every warm load, while a * string collapses to one interned instance per distinct chain. * - * ## Wire format (version `1`) + * ## Wire format (version `2`) * * ``` - * 1|||…[|~] + * 2|||…[|~] * ``` * * - One-character version prefix, then the BASE receiver name, then ordered @@ -29,6 +29,12 @@ * character of the segment and the name follows immediately, so a member * whose name begins with `c` or `f` needs no escaping (`ccount` decodes as a * call to `count`). + * - `a` = await and `i` = index are NAME-FREE and encode as a BARE sigil: an + * awaited call's name already lives on its `c` step, and a subscript's key is + * a value rather than an identifier the resolver could look up. The decoder + * rejects any trailing characters after `a` or `i`, which is what keeps an + * accidentally empty-name `c` or `f` segment refusing instead of decoding as + * one of these. * - A trailing `|~` segment is the TRUNCATION MARKER. NOTE: no current producer * mints one. `extractMixedChain` signals "stopped early" by returning * `baseReceiverName: undefined`, and the encoder requires a base, so a @@ -46,17 +52,44 @@ * different valid chain. * * For `svc.getUser().address.save()`, the receiver of `save` encodes as - * `1|svc|cgetUser|faddress` — 23 bytes. + * `2|svc|cgetUser|faddress` — 23 bytes. For `(await svc.getUserAsync()).save()` + * it is `2|svc|cgetUserAsync|a`. */ import type { MixedChainStep } from 'gitnexus-shared'; import { MAX_CHAIN_DEPTH } from './call-analysis.js'; -const VERSION = '1'; +/** Wire version. Bumped 1 → 2 when the name-free `await` and `index` step kinds + * were added: a v1 decoder reading a v2 payload must REFUSE, not decode the + * prefix it happens to understand, because a chain missing its await or index + * hop decodes cleanly as a different, shorter chain and would type the receiver + * against the wrong member. */ +const VERSION = '2'; const SEPARATOR = '|'; const TRUNCATED = '~'; +/** Sigil per step kind. ONE table, and both directions derive from it — the + * decoder used to hand-write `sigil === 'c' ? 'call' : 'field'` and its own + * await/index comparison, which meant the decoder was exactly the side that + * could drift from the table claiming to prevent drift. */ +const SIGIL_BY_KIND = { + call: 'c', + field: 'f', + await: 'a', + index: 'i', +} as const; + +type StepKind = keyof typeof SIGIL_BY_KIND; + +/** Kinds that encode as a BARE sigil, because they have no member name to + * carry. Derived from the step union rather than listed twice. */ +const NAME_FREE_KINDS = new Set(['await', 'index']); + +const KIND_BY_SIGIL: ReadonlyMap = new Map( + Object.entries(SIGIL_BY_KIND).map(([kind, sigil]) => [sigil, kind as StepKind]), +); + /** Hard cap on the encoded payload. `MAX_CHAIN_DEPTH` already bounds the step * COUNT; this bounds the total bytes so a pathological identifier cannot grow * a shard without limit. Generous against real identifiers — the encoding for @@ -103,8 +136,15 @@ export function encodeReceiverChain( const parts = [VERSION, baseReceiverName]; for (const step of steps) { + // Name-free kinds encode as a bare sigil. They are exempt from the + // non-empty-name guard because they HAVE no name to check — not because the + // guard is relaxed: an empty-name `call` or `field` is still refused below. + if (NAME_FREE_KINDS.has(step.kind)) { + parts.push(SIGIL_BY_KIND[step.kind]); + continue; + } if (!isEncodableSegment(step.name)) return undefined; - parts.push(`${step.kind === 'call' ? 'c' : 'f'}${step.name}`); + parts.push(`${SIGIL_BY_KIND[step.kind]}${step.name}`); } if (options?.truncated === true) parts.push(TRUNCATED); @@ -139,9 +179,20 @@ export function decodeReceiverChain(value: unknown): DecodedReceiverChain | unde for (const part of stepParts) { const sigil = part[0]; const name = part.slice(1); - if (sigil !== 'c' && sigil !== 'f') return undefined; + // Name-free kinds must be EXACTLY their sigil. Rejecting a trailing tail is + // what keeps an accidentally empty-name call or field from decoding as one + // of these: `c` alone stays malformed, it does not become an await. + const kind = sigil === undefined ? undefined : KIND_BY_SIGIL.get(sigil); + if (kind === undefined) return undefined; + if (NAME_FREE_KINDS.has(kind)) { + // Must be EXACTLY the sigil. Rejecting a trailing tail is what keeps an + // accidentally empty-name call or field from decoding as one of these. + if (name.length > 0) return undefined; + steps.push({ kind: kind as 'await' | 'index' }); + continue; + } if (!isEncodableSegment(name)) return undefined; - steps.push({ kind: sigil === 'c' ? 'call' : 'field', name }); + steps.push({ kind: kind as 'call' | 'field', name }); } return { baseReceiverName, steps, truncated }; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 1914b0101..3a80eaa71 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -283,7 +283,10 @@ export interface ExtractedCall { * `svc.getUser().save()` → chain=[{kind:'call',name:'getUser'}], receiverName='svc' * `user.address.save()` → chain=[{kind:'field',name:'address'}], receiverName='user' * `svc.getUser().address.save()` → chain=[{kind:'call',name:'getUser'},{kind:'field',name:'address'}] - * Length is capped at MAX_CHAIN_DEPTH (3). + * Length is capped at MAX_CHAIN_DEPTH. Deliberately NOT restating the number + * here: this comment previously hardcoded `(3)` and would have drifted the + * moment the cap moved, which is exactly the kind of stale doc that reads as + * authoritative. */ receiverMixedChain?: MixedChainStep[]; argTypes?: (string | undefined)[]; diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 68a66aa0e..4444a9687 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -17,8 +17,8 @@ import { createWriteStream, WriteStream } from 'fs'; import path from 'path'; import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; -import { NodeTableName, NODE_TABLES, RELATION_SCHEMA } from './schema.js'; -import { parseRelationSchemaPairs, RelPairRouter } from './rel-pair-routing.js'; +import { NodeTableName, RELATION_SCHEMA } from './schema.js'; +import { VALID_NODE_TABLES, parseRelationSchemaPairs, RelPairRouter } from './rel-pair-routing.js'; import { parseTruthyEnv } from '../ingestion/utils/env.js'; import { SYMBOL_NODE_LABELS } from '../ingestion/utils/symbol-labels.js'; import { applyCjkSegmentationIfEnabled } from '../search/cjk-segmentation.js'; @@ -793,13 +793,13 @@ export const streamAllCSVsToDisk = async ( const relRouter = new RelPairRouter( csvDir, REL_CSV_HEADER, - new Set(NODE_TABLES), + VALID_NODE_TABLES, DECLARED_RELATION_PAIRS, ); try { let emitted = 0; for (const rel of orderedRelationships(graph, sortOutput)) { - const pending = relRouter.route(rel.sourceId, rel.targetId, buildRelRow(rel)); + const pending = relRouter.route(rel.sourceId, rel.targetId, buildRelRow(rel), rel.type); if (pending) await pending; // Periodically hand the event loop back so the overlapped node COPY and // write-stream drains run instead of starving behind this synchronous diff --git a/gitnexus/src/core/lbug/graph-emit-sink.ts b/gitnexus/src/core/lbug/graph-emit-sink.ts index c03bacf01..f1527c6d2 100644 --- a/gitnexus/src/core/lbug/graph-emit-sink.ts +++ b/gitnexus/src/core/lbug/graph-emit-sink.ts @@ -85,9 +85,10 @@ * ## Correctness contract * * Structural sibling of {@link PdgEmitSink}, and reuses its row builder - * (`buildRelRow`), header (`REL_CSV_HEADER`), label derivation (`getNodeLabel`) - * and `RelPairRouter` validity check, so the streamed row SET equals the - * whole-graph emit's and the bulk COPY loads the same rows. Set-level, not + * (`buildRelRow`), header (`REL_CSV_HEADER`) and pair classification + * (`relPairKeyFor`, which is also what `RelPairRouter` routes and skips by), so + * the streamed row SET equals the whole-graph emit's and the bulk COPY loads + * the same rows. Set-level, not * byte-level: rows stream in emit order and are not re-sorted under * `GITNEXUS_SORT_GRAPH_OUTPUT`. */ @@ -96,8 +97,12 @@ import path from 'path'; import type { GraphNode, GraphRelationship, RelationshipType } from 'gitnexus-shared'; import type { KnowledgeGraph } from '../graph/types.js'; import { DECLARED_RELATION_PAIRS, REL_CSV_HEADER, buildRelRow } from './csv-generator.js'; -import { assertDeclaredPair, getNodeLabel } from './rel-pair-routing.js'; -import { NODE_TABLES } from './schema.js'; +import { + VALID_NODE_TABLES, + assertDeclaredPair, + relPairKeyFor, + splitRelPairKey, +} from './rel-pair-routing.js'; import { DEFAULT_EMIT_CHUNK_ROWS, SyncCsvWriter } from './sync-csv-writer.js'; /** @@ -127,8 +132,8 @@ import { DEFAULT_EMIT_CHUNK_ROWS, SyncCsvWriter } from './sync-csv-writer.js'; * `routes`/`tools`, never read back mid-pipeline). * * Adding a relationship type that a phase reads back WITHOUT adding it here is - * a silent-wrong-graph bug, not a crash — and NOTHING automated catches it. - * The differential round-trip test cannot: `addRelationship` partitions edges + * a silent-wrong-graph bug, not a crash. The differential round-trip test cannot: + * `addRelationship` partitions edges * between the graph and the CSVs, and the union of a partition is invariant * under where the partition line falls, so that test stays green no matter how * this set is drawn. Only the read-site audit protects this invariant; re-run it @@ -144,6 +149,8 @@ export const RETAINED_REL_TYPES: ReadonlySet = new Set; private readonly relWriters = new Map(); /** * Ids of relationships already streamed. `KnowledgeGraph.addRelationship` @@ -301,7 +307,6 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { private readonly csvDir: string, private readonly chunkRows: number = DEFAULT_EMIT_CHUNK_ROWS, ) { - this.validTables = new Set(NODE_TABLES as readonly string[]); // Own directory, distinct from the PDG sink's: PdgEmitSink wipes and // recreates its dir on construction and opens with O_EXCL, so a shared dir // would destroy the other sink's manifest on a combined --pdg run. @@ -405,16 +410,24 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { } // Mirror KnowledgeGraph.addRelationship's first-writer-wins dedup. - const fromLabel = getNodeLabel(relationship.sourceId); - const toLabel = getNodeLabel(relationship.targetId); - // Skip edges whose endpoint labels are not valid node tables — mirrors - // `RelPairRouter` exactly so the streamed set matches the whole-graph set. - if (!this.validTables.has(fromLabel) || !this.validTables.has(toLabel)) return; + // Classify + skip via the SHARED `relPairKeyFor`, not a local copy of its + // three lines, so the streamed set cannot drift from the whole-graph set + // `RelPairRouter` produces. `undefined` = an endpoint label is not a node + // table, so the edge is dropped exactly as the router drops it. + const pairKey = relPairKeyFor(relationship.sourceId, relationship.targetId, VALID_NODE_TABLES); + if (pairKey === undefined) return; - const pairKey = `${fromLabel}|${toLabel}`; - assertDeclaredPair(pairKey, DECLARED_RELATION_PAIRS); + assertDeclaredPair( + pairKey, + DECLARED_RELATION_PAIRS, + relationship.type, + relationship.sourceId, + relationship.targetId, + ); let writer = this.relWriters.get(pairKey); if (writer === undefined) { + // Cold: once per pair, so decoding the key back into its labels is free. + const [fromLabel, toLabel] = splitRelPairKey(pairKey); try { writer = new SyncCsvWriter( path.join(this.csvDir, `rel_${fromLabel}_${toLabel}.csv`), diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index d107f5e79..597676455 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -19,6 +19,12 @@ import { STALE_HASH_SENTINEL, NodeTableName, } from './schema.js'; +// Analyze-only, but reached from MCP startup via `pool-adapter.js`. #2802 +// proposed lazy-importing it; rejected — `core/search/bm25-index.ts` statically +// imports `normalizeFtsText` from `csv-generator.js`, and `local-backend.ts` +// dynamically imports bm25-index on the FTS query path, so deferring here +// relocates the startup cost to first query rather than removing it. The +// measured figures live in #2802; they were environment-bound, this is not. import { streamAllCSVsToDisk, type StreamedCSVResult } from './csv-generator.js'; import type { GraphEmitManifest } from './graph-emit-sink.js'; import type { PdgEmitManifest } from './pdg-emit-sink.js'; @@ -65,6 +71,7 @@ import { SPRING_AUTO_CONFIGURATION_REASONS, SPRING_AUTO_CONFIGURATION_SYNTHETIC_ID_PREFIX, } from '../ingestion/frameworks/spring/auto-configuration.js'; +import { SPRING_AOP_EVIDENCE_ID_PREFIX } from '../ingestion/frameworks/spring/aop.js'; // --------------------------------------------------------------------------- // Relationship CSV splitting — extracted for testability (PR #818) // --------------------------------------------------------------------------- @@ -520,6 +527,9 @@ const queryAndDrain = async (targetConn: lbug.Connection, cypher: string): Promi return isSharedSingletonConn(targetConn) ? withConnLock(run) : run(); }; +// determinism: probe — existence only. Every call site runs this through +// `queryAndDrain`, which drains and discards the rows; the ONLY observable is +// whether the read-only shadow replay throws, so no row identity is read. const READ_ONLY_SHADOW_REPLAY_PROBE = 'MATCH (n) RETURN n LIMIT 1'; /** @@ -1828,6 +1838,9 @@ export const loadCachedEmbeddings = async (): Promise<{ // Old schema only had (nodeId, embedding); new schema adds (id, chunkIndex, startLine, endLine, contentHash). // If the query fails (column missing), we return empty cache to force a full rebuild. try { + // determinism: probe — schema probe, not a sample. `readQueryRows` drains + // the result and the rows are dropped on the floor; only whether the + // new-schema columns parse decides the branch. const check = await c.query( `MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex LIMIT 1`, ); @@ -2745,6 +2758,53 @@ export const deleteAllCallSummaries = async (): Promise<{ edgesDeleted: number } export const deleteAllInjects = async (): Promise<{ edgesDeleted: number }> => deleteAllRelationshipsOfType('INJECTS', 'di', 'duplicate INJECTS edges'); +/** + * Drop every Spring AOP `ADVISED_BY` relationship before incremental + * writeback. Pointcut/annotation resolution is whole-program: adding a type in + * a third file can shadow a wildcard annotation import or change a wildcard + * execution match between two otherwise unchanged endpoint files. + */ +export const deleteAllAdvisedBy = async (): Promise<{ edgesDeleted: number }> => + deleteAllRelationshipsOfType('ADVISED_BY', 'spring-aop', 'duplicate ADVISED_BY edges'); + +/** Drop all synthetic Spring AOP evidence nodes before incremental writeback. */ +export const deleteSpringAopEvidenceNodes = async (): Promise<{ nodesDeleted: number }> => { + const c = conn; + if (!c) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + return withConnLock(async () => { + let countResult: lbug.QueryResult | lbug.QueryResult[] | undefined; + const idPrefix = escapeCypherString(SPRING_AOP_EVIDENCE_ID_PREFIX); + const predicate = `n.id STARTS WITH '${idPrefix}'`; + try { + countResult = await c.query( + `MATCH (n:CodeElement) WHERE ${predicate} RETURN count(n) AS cnt`, + ); + const result = Array.isArray(countResult) ? countResult[0] : countResult; + const rows = await result.getAll(); + const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0); + if (count > 0) { + await closeQueryResults( + await c.query(`MATCH (n:CodeElement) WHERE ${predicate} DETACH DELETE n`), + ); + } + if (countResult) await closeQueryResults(countResult); + return { nodesDeleted: count }; + } catch (err) { + if (countResult) await closeQueryResults(countResult); + if (classifyDeleteAllError(err) === 'benign-missing-table') { + return { nodesDeleted: 0 }; + } + const message = err instanceof Error ? err.message : String(err); + throw new Error( + '[spring-aop] failed to clear synthetic evidence before incremental re-write ' + + `(${message}) — aborting to avoid stale advice metadata; the next run will full-rebuild`, + ); + } + }); +}; + /** * Drop Spring-owned auto-configuration `DECLARES` relationships before * incremental writeback. `DECLARES` is generic, so exact reason filtering is @@ -3126,6 +3186,26 @@ export const classifyFtsQueryError = (message: string): FtsQueryFailureClass => return 'other'; }; +/** + * Build the `QUERY_FTS_INDEX` statement shared by BOTH FTS read paths — + * `queryFTS` below and `queryFTSViaExecutor` in `core/search/bm25-index.ts` + * (the MCP connection-pool path). The two ran byte-identical cypher from two + * places, so every change had to be applied twice in lockstep — the `, node.id` + * ORDER BY tiebreak for #2787 being the latest. Lives beside + * {@link classifyFtsQueryError}, which was already shared for exactly this call. + */ +export const buildFtsQueryCypher = ( + tableName: string, + indexName: string, + limit: number, + conjunctive: boolean = false, +): string => ` + CALL QUERY_FTS_INDEX('${tableName}', '${indexName}', $query, conjunctive := ${conjunctive}) + RETURN node, score + ORDER BY score DESC, node.id + LIMIT ${limit} + `; + /** * Query a full-text search index * @param tableName - The node table name @@ -3148,12 +3228,7 @@ export const queryFTS = async ( throw new Error('LadybugDB not initialized. Call initLbug first.'); } - const cypher = ` - CALL QUERY_FTS_INDEX('${tableName}', '${indexName}', $query, conjunctive := ${conjunctive}) - RETURN node, score - ORDER BY score DESC - LIMIT ${limit} - `; + const cypher = buildFtsQueryCypher(tableName, indexName, limit, conjunctive); try { const rows = await executePrepared(cypher, { query }); diff --git a/gitnexus/src/core/lbug/pdg-emit-sink.ts b/gitnexus/src/core/lbug/pdg-emit-sink.ts index cc5560d2a..78d337816 100644 --- a/gitnexus/src/core/lbug/pdg-emit-sink.ts +++ b/gitnexus/src/core/lbug/pdg-emit-sink.ts @@ -24,8 +24,8 @@ * `storage/parsedfile-store.ts`. * * Byte-identity (issue acceptance): the sink reuses the SAME shared row - * builders (`buildBasicBlockRow`, `buildRelRow`) and label derivation - * (`getNodeLabel`) as `streamAllCSVsToDisk`, so the streamed CSV line SET is + * builders (`buildBasicBlockRow`, `buildRelRow`) and pair classification + * (`relPairKeyFor`) as `streamAllCSVsToDisk`, so the streamed CSV line SET is * identical to the whole-graph emit's, and the bulk COPY loads the same rows → * the persisted graph is SET-identical and DB-identical. The guarantee is * set-level, not byte-level on the CSV file: the sink streams rows in emit @@ -54,9 +54,14 @@ import { buildBasicBlockRow, buildRelRow, } from './csv-generator.js'; -import { assertDeclaredPair, getNodeLabel } from './rel-pair-routing.js'; +import { + VALID_NODE_TABLES, + assertDeclaredPair, + relPairKeyFor, + splitRelPairKey, +} from './rel-pair-routing.js'; import { DEFAULT_EMIT_CHUNK_ROWS, SyncCsvWriter } from './sync-csv-writer.js'; -import { NODE_TABLES, type NodeTableName } from './schema.js'; +import { type NodeTableName } from './schema.js'; /** * PDG edge types streamed per-file (all intra-block BasicBlock→BasicBlock). @@ -98,7 +103,6 @@ export interface PdgEmitManifest { * `--pdg` emit, then {@link finalize} once after the last language. */ export class PdgEmitSink implements KnowledgeGraph { - private readonly validTables: Set; private bbWriter: SyncCsvWriter | undefined; /** pairKey (`From|To`) → writer. PDG edges are all `BasicBlock|BasicBlock`, * but the map keeps the sink general and the manifest pair-keyed. */ @@ -129,7 +133,6 @@ export class PdgEmitSink implements KnowledgeGraph { private readonly pdgCsvDir: string, private readonly chunkRows: number = DEFAULT_PDG_EMIT_CHUNK_ROWS, ) { - this.validTables = new Set(NODE_TABLES as readonly string[]); // Clear any streamed CSVs left by a previous (possibly crashed) run so a // later COPY never picks up stale rows. fs.rmSync(pdgCsvDir, { recursive: true, force: true }); @@ -160,15 +163,27 @@ export class PdgEmitSink implements KnowledgeGraph { addRelationship(relationship: GraphRelationship): void { if (PDG_EDGE_TYPES.has(relationship.type)) { - const fromLabel = getNodeLabel(relationship.sourceId); - const toLabel = getNodeLabel(relationship.targetId); - // Skip edges whose endpoint labels are not valid node tables — mirrors - // `RelPairRouter` exactly so the streamed set matches the whole-graph set. - if (!this.validTables.has(fromLabel) || !this.validTables.has(toLabel)) return; - const pairKey = `${fromLabel}|${toLabel}`; - assertDeclaredPair(pairKey, DECLARED_RELATION_PAIRS); + // Classify + skip via the SHARED `relPairKeyFor`, not a local copy of its + // three lines, so the streamed set cannot drift from the whole-graph set + // `RelPairRouter` produces. `undefined` = an endpoint label is not a node + // table, so the edge is dropped exactly as the router drops it. + const pairKey = relPairKeyFor( + relationship.sourceId, + relationship.targetId, + VALID_NODE_TABLES, + ); + if (pairKey === undefined) return; + assertDeclaredPair( + pairKey, + DECLARED_RELATION_PAIRS, + relationship.type, + relationship.sourceId, + relationship.targetId, + ); let writer = this.relWriters.get(pairKey); if (writer === undefined) { + // Cold: once per pair, so decoding the key back into labels is free. + const [fromLabel, toLabel] = splitRelPairKey(pairKey); try { writer = new SyncCsvWriter( path.join(this.pdgCsvDir, `rel_${fromLabel}_${toLabel}.csv`), diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index d88976fa0..be88b2cca 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -460,6 +460,9 @@ const WAITER_TIMEOUT_MS = 15_000; // of the lbug-config retry-budget registry. const LOCK_RETRY_ATTEMPTS = 3; const LOCK_RETRY_DELAY_MS = 2000; +// determinism: probe — existence only. `probeDatabaseForShadowReplay` calls +// `getAll()` purely to force the shadow replay and then discards the result; +// the function returns void, so no row ever reaches a caller. const SHADOW_REPLAY_PROBE_QUERY = 'MATCH (n) RETURN n LIMIT 1'; const poolSidecarLogger = { diff --git a/gitnexus/src/core/lbug/rel-pair-routing.ts b/gitnexus/src/core/lbug/rel-pair-routing.ts index 5a77cec0b..56b95d0aa 100644 --- a/gitnexus/src/core/lbug/rel-pair-routing.ts +++ b/gitnexus/src/core/lbug/rel-pair-routing.ts @@ -31,10 +31,29 @@ import path from 'path'; import { createWriteStream, type WriteStream } from 'fs'; import { once } from 'events'; import { finished } from 'stream/promises'; +import { NODE_TABLES } from 'gitnexus-shared'; +import { findInCauseChain } from '../../lib/utils.js'; /** Injectable for tests (backpressure/error simulation), mirroring split. */ export type WriteStreamFactory = (filePath: string) => WriteStream; +/** + * Every label LadybugDB has a node table for — the filter that decides whether + * an edge is routable at all. + * + * ONE shared instance, deliberately. `RelPairRouter`, `GraphEmitSink` and + * `PdgEmitSink` each used to build their own `new Set(NODE_TABLES)`; three + * copies of the same immutable set are three chances to seed one of them from + * a different source. Typed `ReadonlySet` because that — not `Object.freeze`, + * which does not touch a Set's internal slots — is what actually stops a + * consumer mutating the shared instance. + * + * Imported straight from `gitnexus-shared` rather than `./schema.js`: schema.ts + * imports `parseRelationSchemaPairs` from this module, so the reverse import + * would close a cycle. + */ +export const VALID_NODE_TABLES: ReadonlySet = new Set(NODE_TABLES); + /** * Derive a node's table label from its graph id. Matches the legacy * `getNodeLabel` that lived inline in `loadGraphToLbug`: @@ -48,20 +67,92 @@ export const getNodeLabel = (nodeId: string): string => { return nodeId.split(':')[0]; }; +/** + * Classify one edge into its `From|To` pair key, or `undefined` when the edge + * must be SKIPPED because an endpoint's label is not a real node table. + * + * THE single definition of "which pair does this edge belong to, and is it + * routable at all". `RelPairRouter.route`, `GraphEmitSink.addRelationship`, + * `PdgEmitSink.addRelationship` and the `structural-pair-coverage` corpus guard + * each used to inline the same three lines (label both ends → drop if either + * label is not a node table → join with `|`). The corpus guard's docblock said + * it "mirrors `RelPairRouter.route`" — a mirror is a drift marker: change the + * skip rule here and the guard would keep classifying by the old one, report + * green, and let `analyze` abort on a pair it had already declared covered. + * + * HOT PATH — called once per edge (~1M on a large repo). Returns the key + * string (which every caller needs anyway for its own Map lookup) rather than + * a `{ pairKey, fromLabel, toLabel }` object or a tuple, so the success path + * allocates nothing beyond what `getNodeLabel` already did. Callers that need + * the two labels back — only when opening a new pair's CSV, once per pair — + * decode the key with {@link splitRelPairKey}. + */ +export const relPairKeyFor = ( + fromId: string, + toId: string, + validTables: ReadonlySet, +): string | undefined => { + const fromLabel = getNodeLabel(fromId); + const toLabel = getNodeLabel(toId); + if (!validTables.has(fromLabel) || !validTables.has(toLabel)) return undefined; + return `${fromLabel}|${toLabel}`; +}; + +/** + * Decode a `From|To` pair key back into its two labels. + * + * Safe because `|` cannot occur inside a node label: every label is a + * `NODE_TABLES` identifier (`[A-Za-z][A-Za-z0-9_]*`), so the FIRST `|` is + * always the separator. That invariant was documented in one comment and + * enforced nowhere while every consumer re-derived it with a bare + * `key.split('|')`. + * + * DECODE ONLY — there is deliberately no matching `encode` helper. The key is + * built once per edge inside {@link relPairKeyFor} (~1M edges on a large + * repo), where a function call is a real regression risk; every decode site is + * cold by construction (once per pair when its CSV is opened, or on the + * throw path of {@link assertDeclaredPair}). + */ +export const splitRelPairKey = (key: string): readonly [from: string, to: string] => { + const sep = key.indexOf('|'); + return sep < 0 ? [key, ''] : [key.slice(0, sep), key.slice(sep + 1)]; +}; + +/** + * Build a fresh matcher for the `FROM