diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 6494fac53..77a48e8ce 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "source": { "source": "local", "path": "./gitnexus-claude-plugin" diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 576586d48..717a4c132 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "source": "./gitnexus-claude-plugin", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase." } diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index 853d44860..be02d92fd 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -21,13 +21,20 @@ Run from the project root. This parses all source files, builds the knowledge gr | Flag | Effect | | -------------- | ---------------------------------------------------------------- | +| `--watch` | Keep a Git repository index current with serialized refreshes | +| `--debounce ` | Watch quiet period before refresh (default: 300 ms) | | `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | +| `--spring-actuator ` | Import opt-in Spring Boot Actuator mappings, beans, conditions, configprops, and env snapshots. Forces a full rebuild; unsupported with `--watch`. | **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. +For Spring runtime enrichment, pass a JSON bundle, one endpoint JSON file, or a directory containing endpoint files. Route evidence is authoritative only when `runtimeConfirmed === true`; `runtimeSource` records provenance and may also accompany `handler-conflict`. Env/configprops values are never persisted. + +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Scheduled remote clone/pull is a different command: `gitnexus auto-sync`. Bare `gitnexus watch` is reserved and does not start either job. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. + ### status — Check index freshness ```bash @@ -55,15 +62,19 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. | Flag | Effect | | ------------------- | ----------------------------------------- | -| `--force` | Force full regeneration | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | | `--model ` | LLM model (default: MiniMax-M3) | | `--base-url ` | LLM API base URL | | `--api-key ` | LLM API key | | `--concurrency ` | Parallel LLM calls (default: 3) | +| `--timeout ` | LLM request timeout in seconds (default: disabled) | +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | ### list — Show all indexed repos @@ -82,5 +93,5 @@ Lists all repositories registered in `~/.gitnexus/registry.json`. The MCP `list_ ## Troubleshooting - **"Not inside a git repository"**: Run from a directory inside a git repo -- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server +- **Index is stale after re-analyzing**: Wait for the next MCP tool call to reopen the published index; this normally takes no more than five seconds - **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding diff --git a/.claude/skills/gitnexus-impact-analysis/SKILL.md b/.claude/skills/gitnexus-impact-analysis/SKILL.md index 4fb73f3e6..85d90c90d 100644 --- a/.claude/skills/gitnexus-impact-analysis/SKILL.md +++ b/.claude/skills/gitnexus-impact-analysis/SKILL.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/.gitattributes b/.gitattributes index 5110ebb5d..eeb1a0976 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,3 +15,15 @@ *.so binary *.dll binary *.dylib binary + +# TypeScript sources are always text for diff purposes. Git's binary +# heuristic fires when EITHER blob in a pair carries a NUL, so a source +# file that carried one on a base commit still renders as "Binary files +# differ" — with no hunks and no inline comments — long after the byte +# itself is gone from the working tree. A head-side guard cannot see +# that, by construction. This does not mark the files binary or change +# how they are stored; it only stops the heuristic from hiding a diff. +*.ts diff +*.tsx diff +*.mts diff +*.cts diff diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 047a12a68..4ac6b8845 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -493,6 +493,20 @@ jobs: node --import tsx bench/python-scope/import-target-fingerprint.mjs --check working-directory: gitnexus + - name: Java wildcard-static route constant guards (#3110) + if: ${{ !cancelled() }} + # Build-free: named-import control vs wildcard materialization; + # fingerprints bindings and guards scaling + absolute wall time. + run: node --import tsx bench/java-wildcard-route-constants/measure.mjs --check + working-directory: gitnexus + + - name: Kotlin package-star route constant guards (#3110) + if: ${{ !cancelled() }} + # Build-free: explicit-import control vs package-star folding; + # fingerprints route facts and guards scaling + widening overhead. + run: node --import tsx bench/kotlin-star-route-constants/measure.mjs --check + working-directory: gitnexus + - name: Cross-language scope-capture fingerprint + scaling guards # Runs even after an earlier guard fails (#2895). Every step here was # fail-fast, so the FIRST failing --check aborted the job and every guard @@ -521,6 +535,31 @@ jobs: run: node --import tsx bench/callable-value-flow/measure.mjs --check working-directory: gitnexus + - name: Java Lombok accessor synthesis guards (#2885) + if: ${{ !cancelled() }} + # Build-free: no-Lombok vs Lombok-heavy corpora; fingerprint over + # synthetic Method ids; scaling + widening overhead budgets. + run: node --import tsx bench/java-lombok-synthesis/measure.mjs --check + working-directory: gitnexus + + - name: Kotlin JVM accessor synthesis guards (#2885) + if: ${{ !cancelled() }} + # Build-free: no-property vs data-class corpora; fingerprint over + # synthetic Method ids; scaling + widening overhead budgets. + run: node --import tsx bench/kotlin-jvm-accessors/measure.mjs --check + working-directory: gitnexus + + - name: Kotlin Spring config-consumer capture guards (#2412) + if: ${{ !cancelled() }} + # Build-free: explicit-import control vs wildcard-import feature path; + # fingerprints @Value / @ConfigurationProperties facts and guards scaling + # + widening overhead. The parity check is the regression gate: each file + # declares a sibling nested type named `Value`, which must not suppress + # the imported Spring annotation (file-wide shadowing dropped 2 of every + # 3 facts on this corpus). + run: node --import tsx bench/spring-config-bindings/measure.mjs --check + working-directory: gitnexus + - name: Re-export closure scaling guards (#2864) # Build-free: asserts buildReexportClosures stays linear in chain depth # and within an absolute ceiling on a wide package corpus. #2864 changed @@ -723,20 +762,22 @@ jobs: - name: Cross-language pipeline benchmarks (GITNEXUS_BENCH, serial) if: ${{ !cancelled() }} - # cpp-adl-benchmark.test.ts is not a `*-pipeline-benchmark.test.ts` but - # belongs here for the same reason: it is skipIf-gated on GITNEXUS_BENCH, - # so it had never run in CI and the PR #1990 ADL emit-scaling guard it - # holds was dead. ~45s of test time. + # cpp-adl-benchmark.test.ts and csharp-razor-view-components-benchmark.test.ts + # are not `*-pipeline-benchmark.test.ts` files but belong here for the + # same reason: they are skipIf-gated on GITNEXUS_BENCH, so the scaling + # guards they hold never run in the main coverage job. env: GITNEXUS_BENCH: '1' run: >- npx vitest run --no-file-parallelism test/integration/cobol-pipeline-benchmark.test.ts test/integration/csharp-pipeline-benchmark.test.ts + test/integration/csharp-razor-view-components-benchmark.test.ts test/integration/cpp-adl-benchmark.test.ts test/integration/data-route-table-benchmark.test.ts test/integration/instance-ownership-pipeline-benchmark.test.ts test/integration/spring-bean-resource-benchmark.test.ts + test/integration/spring-dynamic-lookup-benchmark.test.ts test/integration/rust-pipeline-benchmark.test.ts test/integration/php-pipeline-benchmark.test.ts test/integration/ruby-pipeline-benchmark.test.ts diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 16d3e15cc..a768b69e7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -48,7 +48,7 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: languages: ${{ matrix.language }} queries: security-and-quality @@ -71,8 +71,14 @@ jobs: # deliberately contain use-before-init / unused-variable shapes). - '**/test/fixtures/**' - '**/test/**/fixtures/**' + # GET /api/grep intentionally builds RegExp from the query string + # (literal=1 escapes). ReDoS is handled by worker terminate() — + # see SECURITY.md. Inline codeql[] comments do not clear the + # GitHub PR CodeQL gate, so this file is excluded to avoid + # re-filing js/regex-injection on every push of the same line. + - 'gitnexus/src/server/grep-params.ts' - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 73bb80ba5..dac6a7398 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -141,7 +141,7 @@ jobs: uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 04d722160..f6e54108d 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -53,6 +53,6 @@ jobs: retention-days: 5 - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: results.sarif diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 1e2b2d1e3..699289656 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -50,7 +50,7 @@ jobs: persist-credentials: false - name: Setup Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Build image (load locally for scan) uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 @@ -76,7 +76,7 @@ jobs: exit-code: '0' - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: trivy-${{ matrix.image.name }}.sarif category: trivy-${{ matrix.image.name }} diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index f771406a0..9c3b45ab6 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -76,7 +76,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: zizmor.sarif category: zizmor diff --git a/.gitignore b/.gitignore index e16544f71..3d125d475 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ npm-debug.log* # Testing coverage/ +.tmp-test/ +gitnexus/.tmp-test/ # Misc *.local diff --git a/.gitleaksignore b/.gitleaksignore index c713b712a..243cd4410 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -1,2 +1,4 @@ # Deleted README placeholder from PR #2458; no credential was present. c9fdab17f25ebaf332fba6e6ba55ee328f20fe66:README.md:curl-auth-header:348 +# Synthetic Kotlin Actuator fixture value from PR #3107; no credential was present. +3951079300a18b14e79f5b5f5dd778ae19ced6e3:gitnexus/test/integration/spring-actuator-kotlin-runtime-pipeline.test.ts:generic-api-key:8 diff --git a/AGENTS.md b/AGENTS.md index c83a2e909..251f410c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,10 +119,9 @@ This project is indexed by GitNexus as **GitNexus** (248612 symbols, 565510 rela - **MUST run impact analysis before editing.** Use `impact({target: "symbolName", direction: "upstream"})` (MCP) or `node .gitnexus/run.cjs impact "symbolName" --direction upstream --repo .` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis. For unified PDG impact, add `mode: "pdg"` with optional `line: ` — it returns statement-level `affectedStatements` over CDG + REACHING_DEF and inter-procedural symbols in `interproceduralByDepth`/`byDepth`; no-layer/degraded PDG results are UNKNOWN-risk notes (`--pdg` layer). CLI equivalent: `node .gitnexus/run.cjs impact "symbolName" --direction upstream --mode pdg --line --repo .`. - **MUST analyze graph changes before committing.** Use `detect_changes({scope: "all"})` (MCP) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` (CLI fallback). `partial: true` or `truncated: true` is not a clean check — a zero means unseen, not unaffected; re-run it. For regression review: `detect_changes({scope: "compare", base_ref: "main"})` or `node .gitnexus/run.cjs detect-changes --scope compare --base-ref "main" --repo .`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- MUST warn on HIGH/CRITICAL `risk` pre-edit; never use `riskSharedAxes` to waive a HIGH/CRITICAL `risk` warning. Compare File/symbol: MCP File omits axes; Graph-RAG expands File. - **MUST treat `risk: UNKNOWN` as unresolved, not as low.** An empty caller set is not evidence the symbol is unused — it can also mean the callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls). `impact` pairs `UNKNOWN` with a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete; do not proceed on the strength of a zero. -- When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. +- **MUST use `query({search_query: "concept"})` for concepts/flows, `context({name: "symbolName"})` for a named symbol, or `impact` for blast radius, on read-only callers, dependencies, imports, or execution flow.** Graph first; text search only for empty/`UNKNOWN`/literals. - For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). - For control/data dependence, `pdg_query({mode: "controls", target: "fileOrSymbol"})` answers "under what condition does X run?" (CDG, incl. guard clauses) and `pdg_query({mode: "flows", target, variable})` traces "where does variable Y flow?" (REACHING_DEF). `--pdg` layer. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 357d337ae..b1076d1d5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -108,7 +108,7 @@ scan → structure → [springConfig, markdown, cobol] → parse → [routes, to | `pruneLocalSymbols` | `prune-local-symbols.ts` | `scopeResolution` | Drops inert block-local `Const`/`Variable`/`Static` nodes (only a `File→DEFINES` edge) post-resolution | | `mro` | `mro.ts` | `crossFile`, `scopeResolution`, `pruneLocalSymbols`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | | `springAopInheritance` | `spring-aop.ts` | `springAop`, `mro` | Propagates declarative behavior through class/interface inheritance decisions | -| `di` | `di.ts` | `mro` | INJECTS edges from consumer Classes or factory Methods to provider Classes/declaration CodeElements (framework-neutral DI resolution; per-language matchers registered in `di-extractors/`) | +| `di` | `di.ts` | `mro` | INJECTS edges from consumer Classes, factory Methods, or AST-captured programmatic lookup callables to provider Classes/declaration CodeElements (framework-neutral DI resolution; per-language matchers registered in `di-extractors/`) | | `communities` | `communities.ts` | `mro`, `pruneLocalSymbols`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | | `processes` | `processes.ts` | `communities`, `routes`, `tools`, `pruneLocalSymbols`, `structure` | Process nodes + STEP_IN_PROCESS edges | @@ -174,7 +174,7 @@ converging on the routes phase's `(method, url)` registry: | Filesystem convention | path → URL, no parsing | Next.js `app/`, Expo, PHP | | Single-file framework route | `isRouteFile` + worker extraction | Laravel `routes/*.php` | | Cross-file framework route | `discoverRootRouteFiles` + `extractRoutes` | Django `urlpatterns` | -| AST-level route in a normal file | `extractDecoratorRoutes` | Spring, FastAPI, NestJS, **JS/TS dispatch guards and static data route tables** | +| AST-level route in a normal file | `extractDecoratorRoutes` | Spring, FastAPI, NestJS (`@Controller` + `@Get`/`@Post`/…; URLs are controller-relative — `setGlobalPrefix` and URI versioning live in the bootstrap file and are not applied), **JS/TS dispatch guards and static data route tables** | The last row is the one whose name undersells it. A route is DECLARED by a decorator, but it can also be **inferred** from a raw `node:http` server's own @@ -403,6 +403,7 @@ Each language implements `LanguageProvider` (`language-provider.ts`). Key fields | `typeConfig` | Type annotation extraction rules | | `mroStrategy` | `first-wins` / `c3` / `none` | | `descriptionExtractor` | Optional hook returning a symbol's doc-comment text as its `description`; feeds the embedding metadata header so doc-only terms are semantically searchable (issue #2270). Most languages register `createLeadingDocDescriptionExtractor` (shared, language-neutral; per-language comment/wrapper config passed at the call site) | +| `definitionPropertiesExtractor` | Optional language-owned hook for structured, clone-safe definition metadata. Shared ingestion persists these properties opaquely; the owning provider supplies the extraction semantics. | 16 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. diff --git a/CLAUDE.md b/CLAUDE.md index 55b84d583..069163232 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,10 +70,9 @@ This project is indexed by GitNexus as **GitNexus** (248612 symbols, 565510 rela - **MUST run impact analysis before editing.** Use `impact({target: "symbolName", direction: "upstream"})` (MCP) or `node .gitnexus/run.cjs impact "symbolName" --direction upstream --repo .` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis. For unified PDG impact, add `mode: "pdg"` with optional `line: ` — it returns statement-level `affectedStatements` over CDG + REACHING_DEF and inter-procedural symbols in `interproceduralByDepth`/`byDepth`; no-layer/degraded PDG results are UNKNOWN-risk notes (`--pdg` layer). CLI equivalent: `node .gitnexus/run.cjs impact "symbolName" --direction upstream --mode pdg --line --repo .`. - **MUST analyze graph changes before committing.** Use `detect_changes({scope: "all"})` (MCP) or `node .gitnexus/run.cjs detect-changes --scope all --repo .` (CLI fallback). `partial: true` or `truncated: true` is not a clean check — a zero means unseen, not unaffected; re-run it. For regression review: `detect_changes({scope: "compare", base_ref: "main"})` or `node .gitnexus/run.cjs detect-changes --scope compare --base-ref "main" --repo .`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- MUST warn on HIGH/CRITICAL `risk` pre-edit; never use `riskSharedAxes` to waive a HIGH/CRITICAL `risk` warning. Compare File/symbol: MCP File omits axes; Graph-RAG expands File. - **MUST treat `risk: UNKNOWN` as unresolved, not as low.** An empty caller set is not evidence the symbol is unused — it can also mean the callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls). `impact` pairs `UNKNOWN` with a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete; do not proceed on the strength of a zero. -- When exploring unfamiliar code, use `query({search_query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. +- **MUST use `query({search_query: "concept"})` for concepts/flows, `context({name: "symbolName"})` for a named symbol, or `impact` for blast radius, on read-only callers, dependencies, imports, or execution flow.** Graph first; text search only for empty/`UNKNOWN`/literals. - For security review, `explain({target: "fileOrSymbol"})` lists taint findings (source→sink flows; needs `analyze --pdg`). - For control/data dependence, `pdg_query({mode: "controls", target: "fileOrSymbol"})` answers "under what condition does X run?" (CDG, incl. guard clauses) and `pdg_query({mode: "flows", target, variable})` traces "where does variable Y flow?" (REACHING_DEF). `--pdg` layer. diff --git a/Dockerfile.cli b/Dockerfile.cli index b42c22dad..1488bee54 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -51,8 +51,9 @@ RUN npm run postinstall --prefix gitnexus # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime -# curl for the healthcheck; git for cloning; ca-certificates for TLS verification. -RUN apt-get update && apt-get install -y --no-install-recommends curl git ca-certificates && rm -rf /var/lib/apt/lists/* \ +# curl for the healthcheck; git for cloning; procps for watch process identity; +# ca-certificates for TLS verification. +RUN apt-get update && apt-get install -y --no-install-recommends curl git procps ca-certificates && rm -rf /var/lib/apt/lists/* \ && rm -rf /usr/local/lib/node_modules/npm \ && rm -rf /usr/local/lib/node_modules/corepack \ && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack diff --git a/GUARDRAILS.md b/GUARDRAILS.md index e157ade1e..f34cf79d5 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -52,6 +52,12 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m - **Do:** Re-run plain `npx gitnexus analyze` — no `--embeddings` flag needed. A retained `embeddingCheckpoint` in the index metadata forces embedding generation for exactly the pending nodes regardless of flags, and clears once they succeed. `--drop-embeddings` abandons the pending nodes instead of retrying them; `--force` also discards the checkpoint (with a warning) and rebuilds without resuming it. - **Why:** A long analyze run against a flaky HTTP embedding endpoint tolerates bounded sub-batch failures instead of aborting the whole run: it deletes the affected nodes' embedding rows (so they hold zero rows, never a partial set) and records those nodes as pending in `embeddingCheckpoint`. `stats.embeddings` stays an honest, non-zero count of everything that did succeed, so this state never trips the "Embeddings vanished" Sign above — `embedding-checkpoint-pending` is the only reliable signal. +### Scope extraction is incomplete + +- **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["scope-extraction-failed"]` when files were omitted, or `incompleteReasons: ["scope-extraction-unverified"]` when the index predates the completeness receipt or its metadata is unreadable. `impact`/`context` reports the same uncertainty as `epistemic: "lower-bound"`; confirmed omissions set `causes.scopeExtractionFiles > 0`. +- **Do:** Re-run `npx gitnexus analyze` (`--force` for a full graph rebuild). If the reason persists, inspect the scope-extraction warnings and treat impact counts as floors until the affected source is supported or corrected. +- **Why:** Parsing continued, but scope captures for the reported file count could not be produced even after the main-thread fallback. Calls, inheritance, imports, or accesses originating there may therefore be absent from the graph. + ### Analyze reports INCOMPLETE with a collapsed graph write - **Trigger:** `npx gitnexus status` reports `incompleteReasons: ["graph-write-collapsed"]`; the analyze summary printed `Repository indexed INCOMPLETELY` naming an expected and a persisted relationship count, and the CLI exited non-zero. @@ -67,8 +73,8 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Wrong repo in multi-repo setups - **Trigger:** Query/impact results belong to another project. -- **Do:** Call `list_repos`, then pass `repo` on subsequent tools. -- **Why:** Default target is ambiguous when multiple repos are registered. +- **Do:** Confirm an MCP default is configured or the GitNexus process was launched inside the intended registered path without crossing into an unindexed nested Git checkout. Otherwise call `list_repos`, then pass `repo` on subsequent tools; pass it for mutating tools when multiple repos are registered and no MCP default exists. +- **Why:** Read-only tools derive their default from MCP configuration or a process cwd that stays within one registered Git boundary. Outside those paths the target remains ambiguous, and mutating tools stay explicit unless configuration supplies the target. ### LadybugDB lock / "database busy" diff --git a/README.md b/README.md index f4d3c57ea..c255f71e0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# GitNexus (Akon Labs) +# GitNexus (Akon Labs) **⚠️ Important Notice:** GitNexus has NO official cryptocurrency, token, or coin. Any token/coin using the GitNexus name on Pump.fun or any other platform is **not affiliated with, endorsed by, or created by** this project or its maintainers. Do not purchase any cryptocurrency claiming association with GitNexus. @@ -179,7 +179,7 @@ flowchart TB | `group_list` | List configured repository groups | | `group_sync` | Rebuild a group's Contract Registry and cross-repo links | -> Per-repo tools take an optional `repo` parameter (omit it when only one repo is indexed) and an optional `branch` for indexes pinned with `gitnexus analyze --branch`. Omitting `branch` queries the workspace index, which follows your checked-out working tree — switching branches and re-running `gitnexus analyze` updates it incrementally. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. +> Per-repo read-only tools take an optional `repo` parameter. Omit it when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout; otherwise pass it explicitly. Mutating tools require `repo` when multiple repos are indexed and no MCP default exists. Per-repo tools also take an optional `branch` for indexes pinned with `gitnexus analyze --branch`. Omitting `branch` queries the workspace index, which follows your checked-out working tree — switching branches and re-running `gitnexus analyze` updates it incrementally. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. ### Resources for instant context @@ -384,6 +384,7 @@ Everyday commands: ```bash gitnexus setup # Configure MCP for detected editors (one-time; -c to select) gitnexus analyze [path] # Index a repository (or update a stale index) +gitnexus analyze [path] --watch # Watch local files and serialize incremental refreshes gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus eval-server # Start lightweight evaluation HTTP tools (loopback by default) @@ -396,6 +397,28 @@ gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks You can also query the graph directly from the terminal — `gitnexus query`, `context`, `impact`, `trace`, `cypher`, `detect-changes`, and `check` mirror the MCP tools of the same names, and `gitnexus doctor` prints runtime platform capabilities. +`gitnexus analyze --watch` requires a Git repository. It runs one initial +analysis, then debounces scanner-admitted working-tree changes for 300 ms by +default and applies serialized incremental refreshes. Events arriving during a +refresh remain queued, and retryable failures retain the same batch with bounded +backoff. Invalid `.gitnexusrc` or ignore-file reloads pause ordinary refreshes +until the control file is fixed. Stop the watcher with Ctrl+C. + +Watch mode accepts `--debounce`, `--workers`, `--worker-timeout`, +`--max-file-size`, `--branch`, `--pdg`, `--name`, `--allow-duplicate-name`, and +`--verbose`. Explicit one-shot options such as `--force`, `--repair-fts`, +embedding flags, `--skills`, `--self-commit`, `--index-only`, and `--skip-git` +are rejected. Unsupported defaults from `.gitnexusrc` are ignored with a +warning rather than making an otherwise valid repository unwatchable. + +POSIX requests clone-first copy-and-swap publication when the live index has no +orphan sidecars. Windows and sidecar fallback runs update in place: failures +known to occur before writes are retried, while a failure that may have mutated +the live index stops the watcher. Watch mode does not pull remotes. Running MCP +and `serve` processes reopen a newly published index automatically; MCP observes +the replacement on its next tool call, typically within five seconds, so no +restart is required. +
Authenticated eval-server binding @@ -426,10 +449,13 @@ gitnexus analyze --verbose # Log skipped files when parsers are unavailabl gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1, capped at 16, # auto-sized to the repo). 0 is rejected — there is no sequential mode. +gitnexus analyze --spring-actuator ./actuator # Enrich with local Spring Boot Actuator JSON snapshots gitnexus analyze --wal-checkpoint-threshold 67108864 # LadybugDB WAL auto-checkpoint threshold in bytes # (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) ``` +`--spring-actuator` is explicitly opt-in and accepts either a JSON bundle keyed by `mappings`, `beans`, `conditions`, `configprops`, and/or `env`, or a directory containing endpoint-named JSON files. It confirms matching static nodes and adds conservative runtime-only routes, beans, and property keys. The configured input is excluded from source scanning; only normalized repository-relative exclusions are retained for future scans, never absolute paths. Env/configprops values, origins, condition messages, and source names are never persisted or printed. Because snapshots are external runtime state, an enabled run always rebuilds; the first later run without the option rebuilds once to remove runtime evidence. The same path can be set as `springActuator` in `.gitnexusrc`. + If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `--worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. **Embeddings node limit** — `gitnexus analyze --embeddings` generates semantic search vectors with a default 50,000-node safety cap to protect memory on large repositories: @@ -444,6 +470,46 @@ If embeddings are skipped on a large repository, the indexed graph likely exceed
+
+Keep remote repositories indexed with gitnexus auto-sync + +`gitnexus auto-sync` clones or pulls configured repositories, analyzes new commits, and optionally syncs their group. It runs once immediately, then repeats on the configured interval. It runs in the foreground; use your process manager if it must survive a shell session. `gitnexus watch` is reserved and prints this split; it does not start auto-sync or local file watching. + +```bash +# 1. Create the config once. It never overwrites an existing file. +gitnexus auto-sync init + +# 2. Edit $GITNEXUS_HOME/watch_config.yml, then start it. +gitnexus auto-sync start # `gitnexus auto-sync` is equivalent +gitnexus auto-sync status +gitnexus auto-sync restart # Required after config changes +gitnexus auto-sync stop +gitnexus auto-sync reset # Clear failure state; leaves clones and indexes intact +``` + +`GITNEXUS_HOME` defaults to `~/.gitnexus`. A minimal configuration: + +```yaml +sync_interval_minutes: 10 +analyze_timeout: 5m +projects: + - local_path: /absolute/path/to/clones + branches: [main, master] + overwrite_local_changes: false + remote_urls: + - git@github.com:owner/repo.git +``` + +- `sync_interval_minutes` must be at least `5`; `local_path` must be an absolute path. Clones are stored below it as `host/namespace/repo`. +- Remote URLs must use SSH SCP form and are limited to GitHub, GitLab, or Gitee. +- `branches` are tried in order. The legacy `branch` field is supported, but do not set both. +- Analysis runs in an isolated worker; `analyze_timeout` defaults to, and cannot exceed, half of `sync_interval_minutes`. Timeout and `auto-sync stop` request safe cancellation; a worker in native work exits after reaching a JS-visible safe point. Until then, auto-sync reports `cancelling` or `stopping` and retains ownership so another auto-sync cannot take over, for up to 5 seconds — after that the parent stops waiting and leaves the worker to exit on its own rather than killing it mid-write. This behavior is the same on macOS and Windows. `overwrite_local_changes` defaults to `false`, so a dirty local clone is skipped rather than overwritten; setting it to `true` also deletes untracked files in the clone, while keeping ignored paths. +- Add `group_name` only after creating that group with `gitnexus group create `. Partial clone output is isolated and removed after 14 days. + +See the [full auto-sync configuration and runtime reference](gitnexus/README.md#gitnexus-auto-sync) for concurrency, timeouts, failure thresholds, and runtime files. + +
+
Repository groups (multi-repo / monorepo service tracking) @@ -477,6 +543,7 @@ Commit a `.gitnexusrc` JSON file at the repo root to preconfigure recurring `ana "skipContextFiles": true, // alias of skipAgentsMd: keep your own AGENTS.md/CLAUDE.md "skipSkills": true, // don't install standard skill files under .claude/skills/ and .agents/skills/ "embeddings": true, // generate embeddings by default + "springActuator": "./actuator", // optional local runtime snapshot directory or bundle "workerTimeout": 60, } ``` @@ -491,7 +558,7 @@ Notes: - The default branch is resolved as: `--default-branch` > `.gitnexusrc` `defaultBranch`/`branch` > auto-detected `origin/HEAD` > `main`. - `skipContextFiles` / `skipAiContext` are aliases for `skipAgentsMd` — they skip the `AGENTS.md` / `CLAUDE.md` block only. They do **not** imply `skipSkills`. `indexOnly` is the stronger option that skips all file injection. -- Supported keys: `defaultBranch` (`branch`), `skipAgentsMd` (`skipContextFiles`, `skipAiContext`), `skipSkills`, `indexOnly`, `stats`/`noStats`, `embeddings`, `dropEmbeddings`, `name`, `allowDuplicateName`, `maxFileSize`, `workerTimeout`, `walCheckpointThreshold`, `workers`, `embeddingThreads`, `embeddingBatchSize`, `embeddingSubBatchSize`, `embeddingDevice`. +- Supported keys: `defaultBranch` (`branch`), `skipAgentsMd` (`skipContextFiles`, `skipAiContext`), `skipSkills`, `indexOnly`, `stats`/`noStats`, `embeddings`, `dropEmbeddings`, `name`, `allowDuplicateName`, `maxFileSize`, `workerTimeout`, `walCheckpointThreshold`, `workers`, `springActuator`, `embeddingThreads`, `embeddingBatchSize`, `embeddingSubBatchSize`, `embeddingDevice`. - The file is JSON only. Unknown keys and invalid values fail fast with an actionable error before analysis starts.
@@ -501,36 +568,39 @@ Notes: Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max-file-size`, `--verbose`). Use the env-var form when you'd otherwise repeat the same flag every run, or when invoking GitNexus from a long-running host (MCP server, eval-server, CI shell) that already manages its own environment. CLI flags take precedence over env vars; env vars take precedence over built-in defaults. -| Variable | Default | Effect | Tune when… | -| ----------------------------------------------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | -| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | -| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | -| `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | -| `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | -| `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | -| `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | -| `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | -| `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | -| `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget in milliseconds for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. | Slow or heavily loaded hosts where a full pool cold-starting concurrently needs more than 5s, and analyze aborts with "did not report ready within 5000ms". | -| `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | -| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | -| `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). Invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. | -| `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | -| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | -| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Combined with `timeoutBackoffFactor`, prevents exponentially-growing retries from stalling for hours. | Slow files that legitimately need long total retry windows; lower to fail-fast on stalls. | -| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | -| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code. The worker is terminated at its next JS-safe point instead of mid-native-call (which aborts the whole process with `Napi::Error`, #2432); on expiry it is left running, unref'd, and terminated when it surfaces. | Shutdown latency matters more than draining a wedged worker (lower), or a legitimately-slow native grammar needs longer to surface (raise). | -| `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction. On breach the file keeps the captures accumulated so far and logs a warning — the worker returns to JS instead of stalling in native-heavy loops (#2432). `0` expires immediately. | Pathological generated C++ that still exceeds the budget after the indexed lookups; raise for completeness, lower to fail-fast. | -| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | -| `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | -| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | -| `GITNEXUS_MCP_READ_ONLY` | unset | Set to `1` to expose only proven single-repository read tools and resources; `0` disables the policy and any other value fails startup. | The MCP server runs in an environment where graph mutation, raw Cypher, and cross-repository group routing must be unavailable. | -| `GITNEXUS_MCP_ALLOWED_REPOS` | unset | Comma-separated allowlist of canonical indexed repository names or absolute paths. Invalid, ambiguous, or blank entries fail startup. | One MCP process must expose only a bounded subset of the repositories in the global registry. | -| `GITNEXUS_MCP_DEFAULT_REPO` | unset | Canonical indexed repository name or absolute path used when a tool or resource omits its repository. Must belong to the allowlist when one is set. | Several repositories are available but unqualified MCP calls should resolve deterministically. | -| `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` | unset | Default positive-integer response budget for MCP `query`, `context`, and `impact`, estimated at four UTF-8 bytes per token. Explicit `maxTokens` wins. | Long MCP responses consume too much model context and callers cannot reliably add a per-request budget. | -| `GITNEXUS_PUBLIC_ORIGIN` | unset | The single browser origin `serve` is reached through, added to the CORS allowlist and to the write-route origin guard. A wildcard bind (`0.0.0.0`) has no host identity, so without this the server's own UI is refused. **Setting it currently refuses to start:** `serve` has no authentication, requests carrying no `Origin` header already reach `POST /api/analyze` and `DELETE /api/repo`, and this is the setting that would admit browser writes on top of that. Matching rules for when the gate lifts: the hostname must match exactly, and so must the scheme. A value with no scheme (`app.example.com`) means `https`, since a bare host comes from platform service discovery and those terminate TLS; spell out `http://app.example.com` for plain HTTP. An explicit port must match; with no port, any port on that hostname is accepted. Anything that is not one reachable host (a list, `*`, a bare port number, a `:0` port, a trailing dot) warns at startup and allows nothing. | `gitnexus serve` runs behind a reverse proxy or on a wildcard bind, and the UI's index/delete requests return `origin_not_allowed`. | +| Variable | Default | Effect | Tune when… | +| ----------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | +| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | +| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | +| `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. | +| `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. | +| `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | +| `GITNEXUS_MCP_AUTH_TOKEN` | unset | Bearer token for the dedicated `gitnexus mcp --http` server, for a **directly reachable** `gitnexus serve` `/api/mcp` route, and for the `docker-server` / web proxy in front of one. A non-loopback dedicated MCP bind requires it; `serve` enables protocol-layer MCP auth when it is set. Behind a proxy, set the **same** value on both services: the proxy spends the edge `GITNEXUS_SERVE_AUTH_TOKEN`, then replaces `Authorization` with this token on `/api/mcp` only. | Dedicated MCP, a `serve` the client can reach directly, or a proxied deploy (Render Blueprint) where the backend runs protocol-layer MCP auth — configure it on the proxy too. | +| `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | +| `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | +| `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | +| `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | +| `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | +| `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget in milliseconds for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. | Slow or heavily loaded hosts where a full pool cold-starting concurrently needs more than 5s, and analyze aborts with "did not report ready within 5000ms". | +| `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | +| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling in bytes for every GitNexus database (analyze, MCP server, serve, group bridges). `0` restores LadybugDB's native unbounded default of 80% of system RAM; invalid values warn and fall back to the default (#2557). During `analyze` the pool is right-sized to the graph, scaled on non-4 KiB-page hosts by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | A long-lived `gitnexus mcp` or a big incremental `analyze` uses too much memory, or a huge repo's working set genuinely needs a pool larger than 2 GiB. | +| `GITNEXUS_LBUG_MAX_DB_SIZE` | `17179869184` (16 GiB) | Maximum size in bytes of a single LadybugDB database file — an mmap/disk-address-space ceiling, not a memory limit (it does not constrain the buffer pool). Invalid values silently fall back to the default. | Indexing a genuinely huge monorepo whose on-disk graph index approaches 16 GiB. | +| `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | +| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | +| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Combined with `timeoutBackoffFactor`, prevents exponentially-growing retries from stalling for hours. | Slow files that legitimately need long total retry windows; lower to fail-fast on stalls. | +| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | +| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code. The worker is terminated at its next JS-safe point instead of mid-native-call (which aborts the whole process with `Napi::Error`, #2432); on expiry it is left running, unref'd, and terminated when it surfaces. | Shutdown latency matters more than draining a wedged worker (lower), or a legitimately-slow native grammar needs longer to surface (raise). | +| `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction. On breach the file keeps the captures accumulated so far and logs a warning — the worker returns to JS instead of stalling in native-heavy loops (#2432). `0` expires immediately. | Pathological generated C++ that still exceeds the budget after the indexed lookups; raise for completeness, lower to fail-fast. | +| `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Per-bucket byte budget for parse-cache packing. Files are grouped by `(language, hash(path) mod 128)`; packs inside a bucket are cut at this limit. Smaller = finer-grained invalidation and more dispatch. Default is always 2 MiB and no longer scales with worker count. | Tuning incremental-analyze cache invalidation on monorepos without changing `--workers`. | +| `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | +| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | +| `GITNEXUS_MCP_READ_ONLY` | unset | Set to `1` to expose only proven single-repository read tools and resources; `0` disables the policy and any other value fails startup. | The MCP server runs in an environment where graph mutation, raw Cypher, and cross-repository group routing must be unavailable. | +| `GITNEXUS_MCP_ALLOWED_REPOS` | unset | Comma-separated allowlist of canonical indexed repository names or absolute paths. Invalid, ambiguous, or blank entries fail startup. | One MCP process must expose only a bounded subset of the repositories in the global registry. | +| `GITNEXUS_MCP_DEFAULT_REPO` | unset | Canonical indexed repository name or absolute path used when a tool or resource omits its repository. Must belong to the allowlist when one is set. | Several repositories are available but unqualified MCP calls should resolve deterministically. | +| `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` | unset | Default positive-integer response budget for MCP `query`, `context`, and `impact`, estimated at four UTF-8 bytes per token. Explicit `maxTokens` wins. | Long MCP responses consume too much model context and callers cannot reliably add a per-request budget. | +| `GITNEXUS_PUBLIC_ORIGIN` | unset | The single browser origin `serve` is reached through, added to the CORS allowlist and to the write-route origin guard. A wildcard bind (`0.0.0.0`) has no host identity, so without this the server's own UI is refused. **Setting it currently refuses to start:** `serve` has no authentication, requests carrying no `Origin` header already reach `POST /api/analyze` and `DELETE /api/repo`, and this is the setting that would admit browser writes on top of that. Matching rules for when the gate lifts: the hostname must match exactly, and so must the scheme. A value with no scheme (`app.example.com`) means `https`, since a bare host comes from platform service discovery and those terminate TLS; spell out `http://app.example.com` for plain HTTP. An explicit port must match; with no port, any port on that hostname is accepted. Anything that is not one reachable host (a list, `*`, a bare port number, a `:0` port, a trailing dot) warns at startup and allows nothing. | `gitnexus serve` runs behind a reverse proxy or on a wildcard bind, and the UI's index/delete requests return `origin_not_allowed`. | | `GITNEXUS_TRUST_PROXY` | `loopback, linklocal, uniquelocal` | Express `trust proxy` value — which upstream hops may set `X-Forwarded-*`, and so what the per-IP rate limiter reads as the client IP. Set it to the exact number of proxies you control. Every hop past that is one more entry of the chain the caller gets to write. `false`/`no`/`off` (and a `0` hop count) trust no hop; a proxy list Express can compile (`loopback`, `10.0.0.0/8, 127.0.0.1`) names them instead. `true`/`yes`/`on` is **rejected**: it reads the client-controlled leftmost `X-Forwarded-For` entry, so a spoofed chain earns a fresh rate-limit key per request, and express-rate-limit rejects it too (`ERR_ERL_PERMISSIVE_TRUST_PROXY`). Counts above `16` are rejected as well, as a sanity ceiling rather than a safety boundary. Any invalid value warns and falls back to the default. Bind non-loopback with this unset and `serve` warns: a load balancer outside the private ranges is untrusted, so every request keys to the balancer and the per-IP limit becomes one shared limit. | `serve` sits behind a load balancer outside the private ranges (AWS ALB, Cloudflare, CGNAT), where every request otherwise collapses to the proxy hop and rate limiting goes global. | @@ -590,7 +660,7 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas GitNexus uses a **global registry** so one MCP server can serve multiple indexed repos. No per-project MCP config needed — set it up once and it works everywhere. -Each `gitnexus analyze` stores the index in `.gitnexus/` inside the repo (portable, gitignored) and registers a pointer in `~/.gitnexus/registry.json`. When an AI agent starts, the MCP server reads the registry and can serve any indexed repo. LadybugDB connections are opened lazily on first query and evicted after 5 minutes of inactivity (max 5 concurrent). If only one repo is indexed, the `repo` parameter is optional on all tools — agents don't need to change anything. +Each `gitnexus analyze` stores the index in `.gitnexus/` inside the repo (portable, gitignored) and registers a pointer in `~/.gitnexus/registry.json`. When an AI agent starts, the MCP server reads the registry and can serve any indexed repo. LadybugDB connections are opened lazily on first query and evicted after 5 minutes of inactivity (max 5 concurrent). Read-only tools can omit `repo` when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout. Outside those paths—and for mutating tools with multiple indexed repos and no MCP default—pass `repo` explicitly.
Architecture diagram @@ -768,6 +838,7 @@ gitnexus wiki # Use a custom model or provider (default model: minimax/minimax-m2.5) gitnexus wiki --model gpt-4o gitnexus wiki --base-url https://api.anthropic.com/v1 +gitnexus wiki --provider grok # local Grok Build CLI (uses `grok login`, no API key) # Force full regeneration gitnexus wiki --force diff --git a/RUNBOOK.md b/RUNBOOK.md index 0f5c8b7bb..d16ccd52d 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -46,6 +46,17 @@ npx gitnexus status npx gitnexus list ``` +**Scope extraction incomplete:** `npx gitnexus status` reports +`incompleteReasons: ["scope-extraction-failed"]` when one or more files still +lack scope captures after the worker and fallback passes. `impact` and `context` +then report a lower bound with `causes.scopeExtractionFiles` set to the affected +file count. Re-run `npx gitnexus analyze --force`; if the reason remains, inspect +the scope-extraction warnings for the unsupported or malformed source file. +Every pre-existing index remains unverified until it is analyzed once by a +version that writes the completeness receipt. An older index or unreadable completeness record reports +`incompleteReasons: ["scope-extraction-unverified"]`; re-analyze it before treating +empty impact results as exact. + --- ## Embeddings diff --git a/SECURITY.md b/SECURITY.md index d1fbcd051..37d216911 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -59,11 +59,16 @@ The `render.yaml` Blueprint (see the README's **Deploy to Render**) puts `gitnex - **The generated `GITNEXUS_SERVE_AUTH_TOKEN` is the only access control.** The proxy rejects any `/api/*` request without it with a `401` before forwarding. Rotate it by editing the environment variable on the `gitnexus-web` service and redeploying. - **The CSRF guard is inert on this path.** The proxy strips `Origin` before forwarding, so the server's write-origin guard does nothing for proxied traffic — it passes `Origin`-less requests through by design. The token is not a second layer behind the guard. - **Anyone holding the token can read every indexed repo's source.** These routes carry no origin guard, and the first three carry no rate limiter either: `GET /api/repos`, `GET /api/graph`, `POST /api/query`, `GET /api/file`, `GET /api/grep`. Whoever has the token can also index and delete repositories. -- **`POST /api/mcp` rides the same path.** `serve` mounts the MCP handler via `mountMCPEndpoints`, and `createStreamableHttpHandler` is called with no `authToken` — a **pre-existing** gap in `serve` itself, not something this deploy introduces. On Render it is closed only by the edge token and the private network. A `serve` bound directly to a public interface has no such cover. +- **`POST /api/mcp` rides the same path.** When `GITNEXUS_MCP_AUTH_TOKEN` is set on the backend, `serve` protects `/api/mcp` with the same constant-time Bearer check as the dedicated HTTP MCP server, before parsing the request body. The Render Blueprint does not set a backend MCP token by default. To enable it behind the proxy, set the **same** `GITNEXUS_MCP_AUTH_TOKEN` on both the `gitnexus-web` proxy and the `gitnexus-server` backend: the proxy consumes the edge `GITNEXUS_SERVE_AUTH_TOKEN`, then replaces `Authorization` with the MCP token on `/api/mcp` (and its subpaths) only — the edge credential is never forwarded, and other `/api/*` routes stay stripped. Configuring it on the backend alone makes every proxied MCP request `401`. +- **A directly reachable `serve` still needs an explicit control.** If neither `GITNEXUS_MCP_AUTH_TOKEN` nor an authenticated edge/private-network boundary is present, `/api/mcp` is unauthenticated. Do not bind that topology to a LAN or public interface: MCP readers can access indexed source and graph context. - **Rate limits bound cost, not access.** They cap what a token holder can spend; they do not decide who gets in. Do not hand the URL out as a public demo. A token holder has read access to everything the deploy has indexed. +### `/api/grep` regex semantics and residual ReDoS exposure + +`GET /api/grep` executes caller-supplied patterns as real regular expressions (with an optional path-substring `fileFilter` and `caseSensitive` flag) to honor the web chat's grep tool contract; `literal=1` restores the older escaped-substring mode. Mitigations: a 200-character pattern cap, line-by-line matching, a max-200 result cap, and a 5-second wall-clock budget. Matching runs in a `worker_threads` worker so a catastrophic pattern (e.g. `(a+)+$`) can be killed with `terminate()` when the budget expires — the parent event loop (other routes + SSE) stays responsive. A timed-out scan returns partial results with `timedOut: true`; the web grep tool surfaces that flag so an agent does not treat a cut-off scan as exhaustive. CodeQL still flags constructing a `RegExp` from the query string; that is the advertised contract, not accidental injection. Hosted deploys continue to gate the route behind the edge token. + ## Automated Scans Running in CI This repository runs the following scans automatically. Findings appear under the repository's **Security → Code scanning** tab. diff --git a/docker-server.mjs b/docker-server.mjs index e3e9f68ee..3b036f7db 100644 --- a/docker-server.mjs +++ b/docker-server.mjs @@ -112,6 +112,14 @@ const upstreamOrigin = upstreamBase ? new URL(upstreamBase).origin : null; // (gitnexus/src/mcp/http-transport.ts). const authToken = process.env.GITNEXUS_SERVE_AUTH_TOKEN?.trim() || null; +// The protocol-layer credential the upstream `serve` expects on /api/mcp when it +// runs with MCP Bearer auth enabled. Set it to the SAME value on both services: +// the edge token is spent here and replaced with this one for MCP requests only +// (see proxyToUpstream). Unset — the default — means no injection, so a backend +// without MCP auth is unaffected. Blank-is-absent follows resolveAuthToken +// (gitnexus/src/mcp/http-transport.ts). Never logged. +const mcpAuthToken = process.env.GITNEXUS_MCP_AUTH_TOKEN?.trim() || null; + // Mirrors the non-loopback refusal in http-transport.ts (startMcpHttpServer), // relocated because the trust boundary is here: an unguarded `serve` behind a // private service is legitimate, an unguarded public proxy is not. @@ -341,11 +349,17 @@ async function proxyToUpstream(req, res) { // talks to this same-origin web service. delete headers.origin; delete headers.referer; - // The edge token is spent here. `serve` reads no Authorization header - // (gitnexus/src/server/mcp-http.ts mounts /api/mcp unguarded), so forwarding - // it would only copy a live credential into another service's logs. Pinned by - // test. + // The edge token is spent here and must never be forwarded: copying + // Authorization would put a live credential into another service's logs. So + // drop it unconditionally first, then — for the MCP route alone, and only + // when a backend token is configured — replace it with that separate + // protocol credential. Unset GITNEXUS_MCP_AUTH_TOKEN (the default) leaves + // every request stripped, as before. The scope is the normalized pathname, + // so a query string can't widen it and /api/mcpfoo doesn't qualify. delete headers.authorization; + const upstreamPath = upstream.pathname; + const isMcpRoute = upstreamPath === '/api/mcp' || upstreamPath.startsWith('/api/mcp/'); + if (isMcpRoute && mcpAuthToken) headers.authorization = `Bearer ${mcpAuthToken}`; headers.host = upstream.host; // Replace, never forward, the inbound chain (see clientAddressFor). const clientAddress = clientAddressFor(req); diff --git a/docker-server.test.mjs b/docker-server.test.mjs index 80e742f7e..6d2a9f6c2 100644 --- a/docker-server.test.mjs +++ b/docker-server.test.mjs @@ -271,6 +271,12 @@ it('does not inject config into static assets', async () => { const TEST_AUTH_TOKEN = 'proxy-test-token-0123456789abcdefghij'; const TEST_BEARER = `Bearer ${TEST_AUTH_TOKEN}`; +// The protocol token the upstream expects on /api/mcp. Deliberately unlike the +// edge token, so "injected the backend credential" and "forwarded the edge one" +// can never both satisfy an assertion. +const TEST_MCP_TOKEN = 'backend-mcp-token-0123456789abcdefghij'; +const TEST_MCP_BEARER = `Bearer ${TEST_MCP_TOKEN}`; + // rawRequest never sends credentials; apiRequest does. In a file whose subject // is who gets let through, no test should pass because a helper quietly // authenticated for it. @@ -376,6 +382,11 @@ async function withProxy( const proc = spawnServerWithEnv(dir, port, { GITNEXUS_UPSTREAM_URL: schemeless ? target : `http://${target}`, GITNEXUS_SERVE_AUTH_TOKEN: TEST_AUTH_TOKEN, + // An ambient GITNEXUS_MCP_AUTH_TOKEN in the developer's shell would make the + // proxy inject one on /api/mcp, so drop it: spawn omits undefined entries, + // which unsets the inherited value. A test that wants injection sets it via + // `env` below. + GITNEXUS_MCP_AUTH_TOKEN: undefined, ...env, }); proc.stderr.setEncoding('utf8'); @@ -969,8 +980,9 @@ it('forwards an /api/* request that carries the correct token', async () => { }); it('strips the Authorization header instead of forwarding the edge token', async () => { - // The token is spent at this hop. `serve` reads no Authorization header, so - // forwarding would only copy a live credential into another service's logs. + // The edge credential is spent and stripped at this hop. Forwarding it + // would copy a live credential into another service's logs. With no + // GITNEXUS_MCP_AUTH_TOKEN configured — the default — nothing replaces it. await withProxy({}, async (port, ctx) => { const res = await apiRequest(port, '/api/mcp', { method: 'POST', body: '{}' }); assert.equal(res.status, 200, 'the request itself must still be proxied'); @@ -978,6 +990,72 @@ it('strips the Authorization header instead of forwarding the edge token', async }); }); +// -- Upstream MCP token injection (GITNEXUS_MCP_AUTH_TOKEN) ----------------- +// +// A backend running protocol-layer MCP auth expects its own Bearer on +// /api/mcp, and the edge credential can't serve as one. Both services are +// configured with the same GITNEXUS_MCP_AUTH_TOKEN; this hop spends the edge +// token and substitutes the backend one, for that route only. + +// Stands in for a `serve` with MCP Bearer auth enabled: only the exact backend +// credential gets through, so a passing two-hop request proves what was sent. +const mcpBackend = (req, res) => { + if (req.headers.authorization !== TEST_MCP_BEARER) { + res.writeHead(401, { 'Content-Type': 'application/json; charset=utf-8' }); + res.end('{"error":"unauthorized"}'); + return; + } + res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' }); + res.end('{"ok":true}'); +}; + +it('treats a blank GITNEXUS_MCP_AUTH_TOKEN as unset and still strips', async () => { + const env = { GITNEXUS_MCP_AUTH_TOKEN: ' ' }; + await withProxy({ env }, async (port, ctx) => { + const res = await apiRequest(port, '/api/mcp', { method: 'POST', body: '{}' }); + assert.equal(res.status, 200); + assert.equal(ctx.received.headers.authorization, undefined); + }); +}); + +it('replaces the edge credential with the upstream MCP token on /api/mcp', async () => { + const env = { GITNEXUS_MCP_AUTH_TOKEN: TEST_MCP_TOKEN }; + await withProxy({ upstream: mcpBackend, env }, async (port, ctx) => { + const res = await apiRequest(port, '/api/mcp', { method: 'POST', body: '{}' }); + assert.equal(res.status, 200, 'a backend that demands the MCP token must accept this hop'); + assert.equal(ctx.received.headers.authorization, TEST_MCP_BEARER); + assert.notEqual( + ctx.received.headers.authorization, + TEST_BEARER, + 'the edge credential must never be forwarded', + ); + }); +}); + +it('injects the upstream MCP token on /api/mcp subpaths and ignores the query string', async () => { + const env = { GITNEXUS_MCP_AUTH_TOKEN: TEST_MCP_TOKEN }; + await withProxy({ upstream: mcpBackend, env }, async (port, ctx) => { + for (const path of ['/api/mcp/messages', '/api/mcp?session=abc']) { + const res = await apiRequest(port, path, { method: 'POST', body: '{}' }); + assert.equal(res.status, 200, `${path} must reach the MCP backend authenticated`); + assert.equal(ctx.received.headers.authorization, TEST_MCP_BEARER, path); + } + }); +}); + +it('leaves non-MCP routes stripped when an upstream MCP token is configured', async () => { + // /api/mcpfoo shares a prefix with the MCP route but is not it, and a plain + // API route never carries a protocol credential. + const env = { GITNEXUS_MCP_AUTH_TOKEN: TEST_MCP_TOKEN }; + await withProxy({ env }, async (port, ctx) => { + for (const path of ['/api/mcpfoo', '/api/health']) { + const res = await apiRequest(port, path); + assert.equal(res.status, 200); + assert.equal(ctx.received.headers.authorization, undefined, path); + } + }); +}); + it('never gates static assets behind the token', async () => { // The UI has to load before it can prompt for a token. await withProxy({}, async (port, ctx) => { diff --git a/docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md b/docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md new file mode 100644 index 000000000..7e14f2174 --- /dev/null +++ b/docs/plans/2026-08-28-gitnexus-plan-impact-file-risk.md @@ -0,0 +1,312 @@ +# GitNexus Engineering Plan + +> Task: Fix #3075 — File `impact` risk is not comparable to Function/Method risk. +> Evidence verified at commit `6bff33d14cbfe1e7b4f04bca51507e9f64ef579c` (`feat/kotlin-const-resolver`); GitNexus index 129 commits behind, refresh skipped: full-repo `--index-only --pdg` rebuild is impractical this session. Scorer and schema claims are `[verified]` from source; live inversion numbers are `[graph]` on the stale index. + +## 1. Objective + +Make File vs symbol `impact.risk` honest for consumers: either they can tell the scales differ, or they can compare on a shared two-axis score. Do **not** DEFINES-bridge processes/modules onto File targets (issue reporter sampled 8/10 one-importer files jumping to HIGH/CRITICAL). Do **not** retune Function HIGH/CRITICAL thresholds (agent warn-before-edit). + +Acceptance: + +- A File with a wider blast radius than a Function in the same file no longer looks “safer” when a consumer only reads `risk`, **or** the result states that `risk` is not comparable across kinds and offers `riskSharedAxes` for comparison. +- File targets still cannot trip HIGH/CRITICAL via `processes_affected` / `modules_affected` unless those axes become real in the index (they are not today). +- Existing Function/Method labels under the current four-axis ladder stay the same for the same inputs. +- MCP `riskNote` remains UNKNOWN-only (`tools.ts` contract). + +## 2. Current Behaviour + +Callgraph `impact` ends in `LocalBackend._runImpactBFS` (`gitnexus/src/mcp/local/local-backend.ts`). After BFS it enriches impacted ids with `STEP_IN_PROCESS` and `MEMBER_OF`, then scores: + +```7720:7738:gitnexus/src/mcp/local/local-backend.ts + } else if ( + directCount >= 30 || + processCount >= 5 || + moduleCount >= 5 || + impacted.length >= 200 + ) { + risk = 'CRITICAL'; + } else if ( + directCount >= 15 || + processCount >= 3 || + moduleCount >= 3 || + impacted.length >= 100 + ) { + risk = 'HIGH'; + } else if (directCount >= 5 || impacted.length >= 30) { + risk = 'MEDIUM'; + } else { + risk = 'LOW'; + } +``` + +Empty upstream → `UNKNOWN` + `riskNote`. Downstream empty stays LOW. `skipEnrichment` (ambiguous probes) already scores on direct+total only. PDG mode forces `risk: UNKNOWN` (`composeUnifiedPdgImpactResult`) — out of scope. + +File BFS walk is mostly File←IMPORTS File. Enrichment queries those File ids. Processes are CALLS traces (`process-processor.ts`); communities admit only Function/Class/Method/Interface (`isCommunitySymbol` in `community-processor.ts:412-416`). File is not in that set. `enrichCandidateLabels` UNION also **omits File**, so File `target.type` is often `""`; detect File via `id` prefix `File:`. + +Web Graph RAG (`gitnexus-web/src/core/llm/tools.ts` ~1331–1346) duplicates the same ladder. + +## 3. Relevant Architecture + +| Layer | Role | +|---|---| +| Index | File never sources `STEP_IN_PROCESS` / `MEMBER_OF` by construction | +| MCP `_runImpactBFS` | Blast radius + four-axis `risk` | +| Ambiguous probes | `skipEnrichment` → 2-axis `risk` already | +| `mergeRisk` | Group overlay; monotone in crossings; does not know target kind | +| CLI `formatImpactResult` | Prints counts; **does not print `risk`** on the resolved callgraph path; JSON `impactCommand` still ships `risk` | +| `ai-context.ts` / `tools.ts` | Agent contract: warn on HIGH/CRITICAL; `riskNote` UNKNOWN-only | +| Web LLM `impact` | Same formula, prose `RISK:` line | + +Modules: Local (MCP), Cli (format/docs), Group (`mergeRisk`), gitnexus-web LLM tools. Shared package `gitnexus-shared` is already a dependency of both CLI and web. + +## 4. GitNexus Findings + +- Primary: `_runImpactBFS` — d=1 `[graph]` `impact(target:_runImpactBFS, maxDepth:1, includeTests:true)`: `_impactImpl`, `impactByUid`. Production chain `[verified]`: `impact` → `_impactImpl` → `_runImpactBFS`; `impactByUid` skips per-symbol process lists but **not** aggregation (`skipPerSymbolEnrichment` only). +- `LocalBackend.impact` d=1 `[graph]` `context`: `callTool`. +- Duplicate scorer `[verified]` grep: `gitnexus-web/src/core/llm/tools.ts`. +- `mergeRisk` `[verified]` callers in `src/`: only `runGroupImpact` (`cross-impact.ts:907`). Graph d=1 listed a test File (`impact-pdg-shape.test.ts`) and missed `runGroupImpact` — trust source. +- Schema `[verified]`: `isCommunitySymbol` excludes File; `schema.ts` documents MEMBER_OF as Function/Class/Method/Interface only. +- Live inversion `[graph]` stale index, `impact summaryOnly` on GitNexus: + +| target | kind | impacted | direct | processes | modules | risk | +|---|---|---|---|---|---|---| +| `lbug-config.ts` | File | 54 | 12 | 0 | 0 | MEDIUM | +| `openLbugConnection` | Function | 16 | 9 | 3 | 2 | HIGH | +| `local-backend.ts` | File | 12 | 10 | 0 | 0 | MEDIUM | +| `refreshRepos` | Method | 50 | 5 | 4 | 7 | CRITICAL | + +- Clusters/processes resources `[graph]`: Local/Cli/Group sit in the impact path; process traces are function-stepped, not File-stepped. +- Related tests `[verified]`: `test/unit/impact-pagination.test.ts` (CRITICAL from `direct=400`); `test/integration/impact-zero-caller-risk.test.ts` (`withTestLbugDB` seed — pattern to extend); `test/unit/eval-formatters.test.ts` (`formatImpactResult`); group `mergeRisk` tests. + +## 5. Statement-Level PDG Findings + +PDG unavailable (`pdg_query` on `_runImpactBFS`: “no PDG layer”). Recommend `node .gitnexus/run.cjs analyze --index-only --pdg` before any future statement-slice work. Control flow of the scorer is a straight if/else after enrichment; no hidden guards. `skipEnrichment` is the only branch that structurally zeros process/module counts besides File ids. + +## 6. Proposed Changes + +### 6.1 Extract `scoreImpactRisk` — `gitnexus-shared/src/impact-risk.ts` (new) + +- **Responsibility:** Pure function: `{ direction, directCount, processCount, moduleCount, impactedCount, unusedAxes }` → `{ risk, riskSharedAxes, riskScale }`. +- **Behaviour:** Existing UNKNOWN/CRITICAL/HIGH/MEDIUM/LOW thresholds unchanged when `unusedAxes` is empty. `riskSharedAxes` always scores as if `processCount=0` and `moduleCount=0` (UNKNOWN rule still applies). `riskScale.comparableAcrossKinds` is false iff `unusedAxes` is non-empty. `riskScale.unusedAxes` lists `{ axis, reason }`. +- **Constraints:** Zero deps. Export from `gitnexus-shared/src/index.ts`. Do not put MCP types here. +- **File detection:** caller passes unused axes; helper does not parse UIDs. + +### 6.2 Wire MCP — `_runImpactBFS` in `local-backend.ts` + +- After computing `processCount`/`moduleCount`, set `unusedAxes`: + - target `id` starts with `File:` **or** `symType === 'File'` → processes + modules, reason `file-nodes-have-no-process-or-community-membership`; + - `skipEnrichment` → same axes, reason `enrichment-skipped` (ambiguous probes). +- Replace inline ladder with `scoreImpactRisk`. +- Spread `riskScale` and `riskSharedAxes` on the result next to `risk`. Do **not** set `riskNote` for File. +- Ambiguous candidate summaries: forward the new fields (probes already skip enrichment). +- `target.type` for File: if still `""`, prefer `'File'` when `id` starts with `File:` (display-only; helps CLI). + +### 6.3 Web duplicate — `gitnexus-web/src/core/llm/tools.ts` + +- Import `scoreImpactRisk` from `gitnexus-shared`. Print `RISK:` from `risk`; if `!comparableAcrossKinds`, one extra line: not comparable to Function risk; shared-axes label is `riskSharedAxes`. + +### 6.4 Agent/MCP contract copy + +- `gitnexus/src/mcp/tools.ts` impact description: document `riskScale` / `riskSharedAxes`; keep `riskNote` UNKNOWN-only; say File `risk` is not comparable to symbol `risk`. +- `gitnexus/src/cli/ai-context.ts`: HIGH/CRITICAL warning still applies; add: do not rank a File `MEDIUM` below a contained Function `HIGH` without `riskSharedAxes`. +- `formatImpactResult`: on resolved callgraph results with `risk`, print `Risk: {risk}` and, when incomparable, `Shared-axes risk: {riskSharedAxes} (File/process axes unused)`. + +### 6.5 Explicitly not changing + +- DEFINES-bridge, community/process indexers, `mergeRisk` formula, PDG `UNKNOWN`, `detectChanges` `risk_level`, Function thresholds. + +## 7. Implementation Sequence + +1. Add `gitnexus-shared` helper + unit table (issue-shaped inputs + UNKNOWN + skipEnrichment). Shared package tests if present; otherwise `gitnexus/test/unit/impact-risk.test.ts` importing the helper. +2. Switch `_runImpactBFS` + candidate probe payload. Tree still coherent: old `risk` values identical for Function fixtures. +3. Integration seed in `impact-zero-caller-risk.test.ts` **or** new `impact-file-risk-scale.test.ts`: File with ≥5 File IMPORTS (MEDIUM on direct) vs Function with 3 process-member callers (HIGH); assert File `riskScale.comparableAcrossKinds === false`, Function true, File `riskSharedAxes === risk`, Function `riskSharedAxes` is LOW/MEDIUM while `risk` is HIGH. +4. CLI formatter + `eval-formatters.test.ts`. +5. `tools.ts` + `ai-context.ts` wording. +6. Web import + a unit assertion on the printed RISK block if a test already covers that tool. +7. `npx tsc --noEmit` in `gitnexus/` and `gitnexus-web/`; `cd gitnexus && npm run test:unit -- test/unit/impact-risk.test.ts test/unit/eval-formatters.test.ts`; integration file from step 3. + +## 8. Test Strategy + +| File | Scenarios | +|---|---| +| `gitnexus/test/unit/impact-risk.test.ts` (new) | Issue table: File(25,13,0,0)→MEDIUM; Function(15,2,4,2)→HIGH; shared-axes File MEDIUM vs Function LOW; empty upstream UNKNOWN; downstream empty LOW; skipEnrichment unused axes; CRITICAL via direct≥30 still works with unused process axes | +| `gitnexus/test/integration/impact-file-risk-scale.test.ts` (new) | `withTestLbugDB` seed: `File:src/crypto.ts` ← 13 File IMPORTS, no File STEP_IN_PROCESS; `getEncryptionKey` with 2 CALLS from functions that have STEP_IN_PROCESS to 4 distinct Process nodes — reproduce inversion; assert new fields | +| `gitnexus/test/integration/impact-zero-caller-risk.test.ts` | Unchanged UNKNOWN/`riskNote`; candidates may grow `riskScale` — assert still present only when UNKNOWN for `riskNote` | +| `gitnexus/test/unit/impact-pagination.test.ts` | Hub CRITICAL unchanged | +| `gitnexus/test/unit/eval-formatters.test.ts` | Resolved result prints Risk + shared-axes line for File-shaped `riskScale` | +| Web | Only if an existing Graph RAG impact test snapshots `RISK:` | + +Commands (exist in `gitnexus/package.json`): `npm run test:unit`, `npm test` (full vitest), `npx tsc --noEmit`. Web: `npm test`, `npx tsc -b --noEmit`. Integration needs `pretest:integration` / `npm run test:integration` (runs `scripts/build.js`). + +## 9. Risk and Impact Analysis + +Direct dependents of `_runImpactBFS` `[graph]`: `_impactImpl`, `impactByUid`. `_impactImpl` is the only d=1 of `impact` besides the method’s own class. Any JSON consumer of `impact` (MCP, CLI `output(result)`, group local leg) sees additive fields — compatible if they ignore unknowns. + +- **HIGH workflow:** Function HIGH/CRITICAL unchanged. File still cannot reach HIGH via processes; a File with `direct≥15` or `total≥100` still can. Agents that compare File MEDIUM vs Function HIGH must start using `riskSharedAxes` or `riskScale`. +- **Ambiguous `maxRisk`:** probes skip enrichment, so File vs Function candidates are already 2-axis there — inversion is weaker on that path. +- **Group `mergeRisk`:** still compares incomparable File local `risk` to crossing count. Do not retune this PR; if a group File target is common, follow-up. +- **Web:** browser bundle picks up `gitnexus-shared` export — confirm `gitnexus-shared` build/exports include the new file. +- **Performance:** none (pure arithmetic after existing enrichment). +- **Ladybug empty labels:** File detection must not rely on `symType` alone. + +## 10. Files Expected to Change + +| File | Symbols | Reason | +|---|---|---| +| `gitnexus-shared/src/impact-risk.ts` | `scoreImpactRisk` | New shared scorer | +| `gitnexus-shared/src/index.ts` | exports | Public helper | +| `gitnexus/src/mcp/local/local-backend.ts` | `_runImpactBFS`, ambiguous candidate map | Wire scorer + File unused axes | +| `gitnexus/src/mcp/tools.ts` | `impact` description | Contract | +| `gitnexus/src/cli/ai-context.ts` | generated Always Do | Agent warning | +| `gitnexus/src/cli/eval-server.ts` | `formatImpactResult` | Print scale | +| `gitnexus-web/src/core/llm/tools.ts` | web `impact` | Same formula | +| `gitnexus/test/unit/impact-risk.test.ts` | — | Table tests | +| `gitnexus/test/integration/impact-file-risk-scale.test.ts` | — | Seeded inversion | +| `gitnexus/test/unit/eval-formatters.test.ts` | `formatImpactResult` | Formatter | + +## 11. Reusable Implementation Context + +```yaml +implementation_context: + task_summary: "Fix #3075: File impact.risk is a 2-axis score silently labelled on a 4-axis scale. Extract scoreImpactRisk; mark File/skipEnrichment axes unused; add riskScale + riskSharedAxes; do not DEFINES-bridge or retune Function thresholds." + acceptance_criteria: + - "File vs Function comparison is either labelled incomparable (riskScale) or done via riskSharedAxes" + - "Function/Method risk for identical four-axis inputs unchanged" + - "riskNote still UNKNOWN-only" + - "Integration seed reproduces crypto.ts-style inversion and asserts the new fields" + primary_symbols: + - symbol: "_runImpactBFS" + file: "gitnexus/src/mcp/local/local-backend.ts" + lines: "6991-7888" + role: "BFS + enrichment + inline risk ladder (replace ladder only)" + - symbol: "scoreImpactRisk" + file: "gitnexus-shared/src/impact-risk.ts" + lines: "new" + role: "Pure scorer + shared-axes + riskScale" + - symbol: "formatImpactResult" + file: "gitnexus/src/cli/eval-server.ts" + lines: "305-641" + role: "Human/LLM text surface for impact JSON" + related_symbols: + - symbol: "_impactImpl" + relationship: "CALLS" + relevance: "Resolves target, PDG vs callgraph, ambiguous skipEnrichment probes" + - symbol: "impactByUid" + relationship: "CALLS" + relevance: "Group fan-out; keep skipPerSymbolEnrichment; still run aggregation" + - symbol: "mergeRisk" + relationship: "consumes risk string" + relevance: "Do not change this PR" + - symbol: "isCommunitySymbol" + relationship: "index gate" + relevance: "Why File modules_affected is always 0" + - symbol: "composeUnifiedPdgImpactResult" + relationship: "separate path" + relevance: "PDG risk stays UNKNOWN" + execution_path: + - "impact / callTool → _impactImpl (resolve symbol, File id prefix File:)" + - "_runImpactBFS: IMPORTS-heavy walk for File; CALLS walk for Function" + - "Enrich STEP_IN_PROCESS / MEMBER_OF on impacted ids (empty for File ids)" + - "scoreImpactRisk with unusedAxes for File or skipEnrichment" + - "JSON to MCP/CLI; formatImpactResult for eval text; web LLM tools parallel path" + pdg_constraints: + - description: "No PDG layer on the planning index; scorer is post-enrichment arithmetic" + affected_statements: [] + implementation_consequence: "Do not wait on PDG; do not change pdg impact risk" + architectural_patterns: + - pattern: "Additive optional JSON fields on impact (riskNote, epistemic, partial)" + example_location: "gitnexus/src/mcp/local/local-backend.ts _runImpactBFS base object ~7754" + usage_guidance: "Add riskScale/riskSharedAxes the same way; never overload riskNote" + - pattern: "withTestLbugDB CREATE seed for impact contract" + example_location: "gitnexus/test/integration/impact-zero-caller-risk.test.ts" + usage_guidance: "Seed File IMPORTS + Function CALLS + Process membership separately" + files_to_modify: + - file: "gitnexus-shared/src/impact-risk.ts" + symbols: ["scoreImpactRisk"] + intended_change: "new pure scorer" + - file: "gitnexus-shared/src/index.ts" + symbols: [] + intended_change: "re-export" + - file: "gitnexus/src/mcp/local/local-backend.ts" + symbols: ["_runImpactBFS"] + intended_change: "unusedAxes + helper; File type display" + - file: "gitnexus/src/mcp/tools.ts" + symbols: [] + intended_change: "document fields" + - file: "gitnexus/src/cli/ai-context.ts" + symbols: [] + intended_change: "agent comparability note" + - file: "gitnexus/src/cli/eval-server.ts" + symbols: ["formatImpactResult"] + intended_change: "print risk + shared-axes when incomparable" + - file: "gitnexus-web/src/core/llm/tools.ts" + symbols: [] + intended_change: "import helper; extra prose line" + tests: + - file: "gitnexus/test/unit/impact-risk.test.ts" + scenarios: + - "File(25,13,0,0)+unused process/module → risk MEDIUM, comparableAcrossKinds false, riskSharedAxes MEDIUM" + - "Function(15,2,4,2) → HIGH, riskSharedAxes LOW (direct 2, total 15)" + - "upstream impactedCount 0 → UNKNOWN both fields" + - "direct 400 → CRITICAL even with unused process axes" + - file: "gitnexus/test/integration/impact-file-risk-scale.test.ts" + scenarios: + - "Seed File crypto.ts with 13 File importers vs getEncryptionKey with process-rich callers → inversion on risk, File incomparable, Function comparable" + - file: "gitnexus/test/unit/eval-formatters.test.ts" + scenarios: + - "formatImpactResult includes Shared-axes risk when riskScale.comparableAcrossKinds is false" + verification_commands: + - "cd gitnexus && npx tsc --noEmit" + - "cd gitnexus && npm run test:unit -- test/unit/impact-risk.test.ts test/unit/eval-formatters.test.ts test/unit/impact-pagination.test.ts" + - "cd gitnexus && npm run test:integration -- test/integration/impact-file-risk-scale.test.ts test/integration/impact-zero-caller-risk.test.ts" + - "cd gitnexus-web && npx tsc -b --noEmit" + risks: + - "Consumers that only read risk still see the inversion unless they adopt riskScale/riskSharedAxes — that is the chosen (explicit-scale) fix" + - "File type often empty; must key unusedAxes off File: id prefix" + - "gitnexus-shared export must reach the web bundle" + assumptions: + - "WHAT: File nodes never gain STEP_IN_PROCESS/MEMBER_OF without an indexer change. HOW: keep isCommunitySymbol and process traces as-is; tests seed File with zero such edges" + - "WHAT: Additive JSON fields are backward compatible. HOW: existing tests that exact-match the full impact object may need to allow extra keys — grep expect(res).toEqual on impact results before landing" + - "WHAT: HEAD 6bff33d is the pin; scorer line numbers ~7720. HOW: re-read the ladder if that hunk moved" + open_questions: + - "Whether GroupImpactResult should copy riskScale from local File targets (deferred unless tests already snapshot the full group object)" + avoid: + - "Do not DEFINES-bridge File→symbol processes/modules" + - "Do not lower Function process/module HIGH/CRITICAL thresholds" + - "Do not reuse riskNote for File incomparability" + - "Do not change PDG impact risk or detectChanges risk_level" + - "Do not treat labels(n)[0] or empty target.type as proof the node is not a File" + - "Do not repeat full repository discovery" +``` + +## 12. Assumptions and Open Questions + +**Assumptions** + +- Indexer will not start attaching File→Process/Community in this change (`isCommunitySymbol` stays). `[verified]` source; `[assumed]` future indexers. +- Ignoring unknown JSON keys is safe for MCP clients; any `toEqual` goldens in-repo must be updated. `[assumed]` — grep during implement. +- Stale-index inversion (`lbug-config.ts` vs `openLbugConnection`) is illustrative; the integration seed is the regression lock. `[graph]` vs `[verified]` seed. + +**Open questions** + +- Group `mergeRisk` + File local risk: copy `riskScale` onto `GroupImpactResult`? Default **no** unless a test breaks. +- Class/Interface STEP_IN_PROCESS sparsity: out of scope (#3075 is File). +- Printing `risk` on CLI formatted output is new (JSON already has it). Keep the extra lines short. + +**Deferred** + +- Recalibrated File-only HIGH thresholds. +- Indexing File community membership. +- DEFINES-bridge after a threshold RFC. +- Related #2975 (docs vs scorer wording) except as touched by `tools.ts`. + +## 13. Definition of Done + +- [ ] `scoreImpactRisk` is the only callgraph ladder in MCP and web. +- [ ] File (and skipEnrichment) results include `riskScale.comparableAcrossKinds === false` and `riskSharedAxes`. +- [ ] Function four-axis HIGH/CRITICAL cases in unit tests still pass with the same labels. +- [ ] Integration seed proves wider File blast + lower `risk` than a contained Function, and `riskSharedAxes` orders them without pretending processes existed on the File. +- [ ] `riskNote` still absent unless `risk === 'UNKNOWN'`. +- [ ] `tools.ts` + `ai-context.ts` state that File `risk` is not comparable to symbol `risk`. +- [ ] `cd gitnexus && npx tsc --noEmit` and the named unit/integration commands pass; web typecheck passes. diff --git a/eslint-rules/require-safe-parse.mjs b/eslint-rules/require-safe-parse.mjs index 4ab9dbd8a..4bad4280d 100644 --- a/eslint-rules/require-safe-parse.mjs +++ b/eslint-rules/require-safe-parse.mjs @@ -19,14 +19,14 @@ * * False-positive suppression: * - Skips calls whose receiver is a known non-tree-sitter library (`JSON`, - * `URL`, `marked`, `Number`). + * `URL`, `marked`, `Number`, `path`). * - Skips calls whose first argument is a string-literal (grammar-load smoke * tests like `_testParser.parse('service X { rpc Y (R) returns (R); }')`). * - Skips test files (`.test.ts`/`.test.tsx`/`.spec.ts`). * - Skips the `safe-parse.ts` helper itself. */ -const SKIPPED_RECEIVERS = new Set(['JSON', 'URL', 'marked', 'Number', 'Math']); +const SKIPPED_RECEIVERS = new Set(['JSON', 'URL', 'marked', 'Number', 'Math', 'path']); export default { meta: { @@ -74,7 +74,7 @@ export default { // Receiver-text-shape skip: anything matching well-known JS APIs that // happen to have a `.parse()` shape but aren't tree-sitter. if ( - /^(JSON|URL|marked|Number|Math|Date|globalThis\.JSON)\b/.test(receiverText) || + /^(JSON|URL|marked|Number|Math|Date|path|globalThis\.JSON)\b/.test(receiverText) || /\bjson\.parse\b/i.test(receiverText) ) { return; diff --git a/eval/workflow_bench/runtime_mounts.py b/eval/workflow_bench/runtime_mounts.py index 3f2e6fcf2..c532ac1b8 100644 --- a/eval/workflow_bench/runtime_mounts.py +++ b/eval/workflow_bench/runtime_mounts.py @@ -28,8 +28,10 @@ from .proposer_sandbox import ( SandboxError, ) -PINNED_GITNEXUS_VERSION = "1.6.9" HARNESS_ROOT = Path(__file__).resolve().parents[2] +# The mounted runtime is built from this checkout, so the pin tracks the harness' +# own package version. A hardcoded copy only drifts on release day (#3064). +PINNED_GITNEXUS_VERSION = json.loads((HARNESS_ROOT / "gitnexus" / "package.json").read_text())["version"] CE_ARMS = frozenset({"ce_workflow", "ce_workflow_direct", "ce_review"}) SANDBOX_CE_PLUGIN = "/opt/compound-engineering-plugin" diff --git a/gitnexus-claude-plugin/.claude-plugin/plugin.json b/gitnexus-claude-plugin/.claude-plugin/plugin.json index ace058dad..9e9372dea 100644 --- a/gitnexus-claude-plugin/.claude-plugin/plugin.json +++ b/gitnexus-claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.9", + "version": "1.6.10", "author": { "name": "GitNexus" }, diff --git a/gitnexus-claude-plugin/.codex-plugin/plugin.json b/gitnexus-claude-plugin/.codex-plugin/plugin.json index c9a03db4d..67ef62af2 100644 --- a/gitnexus-claude-plugin/.codex-plugin/plugin.json +++ b/gitnexus-claude-plugin/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.9", + "version": "1.6.10", "skills": "./skills", "mcpServers": "./.mcp.json", "hooks": "./hooks/hooks.json", diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index 9c7a1b599..be02d92fd 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -19,14 +19,21 @@ node .gitnexus/run.cjs analyze Run from the project root. This parses all source files, builds the knowledge graph, writes it to `.gitnexus/`, and generates CLAUDE.md / AGENTS.md context files. -| Flag | Effect | -|------|--------| -| `--force` | Force full re-index even if up to date | +| Flag | Effect | +| -------------- | ---------------------------------------------------------------- | +| `--watch` | Keep a Git repository index current with serialized refreshes | +| `--debounce ` | Watch quiet period before refresh (default: 300 ms) | +| `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | +| `--spring-actuator ` | Import opt-in Spring Boot Actuator mappings, beans, conditions, configprops, and env snapshots. Forces a full rebuild; unsupported with `--watch`. | -**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. +**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. + +For Spring runtime enrichment, pass a JSON bundle, one endpoint JSON file, or a directory containing endpoint files. Route evidence is authoritative only when `runtimeConfirmed === true`; `runtimeSource` records provenance and may also accompany `handler-conflict`. Env/configprops values are never persisted. + +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Scheduled remote clone/pull is a different command: `gitnexus auto-sync`. Bare `gitnexus watch` is reserved and does not start either job. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. ### status — Check index freshness @@ -44,10 +51,10 @@ node .gitnexus/run.cjs clean Deletes the `.gitnexus/` directory and unregisters the repo from the global registry. Use before re-indexing if the index is corrupt or after removing GitNexus from a project. -| Flag | Effect | -|------|--------| -| `--force` | Skip confirmation prompt | -| `--all` | Clean all indexed repos, not just the current one | +| Flag | Effect | +| --------- | ------------------------------------------------- | +| `--force` | Skip confirmation prompt | +| `--all` | Clean all indexed repos, not just the current one | ### wiki — Generate documentation from the graph @@ -55,19 +62,21 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. -| Flag | Effect | -|------|--------| -| `--force` | Force full regeneration, also required to re-gerenate an existing wiki in a different language | -| `--model ` | LLM model (default: MiniMax-M3) | -| `--base-url ` | LLM API base URL | -| `--api-key ` | LLM API key | -| `--concurrency ` | Parallel LLM calls (default: 3) | -| `--gist` | Publish wiki as a public GitHub Gist | +| Flag | Effect | +| ------------------- | ----------------------------------------- | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | +| `--model ` | LLM model (default: MiniMax-M3) | +| `--base-url ` | LLM API base URL | +| `--api-key ` | LLM API key | +| `--concurrency ` | Parallel LLM calls (default: 3) | | `--timeout ` | LLM request timeout in seconds (default: disabled) | -| `--retries ` | Max LLM retry attempts per request (default: 3) | -| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese)| +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | +| `--gist` | Publish wiki as a public GitHub Gist | + ### list — Show all indexed repos ```bash @@ -84,5 +93,5 @@ Lists all repositories registered in `~/.gitnexus/registry.json`. The MCP `list_ ## Troubleshooting - **"Not inside a git repository"**: Run from a directory inside a git repo -- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server +- **Index is stale after re-analyzing**: Wait for the next MCP tool call to reopen the published index; this normally takes no more than five seconds - **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-debugging/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-exploring/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md index 4fb73f3e6..85d90c90d 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-lfg/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-plan/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-refactoring/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-review/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json b/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json index 4fe6590dd..1af6255fb 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json +++ b/gitnexus-claude-plugin/skills/gitnexus-work/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "gitnexus": { "command": "npx", - "args": ["-y", "gitnexus@1.6.9", "mcp"] + "args": ["-y", "gitnexus@1.6.10", "mcp"] } } } diff --git a/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs b/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs index e68aca1de..564384f83 100644 --- a/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs +++ b/gitnexus-cursor-integration/hooks/gitnexus-hook.cjs @@ -85,40 +85,241 @@ function findGitNexusDir(startDir) { return null; } +function tokenizeShellWords(command) { + const tokens = []; + let current = ''; + let quote = null; + let escaped = false; + let hasToken = false; + + for (let index = 0; index < command.length; index += 1) { + const char = command[index]; + if (escaped) { + current += char; + escaped = false; + hasToken = true; + continue; + } + + if (quote === "'") { + if (char === "'") quote = null; + else current += char; + hasToken = true; + continue; + } + + if (quote === '"') { + if (char === '"') { + quote = null; + } else if (char === '\\') { + const next = command[index + 1]; + if (next === '$' || next === '`' || next === '"' || next === '\\') { + escaped = true; + } else { + current += '\\'; + } + } else { + current += char; + } + hasToken = true; + continue; + } + + if (char === '\\') { + const next = command[index + 1]; + if (next === undefined || /\s/.test(next) || next === "'" || next === '"' || next === '\\') { + escaped = true; + } else { + current += '\\' + next; + index += 1; + } + hasToken = true; + } else if (char === "'" || char === '"') { + quote = char; + hasToken = true; + } else if (/\s/.test(char)) { + if (hasToken) tokens.push(current); + current = ''; + hasToken = false; + } else if (char === ';' || char === '|' || char === '&') { + if (hasToken) tokens.push(current); + current = ''; + hasToken = false; + const next = command[index + 1]; + if ((char === '|' || char === '&') && next === char) { + tokens.push(char + char); + index += 1; + } else { + tokens.push(char); + } + } else { + current += char; + hasToken = true; + } + } + + if (escaped) current += '\\'; + if (hasToken) tokens.push(current); + return tokens; +} + function parseRgGrepPattern(cmd) { - const tokens = cmd.split(/\s+/); + const tokens = tokenizeShellWords(cmd); let foundCmd = false; let skipNext = false; + let skipNextAsPattern = false; + let endOfOptions = false; + let explicitPatternSeen = false; + let patternFileSeen = false; const flagsWithValues = new Set([ '-e', '-f', + '--file', '-m', + '--max-count', '-A', '-B', '-C', '-g', '--glob', + '--iglob', '-t', '--type', '--include', '--exclude', + '--encoding', + '--path', ]); + const rgValueFlags = new Set(['-r', '--replace']); + const patternFlags = new Set(['-e', '--regexp']); + const connectors = new Set(['&&', '||', ';', '|', '&']); + const wrappers = new Set([ + 'npx', + 'bunx', + 'pnpm', + 'yarn', + 'npm', + 'sudo', + 'env', + 'command', + 'time', + 'nice', + 'xargs', + 'dlx', + 'exec', + 'run', + 'git', + ]); + const wrapperFlagsWithValues = new Set([ + '--package', + '-p', + '--call', + '--prefix', + '--shell', + '--filter', + '--workspace', + '--dir', + '--cwd', + ]); + const basename = (token) => + token + .split(/[\\/]/) + .pop() + ?.replace(/\.(exe|cmd|bat)$/i, ''); + let previousToken; + let seenWrapper = false; + let searchCommand = null; for (const token of tokens) { if (skipNext) { skipNext = false; + if (skipNextAsPattern) { + skipNextAsPattern = false; + if (token.length >= 3) return token; + } + previousToken = token; continue; } if (!foundCmd) { - if (/\brg$|\bgrep$/.test(token)) foundCmd = true; + if (connectors.has(token)) { + seenWrapper = false; + previousToken = token; + continue; + } + const commandName = basename(token); + if (wrappers.has(commandName)) { + seenWrapper = true; + previousToken = token; + continue; + } + if (seenWrapper && token.startsWith('-')) { + const flagName = token.split('=', 1)[0]; + if (!token.includes('=') && wrapperFlagsWithValues.has(flagName)) skipNext = true; + previousToken = token; + continue; + } + if (seenWrapper && /^[A-Za-z_][A-Za-z0-9_]*=/.test(token)) { + previousToken = token; + continue; + } + const atCommandPosition = + previousToken === undefined || + connectors.has(previousToken) || + wrappers.has(basename(previousToken)) || + seenWrapper; + if (atCommandPosition && (commandName === 'rg' || commandName === 'grep')) { + foundCmd = true; + searchCommand = commandName; + } else if (seenWrapper) { + seenWrapper = false; + } + previousToken = token; + continue; + } + previousToken = token; + if (endOfOptions) { + if (explicitPatternSeen || patternFileSeen) continue; + return token.length >= 3 ? token : null; + } + if (token === '--') { + endOfOptions = true; continue; } if (token.startsWith('-')) { - if (flagsWithValues.has(token)) skipNext = true; + if (token === '-f' || token === '--file') { + skipNext = true; + patternFileSeen = true; + continue; + } + if (token.startsWith('--file=')) { + patternFileSeen = true; + continue; + } + if (token.startsWith('--regexp=')) { + explicitPatternSeen = true; + const value = token.slice('--regexp='.length); + if (value.length >= 3) return value; + continue; + } + const attachedPattern = token.match(/^-e(.+)$/); + if (attachedPattern) { + explicitPatternSeen = true; + if (attachedPattern[1].length >= 3) return attachedPattern[1]; + continue; + } + if ( + flagsWithValues.has(token) || + patternFlags.has(token) || + (searchCommand === 'rg' && rgValueFlags.has(token)) + ) { + skipNext = true; + skipNextAsPattern = patternFlags.has(token); + if (skipNextAsPattern) explicitPatternSeen = true; + } continue; } - const cleaned = token.replace(/['"]/g, ''); - return cleaned.length >= 3 ? cleaned : null; + if (explicitPatternSeen || patternFileSeen) continue; + return token.length >= 3 ? token : null; } return null; } @@ -179,12 +380,6 @@ function extractPattern(toolName, toolInput) { if (t === 'shell') { const cmd = toolInput.command || ''; if (!/\brg\b|\bgrep\b/.test(cmd)) return null; - // NOTE: parseRgGrepPattern uses split(/\s+/) and cannot handle shell - // quoting. `rg "User Service" src/` returns "User" (the first token - // after the rg/grep arg, with surrounding quotes stripped) — the - // multi-word pattern is intentionally not reconstructed since BM25 is - // already token-tolerant. Quoted single tokens (`rg "validateUser"`) - // work fine. return parseRgGrepPattern(cmd); } @@ -282,4 +477,6 @@ function main() { } } -main(); +if (require.main === module) main(); + +module.exports = { parseRgGrepPattern, tokenizeShellWords }; diff --git a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md index 4fb73f3e6..85d90c90d 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/gitnexus-shared/src/graph/types.ts b/gitnexus-shared/src/graph/types.ts index d9d916e0d..8b46113df 100644 --- a/gitnexus-shared/src/graph/types.ts +++ b/gitnexus-shared/src/graph/types.ts @@ -45,6 +45,21 @@ export type NodeLabel = | 'Section' | 'Route' | 'Tool' + /** + * A message-broker destination — a Kafka topic, a Rabbit exchange/routing + * key, a JMS queue, a Spring Cloud Stream binding. The framework overlay for + * ASYNCHRONOUS entry/exit points, symmetric to `Route` for HTTP. + * + * Identity is `(broker, resolved ADDRESS)`, so a publisher and a consumer of + * the same address on the same broker land on one node and the connection is + * a single hop — while a Kafka topic and a Rabbit queue that share a name + * stay two nodes, the same way `GET /x` and `POST /x` are two Routes. A + * destination whose address could NOT be resolved is keyed by its source + * location instead and carries no `address` property at all. See + * `pipeline-phases/spring-destinations.ts` for why an unresolved spelling may + * not key a node, and `ingestion/destination-key.ts` for why the broker may. + */ + | 'Destination' // Taint/PDG substrate (issue #2080). Intra-procedural control-flow node. // Emitted by no phase yet — M1 (#2081) populates these behind an opt-in. | 'BasicBlock'; @@ -95,6 +110,30 @@ export type NodeProperties = { responseKeys?: string[]; errorKeys?: string[]; middleware?: string[]; + /** Route runtime evidence is authoritative only when this is exactly true. */ + runtimeConfirmed?: boolean; + /** Provenance of runtime evidence; presence alone does not imply confirmation. */ + runtimeSource?: string; + /** Runtime result such as runtime-confirmed or handler-conflict. */ + runtimeStatus?: string; + // Destination (async messaging overlay). See the `Destination` label above. + /** The RESOLVED broker address. Together with `broker` it is the key a + * cross-repository pass joins on. Present only when the address resolved: + * absent is the load-bearing state, because an absent property cannot match + * another absent property. */ + address?: string; + /** Broker family the syntax attests to (`kafka`, `rabbit`, `jms`, …). Part + * of the node's identity alongside `address`, not a label on it. */ + broker?: string; + /** How the address was arrived at (`literal`, `constant`) when it resolved, + * or the named reason it did not. */ + resolution?: string; + /** Configuration key named by an unresolvable `${…}` placeholder. The key + * only — configuration VALUES are deliberately absent from this graph. */ + configKey?: string; + /** The `${key:default}` default text. Not an address: configuration can + * override it and the graph cannot see whether it did. */ + configDefault?: string; // BasicBlock (taint/PDG substrate, issue #2080) — reuses filePath/startLine/endLine. text?: string; /** BasicBlock: space-joined leaf callee names invoked in the block — the @@ -122,6 +161,19 @@ export type RelationshipType = | 'MEMBER_OF' | 'STEP_IN_PROCESS' | 'HANDLES_ROUTE' + /** Outbound async messaging. Source = the callable that performs the publish + * (or its File); target = the `Destination` it publishes to. Emitted by + * `pipeline-phases/spring-destinations.ts` from Spring messaging-template + * calls (`kafkaTemplate.send(...)`, `rabbitTemplate.convertAndSend(...)`). + * One edge per address: a publish that names two destinations yields two + * edges, and `reason` records which argument each came from. */ + | 'PUBLISHES_TO' + /** Inbound async messaging — the mirror of `PUBLISHES_TO`. Source = the + * annotated handler callable (or its File); target = the `Destination` it + * subscribes to. Emitted from `@KafkaListener` / `@RabbitListener` / + * `@JmsListener` and their siblings. Together the two types make + * "who else reads what this service writes" a two-hop traversal. */ + | 'CONSUMES_FROM' | 'FETCHES' | 'HANDLES_TOOL' | 'ENTRY_POINT_OF' diff --git a/gitnexus-shared/src/impact-risk.ts b/gitnexus-shared/src/impact-risk.ts new file mode 100644 index 000000000..413d02f76 --- /dev/null +++ b/gitnexus-shared/src/impact-risk.ts @@ -0,0 +1,151 @@ +export type ImpactRisk = 'LOW' | 'MEDIUM' | 'HIGH' | 'CRITICAL' | 'UNKNOWN'; + +export type ImpactRiskAxis = 'processes' | 'modules'; + +export type UnusedImpactRiskReason = + | 'file-nodes-have-no-process-or-community-membership' + | 'enrichment-skipped' + | 'enrichment-budget-exhausted' + | 'enrichment-truncated' + | 'enrichment-query-failed'; + +export interface UnusedImpactRiskAxis { + axis: ImpactRiskAxis; + reason: UnusedImpactRiskReason; +} + +export interface ImpactRiskInput { + direction: 'upstream' | 'downstream'; + directCount: number; + processCount: number; + moduleCount: number; + impactedCount: number; + unusedAxes?: readonly UnusedImpactRiskAxis[]; +} + +export interface ImpactRiskResult { + risk: ImpactRisk; + riskSharedAxes: ImpactRisk; + riskScale: { + comparableAcrossKinds: boolean; + unusedAxes: readonly UnusedImpactRiskAxis[]; + }; +} + +function score( + input: Pick< + ImpactRiskInput, + 'direction' | 'directCount' | 'processCount' | 'moduleCount' | 'impactedCount' + >, +): ImpactRisk { + const { direction, directCount, processCount, moduleCount, impactedCount } = input; + + if (direction === 'upstream' && impactedCount === 0) return 'UNKNOWN'; + if (directCount >= 30 || processCount >= 5 || moduleCount >= 5 || impactedCount >= 200) { + return 'CRITICAL'; + } + if (directCount >= 15 || processCount >= 3 || moduleCount >= 3 || impactedCount >= 100) { + return 'HIGH'; + } + if (directCount >= 5 || impactedCount >= 30) return 'MEDIUM'; + return 'LOW'; +} + +const UNMEASURED_REASONS: ReadonlySet = new Set([ + 'file-nodes-have-no-process-or-community-membership', + 'enrichment-skipped', + 'enrichment-budget-exhausted', +]); + +function unusedPair(reason: UnusedImpactRiskReason): UnusedImpactRiskAxis[] { + return [ + { axis: 'processes', reason }, + { axis: 'modules', reason }, + ]; +} + +function countsWithUnmeasuredAxesZeroed( + input: ImpactRiskInput, +): Pick< + ImpactRiskInput, + 'direction' | 'directCount' | 'processCount' | 'moduleCount' | 'impactedCount' +> { + let processCount = input.processCount; + let moduleCount = input.moduleCount; + for (const unused of input.unusedAxes ?? []) { + if (!UNMEASURED_REASONS.has(unused.reason)) continue; + if (unused.axis === 'processes') processCount = 0; + if (unused.axis === 'modules') moduleCount = 0; + } + return { + direction: input.direction, + directCount: input.directCount, + processCount, + moduleCount, + impactedCount: input.impactedCount, + }; +} + +/** Map walk outcomes to unused process/module axes so comparability matches what was sampled. */ +export function unusedAxesForImpactWalk(input: { + isFileTarget: boolean; + skipEnrichment: boolean; + maxChunks: number; + processQueryFailed: boolean; + moduleQueryFailed: boolean; + /** When 0, a zero chunk budget is not an unused-axis event — there was nothing to enrich. */ + impactedCount: number; + /** True when process/module queries ran on a strict subset of impacted symbols. */ + enrichmentTruncated?: boolean; +}): UnusedImpactRiskAxis[] { + if (input.isFileTarget) { + return unusedPair('file-nodes-have-no-process-or-community-membership'); + } + if (input.skipEnrichment) { + return unusedPair('enrichment-skipped'); + } + if (input.maxChunks === 0 && input.impactedCount > 0) { + return unusedPair('enrichment-budget-exhausted'); + } + const unused: UnusedImpactRiskAxis[] = []; + if (input.enrichmentTruncated) { + unused.push(...unusedPair('enrichment-truncated')); + } + if (input.processQueryFailed) { + unused.push({ axis: 'processes', reason: 'enrichment-query-failed' }); + } + if (input.moduleQueryFailed) { + unused.push({ axis: 'modules', reason: 'enrichment-query-failed' }); + } + return unused; +} + +const INCOMPLETE_SAMPLE_REASONS: ReadonlySet = new Set([ + 'enrichment-query-failed', + 'enrichment-truncated', +]); + +export function scoreImpactRisk(input: ImpactRiskInput): ImpactRiskResult { + const unusedAxes = input.unusedAxes ?? []; + const observedRisk = score(countsWithUnmeasuredAxesZeroed(input)); + const incompleteSample = unusedAxes.some((unused) => + INCOMPLETE_SAMPLE_REASONS.has(unused.reason), + ); + // Failed queries and truncated samples make observed process/module counts + // lower bounds. Preserve any HIGH/CRITICAL warning already proved by those + // counts, but never emit a confident LOW/MEDIUM edit gate from an incomplete + // enrichment pass. + const risk = + incompleteSample && (observedRisk === 'LOW' || observedRisk === 'MEDIUM') + ? 'UNKNOWN' + : observedRisk; + + return { + risk, + riskSharedAxes: score({ ...input, processCount: 0, moduleCount: 0 }), + riskScale: { + comparableAcrossKinds: unusedAxes.length === 0, + unusedAxes, + }, + }; +} diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index 13c2eac5a..9857a60cc 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -25,6 +25,17 @@ export { } from './language-detection.js'; export type { MroStrategy } from './mro-strategy.js'; +// Impact risk scoring +export { scoreImpactRisk, unusedAxesForImpactWalk } from './impact-risk.js'; +export type { + ImpactRisk, + ImpactRiskAxis, + ImpactRiskInput, + ImpactRiskResult, + UnusedImpactRiskAxis, + UnusedImpactRiskReason, +} from './impact-risk.js'; + // Pipeline progress export type { PipelinePhase, PipelineProgress } from './pipeline.js'; diff --git a/gitnexus-shared/src/lbug/schema-constants.ts b/gitnexus-shared/src/lbug/schema-constants.ts index 350aa273d..217c382a6 100644 --- a/gitnexus-shared/src/lbug/schema-constants.ts +++ b/gitnexus-shared/src/lbug/schema-constants.ts @@ -40,6 +40,8 @@ export const NODE_TABLES = [ 'Module', 'Route', 'Tool', + // Async messaging overlay — the broker-side counterpart of `Route`. + 'Destination', // Taint/PDG substrate (issue #2080) — inert until M1 (#2081) emits blocks. 'BasicBlock', ] as const; @@ -64,6 +66,8 @@ export const REL_TYPES = [ 'MEMBER_OF', 'STEP_IN_PROCESS', 'HANDLES_ROUTE', + 'PUBLISHES_TO', + 'CONSUMES_FROM', 'FETCHES', 'HANDLES_TOOL', 'ENTRY_POINT_OF', diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index a0162ee9c..53e1b5c69 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -36,28 +36,28 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.11", + "react-i18next": "^17.0.12", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", "remark-gfm": "^4.0.1", "sigma": "^3.0.3", "tailwindcss": "^4.3.3", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "zod": "^4.4.3" }, "devDependencies": { "@babel/types": "^8.0.4", "@playwright/test": "^1.62.0", - "@testing-library/jest-dom": "^6.9.1", + "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.1", + "@testing-library/user-event": "^14.6.6", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.9.9", + "@vercel/node": "^5.10.2", "@vitejs/plugin-react": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", "jsdom": "^29.1.1", @@ -246,9 +246,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -307,13 +307,6 @@ "specificity": "bin/cli.js" } }, - "node_modules/@bytecodealliance/preview2-shim": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@bytecodealliance/preview2-shim/-/preview2-shim-0.17.6.tgz", - "integrity": "sha512-n3cM88gTen5980UOBAD6xDcNNL3ocTK8keab21bpx1ONdA+ARj7uD1qoFxOWCyKlkpSi195FH+GeAut7Oc6zZw==", - "dev": true, - "license": "(Apache-2.0 WITH LLVM-exception)" - }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", @@ -1419,17 +1412,6 @@ "node": ">=20" } }, - "node_modules/@renovatebot/pep440": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@renovatebot/pep440/-/pep440-4.2.1.tgz", - "integrity": "sha512-2FK1hF93Fuf1laSdfiEmJvSJPVIDHEUTz68D3Fi9s0IZrrpaEcj6pTFBTbYvsgC5du4ogrtf5re7yMMvrKNgkw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.9.0 || ^22.11.0 || ^24", - "pnpm": "^10.0.0" - } - }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", @@ -1517,9 +1499,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1536,9 +1515,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1555,9 +1531,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1574,9 +1547,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1593,9 +1563,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1612,9 +1579,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1865,9 +1829,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1884,9 +1845,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1903,9 +1861,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1922,9 +1877,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2091,9 +2043,9 @@ } }, "node_modules/@testing-library/jest-dom": { - "version": "6.9.1", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", - "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-7.0.0.tgz", + "integrity": "sha512-HKAH9C6mBo5yBG6yRO5i43L2iisencAo5z+o5P/saHUoY+miC5ivXRxHBJcFyB5ypPNxHJdK3BoF/3O4DIptMg==", "dev": true, "license": "MIT", "dependencies": { @@ -2105,9 +2057,12 @@ "redent": "^3.0.0" }, "engines": { - "node": ">=14", + "node": ">=22", "npm": ">=6", "yarn": ">=1" + }, + "peerDependencies": { + "@testing-library/dom": ">=10 <11" } }, "node_modules/@testing-library/jest-dom/node_modules/dom-accessibility-api": { @@ -2146,9 +2101,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.1", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", - "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "version": "14.6.6", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", + "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", "dev": true, "license": "MIT", "engines": { @@ -2638,13 +2593,12 @@ } }, "node_modules/@vercel/build-utils": { - "version": "14.0.5", - "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-14.0.5.tgz", - "integrity": "sha512-ChbTraIvChbcFXMwDPLE8MoWpNGSRhJ2cXsE0V3iJQIVYDRgjFoT6JzWfkuc7w/3ojLLr8eMoae7M1v6OXoC5Q==", + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-14.2.0.tgz", + "integrity": "sha512-GwmtB31tBXQEzFw11grr8BKFCBdUORmYeooB0ZtonaCXZMZaPCHLBFTMFKsvaV6ZciQORPInRwXShbFvmnjqtg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@vercel/python-analysis": "0.13.2", "cjs-module-lexer": "1.2.3", "es-module-lexer": "1.5.0" } @@ -2691,9 +2645,9 @@ } }, "node_modules/@vercel/node": { - "version": "5.9.9", - "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.9.9.tgz", - "integrity": "sha512-jaMocJLa+rP3WpwYrbx2kUpHObjXK/JZOsbtmodDMAtfXbwl7niPNcEbdYYj/fBPSX8yRUXBF3tQsasocbjD5Q==", + "version": "5.10.2", + "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.10.2.tgz", + "integrity": "sha512-YBXcoQVOh5O2ySXvzE+POhPEQEPMJJo4ctlMMdp5why/NIoa8m6gotv14j8Uo6D5qyZsnc+0+++JgUiV4mYB6w==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2701,7 +2655,7 @@ "@edge-runtime/primitives": "4.1.0", "@edge-runtime/vm": "3.2.0", "@types/node": "20.11.0", - "@vercel/build-utils": "14.0.5", + "@vercel/build-utils": "14.2.0", "@vercel/error-utils": "2.2.1", "@vercel/nft": "1.10.0", "@vercel/static-config": "3.4.1", @@ -2738,32 +2692,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@vercel/python-analysis": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/@vercel/python-analysis/-/python-analysis-0.13.2.tgz", - "integrity": "sha512-IEr5K2gvX143NBoQc1W4BWrdDWjZwxnIT6UrL5Y1dnyH7Cqc4AV00FIAddB1YpnIZBJwT4ZhE8QbgqBeO6C9Zw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bytecodealliance/preview2-shim": "0.17.6", - "@renovatebot/pep440": "4.2.1", - "fs-extra": "11.1.1", - "js-yaml": "4.1.1", - "minimatch": "10.1.1", - "smol-toml": "1.5.2", - "zod": "3.22.4" - } - }, - "node_modules/@vercel/python-analysis/node_modules/zod": { - "version": "3.22.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", - "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, "node_modules/@vercel/static-config": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/@vercel/static-config/-/static-config-3.4.1.tgz", @@ -3051,13 +2979,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -4511,21 +4432,6 @@ "node": ">=0.4.x" } }, - "node_modules/fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -5197,19 +5103,6 @@ "license": "MIT", "peer": true }, - "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/jsdom": { "version": "29.1.1", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", @@ -5265,9 +5158,9 @@ } }, "node_modules/jsdom/node_modules/undici": { - "version": "7.25.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz", - "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { @@ -5319,19 +5212,6 @@ "dev": true, "license": "MIT" }, - "node_modules/jsonfile": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", - "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, "node_modules/katex": { "version": "0.16.47", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", @@ -6884,9 +6764,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", @@ -7414,12 +7294,12 @@ } }, "node_modules/react-i18next": { - "version": "17.0.11", - "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.11.tgz", - "integrity": "sha512-cDtkXgxjuFTWUH6V+aQn1Ve5vDiUztCNPWW5GtSHDccsgRXO1nE6QFWCEmc1KAutrb3OUv87wFShJL5RhUwPXg==", + "version": "17.0.12", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.12.tgz", + "integrity": "sha512-lFWPEGkxQ6RhusdUkysFBD58VHfSSzvHBzqMgN0SvfVpdQGfwtNkStTqdy08/sJd7s807qqutgx93fRpD0DJ3Q==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.29.2", + "@babel/runtime": "^7.29.7", "html-parse-stringify": "^4.0.1", "use-sync-external-store": "^1.6.0" }, @@ -7805,19 +7685,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/smol-toml": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", - "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">= 18" - }, - "funding": { - "url": "https://github.com/sponsors/cyyynthia" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -7952,9 +7819,9 @@ } }, "node_modules/tar": { - "version": "7.5.20", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.20.tgz", - "integrity": "sha512-9FcyK4PA6+WbzlTM9WhQm6vB5W7cP7dUiPsv1g7YDwEQnQ1CGpK3MGlKk/ITVWMk05kHZuBhmVhiv8LZoy/PFQ==", + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { @@ -8190,9 +8057,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "6.24.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.24.0.tgz", - "integrity": "sha512-lVLNosgqo5EkGqh5XUDhGfsMSoO8K0BAN0TyJLvwNRSl4xWGZlCVYsAIpa/OpA3TvmnM01GWcoKmc3ZWo5wKKA==", + "version": "6.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", "dev": true, "license": "MIT", "engines": { @@ -8293,16 +8160,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/use-sync-external-store": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", @@ -8313,9 +8170,9 @@ } }, "node_modules/uuid": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", - "integrity": "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 046bab225..09ac0b640 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -46,28 +46,28 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.11", + "react-i18next": "^17.0.12", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", "remark-gfm": "^4.0.1", "sigma": "^3.0.3", "tailwindcss": "^4.3.3", - "uuid": "^14.0.1", + "uuid": "^14.0.2", "zod": "^4.4.3" }, "devDependencies": { "@babel/types": "^8.0.4", "@playwright/test": "^1.62.0", - "@testing-library/jest-dom": "^6.9.1", + "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.1", + "@testing-library/user-event": "^14.6.6", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.9.9", + "@vercel/node": "^5.10.2", "@vitejs/plugin-react": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", "jsdom": "^29.1.1", @@ -83,7 +83,7 @@ }, "@vercel/node": { "path-to-regexp": "6.3.0", - "undici": "6.24.0" + "undici": "6.28.0" }, "@vercel/python-analysis": { "minimatch": "10.2.3", diff --git a/gitnexus-web/src/core/llm/tools.ts b/gitnexus-web/src/core/llm/tools.ts index a702e7db8..421725be3 100644 --- a/gitnexus-web/src/core/llm/tools.ts +++ b/gitnexus-web/src/core/llm/tools.ts @@ -13,8 +13,12 @@ import { tool } from '@langchain/core/tools'; import { z } from 'zod'; -import { NODE_TABLES, REL_TYPES } from 'gitnexus-shared'; -import type { EnrichedSearchResult, GrepResult } from '../../services/backend-client'; +import { NODE_TABLES, REL_TYPES, scoreImpactRisk, unusedAxesForImpactWalk } from 'gitnexus-shared'; +import type { + EnrichedSearchResult, + GrepOptions, + GrepResponse, +} from '../../services/backend-client'; /** * Tool names registered by createGraphRAGTools — kept in sync with each tool's `name` @@ -44,7 +48,7 @@ export interface GraphRAGBackend { query: string, opts?: { limit?: number; mode?: 'hybrid' | 'semantic' | 'bm25'; enrich?: boolean }, ) => Promise; - grep: (pattern: string, limit?: number) => Promise; + grep: (pattern: string, limit?: number, opts?: GrepOptions) => Promise; readFile: (filePath: string) => Promise; } @@ -375,20 +379,22 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, } const limit = maxResults ?? 100; - const fullPattern = fileFilter - ? `(?=.*${fileFilter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}).*${pattern}` - : pattern; - - const results = await backendGrep(fullPattern, limit); + const { results, timedOut } = await backendGrep(pattern, limit, { + fileFilter, + caseSensitive, + }); + const timeoutMsg = timedOut + ? '\n\n(Scan timed out after a few seconds — results may be incomplete)' + : ''; if (results.length === 0) { - return `No matches for "${pattern}"${fileFilter ? ` in files matching "${fileFilter}"` : ''}`; + return `No matches for "${pattern}"${fileFilter ? ` in files matching "${fileFilter}"` : ''}${timeoutMsg}`; } const formatted = results.map((r) => `${r.filePath}:${r.line}: ${r.text}`).join('\n'); const truncatedMsg = results.length >= limit ? `\n\n(Showing first ${limit} results)` : ''; - return `Found ${results.length} matches:\n\n${formatted}${truncatedMsg}`; + return `Found ${results.length} matches:\n\n${formatted}${truncatedMsg}${timeoutMsg}`; } catch (error) { return `Grep error: ${error instanceof Error ? error.message : String(error)}`; } @@ -396,16 +402,20 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, { name: 'grep', description: - 'Search for exact text patterns across all files using regex. Use for finding specific strings, error messages, TODOs, variable names, etc.', + 'Search file contents with a regular expression (server executes it as a real regex — alternation like "sign|Sign" works). Matches are case-insensitive unless caseSensitive is set. fileFilter keeps only files whose path contains the substring. Each call caps at maxResults matches (default 100) and the server stops after a few seconds (the tool will say so if the scan was incomplete), so prefer precise patterns over catch-alls.', schema: z.object({ pattern: z .string() - .describe('Regex pattern to search for (e.g., "TODO", "console\\.log", "API_KEY")'), + .describe( + 'Regex pattern to search for (e.g., "TODO|FIXME", "console\\.log", "signOrder")', + ), fileFilter: z .string() .optional() .nullable() - .describe('Only search files containing this string (e.g., ".ts", "src/api")'), + .describe( + 'Only search files whose path contains this substring (e.g., ".ts", "src/api", "Controller.java")', + ), caseSensitive: z .boolean() .optional() @@ -1219,7 +1229,7 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, const targetFileName = (targetFilePath || target).split('/').pop() || target; const baseName = targetFileName.replace(/\.[^/.]+$/, ''); try { - const hints = await backendGrep(`\\b${escapeRegex(baseName)}\\b`, 15); + const { results: hints } = await backendGrep(`\\b${escapeRegex(baseName)}\\b`, 15); const filtered = hints.filter((h) => h.filePath !== targetFilePath); if (filtered.length > 0) { @@ -1275,6 +1285,9 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, stepCount: number | null; }> = []; let affectedClusters: Array<{ label: string; hits: number; impact: string }> = []; + let processQueryFailed = false; + let clusterQueryFailed = false; + let clusterClassificationFailed = false; if (trimmedIds.length > 0) { const processQuery = ` @@ -1302,9 +1315,23 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, : ''; const [processRes, clusterRes, directClusterRes] = await Promise.all([ - executeQuery(processQuery), - executeQuery(clusterQuery), - directClusterQuery ? executeQuery(directClusterQuery) : Promise.resolve([]), + executeQuery(processQuery).catch((err) => { + processQueryFailed = true; + if (import.meta.env.DEV) console.warn('Impact process enrichment failed:', err); + return []; + }), + executeQuery(clusterQuery).catch((err) => { + clusterQueryFailed = true; + if (import.meta.env.DEV) console.warn('Impact cluster enrichment failed:', err); + return []; + }), + directClusterQuery + ? executeQuery(directClusterQuery).catch((err) => { + clusterClassificationFailed = true; + if (import.meta.env.DEV) console.warn('Impact cluster enrichment failed:', err); + return []; + }) + : Promise.resolve([]), ]); const directClusterSet = new Set(); @@ -1323,7 +1350,11 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, affectedClusters = clusterRes.map((row: any) => { const label = Array.isArray(row) ? row[0] : row.label; const hits = Array.isArray(row) ? row[1] : row.hits; - const impact = directClusterSet.has(label) ? 'direct' : 'indirect'; + const impact = clusterClassificationFailed + ? 'classification-unavailable' + : directClusterSet.has(label) + ? 'direct' + : 'indirect'; return { label, hits, impact }; }); } @@ -1331,19 +1362,25 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, const directCount = depth1.length; const processCount = affectedProcesses.length; const clusterCount = affectedClusters.length; - let risk = 'LOW'; - if (directCount >= 30 || processCount >= 5 || clusterCount >= 5 || totalAffected >= 200) { - risk = 'CRITICAL'; - } else if ( - directCount >= 15 || - processCount >= 3 || - clusterCount >= 3 || - totalAffected >= 100 - ) { - risk = 'HIGH'; - } else if (directCount >= 5 || totalAffected >= 30) { - risk = 'MEDIUM'; - } + const enrichmentCapped = allNodeIds.length > maxIdsForContext; + const unusedAxes = unusedAxesForImpactWalk({ + isFileTarget: false, + skipEnrichment: false, + maxChunks: 10, + processQueryFailed, + moduleQueryFailed: clusterQueryFailed, + impactedCount: totalAffected, + enrichmentTruncated: enrichmentCapped, + }); + const scored = scoreImpactRisk({ + direction, + directCount, + processCount, + moduleCount: clusterCount, + impactedCount: totalAffected, + unusedAxes, + }); + const { risk, riskSharedAxes, riskScale } = scored; // ===== COMPACT TABULAR OUTPUT ===== const lines: string[] = [ @@ -1351,22 +1388,42 @@ MATCH (n:Function {id: emb.nodeId}) RETURN n`, `Confidence: High ${confidenceBuckets.high} | Medium ${confidenceBuckets.medium} | Low ${confidenceBuckets.low}`, ``, `AFFECTED PROCESSES:`, - ...(affectedProcesses.length > 0 - ? affectedProcesses.map( - (p) => - `- ${p.label} - BROKEN at step ${p.minStep ?? '?'} (${p.hits} symbols, ${p.stepCount ?? '?'} steps)`, - ) - : ['- None found']), + ...(processQueryFailed + ? ['- Unavailable (enrichment query failed)'] + : affectedProcesses.length > 0 + ? affectedProcesses.map( + (p) => + `- ${p.label} - BROKEN at step ${p.minStep ?? '?'} (${p.hits} symbols, ${p.stepCount ?? '?'} steps)`, + ) + : ['- None found']), ``, `AFFECTED CLUSTERS:`, - ...(affectedClusters.length > 0 - ? affectedClusters.map((c) => `- ${c.label} (${c.impact}, ${c.hits} symbols)`) - : ['- None found']), + ...(clusterQueryFailed + ? ['- Unavailable (enrichment query failed)'] + : affectedClusters.length > 0 + ? affectedClusters.map((c) => `- ${c.label} (${c.impact}, ${c.hits} symbols)`) + : ['- None found']), ``, - `RISK: ${risk}`, + `RISK: ${risk} (edit gate — warn on HIGH/CRITICAL)`, + `Shared-axes: ${riskSharedAxes} (File vs symbol compare only; do not waive a HIGH risk warning)`, + `Note: this Graph-RAG surface expands File targets to in-file symbols before enrichment, so process/cluster axes are comparable here when enrichment succeeds. MCP File impact does not.`, + ...(riskScale.comparableAcrossKinds + ? [] + : [ + `Note: process/module axes were unused (${riskScale.unusedAxes.map((a) => a.reason).join(', ')}).`, + ]), + ...(risk === 'UNKNOWN' && (processQueryFailed || clusterQueryFailed) + ? ['Note: risk is unresolved because enrichment failed; retry before editing.'] + : []), + ...(enrichmentCapped + ? [`Note: process/cluster enrichment is partial (first ${maxIdsForContext} symbols).`] + : []), + ...(clusterClassificationFailed + ? ['Note: direct/indirect cluster classification is unavailable.'] + : []), `- Direct callers: ${directCount}`, - `- Processes affected: ${processCount}`, - `- Clusters affected: ${clusterCount}`, + `- Processes affected: ${processQueryFailed ? 'unavailable' : processCount}`, + `- Clusters affected: ${clusterQueryFailed ? 'unavailable' : clusterCount}`, ``, ]; @@ -1472,7 +1529,9 @@ relationTypes filter (optional): Additional output sections: - Affected processes (with step impact) - Affected clusters (direct/indirect) -- Risk summary (based on direct callers, processes, clusters)`, +- RISK is the edit gate: warn before edits on HIGH/CRITICAL; UNKNOWN requires retry or corroboration +- Shared-axes risk compares File and symbol targets using direct/total counts only; it never waives the RISK gate +- riskScale notes unavailable process/module axes. This Graph-RAG tool expands File targets to in-file symbols; MCP File impact does not`, schema: z.object({ target: z.string().describe('Name of the function, class, or file to analyze'), direction: z diff --git a/gitnexus-web/src/hooks/useAppState.tsx b/gitnexus-web/src/hooks/useAppState.tsx index d698b87d5..deeda86bf 100644 --- a/gitnexus-web/src/hooks/useAppState.tsx +++ b/gitnexus-web/src/hooks/useAppState.tsx @@ -40,6 +40,7 @@ import { repoIdentity as repoIdentityOf, type BackendRepo, type ConnectResult, + type GrepOptions, type JobProgress, } from '../services/backend-client'; import { ERROR_RESET_DELAY_MS } from '../config/ui-constants'; @@ -671,7 +672,8 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { const backend = { executeQuery, search: (query: string, opts?: any) => backendSearch(query, { ...opts, repo }), - grep: (pattern: string, limit?: number) => backendGrep(pattern, repo, limit), + grep: (pattern: string, limit?: number, opts?: GrepOptions) => + backendGrep(pattern, repo, limit, opts), readFile: (filePath: string) => backendReadFile(filePath, { repo }).then((r) => r.content), }; diff --git a/gitnexus-web/src/lib/constants.ts b/gitnexus-web/src/lib/constants.ts index 2f717cab3..5a85754cf 100644 --- a/gitnexus-web/src/lib/constants.ts +++ b/gitnexus-web/src/lib/constants.ts @@ -37,6 +37,7 @@ export const NODE_COLORS: Record = { Constructor: '#10b981', // Emerald - like Function Template: '#a78bfa', // Violet light - like Type Route: '#f43f5e', // Rose - like Process + Destination: '#fb7185', // Rose light - like Route, the broker-side counterpart Tool: '#a855f7', // Purple - like Project BasicBlock: '#475569', // Slate darker - control-flow node (muted, taint/PDG substrate) }; @@ -79,6 +80,7 @@ export const NODE_SIZES: Record = { Constructor: 4, // Like Function Template: 3, // Like Type Route: 5, // Like Enum + Destination: 5, // Like Route - the broker-side counterpart Tool: 5, // Like Enum BasicBlock: 2, // Tiny - control-flow node (taint/PDG substrate) }; diff --git a/gitnexus-web/src/lib/upload-filter.test.ts b/gitnexus-web/src/lib/upload-filter.test.ts index e97b9ec2c..1943720ba 100644 --- a/gitnexus-web/src/lib/upload-filter.test.ts +++ b/gitnexus-web/src/lib/upload-filter.test.ts @@ -31,6 +31,22 @@ describe('filterRepoFiles', () => { expect(r.droppedCount).toBe(4); }); + it('excludes emitted _next output, including the Capacitor/Cordova copy', () => { + // `.next` was listed but `_next` was not, so a mobile-wrapped Next.js app + // uploaded its whole minified bundle against the server's caps for files + // the analyzer then discards anyway (#3007). + const input = [ + f('repo/android/app/src/main/assets/public/_next/static/chunks/main.js'), + f('repo/ios/App/App/public/_next/static/chunks/framework.js'), + f('repo/_next/static/chunks/x.js'), + f('repo/src/index.ts'), + f('repo/src/_nextgen/index.ts'), + ]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/src/_nextgen/index.ts']); + expect(r.droppedCount).toBe(3); + }); + it('drops files over the per-file size cap', () => { const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)]; const r = filterRepoFiles(input); diff --git a/gitnexus-web/src/lib/upload-filter.ts b/gitnexus-web/src/lib/upload-filter.ts index a24520f74..9b3fb30db 100644 --- a/gitnexus-web/src/lib/upload-filter.ts +++ b/gitnexus-web/src/lib/upload-filter.ts @@ -22,6 +22,17 @@ export const EXCLUDED_DIRS = new Set([ 'build', 'out', '.next', + // `.next` is the build CACHE, `_next` the EMITTED output — different + // directories. A Capacitor/Cordova shell leaves the emitted bundle at + // `/app/src/main/assets/public/_next/`, so without this the whole + // minified tree is uploaded against the server's file/byte caps only to be + // discarded by the analyzer's own ignore list (#3007). + // + // This pre-filter reads no repository ignore rules, so unlike the CLI walker + // a `.gitnexusignore` negation cannot recover anything dropped here. Names + // added below must therefore stay a subset of the analyzer's own list; see + // `gitnexus/test/unit/upload-filter-ignore-drift.test.ts`. + '_next', '.nuxt', '.cache', 'coverage', diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index 52f8c65c9..706b6d90e 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -64,6 +64,12 @@ export interface GrepResult { text: string; } +/** Full `/api/grep` payload — `timedOut` is true when the 5s budget cut the scan short. */ +export interface GrepResponse { + results: GrepResult[]; + timedOut: boolean; +} + export interface JobProgress { phase: string; percent: number; @@ -869,23 +875,37 @@ export const search = async ( return (body.results ?? []) as EnrichedSearchResult[]; }; -/** Grep across file contents in the indexed repo. */ +/** Options for {@link grep} beyond pattern/repo/limit. */ +export interface GrepOptions { + /** Only search files whose path contains this substring (case-insensitive). */ + fileFilter?: string | null; + /** Case-sensitive matching (default: insensitive). */ + caseSensitive?: boolean; +} + +/** Grep across file contents in the indexed repo. Regex semantics server-side. */ export const grep = async ( pattern: string, repo?: string, limit?: number, -): Promise => { + opts?: GrepOptions, +): Promise => { const params = [ `pattern=${encodeURIComponent(pattern)}`, repoParam(repo), limit ? `limit=${limit}` : '', + opts?.fileFilter ? `fileFilter=${encodeURIComponent(opts.fileFilter)}` : '', + opts?.caseSensitive ? 'caseSensitive=1' : '', ] .filter(Boolean) .join('&'); const response = await fetchWithTimeout(`${_backendUrl}/api/grep?${params}`); await assertOk(response); - const body = await response.json(); - return (body.results ?? []) as GrepResult[]; + const body = (await response.json()) as Partial; + return { + results: body.results ?? [], + timedOut: body.timedOut === true, + }; }; /** Result from reading a file, optionally with line range. */ diff --git a/gitnexus-web/test/unit/agent-prompt.test.ts b/gitnexus-web/test/unit/agent-prompt.test.ts index c2cb1392f..8bf69a513 100644 --- a/gitnexus-web/test/unit/agent-prompt.test.ts +++ b/gitnexus-web/test/unit/agent-prompt.test.ts @@ -43,7 +43,7 @@ const FORBIDDEN_TOOL_NAMES = [ const stubBackend: GraphRAGBackend = { executeQuery: async () => [], search: async () => [], - grep: async () => [], + grep: async () => ({ results: [], timedOut: false }), readFile: async () => '', }; diff --git a/gitnexus-web/test/unit/backend-client-grep.test.ts b/gitnexus-web/test/unit/backend-client-grep.test.ts new file mode 100644 index 000000000..183068d5e --- /dev/null +++ b/gitnexus-web/test/unit/backend-client-grep.test.ts @@ -0,0 +1,75 @@ +/** + * `/api/grep` client: query params and `timedOut` must reach callers. + * Dropping `timedOut` made a 5s partial scan look like a complete miss. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers'; +import { grep, setBackendUrl } from '../../src/services/backend-client'; + +const BASE = 'http://grep-client.test:4747'; + +const jsonOk = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + +describe('backend-client grep', () => { + beforeEach(() => { + __resetBreakerRegistry__(); + setBackendUrl(BASE); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('forwards fileFilter and caseSensitive and returns timedOut', async () => { + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + expect(url).toContain('/api/grep?'); + expect(url).toContain(`pattern=${encodeURIComponent('sign|Sign')}`); + expect(url).toContain(`fileFilter=${encodeURIComponent('src/api')}`); + expect(url).toContain('caseSensitive=1'); + expect(url).toContain('limit=12'); + return jsonOk({ + results: [{ filePath: 'src/api.ts', line: 3, text: 'signOrder()' }], + timedOut: true, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const body = await grep('sign|Sign', '/repo', 12, { + fileFilter: 'src/api', + caseSensitive: true, + }); + expect(body.results).toEqual([{ filePath: 'src/api.ts', line: 3, text: 'signOrder()' }]); + expect(body.timedOut).toBe(true); + }); + + it('reports timedOut false when the server completed the scan', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return jsonOk({ results: [] }); + }), + ); + + const body = await grep('TODO'); + expect(body).toEqual({ results: [], timedOut: false }); + }); + + it('does not send fileFilter when it is null or empty', async () => { + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + expect(url).not.toContain('fileFilter='); + return jsonOk({ results: [] }); + }); + vi.stubGlobal('fetch', fetchMock); + + for (const fileFilter of ['', null] as const) { + await grep('x', undefined, undefined, { fileFilter }); + } + expect(fetchMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/gitnexus-web/test/unit/grep-tool.test.ts b/gitnexus-web/test/unit/grep-tool.test.ts new file mode 100644 index 000000000..8701ca665 --- /dev/null +++ b/gitnexus-web/test/unit/grep-tool.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createGraphRAGTools, type GraphRAGBackend } from '../../src/core/llm/tools'; + +const noOpBackend: GraphRAGBackend = { + executeQuery: async () => [], + search: async () => [], + grep: async () => ({ results: [], timedOut: false }), + readFile: async () => '', +}; + +function grepTool(backend: GraphRAGBackend) { + return createGraphRAGTools(backend).find((candidate) => candidate.name === 'grep')!; +} + +describe('grep tool timeout contract', () => { + it('says the scan was incomplete when the server sets timedOut with no hits', async () => { + const grep = vi.fn(async () => ({ results: [], timedOut: true })); + const output = await grepTool({ ...noOpBackend, grep }).invoke({ pattern: 'signOrder' }); + expect(output).toContain('No matches for "signOrder"'); + expect(output).toContain('results may be incomplete'); + }); + + it('still warns when a timed-out scan returned some hits below the limit', async () => { + const grep = vi.fn(async () => ({ + results: [{ filePath: 'a.ts', line: 1, text: 'signOrder()' }], + timedOut: true, + })); + const output = await grepTool({ ...noOpBackend, grep }).invoke({ + pattern: 'signOrder', + maxResults: 100, + }); + expect(output).toContain('Found 1 matches'); + expect(output).toContain('results may be incomplete'); + expect(output).not.toContain('Showing first'); + }); +}); diff --git a/gitnexus-web/test/unit/impact-tool.test.ts b/gitnexus-web/test/unit/impact-tool.test.ts new file mode 100644 index 000000000..704240e6e --- /dev/null +++ b/gitnexus-web/test/unit/impact-tool.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createGraphRAGTools, type GraphRAGBackend } from '../../src/core/llm/tools'; + +const noOpBackend: GraphRAGBackend = { + executeQuery: async () => [], + search: async () => [], + grep: async () => ({ results: [], timedOut: false }), + readFile: async () => '', +}; + +function impactTool(backend: GraphRAGBackend) { + return createGraphRAGTools(backend).find((candidate) => candidate.name === 'impact')!; +} + +describe('Graph-RAG impact risk contract', () => { + it('advertises the edit gate, shared axes, and MCP File difference', () => { + const description = impactTool(noOpBackend).description; + expect(description).toContain('RISK is the edit gate'); + expect(description).toContain('Shared-axes risk'); + expect(description).toContain('riskScale'); + expect(description).toContain('MCP File impact does not'); + }); + + it('renders failed enrichment as unavailable and fails the risk gate closed', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + if (query.includes('MATCH (affected)-[r:CodeRelation]->(target)')) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + startLine: 4, + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) throw new Error('process query failed'); + if (query.includes('MEMBER_OF')) return []; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('AFFECTED PROCESSES:\n- Unavailable (enrichment query failed)'); + expect(output).not.toContain('AFFECTED PROCESSES:\n- None found'); + expect(output).toContain('RISK: UNKNOWN'); + expect(output).toContain('risk is unresolved because enrichment failed'); + expect(output).toContain('- Processes affected: unavailable'); + }); + + it('preserves proved CRITICAL risk when the cluster query fails', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + if (query.includes('MATCH (affected)-[r:CodeRelation]->(target)')) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) { + return Array.from({ length: 5 }, (_, index) => ({ + label: `process-${index}`, + hits: 1, + minStep: index + 1, + stepCount: 5, + })); + } + if (query.includes('MEMBER_OF') && query.includes('COUNT(DISTINCT s.id)')) { + throw new Error('cluster query failed'); + } + if (query.includes('MEMBER_OF')) return []; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('RISK: CRITICAL'); + expect(output).toContain('AFFECTED CLUSTERS:\n- Unavailable (enrichment query failed)'); + expect(output).toContain('- Processes affected: 5'); + expect(output).toContain('- Clusters affected: unavailable'); + }); + + it('does not invent direct/indirect cluster classification after its query fails', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + if (query.includes('MATCH (affected)-[r:CodeRelation]->(target)')) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) return []; + if (query.includes('MEMBER_OF') && query.includes('RETURN DISTINCT')) { + throw new Error('classification query failed'); + } + if (query.includes('MEMBER_OF')) return [{ label: 'Core', hits: 1 }]; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('- Core (classification-unavailable, 1 symbols)'); + expect(output).toContain('direct/indirect cluster classification is unavailable'); + expect(output).not.toContain('process/module axes were unused'); + }); + + it('treats successful File expansion as comparable because enrichment runs on member symbols', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("n.filePath CONTAINS 'src/target.ts'")) { + return [{ id: 'file-id', nodeType: 'File', filePath: 'src/target.ts' }]; + } + if (query.includes("callee.filePath = 'src/target.ts'")) { + return [ + { + id: 'caller-id', + name: 'caller', + nodeType: 'Function', + filePath: 'src/caller.ts', + edgeType: 'CALLS', + confidence: 1, + }, + ]; + } + if (query.includes('STEP_IN_PROCESS')) { + return [{ label: 'Build', hits: 1, minStep: 1, stepCount: 1 }]; + } + if (query.includes('MEMBER_OF') && query.includes('RETURN DISTINCT')) { + return [{ label: 'Core' }]; + } + if (query.includes('MEMBER_OF')) return [{ label: 'Core', hits: 1 }]; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'src/target.ts', + direction: 'upstream', + maxDepth: 1, + }); + + expect(output).toContain('process/cluster axes are comparable here when enrichment succeeds'); + expect(output).toContain('- Processes affected: 1'); + expect(output).toContain('- Clusters affected: 1'); + expect(output).not.toContain('process/module axes were unused'); + }); + + it('surfaces the 500-symbol enrichment cap as partial', async () => { + const executeQuery = vi.fn(async (query: string) => { + if (query.includes("WHERE n.name = 'target'")) { + return [{ id: 'target-id', nodeType: 'Function', filePath: 'src/target.ts' }]; + } + const depth = query.includes('3 AS depth') ? 3 : query.includes('2 AS depth') ? 2 : 1; + if (query.includes('CodeRelation') && query.includes(` ${depth} AS depth`)) { + return Array.from({ length: 200 }, (_, index) => ({ + id: `d${depth}-${index}`, + name: `node-${depth}-${index}`, + nodeType: 'Function', + filePath: `src/d${depth}-${index}.ts`, + edgeType: 'CALLS', + confidence: 1, + })); + } + if (query.includes('STEP_IN_PROCESS') || query.includes('MEMBER_OF')) return []; + return []; + }); + + const output = await impactTool({ ...noOpBackend, executeQuery }).invoke({ + target: 'target', + direction: 'upstream', + maxDepth: 3, + }); + + expect(output).toContain('process/cluster enrichment is partial (first 500 symbols)'); + expect(output).toContain('enrichment-truncated'); + expect(output).not.toContain('enrichment-budget-exhausted'); + }); +}); diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index def77dce8..6b4520d63 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -4,6 +4,97 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +## [1.6.10] - 2026-08-27 + +### Added + +- **Spring framework modeling expanded end to end** — AOP transactions, caching and security (#2783), `@Bean` factories and `@Resource` injection (#2740), profiles/conditions/auto-configuration (#2678), constructor and standard injection (#2632), bean candidate inventory (#2494), configuration-property consumers, and non-HTTP handler entry points (#2891) +- **Receiver chains typed from AST structure across all 14 languages**, with an explicit epistemic lower bound on what the graph can claim (#2708, #2744, #2747) +- **Java enum constant bodies modeled as first-class instances**, with JLS 13.1 anonymous-class naming (#2558) +- **More route surfaces indexed** — Java constant-based route paths such as `@PostMapping(ApiPathConstants.X)` (#2980) and JavaScript data route tables (#2972) +- **MCP server hardening** — repository allowlist, fail-closed read-only mode, deterministic output budgets, and normalized `impact`/`context` aliases +- **`bunx` lane so bun-only machines can run GitNexus** (#2765) +- **Codex support** — hooks, plugin marketplace and setup (#2328, #2369) — plus CodeBuddy and Qoder coding-agent integrations (#2368) +- **Skills mirrored to `.agents/skills/`** when an `.agents/` directory exists +- **One-click Render deploy** (#2804) +- **`serve` origin/proxy configuration is validated and port-scoped** (#2820) +- **Expanded TypeScript/JavaScript taint sink model** (#2490) +- **Wiki generation accepts explicit HTTP LLM hosts** (#2491) +- **Embedding request-body dimensions configurable** via `GITNEXUS_EMBEDDING_REQUEST_DIMS` (#2574) +- **Refreshed MiniMax model and endpoint configuration** (#2780) +- **`MAX_CALLABLE_VALUE_TARGETS` and `MAX_PROPERTY_DISPATCH_FANOUT` configurable via env** (#2725, #2726) +- **Opt-in `analyze --self-commit`** for AGENTS.md/CLAUDE.md churn (#2640) +- **Buffer pool sized to the graph before the database opens**, with an adaptive size hint +- **CI review agent runs as a coordinated reviewer swarm** on Sonnet 5 with structured, linked reviews (#2570, #2572), alongside the GitNexus Engineering Tool Kit skills (#2566) and an online skill-evolution loop (#2571) +- **Icebug community-engine prototype behind a gate** (#2376) + +### Fixed + +- **`group sync` stops claiming matching it never did** — the advertised BM25/embedding cascade was config, help text and MCP schema with no matcher behind it; the unread `matching.bm25_threshold`, `matching.embedding_threshold`, `detect.embedding_fallback` and `--skip-embeddings` surfaces are removed (#3020) +- **Emitted Next.js build output is ignored during ingestion**, and the inert `public/build` entry is deleted (#3018) +- **NestJS decorator routes are indexed** so `api_impact` and `route_map` stop reporting live endpoints as non-existent (#3017) +- **Import resolution gated by real module configuration** instead of path-suffix guessing — TypeScript config (#2953, #2956), Java and Kotlin declared packages (#2955, #2990), Go module paths (#2984), PHP Composer autoload maps (#2987), Python `__init__.py` re-exports (#2864) and unaliased dotted namespace imports (#2826, #2828), and JavaScript module extensions (#3034) +- **Interface dispatch is generic-instantiation aware** (#2912, #2939), fans out from Case 3b receivers (#2832, #2842) and from C# record interface calls (#2904), and resolves through generic-typed field receivers in every language (#2833, #2855) +- **Go method sets modeled exactly** so interface satisfaction is decidable (#2813, #2829), out-of-repo package qualifiers resolve, and an undecided interface check is no longer reported as a decided negative (#2873, #2921) +- **Go pointer-receiver calls resolve**, reporting the program boundary instead of hedging (#2766, #2782) +- **Java record support** — graph nodes for `record_declaration`, component accessors, enum and record interface heritage (#2564, #2916, #2935, #2936), plus `E.CONST.method()` enum-constant receiver dispatch (#2561) and JLS binary-name identities for local classes, enums, records and interfaces (#2562, #2653) +- **Rust module-qualified calls resolve against the module tree** (#2730, #2741), items are qualified by their enclosing `mod` chain (#2742, #2745), duplicate type names stay ambiguous in range binding (#2514, #2652), and `Box` names normalize +- **Closure bindings are call sources in every language**, and function-local values carry their own identity (#2693, #2695, #2699, #2718) +- **A named receiver's member never resolves lexically** (#2714), platform builtins stop resolving to unrelated same-file symbols (#2549), and inline constructor receivers are typed in every spelling (#2708, #2737) +- **Python calls resolve through constructor-injected fields** (#2628) and module-imported classes (#2770) +- **Package directories that repeat higher in the path resolve correctly** (#2881, #2929) +- **`check` stops reporting erased and deferred imports as initialization cycles** (#2934) +- **`detect_changes` no longer scales its query with the diff's hunk count** (#2915, #2930), and CR-only line-ending diffs are ignored (#2839) +- **`group` stops reporting what could not be measured as a measurement of zero** (#3012), resolves HTTP consumers through configured clients and constant route tables (#3008), and preserves manifest-only impact crossings (#2784) +- **`impact` and `context` are reproducible** — deterministic ordering on every capped query (#2787, #2796) — and Convex caller results are marked incomplete rather than empty (#3044) +- **Object handler identity is preserved** during ingestion (#3046), nested source directories are discovered (#3043), and parse-node insertion is canonicalized +- **Large-repo analyze OOM and the false worker-timeout cascade are fixed** (#2649, #2679) +- **Single-writer lock on the index write path** (#2658, #2677), atomic index swap with read-pool staleness invalidation (#2614), and reliable large incremental writeback commits (#2409, #2425) +- **Remote URLs are stripped of credentials before they are persisted** (#2914, #2928), and every registry write gets its own tmp path (#2888, #2920) +- **Schema version derived from a DDL fingerprint** instead of a hand-incremented constant (#2798, #2808), and the scope-resolution relation cross product is fully declared (#2792, #2793) +- **FTS reliability** — binary payloads stay out of the description column and an unbuildable index is confined to its own table (#2919), FTS-indexed DML is gated before the incremental writeback (#2841, #2854), analyze degrades instead of aborting on index-build failure (#2548), real LOAD errors surface and broken extension files self-heal (#2374, #2375), and Windows missing-dependency load failures are diagnosed (#2383) +- **`VECTOR` is loaded only when needed** (#3045) and before the incremental writeback touches embedding rows (#2623, #2624) +- **Buffer pool bounded instead of taking the native 80%-of-RAM default** (#2560), scaled by the OS page-size granule ratio (#2631, #2636), with a COPY-safe floor and actionable diagnostics for non-4K page sizes (#2424) +- **`Napi::Error` SIGABRT on analyze eliminated** — C++ type lookups are indexed and workers terminate only at JS-safe points (#2432, #2436) +- **Native-load failures fail closed**, including truncated-binary SIGBUS (#2441, #2651), and glibc-too-old loads are no longer misdiagnosed (#2672, #2689) +- **Index staleness reporting fixed** — no false-stale status after analyze, with inline staleness in `query`/`context`/`impact`/`cypher` (#2655, #2668, #2683) +- **Windows path handling** — the `\\?\` long-path prefix no longer breaks repo path matching (#2667, #2700), `parts` negation is honored (#2720), and missing-shadow errors let `serve` repo-switch recover (#2382, #2387) +- **Embeddings survive partial failures** — unparseable 200 responses are retried (#2790, #2795), batch inserts are retry-safe (#2453), HTTP generation is resumable, resume checkpoints bind to their provider, and proxy-blocked installs self-heal (#2370, #2372) +- **Custom HTTP embedding endpoint failures are reported as themselves**, not as Hugging Face download errors (#2385, #2386) +- **Exact symbol content with 0-based line storage and 1-based MCP display** (#2377, #2379, #2380) +- **`rename` reports every edit that apply writes** and reconciles its report on partial failure (#2605) +- **Global registry transactions serialized across processes** (#2716) +- **Swift indented conditional directives are preprocessed** so class bodies survive parsing (#2771), and Swift member-containment pairs are declared in the `CONTAINS` DDL (#2769) +- **JavaScript `exports.foo = function () {}` CommonJS exports are indexed** (#2723, #2729), and `const X = () => {}` is no longer double-indexed as a Function plus an edgeless Const twin (#2687, #2691) +- **JVM sibling injection is proximity-bounded** (#2732), and C#/Kotlin free calls are gated by instance ownership (#2563, #2654) +- **Dart extension type symbols are extracted** (#2539), and declarations recover after embedded NUL bytes (#2430) +- **CLI and hooks fail loudly on backend error payloads**, with an MCP query hint when the server owns the DB lock (#2396, #2397) +- **Committed agent guides stop churning**, with an `--index-only` nudge (#2907, #2927), and `gitnexus-plan` artifacts publish on macOS without an interpreter (#2905, #2922) +- **The 300-flows cap is removed for large repositories** (#2198) + +### Changed + +- **BREAKING: Node `^22.18.0 || >=24.11.0` is now the supported floor**; the `@types/uuid` stub is dropped +- **BREAKING: the non-functional `group` matching knobs are gone** — `matching.bm25_threshold`, `matching.embedding_threshold`, `detect.embedding_fallback` in `group.yaml`, the `gitnexus group sync --skip-embeddings` flag, and the MCP `group_sync` `skipEmbeddings` argument (#3020) +- **Structural relationships are held out of the JS heap by default** during analyze (#2680, #2685) +- **Global ignore support** — `core.excludesFile`, `.git/info/exclude`, and a user-level global ignore file are honored (#2606) +- **Plugin manifests sync on every version bump** (#2445), and planning output under `docs/plans` is no longer tracked + +### Performance + +- **Import resolution indexed instead of scanned** — every scanning resolver with a consolidated memo (#2911), a per-run workspace index for Go/C#/Dart/Ruby (#2898), and Kotlin import resolution (#2872) +- **MCP server startup drops the analyze-only language-provider closure** (#2802, #2806) +- **C++ qualified namespace members indexed once per pipeline run** (#2788, #2794) +- **Vendored Leiden O(communities × N) copy removed**, with Icebug wired to its real API (#2337, #2692) +- **`core.excludesFile` / `info/exclude` resolution memoized** (#2606) + +### Chore / Dependencies + +- **`@ladybugdb/core` bumped to ^0.18.3** for the rel-property IN-predicate fix (#2508, #2634) +- **Security overrides** — `sharp` >=0.35.0 for libvips vulnerabilities (#2993) and `adm-zip` >=0.6.0 for a memory-allocation vulnerability (#2992) +- **~130 dependency bumps** across the CLI, web app and GitHub Actions, including `@modelcontextprotocol/sdk`, LangChain, Vite, Vitest, TypeScript, React and the Docker/CodeQL action suite +- **CI hardening** — Windows shard watchdog widened with exit diagnostics (#2449), platform-sensitive matrix sharded to fix the Windows cross-platform timeout (#2394), and CI Report no longer dies silently when the tests job fails (#2728) + ## [1.6.9] - 2026-07-04 ### Added diff --git a/gitnexus/README.md b/gitnexus/README.md index 45a6f8ed3..7be308e0f 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -204,7 +204,7 @@ Your AI agent gets **17 tools** (15 per-repo + 2 group) automatically: | `group_list` | List configured repository groups | | `group_sync` | Rebuild a group's Contract Registry and cross-repo links | -> With one indexed repo, the `repo` param is optional. With multiple, specify which: `query({search_query: "auth", repo: "my-app"})`. Per-repo tools also take an optional `branch` for indexes pinned with `gitnexus analyze --branch`; omitting it queries the workspace index, which follows your checked-out working tree. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. +> Read-only tools can omit `repo` when one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout. Otherwise—and for mutating tools with multiple indexed repos and no MCP default—specify it explicitly: `query({search_query: "auth", repo: "my-app"})`. Per-repo tools also take an optional `branch` for indexes pinned with `gitnexus analyze --branch`; omitting it queries the workspace index, which follows your checked-out working tree. `explain` and `pdg_query` need an index built with `gitnexus analyze --pdg`. ## MCP Resources @@ -234,19 +234,22 @@ Your AI agent gets **17 tools** (15 per-repo + 2 group) automatically: gitnexus setup # Configure MCP for detected editors (one-time; use -c to select) gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks (add --force to apply) gitnexus analyze [path] # Index a repository (or update stale index) +gitnexus analyze [path] --watch # Watch local files and serialize incremental refreshes gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild gitnexus analyze --embeddings # Enable embedding generation (slower, better search) gitnexus embeddings install # Fetch the optional local embedding stack on demand (--cuda, --force) gitnexus analyze --skills # Generate repo-specific skill files from detected communities -gitnexus analyze --skip-agents-md # Preserve custom AGENTS.md/CLAUDE.md gitnexus section edits +gitnexus analyze --skip-agents-md # Preserve custom AGENTS.md/CLAUDE.md gitnexus section edits (does not skip standard skills; use --skip-skills; community --skills files are unaffected) gitnexus analyze --skip-skills # Skip installing standard .claude/skills/gitnexus-* skill files gitnexus analyze --skip-git # Index folders that are not Git repositories gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1, capped at 16) +gitnexus analyze --spring-actuator ./actuator # Enrich with local Spring Boot Actuator JSON snapshots gitnexus analyze --verbose # Log skipped files when parsers are unavailable gitnexus analyze --max-file-size 1024 # Skip files larger than N KB (default: 512, cap: 32768) gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) +gitnexus auto-sync [init|start|restart|stop|status|reset] # Scheduled remote clone/pull + analyze from GITNEXUS_HOME/watch_config.yml gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI gitnexus index # Register an existing .gitnexus/ folder into the global registry @@ -256,6 +259,7 @@ gitnexus clean # Delete index for current repo gitnexus clean --all --force # Delete all indexes gitnexus wiki [path] # Generate LLM-powered docs from knowledge graph gitnexus wiki --model # Wiki with custom LLM model (default: minimax/minimax-m2.5) +gitnexus wiki --provider grok # Local Grok Build CLI (uses `grok login`, no API key) gitnexus wiki --base-url http://llama-box.local:8080/v1 --allow-insecure-connection llama-box.local # Allow an exact LAN/self-hosted HTTP LLM host; env: GITNEXUS_ALLOW_INSECURE_CONNECTION gitnexus doctor # Show runtime platform capabilities and embedding configuration @@ -281,6 +285,70 @@ gitnexus group status # Check staleness of repos in a group gitnexus group impact --target --repo # Cross-repo blast radius ``` +`gitnexus analyze --watch` requires a Git repository. It performs an initial +analysis and then debounces scanner-admitted working-tree changes for 300 ms by +default into serialized incremental refreshes. Events arriving during a run +remain queued, and retryable failures retain the same batch with bounded +backoff. Invalid `.gitnexusrc` or ignore-file reloads pause ordinary refreshes +until the control file is fixed. Watch refreshes update only the graph: they +intentionally skip AGENTS.md / CLAUDE.md injection and standard skill +installation. Run a one-shot `gitnexus analyze` when those generated files need +updating. Stop watch mode with Ctrl+C. + +Watch mode accepts `--debounce`, `--workers`, `--worker-timeout`, +`--max-file-size`, `--branch`, `--pdg`, `--name`, `--allow-duplicate-name`, and +`--verbose`. Explicit one-shot options such as `--force`, `--repair-fts`, +embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, +`--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git` +are rejected. Unsupported defaults from `.gitnexusrc` are ignored with a warning. + +POSIX requests clone-first copy-and-swap publication when the live index has no +orphan sidecars. Windows and sidecar fallback runs update in place: failures +known to occur before writes are retried, while a failure that may have mutated +the live index stops the watcher. Watch mode does not pull remotes. Running MCP +and `serve` processes periodically check for a newly published index and reopen +it without a restart. MCP checks are throttled to once every five seconds, so a +tool call before the next check can briefly use the previous index. + +### `gitnexus auto-sync` + +`gitnexus auto-sync` is a different product from `gitnexus analyze --watch`. It is the explicit long-running auto-sync entrypoint that clones or pulls configured remotes. `gitnexus watch` is reserved and does not start either job: it prints this split. `GITNEXUS_HOME` defaults to `~/.gitnexus`; `gitnexus auto-sync init` creates its default `$GITNEXUS_HOME/watch_config.yml`. Bare `gitnexus auto-sync` is the same as `gitnexus auto-sync start`; `restart`, `stop`, `status`, and `reset` manage the same `GITNEXUS_HOME` instance. `reset` removes only the derived analysis state and commit snapshot; clones, indexes, and registry entries are untouched. `start` runs in the foreground, reads the configuration once at startup, runs once immediately, then repeats on `sync_interval_minutes`; restart it after changing the configuration. Watch runtime artifacts live under `$GITNEXUS_HOME/watch/`: `project_commit_info.txt` is the human-readable per-loop snapshot, `auto-sync-state.json` is the machine state used for commit skipping and analyze failure thresholds, `watch.mutex` prevents multiple auto-sync processes for one home, `watch.owner.json` records ownership metadata, `watch.pid` plus `watch.status.json` expose process state, `watch.stop..json` is a temporary owner-fenced stop request, and `quarantine/` stores partial clone output before entries are removed after 14 days, keeping at most the five newest entries per repository regardless of age. Mutexes with verified dead owners are reclaimed automatically after an abnormal exit. Invalid or legacy mutexes fail closed; confirm no auto-sync process is running before manually removing `watch.mutex` and stale `watch.pid` / `watch.owner.json`. + +```yaml +sync_interval_minutes: 10 +max_concurrency: 1 +repo_git_timeout: 10s +analyze_timeout: 5m +analyze_failure_threshold: 3 +projects: + - local_path: /abs/path/to/repos + branches: [master, main] + overwrite_local_changes: false + remote_urls: + - git@github.com:owner/repo.git + - git@gitlab.com:group/repo.git + - git@gitee.com:owner/repo.git +``` + +`sync_interval_minutes` must be an integer of at least `5`. `local_path` must be an absolute path without traversal; each remote is cloned below it as `host/namespace/repo`, preventing same-basename repositories from colliding. `remote_urls` must use SSH SCP form for github.com, gitlab.com, or gitee.com. `repo_git_timeout` applies to each repo clone/pull and defaults to `10s`; a bare number such as `10` is interpreted as seconds, while `10000ms`, `10s`, and `1m` keep their explicit units. It must not exceed one hour or `sync_interval_minutes`, whichever is smaller — so a bare `600000` is rejected, because it means 600000 seconds rather than milliseconds. `analyze_timeout` applies to each isolated analysis worker, defaults to half of `sync_interval_minutes`, and cannot exceed that value; this keeps it within Node's timer range. Timeout and `auto-sync stop` request safe cancellation; a worker already in native work exits after it returns to a JS-visible safe point. While waiting, auto-sync reports `cancelling` or `stopping` and keeps its ownership files so another auto-sync cannot take over. The parent waits up to 5 seconds for the worker to exit; after that it stops waiting, releases its ownership files, and leaves the worker to finish and exit on its own rather than killing it mid-write. `auto-sync stop` uses this same control path on macOS and Windows. `overwrite_local_changes` defaults to `false`; a dirty local clone is skipped with an error log, while `true` allows branch fallback to replace local changes and additionally discards untracked files and directories in the clone after checkout — ignored paths, including GitNexus's own `.gitnexus/` storage, are preserved. `max_concurrency` defaults to `1` and is capped at runtime by `floor(availableMemoryGB / 2)` with a minimum of `1`; the effective value is printed at the start of each loop. Each analysis worker's heap cap is the machine-wide cap divided by the number of repositories analyzed in parallel, so concurrent workers share one memory budget instead of each claiming the whole machine. `analyze_failure_threshold` defaults to `3`, must be at least `2`, and pauses repeated failures only for the same repo branch and commit; a new commit or `gitnexus auto-sync reset` clears the block and allows analysis again. Repositories are registered and added to groups by their full remote identity (`host/namespace/repo`), so repositories with the same basename remain distinct. Use `branches` to try branches in order; legacy `branch` remains supported, but the two fields cannot be set together. If all branches are unavailable or time out, watch logs an error, records the repo status, and skips that repo for the loop. Leave `group_name` empty or omit it to skip group add/sync for that project; otherwise create the group first with `gitnexus group create `. `$GITNEXUS_HOME/watch/project_commit_info.txt` is for inspection only; GitNexus stores machine state separately in `$GITNEXUS_HOME/watch/auto-sync-state.json`. + +GraphQL contract matching is opt-in in the group's `group.yaml`: + +```yaml +detect: + graphql: true +``` + +The initial exact-only slice matches methods and properties on top-level NestJS `@Resolver` +classes using imported `@Query`, `@Mutation`, and `@Subscription` decorators. Named +`.graphql`/`.gql` operations are anchored by generated `Document` declarations; +object, static `gql` template, and `TypedDocumentString` initializers must prove the operation name +and root fields. Dynamic decorator names, anonymous operations, and ambiguous or missing graph +anchors are deliberately omitted. Add common infrastructure fields such as `/health` to +`matching.exclude_links_paths` to keep those GraphQL contracts visible without cross-linking them. + +`--spring-actuator` is explicitly opt-in. The path may be a JSON bundle keyed by `mappings`, `beans`, `conditions`, `configprops`, and/or `env`, or a directory containing endpoint-named JSON files. Runtime mappings and beans confirm matching static nodes; conditions and configuration property keys enrich existing evidence, with conservative runtime-only nodes added when no match exists. The configured input is excluded from source scanning; only normalized repository-relative exclusions are retained for future scans, never absolute paths. Env/configprops values, origins, condition messages, and source names are never persisted or printed. Enabled runs always rebuild because runtime snapshots are external to git freshness; omitting the option later rebuilds once to remove runtime evidence. Project config can set the same path with `springActuator` in `.gitnexusrc`. + > **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. ## Remote Embeddings @@ -390,7 +458,7 @@ Installed automatically by both `gitnexus analyze` (per-repo) and `gitnexus setu LadybugDB native binary ships as a prebuild against that floor, so on an older host it cannot load and reinstalling does not help — see [Linux: `GLIBC_2.34' not found`](#linux-glibc_234-not-found). -- **Windows, for full-text search:** the Microsoft Visual C++ 2015-2022 Redistributable (x64) *and* +- **Windows, for full-text search:** the Microsoft Visual C++ 2015-2022 Redistributable (x64) _and_ OpenSSL 3 (`libssl-3-x64.dll`, `libcrypto-3-x64.dll`) resolvable on `PATH` — see [Windows: full-text search unavailable](#windows-full-text-search-unavailable). @@ -667,17 +735,17 @@ For repositories with very large source files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BY Four env vars expose the pool's resilience layers (respawn budget, cumulative-timeout cap, circuit breaker, startup handshake). Defaults are tuned for typical repos; bump them when an analyze legitimately needs more retries, or lower them to fail-fast on a known-bad shape. -| Variable | Default | Effect | -| ----------------------------------------------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per slot before the slot is dropped from the active rotation. | -| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. | -| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. | -| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). | -| `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. Raise it on a slow or heavily loaded host where a full pool cold-starting concurrently needs more than 5s. | -| `GITNEXUS_MEMORY` | `off` | unset (autopilot on) | `off` declines GitNexus's memory autopilot: analyze will neither re-run itself with a RAM-aware heap cap nor abort the parse before V8 enters its ineffective-mark-compact death spiral. Use it when you want to drive memory manually; to simply pin a heap size, pass Node's own `--max-old-space-size`, which is already honoured as your decision. | -| `GITNEXUS_WORKER_HEAP_MB` | `clamp(512, RAM/2/poolSize, 4096)` | Per-worker V8 old-generation heap cap (#2649). Bounds pool RSS on large repos; a worker exceeding it dies with a real heap error handled by quarantine/respawn. | -| `GITNEXUS_SERVER_ANALYZE_HEAP_MB` | `min(8192, auto cap)` | Heap for the web/MCP server's forked analyze worker (#2649). Defaults to the historical 8192 MB bounded by the machine/container's RAM-aware auto cap; set an absolute MB value to override. | -| `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning (#2432). `0` expires immediately. | +| Variable | Default | Effect | +| ----------------------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per slot before the slot is dropped from the active rotation. | +| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. | +| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. | +| `GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS` | `30000` | Max wait at pool shutdown for a retired worker still inside native code — terminated at its next JS-safe point instead of mid-native-call, which would abort the process (`Napi::Error`, #2432). | +| `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. Raise it on a slow or heavily loaded host where a full pool cold-starting concurrently needs more than 5s. | +| `GITNEXUS_MEMORY` | `off` | unset (autopilot on) | `off` declines GitNexus's memory autopilot: analyze will neither re-run itself with a RAM-aware heap cap nor abort the parse before V8 enters its ineffective-mark-compact death spiral. Use it when you want to drive memory manually; to simply pin a heap size, pass Node's own `--max-old-space-size`, which is already honoured as your decision. | +| `GITNEXUS_WORKER_HEAP_MB` | `clamp(512, RAM/2/poolSize, 4096)` | Per-worker V8 old-generation heap cap (#2649). Bounds pool RSS on large repos; a worker exceeding it dies with a real heap error handled by quarantine/respawn. | +| `GITNEXUS_SERVER_ANALYZE_HEAP_MB` | `min(8192, auto cap)` | Heap for the web/MCP server's forked analyze worker (#2649). Defaults to the historical 8192 MB bounded by the machine/container's RAM-aware auto cap; set an absolute MB value to override. | +| `GITNEXUS_CPP_CAPTURE_BUDGET_MS` | `20000` | Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning (#2432). `0` expires immediately. | ### Graph cleanup tuning @@ -691,8 +759,8 @@ Programmatic callers can pass `keepLocalValueSymbols: true` in `PipelineOptions` ### Scope-resolution property-key dispatch cap -During scope resolution GitNexus synthesizes CALLS edges through *property-key -dispatch* — call sites like `hooks.emitScopeCaptures()` where a property key is +During scope resolution GitNexus synthesizes CALLS edges through _property-key +dispatch_ — call sites like `hooks.emitScopeCaptures()` where a property key is registered by multiple definitions across the codebase. To keep this fan-in bounded, each property key is capped at **32 registrations**: a key registered by more than 32 distinct functions is skipped entirely (no CALLS are synthesized @@ -700,8 +768,8 @@ through it), and the dropped key names are surfaced in the analyze log for operator visibility. The cap is calibrated at 2× this repo's own provider table (16 legitimate registrations, one per language provider). -| Variable | Default | Effect | -| --------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Variable | Default | Effect | +| --------------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `GITNEXUS_MAX_PROPERTY_DISPATCH_FANOUT` | `32` | Per-property-key registration cap in the property-dispatch scope-resolution pass. Set to a positive integer to raise it for repositories whose provider/hook tables exceed the default and lose CALLS coverage on a legitimate key; non-integer or `< 1` values fall back to `32`. Lowering it tightens the overflow budget. | ```bash @@ -714,11 +782,11 @@ npx gitnexus analyze --force ### Scope-resolution dispatch-target cap -During scope resolution GitNexus resolves calls that flow through *callable -values* — function/method references bound to variables, passed as arguments, +During scope resolution GitNexus resolves calls that flow through _callable +values_ — function/method references bound to variables, passed as arguments, or stored in maps/tables. To keep that inclusion-based resolution finite, each callable site is capped at **32 dispatch targets**. When a site gathers more -candidates than the cap it is treated as **overflowed** and *all* of its call +candidates than the cap it is treated as **overflowed** and _all_ of its call edges are dropped — a cliff, not a tail, so a repository with a legitimately wide dispatch table (a single callable site resolving to 33+ targets) loses that site's whole call chain. In that case `analyze` logs @@ -728,8 +796,8 @@ candidate count, and the cap (32). Raise the cap for such repositories: -| Variable | Default | Effect | -| ------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Variable | Default | Effect | +| ------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `GITNEXUS_MAX_CALLABLE_VALUE_TARGETS` | `32` | Per-callable-site dispatch-target cap in the callable-value-flow scope-resolution pass. Set to a positive integer to raise it for repositories whose wide dispatch tables overflow the default and lose a whole call chain; non-integer or `< 1` values fall back to `32`. Lowering it tightens the overflow budget. | ```bash diff --git a/gitnexus/bench/cross-repo-trace/verify.mjs b/gitnexus/bench/cross-repo-trace/verify.mjs index 8497af38d..e965bfc07 100644 --- a/gitnexus/bench/cross-repo-trace/verify.mjs +++ b/gitnexus/bench/cross-repo-trace/verify.mjs @@ -58,9 +58,6 @@ packages: {} detect: http: true matching: - bm25_threshold: 0.7 - embedding_threshold: 0.65 - max_candidates_per_step: 3 `; } diff --git a/gitnexus/bench/emit-persistence/baselines.json b/gitnexus/bench/emit-persistence/baselines.json index 1d295bd19..5b3568256 100644 --- a/gitnexus/bench/emit-persistence/baselines.json +++ b/gitnexus/bench/emit-persistence/baselines.json @@ -1,7 +1,11 @@ { - "fingerprint": "4ee15e742a9839671a900df4f57c1c91196c64256c8cab2ac445bec605a092d5", + "fingerprint": "72096279092d4f118de7e179333705c19c9aff2664f77d71a7da48cd9f73fb5a", "scaling_budget": 1.8, "max_ms_large": 1000, + "_rebaselined_destination_broker_conflict_column_removed": "The `Destination` node table lost its trailing `brokerConflict` STRING column (DESTINATION_SCHEMA in src/core/lbug/schema.ts, the COPY statement in lbug-adapter.ts, and the `destinationWriter` header plus its row cell in csv-generator.ts). The column existed only to say WHY a destination's address had been withdrawn when two brokers claimed it; the address is no longer withdrawn — a resolved `Destination` is now keyed by `(broker, address)` via `ingestion/destination-key.ts`, so two brokers on one name are two ordinary joinable nodes and there is nothing to diagnose. That makes this a header-only shrink, and it was verified as one rather than assumed: dumping every CSV this bench emits with csv-generator.ts at the merge base and again on this branch, then diffing per file by filename, byte length and sha256, shows the file SET identical at 36 CSVs on both sides, 35 of the 36 byte-IDENTICAL (same sha256, not merely same length), and the sole difference `destination.csv` shrinking 112 -> 97 bytes: `id,name,filePath,startLine,endLine,address,broker,resolution,configKey,configDefault,brokerConflict,description` -> the same list without `brokerConflict`. That file is header-only on both sides — the synthetic benchmark graph contains no Destination nodes — so no row moved, was re-routed to another pair file, or reordered, which is the class of change this fingerprint exists to catch. Prior 4b339233662b0eebb236738abad8f7c039b9930bc1222dcad7b814afa6332fbf -> 72096279092d4f118de7e179333705c19c9aff2664f77d71a7da48cd9f73fb5a, reproduced identically across two consecutive runs. Both timing gates passed while the guard was red (scaling_ratio 0.818 and 0.751 across those two runs against the 1.8 budget; elapsed_ms_large 65.64ms and 59.32ms against the 1000ms backstop), so no throughput claim is being rebaselined away.", + "_rebaselined_3132_destination_node_table": "A new `Destination` node table (async messaging overlay; see DESTINATION_SCHEMA in src/core/lbug/schema.ts) means `streamAllCSVsToDisk` writes one more FILE, not one more column — the first rebaseline here that changes the file SET rather than a header. That makes the usual evidence more important, not less, so it was gathered the same way: dump every CSV this bench emits on the merge base and on this branch, then diff per-file by filename, byte length and sha256. Result: 35 files -> 36, the sole addition is `destination.csv`, and ALL 35 pre-existing files are byte-IDENTICAL — not merely same-length, same sha256. So nothing was re-routed into the new file and nothing reordered, which is exactly what this fingerprint exists to catch. `destination.csv` is 112 bytes, header only: the synthetic benchmark graph has no Destination nodes, so no row exists to move. Prior 7b2ec01a110dcbc66868fba2c97714aaece8c3864c2eba00df68b5c027f034d2 -> 4b339233662b0eebb236738abad8f7c039b9930bc1222dcad7b814afa6332fbf, reproduced identically across two runs. Both timing gates passed while this was red (scaling_ratio 0.711 vs the 1.8 budget, elapsed_ms_large 58.88ms vs the 1000ms backstop), so no throughput claim is being rebaselined away.", "_rebaselined_2856_property_is_detail": "Third and last of the bench guards this branch left red. The Property node table gained an `isDetail` BOOLEAN column (see PROPERTY_SCHEMA in src/core/lbug/schema.ts), so `streamAllCSVsToDisk` writes one more header field and one more cell per Property row — csv-generator.ts `propertyHeader` and the `node.label === 'Property'` tail. Verified to be header-only drift rather than a change in what is emitted: dumping every CSV this bench produces on `origin/main` and on this branch and diffing per-file (filename, byte length, sha256) shows the file SET is identical at 35 CSVs on both sides, 34 of the 35 are byte-identical, and the sole difference is `property.csv` growing 68 -> 77 bytes, `id,name,filePath,startLine,endLine,content,description,declaredType` -> `...,declaredType,isDetail`. The synthetic graph has no Property nodes, so no ROW moved at all. That is the check that matters here: a row routed to the wrong pair file, or a within-file reordering, is what this fingerprint exists to catch, and neither happened. Prior 69e9182ae205183ade24c3d8ad5d7292aea677144b1cbe443dd631bc25b0cafe -> 4ee15e742a9839671a900df4f57c1c91196c64256c8cab2ac445bec605a092d5. Both timing gates passed unchanged while this was red (scaling_ratio 0.783 vs budget 1.8, elapsed_ms_large 229ms vs the 1000ms backstop), so no throughput claim is being rebaselined away.", + "_rebaselined_3040_convex_endpoint_factory": "Const and Function gained a trailing convexEndpointFactory column. A deterministic 2,400-entity emit produced the same 35 CSV files and fingerprint c4d799c5336d616955b3530ba051b7dca300d1a0e412a66741cf2f27e04c533e. Removing the new Const and Function header fields plus the new trailing empty Function cell from each of 4,800 Function rows restored the exact prior fingerprint 4ee15e742a9839671a900df4f57c1c91196c64256c8cab2ac445bec605a092d5. No file or row moved or reordered. The measured scaling ratio remained 0.826 against the 1.8 budget and elapsed_ms_large was 307.75ms against the 1000ms backstop.", + "_rebaselined_3107_route_runtime_evidence": "Route gained trailing runtimeConfirmed BOOLEAN, runtimeSource STRING, and runtimeStatus STRING columns in its schema, CSV header/rows, COPY statement, and graph API projection. The deterministic emit still produces the same 35 CSV files; the synthetic benchmark graph has no Route rows, so the only byte drift is the Route CSV header and no row moved or reordered. Prior c4d799c5336d616955b3530ba051b7dca300d1a0e412a66741cf2f27e04c533e -> 7b2ec01a110dcbc66868fba2c97714aaece8c3864c2eba00df68b5c027f034d2. While the guard was red, scaling_ratio was 1.044 against the 1.8 budget and elapsed_ms_large was 121.08ms against the 1000ms backstop.", "_note": "fingerprint = sha256 over per-file digests (filename + sha256(file bytes)), entry list sorted — binds each emitted line to its file so a row routed to the WRONG pair file changes the hash, AND catches within-file row reordering (file bytes hashed as-written). Byte-identity gate for #2203 U2/U3. NOTE: a future change that legitimately reorders emit (without changing the node/edge SET) will trip --check; regenerate then, and record WHY in a `_rebaselined_` key alongside — bench/scope-capture/baselines.json sets that convention and it is what makes a regenerated hash reviewable. scaling_budget bounds (t_large/t_small)/(LARGE/SMALL): observed ~0.95-1.05 (linear); 1.8 tolerates disk-I/O timing noise on CI while still catching an O(n^2) re-regression (~4x). max_ms_large=1000ms is a coarse absolute backstop (observed ~200ms) that catches a gross uniform slowdown the ratio gate misses; generous so CI host noise won't flake it. Regenerate via `node --import tsx bench/emit-persistence/measure.mjs`." } diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index c01881c3d..d8d50b015 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -1,9 +1,10 @@ { - "_what": "Baselines for bench/import-target/measure.mjs \u2014 EVERY import-target resolver registered in SCOPE_RESOLVERS, on one shared corpus, plus csharp a second time WITH csproj configs. One entry per registered language and one more for the csproj arm, no registered language ungated \u2014 and that is ASSERTED rather than asserted-in-a-comment, which is also why no roster of language names is kept in this prose to go stale: measure.mjs derives its language list from a LANG_REGISTRY table and a --check inventory arm reconciles that table against SCOPE_RESOLVERS in both directions. A C/C++ #include is an import site for this purpose and is gated like every other registered language. csharp and csharp_csproj resolve the IDENTICAL file corpus (buildFiles aliases the two) and differ in exactly one thing: whether csharpConfigs is supplied. Without that second arm the csproj namespace-directory index ships unmeasured, because every C# import in the no-csproj arm returns before reaching it. C and C++ follow that same precedent for a different context \u2014 their HEADERS arrive through resolutionConfig rather than through allFilePaths, and augmentedFilePaths unions the two once per pass, so the corpus is split at newPass rather than pre-merged. The first nine were added as their own O(imports x files) scans were indexed away (#2877/#2878/#2879/#2880, #2872, #2901, #2902, #2908) and this is the forward guard on each; the other eight were ungated until now, and PR #2911 \u2014 JavaScript reaching suffixResolve with no index at all, 25972 us per import at 8000 files \u2014 is what that costs.", + "_what": "Baselines for bench/import-target/measure.mjs cover every import-target resolver registered in SCOPE_RESOLVERS on a shared corpus, plus a configured C# arm for the branch the default call cannot reach. measure.mjs derives its arm inventory from LANG_REGISTRY and --check reconciles the registered languages in both directions. The single PHP arm supplies its production PSR-4 Composer mapping; csharp_csproj supplies csproj configuration. C and C++ also receive their production resolutionConfig header corpus. The timing, shape, fingerprint, context, and retained-heap gates therefore cover each production resolver path without splitting PHP into configured and unconfigured identities.", "_rebaselined_2960_kotlin_declared_packages": "Kotlin now resolves only from parsed package facts and local module bindings. This deliberately changes its five fingerprints, removes path-depth sensitivity, adds the context probe, and reduces the 32000-file retained index from 40.82 MiB to 4.31 MiB. External same-name path decoys now remain unresolved.", + "_php_composer_gate_2962": "The canonical PHP arm supplies an authoritative App PSR-4 mapping. A deterministic Vendor0 suffix decoy makes deletion of the external gate change every timing fingerprint, while the heap arm separately pins a mapped miss, the rendered mapping, and the external null result. Three serial samples measured depth ratios 1.058, 1.114, and 1.158; the 1.8 budget is 1.55x the observed maximum. The mapped-miss heap reading peaked at 39607216 retained bytes at 32000 files.", "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the languages these PRs changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files \u2014 that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) for Kotlin's current declared-package behavior in test/unit/kotlin-module-resolution.test.ts, and for the four resolvers added there in test/unit/scope-resolution/{php,java,cobol}-import-target-parity.test.ts and test/unit/import-resolvers/csharp-csproj-parity.test.ts, and for JavaScript in test/unit/scope-resolution/javascript-import-target-parity.test.ts (a differential over 211200 old-vs-new pairs, PR #2911). The eight languages added last have no per-language parity harness against a pre-change implementation and do NOT need one: nothing about their resolution changed, so there is no before to diff against. Their fingerprints are pure forward guards, minted from the current implementations, and their adapter-boundary index reuse is covered for every registered language at once by test/unit/scope-resolution/import-target-index-reuse.contract.test.ts. NOTE for csharp_csproj: on this corpus the #2902 indexed leg (step 3 of resolveCSharpImportInternal) is reached by 2221 of the 3200 small-arm imports but answers null for every one of them \u2014 the 979 that resolve do so at step 2 \u2014 so this fingerprint pins that legs cost and its null answers, while its positive tie-breaks (unanchored substring, iteration order) are pinned by csharp-csproj-parity.test.ts. NOTE for kotlin, go, csharp and java: twenty fingerprints across these four languages were re-baselined in #2881, the one deliberate behaviour change any language in this file has had. It landed in two steps and the second is the reason the first is not a special case: Kotlin first, then the shared package-dir-index (go, java, csharp) and the csproj namespace index once the same rule was found live there. `getKotlinFileIndex` no longer requires a file's package directory to be the FIRST occurrence of that name in its own path, so the unique arm's `d % 7` nested slice (`mod{d}/src/main/kotlin/com/example/pkg{d}/inner/pkg{d}`) now belongs to package `pkg{d}` and its wildcard imports resolve: resolved 1100 -> 1153 small and deep, 4456 -> 4681 large. The collide arm needed a CORPUS edit alongside it, not just a new number \u2014 its `d % 7` slice deliberately imported `com.example.vendor{d}`, a package that exists nowhere, purely to mirror the unique arm's nested-slice MISS, so leaving it would have left collide at 1100 against small's 1153 and broken the same-workload invariant the arm is built on (that assertion is what caught it). It now uses the same `com.example.models.*` spelling as the rest of the arm, which is why its distinct_outcomes fell (2775 -> 2744, 11087 -> 10961): one shared target instead of one per d. The record-level evidence for the resolver change \u2014 235 of 19968 records moved, 54 null -> resolved, 0 buckets losing a member \u2014 is in bench/kotlin-import-target/baselines.json `_provenance`. The kotlin heap_reading_bytes and heap_ceiling_bytes moved with it, together as `_heap_reading_note` requires: 48073096 -> 48200224 bytes_large (+127128, +0.264%), ceiling still exactly 1.5x. Small, and it is worth saying WHY it is small rather than reading the number as evidence that the change is cheap. `dirChildren` grows by one entry per component-suffix the old rule used to skip, and this arm can only see part of that: the heap corpus is built with HEAP_PAD 8, which prefixes every path with `d0/\u2026/d7/`, so no path can begin with a suffix of its own directory and the leading-segment half of the old rule is structurally invisible here. What moves the reading is the `d % 7` nested slice alone. Read +0.264% as this arm's ceiling on the effect, not as the effect. GO NEEDED A CORPUS EDIT TO BE GATED AT ALL. Its nested slice was `src/pkg{d}/internal/pkg{d}`, repeating only the LAST segment, while a Go query addresses the whole package path `src/pkg{d}` \u2014 so the directory never even ended with the query and the first-occurrence rule was never reached. Every go arm sat unchanged through the resolver fix. `uniqueDir`/`collideDir` now repeat the shape at the granularity Go actually queries (`src/pkg{d}/internal/src/pkg{d}`, `svc{d}/internal/sub/svc{d}/internal`), which is what moved go from 979 to 1153 resolved and bumped `languages.go.heap.path_segments` 13 -> 14. The general lesson: a corpus that carries a shape the QUERY cannot express does not gate that shape. CSHARP AND JAVA HIT THE SAME COLLIDE-ARM TRAP AS KOTLIN. Both collide arms sent their `d % 7` slice to a namespace that exists nowhere (`App.Src{d}.Vendor`, `com.svc{d}.vendor`) purely to MIRROR the unique arm's nested-slice miss; once that miss became a hit, collide sat at 979/1100 against small's 1153 and the same-workload assertion failed. Both now use the same spelling as the rest of their arm. HEAP: no reading here moved for the resolver change. An earlier revision of this branch re-recorded `csharp_csproj` 73703384 -> 73116520 as a -0.79% effect of the step-2 filter; review measured base and branch three times each and got the same 73.10e6 on BOTH sides \u2014 the recorded 73703384 was simply not reproducible on this box, and re-recording it would have dropped that language's derived floor by 0.8% for no reason belonging to this change. Reverted. Everything else sat within +/-0.03%. Note that `_heap_reading_note`'s claim that these readings 'reproduce to the byte across processes on one box' did NOT hold on the box this was measured on: go, dart, ruby, python, php and cpp all wandered by a few hundred to a few thousand bytes between processes with no code change touching them. Treat sub-0.05% movement as jitter, not signal. HEAP, kotlin, second movement: 48200224 -> 42802456 (-11.20%), re-recorded with its ceiling. `getKotlinFileIndex` now compacts each `dirChildren` bucket as it freezes it. `addChild` mints a bucket as `[raw]` and pushes the rest, and V8 grows a backing store by `old + old/2 + 16`, so the second child takes a 1-slot store to 17: 61144 buckets, 52.9% of their slots empty, 88 B each. Same fix and same accounting as the python `byBasename` sentence above. Note what this means for the gate: a memory WIN of this size passes every arm \u2014 it is under the ceiling and over the 0.5x floor \u2014 so it is recorded because the convention says a reading and its ceiling move together, not because anything went red. kotlin now reads 40.82 MiB. The prose in measure.mjs calling it '45.85 MiB, the second-largest reading in this file' is corrected with it \u2014 and was already wrong on the ranking before this change, since csharp_csproj (69.73) and php (47.28) both read higher; kotlin was third. A measurement written into prose is not re-taken, which is the finding `_heap_bound_note` records about this very file. One further corpus edit, made in review and MEASURED rather than assumed: kotlin's collide layout repeated only the `models` leaf (`\u2026/com/example/models/inner/models`) while a Kotlin query addresses the whole dotted path, so a full revert of the Kotlin guards left both collide fingerprints UNMOVED \u2014 the arm was blind to the rule it was re-baselined for. Deepening it to `\u2026/models/inner/com/example/models` makes the revert move both, and those two fingerprints are the only ones that changed for it. The same deepening was applied to the java and kotlin UNIQUE arms and REVERTED: it moved ten more fingerprints, grew java's heap reading 43%, and bought nothing \u2014 progressive stripping lands those queries on the same file with or without the rule, so the control still failed only on go.", "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 \u2014 which deletes the entire depth arm \u2014 moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir. THE HEAP ARM IS ASSERTED THE SAME WAY, by the same loop, and was not before: files_small, files_large, path_segments and probe decide WHAT it measures, and every one of them was reported and compared to nothing. Swapping HEAP_PROBE_TARGET.csharp_csproj for a target matching no CSPROJ_CONFIGS rootNamespace skips the whole config loop, so the getFilesInDir and getInsensitive legs never run and the arm the header calls the witness that the read pattern IS the footprint quietly becomes a two-map arm \u2014 73703384 -> 59921216 B, ratio 1.017 -> 1.011, ceiling and floor both still passing and --check still exiting 0. Setting HEAP_SMALL equal to HEAP_LARGE is the same hole from the other side: ratio goes to ~1.0 by construction and bytes_large never moves. bytes_small and bytes_large are deliberately NOT asserted for equality \u2014 heap_ceiling_bytes and the heap_reading_bytes floor bound them with ~50% either way, because heapUsed accounting moves across platforms and Node majors and an exact byte assertion would be a re-baseline per runner. THE CONTEXT ARM IS ASSERTED THE SAME WAY, by the same loop, and more strictly than either: target, with_context and without_context are exact strings with no tolerance at all, because the arm resolves one import over a three-file corpus and has no measurement noise to tolerate. A separate check requires the last two to DIFFER, for the same reason deep.fingerprint must differ from small.fingerprint \u2014 a probe on which both call shapes agree asserts one number twice. Both halves run through resolveOne, so what the arm gates is this bench threading run.ts's fifth argument, not the resolvers' behaviour.", - "_arms_note": "Five timing arms, one memory arm and one deterministic arm elsewhere, because none of them gates alone. scaling_ratio (t_large/t_small)/(1600/400) catches cost growing with FILE COUNT \u2014 the #2877-#2880, #2901, #2902 and #2908 regressions themselves; every one of those legs was Theta(files) per import, so a revert scores ~4 here by construction. depth_ratio (t_deep/t_small at a FIXED file count, ~6x the path components) catches cost growing with path DEPTH, which scaling_ratio divides out and structurally cannot see; buildSuffixIndex (C#, Ruby, PHP, Java) emits one entry per component, while Kotlin's declared-package index is depth-free while Go, Dart and COBOL, whose indexes are depth-free, sit at ~1.0. csharp's depth_budget has now been retightened twice for the same reason, and the second time it did lock the win in. It was 5 against a then-measured 3.318; #2903 made buildSuffixIndex's dirMap lazy and it became 3.5 against 2.31, with the file stating plainly that 3.5 did NOT lock that win in because a revert to an eager dirMap scores 3.318 and passes. Extending the laziness to the two SUFFIX maps drops it again, to 1.438 (java likewise 2.214 -> 1.402), because the deep arm has ~6x the path components and an O(files x depth) build of a map the no-csproj leg never reads is exactly the cost that scales with depth. Both are now 2.2, which is this file's 1.5x convention against measurements whose own peak-to-peak over 4 runs is 1.04x and 1.07x \u2014 and 2.2 DOES lock it in: an eager rebuild scores 2.3+ and fails. The other fifteen depth budgets sit at 1.37-1.75x measured and are unchanged. collide_scaling_ratio is the same measurement on a SHARED-LEAF layout (svcN/internal, SrcN/Models, com/example/model in every service, a repeated mod0.dart/mod0.rb/Mod0.cpy basename) carrying an identical file, import and resolved count: the small/large/deep arms mint one directory name per index, so every index bucket in them holds exactly ONE entry (measured: max last-segment bucket 1 and max matching directories 1 for go and csharp at 400 and 1600 files; max basename bucket 1 for dart and ruby), and bucket cardinality is the only non-constant term the new indexes have. On the shared-leaf shape go, csharp, dart and java legitimately score 2.1-3.9 because the bucket grows with the file count BY CONSTRUCTION \u2014 this is a limit on the SCOPE of the \"independent of corpus size\" claim, not a regression (the indexed code is still faster there than the pre-change full scan); their collide budgets say so honestly instead of pretending 1.8. Ruby, Kotlin, PHP and COBOL answer from keyed maps and are collision-immune, so they keep the linear 1.8 budget and that immunity is the assertion. csharp_csproj is the one arm that runs the other way: its shared leaf collapses dirsByLastSegment to the single key Models, so the slash-free sweep (see CSPROJ_CONFIGS) is CHEAPER on the collide layout than on the unique one and its expensive scale arm is large, not collide_large. Its 1.8 collide budget is therefore the linear one, and the arm that carries its real cost is the unique one. The collide arm is also the only arm that reaches filesDirectlyInPkgDir's dirCount > 1 merge (go: 388 multi-directory calls at 400 files, up to 9 directories; 1517 at 1600 files, up to 34) and the only one that reaches COBOL's copybook-over-source tier tie-break, which needs one bookname to name two files. small_ms_ceiling and collide_ms_ceiling are ABSOLUTE (~4x the measured arm), because a constant-factor regression that grows both scale arms equally passes every ratio. The five arms added here use 4.2x, the middle of the 3.7-4.6x the original five already carry; the two COBOL arms use ~5x, the multiplier dart's sub-1 ms arm has always carried, because a fixed scheduler hiccup is a larger fraction of a smaller number \u2014 measured over 8 runs they sat at 0.25-0.37 ms and 0.18-0.30 ms, and the pre-#2908 two-scans-per-COPY implementation costs ~300 ms on the same arm, so 2.0 and 1.5 still separate fixed from broken by two orders of magnitude. NOISE, measured rather than assumed: depth_ratio divides two sub-3 ms numbers (Dart's are sub-1 ms) and is by far the noisiest arm here, so it set N for the whole file. fastest() is a min-of-N estimator, so N is the knob. Over 22 --check runs on an idle box, peak-to-peak: at N=5 go ran 0.757-1.748 (2.31x) and tripped its own 1.6 budget about 1 run in 20; at N=7 (the kotlin-import-target setting) Dart still ran 0.678-2.043 (3.01x) and tripped once; at N=15 (bench/cfg, bench/schema-pairs, bench/callable-value-flow) every language collapsed to a 1.13-1.26x swing with 22/22 passing. The budgets were NOT widened; the estimator was fixed instead, which is why the headroom above is real rather than granted. N IS NOW PER LANGUAGE, and that is a refinement of the same finding rather than a retreat from it. The overshoot of min-of-K against min-of-15 is a function of the CELL's absolute duration, not of the language: replayed against two independent runs' full sample sets, the worst overshoots at K=7 land on swift.small (0.43 ms, 31.8%) and dart.collide (1.5 ms, 37.6%), while every cell at or above 10 ms overshoots by at most 6.3%. So repsFor() keeps 15 while a language's cheapest arm is under 5 ms and otherwise spends ~150 ms per cell, floored at 7 \u2014 15 for go, csharp, dart, kotlin, java, cobol, swift, rust, python, c and cpp (every language the flakiness above was ever about, cheapest arm 0.19-3.2 ms) and 7-8 for csharp_csproj, ruby, php, javascript, typescript and vue (cheapest arm 20-28 ms). Per LANGUAGE, not per cell, so all five arms of a language share one estimator and the four ratios stay comparisons of like with like. The replay passed all 85 cells on all five gates at 0.4-0.7 of budget and saved 12.8 s and 12.4 s of a 46 s run; min-of-7 also reads slightly HIGHER than min-of-15, so the ceilings get marginally more sensitive rather than less. Confirmed on 4 fresh runs with the adaptive estimator live: every small arm inside 1.12x peak-to-peak and every collide arm inside 1.07x, with the six 7-8 rep languages at 1.008-1.071 \u2014 no worse than the 11 that kept 15. The chosen N is reported per language as `reps`. heap_ceiling_bytes bounds the retained per-pass import index, the only arm here that can see memory: buildSuffixIndex emits maps at O(files x depth), the profile package-dir-index.ts cites #2649 to avoid for itself, and csharp, ruby, php and java all retained NOTHING across imports at BASE (C#'s no-csproj leg and PHP's and Java's every leg re-scanned the raw Set; Ruby rebuilt and discarded a suffix index per require). It is measured at 8000 and 32000 files at HEAP_PAD depth rather than at the timing arms' sizes, because the finding is an ABSOLUTE footprint at repository scale. THE ARM NOW READS WHAT THE LANGUAGE READS, and that change is the whole reason this file was re-baselined. Four of these arms used to call getWorkspaceFileIndex(set) directly and then read index.all.length, which asks no suffix question at all \u2014 harmless only while buildSuffixIndex built both maps eagerly. The moment they went lazy the direct call built NO map, csharp, ruby, php and java each reported 0 B at 32000 files, and 0 B is under every ceiling: --check printed PASS over four gates that had silently become ceilings over nothing, which is precisely the failure this file's own header warns about for rust and cobol. Every arm now resolves a real MISSING import through the real resolver (HEAP_PROBE_TARGET, asserted to miss), so the maps it forces are the maps production forces, and a resolver that starts asking a new question moves the number without anyone editing the bench. That makes the READ PATTERN the dominant term, and the eight numbers say so: java 34958600 B and csharp 29862200 B ask index.get and never getInsensitive; php 37579888 B asks getInsensitive and never get, plus its own first-proper-suffix map; ruby 41025360 B and javascript 26745296 B read get(s) || getInsensitive(s) and pay for both, the second DERIVED from the first; and csharp_csproj 73705944 B additionally asks getFilesInDir. csharp_csproj IS NOW GATED, reversing the earlier decision that it would be 'a ceiling on a duplicate': at +20.8% of the C# index it was one, and at 2.47x of it \u2014 same corpus, same getWorkspaceFileIndex, three maps instead of one \u2014 it is the witness that the read pattern is the footprint. The old RESIDUAL note is superseded by that number: a dirMap-sized addition is no longer +18%, and a consumer that asks all three questions blows csharp's ceiling by 1.64x rather than sliding under it. A SECOND MEASUREMENT BIAS was removed at the same time and it moved every figure here, so do not read these against the old ones as if only the read pattern changed. buildFiles mints paths with template literals, which V8 keeps as ropes; the first traversal that slices one flattens it, allocating the flat string and dropping the rope's pieces, so a build measured over an unflattened corpus reports the index MINUS that net release \u2014 11% low, uniformly. bytes_small was read over a corpus a discarded warm-up pass had already flattened and bytes_large over a fresh one, so every ratio read ~0.85-0.89 for structures that are exactly linear in the file count. measureHeap now flattens each corpus before measuring it; all eight ratios read 0.998-1.017, and the warm-up pass is gone because with the corpus flat a language's first and second reads agree to within 0.3%. python's figure rises from 7624992 to 10362976 for this reason and not because anything regressed, and then to 10543152 (+1.7%) because #2913's nestedDirNames set is retained for the pass, and then FALLS to 6360936 (-39.7%) for a reason worth knowing: byBasename holds roughly one bucket per file, and building each with `[]` followed by `push` made V8 grow the backing store to its 16-slot minimum, so every single-file bucket retained 15 empty pointer slots. Constructing the one-element buckets directly (`set(base, [entry])`) is byte-identical in contents and 3.9 MiB smaller at 32000 paths \u2014 37% of what this arm used to read was empty array slots \u2014 the ancestorsByDir memo itself is NOT in this reading, because python's probe target misses at the nested-name rejection and never reaches the walk, so this arm does not bound that memo; measured separately with a probe that does reach it, a 32000-file corpus with every file in its own 10-deep directory retains ~19 MB, which would clear this ceiling, so repointing python's heap probe at a walking spelling means re-recording the ceiling in the same change, and c is unchanged at 10018816 because its basename map does not slice paths. Its ceiling is 1.5x the measured arm, and the DIFFERENCE FROM THE 4x TIMING CONVENTION IS DELIBERATE \u2014 do not harmonise it back. 4x exists because runner contention dominates a wall-clock number; this one has essentially no measurement noise (across 4 runs the widest spread was 0.11% on python, 0.03% on csharp_csproj and 0.00% \u2014 identical to the byte \u2014 on ruby, php, java, javascript and c, and the same holds across separate processes), so 4x would throw away almost all of the gate's power and sail straight past the regression this arm exists to catch. 1.5x still tolerates ~50% of cross-platform and Node-version drift, far more than a Node major bump plausibly moves heapUsed accounting; it catches a duplicated index (+100%) or a second exactMap-sized suffix map (+~85%). heap_floor_fraction is the arm the 0 B incident proved was missing. A ceiling can only say 'not too big'; nothing said 'still measuring something', which is why four dead arms passed. The floor is 0.5 x each language's RECORDED READING (heap_reading_bytes), which is half the measured size and says so. It used to be 0.33 x the CEILING, described the same way \u2014 true only while every ceiling stayed at exactly 1.5x its reading, a convention this file states and nothing enforces, so re-tuning one ceiling upward would have loosened that language's floor by the same factor in the one direction a floor exists to watch. The two forms agree to within 0.8% for all eight today, so this is a correction of derivation, not of strength. It sits ~400x above the readings' own reproducibility and far below any collapse. A genuine 2x memory WIN trips it too, and that is intended: like a fingerprint move, it must be explained and re-baselined rather than absorbed. COBOL is left out for the opposite reason: its index is two Map, O(files) with no depth term, and at 32000 files its retained delta does not clear the noise of the measurement itself. heap_ratio_budget, the linear-growth check across the 4x file-count gap, is the orthogonal arm: it sees per-file and per-depth growth but not a constant factor. ---- THE EIGHT LANGUAGES ADDED LAST (swift, rust, python, javascript, typescript, vue, c, cpp) ---- They carry the SAME five arms and the same gates; what differs is which arm can actually fail for each, because each resolver has a different cost axis, and the budgets below say so instead of copying a number across. Every figure quoted is the MAXIMUM over 5 full runs on an idle box, and the peak-to-peak of every one of these arms stayed inside 1.10x over those runs \u2014 tighter than the 1.13-1.26x the original nine record, because none of these arms divides two sub-1 ms numbers the way dart depth_ratio does. depth_budget is ~1.5x measured throughout: swift 2.3 (1.487), rust 2.1 (1.377), javascript 2.1 (1.376), typescript 2.1 (1.381), vue 2.3 (1.563), c 3.0 (1.990), cpp 3.0 (1.999). PYTHON WAS 11 AGAINST 7.389 AND IS NOW 2.6 AGAINST 1.872, because #2913 fixed the resolver rather than the budget. Its INDEX was always depth-free; hasRepoCandidate and resolveAbsoluteFromFiles each rebuilt one ancestor prefix per directory component of the importer on EVERY import, and the index's own dirPrefixes build inserted one entry per component per file, so the resolver was quadratic in path depth where every other language here is linear or flat. The prefixes are a pure function of the importer's DIRECTORY, so they are now memoized per directory inside getPythonFileIndex (ancestorsByDir), the leading segment is rejected up front against a set of nested directory names, the module and package buckets are consulted before the walk rather than inside it, and the dirPrefixes build stops at the first ancestor already stored. All five fingerprints are byte-identical, so it is a hoist. The budget is 2.2, and BOTH numbers behind it were re-measured on a quiet box AFTER the context leg below started being measured, because that change moved the arm: the work it adds is depth-FLAT, so python's absolute cost more than doubled while depth_ratio FELL to 1.405-1.563 over 5 serial runs (peak-to-peak 1.11x). A budget carried over from before that change would have been slack against a smaller ratio. 2.2 is 1.41x the measured maximum, inside the 1.37-1.75x band the other fifteen sit in, and it LOCKS THE WIN IN: reverting the per-directory ancestor memo alone scores 2.524 and reverting the nested-name rejection alone scores 2.553, both measured under the current call shape, so each fails at 2.2 with 13% to spare. Do not read those two figures as the pre-#2913 cost \u2014 7.239 was that, and the gap closed because the bare-import tier stopped walking at all (see below). The other two parts of the fix are not gated by this arm and are not meant to be: reverting the bucket prune or the dirPrefixes early break lands under any budget this arm's noise supports, so they are gated deterministically instead, by the prefix-parity and package-probe arms of test/unit/scope-resolution/python/python-importer-ancestors.test.ts and python-import-target-parity.test.ts, which go red on exactly those two mutations. A timing budget catches what it can measure; the counts catch the rest. THE BARE-IMPORT TIER (`import os`, single segment, no dot) was a separate O(depth) walk in import-resolvers/python.ts that this bench cannot see at all, because every python arm here spells its imports with a dot and returns at the `pathLike.includes('/')` guard before reaching it. It ran TWICE per `from x import y` \u2014 the package probe's recursion re-ran the whole tail on identical inputs \u2014 and is now one memoized chain plus an O(1) proof-of-absence against the index's basename buckets: 12/24/72 Set probes at depth 1/4/16 became a flat 2, and 11.615 us/import at 18 path components became 0.740. Gated by probe COUNT in test/unit/scope-resolution/python/python-import-probe-count.test.ts, not here. collide_scaling_budget splits three ways. Three languages scan a bucket that grows with the corpus and get their measured value x1.5: swift 4.9 (3.279 \u2014 its bucket is the module file list it RETURNS, and its collide arm is four modules instead of dirs of them so that bucket is fileCount/4, i.e. 100 files at 400 and 400 at 1600), c 3.8 (2.535) and cpp 4.0 (2.639, the same basename bucket its suffix fallback walks). Four answer from keyed maps and keep the linear 1.8 \u2014 python 1.097, javascript 1.083, typescript 1.053, vue 1.079 \u2014 and that immunity IS the assertion, exactly as for ruby, kotlin, php and cobol. RUST IS THE ONE ARM THAT WAS REDESIGNED RATHER THAN BUDGETED. It resolves by probing candidate paths with allFilePaths.has(...) and never searches, so its cost is O(path segments) and provably flat in the file count (1.095 scaling, 1.061 collide scaling): a shared-leaf collide arm for rust would have asserted nothing, which is worse than no arm. Its collide corpus is instead a deep module tree (src/l0/l1/l2/l3/l4/mod{d}) whose targets carry ~2x the :: segments, so the arm exercises the axis that CAN grow, its 1.8 budget asserts the flatness across file counts, and collide_ms_ceiling 19 bounds the absolute cost of the long-path probe. small_ms_ceiling and collide_ms_ceiling are ~4x measured as everywhere else: rust 10/19 (2.609/4.704), python 7/8 (1.76/1.929, retightened from 12/15 against 3.044/3.771 by #2913), javascript 85/89 (21.254/22.145), typescript 85/86 (21.250/21.464), vue 81/93 (20.164/23.227), c 7/11 (1.620/2.850), cpp 7/12 (1.581/3.009). Swift takes ~5x (2 against 0.421 and 4 against 0.821) \u2014 the multiplier dart and cobol already carry, because a fixed scheduler hiccup is a larger fraction of a sub-1 ms number. ONE CAVEAT ON THE THREE ts-FAMILY MS NUMBERS, stated because nothing else in this file would reveal it: resolveTsTarget carries a per-pass resolveCache keyed currentFile::importPath, which no other resolver here has, and ~10% of this corpus is repeat pairs. Their us/import is therefore a slight underestimate of a cold resolve. It is left in rather than defeated because it is what the real pipeline does, and it is identical across all three so the arms stay comparable. HEAP for the eight: rust, swift, typescript, vue, cpp and cobol are still NOT gated, all of them measured before being left out. rust builds no index on this hook (16 B at 8000 files, 0 B at 32000); swift holds one pointer per file-times-segment and mints no strings, reading 0.98 MB at 8000 files against 0.29 MB at 32000 \u2014 a 4x larger corpus reading 3x SMALLER, which is what a measurement below its own noise floor looks like, and the same reading cobol gives (0.54 MB then 0 B); typescript and vue duplicate javascript through the same builder over the same-shaped corpus, and cpp duplicates c (10021320 against 10016960, 0.04% apart). Those four duplications are the ONLY exclusions that still rest on 'it would be a duplicate', and they are duplicates of a builder AND of a read pattern, which is the pairing csharp_csproj failed once the read pattern started to matter \u2014 if any of the four ever diverges in what it ASKS the index, it earns an arm the same way csharp_csproj just did. All eight gated arms are read the same way now (retainedPassBytes, one real import), so unlike before they are directly comparable to one another. WALL CLOCK \u2014 ~33-35 s in report mode, down from ~46 s, and ~44-45 s for --check, which is essentially UNCHANGED from ~46 s. Only report mode got faster; do not read the pair as 46 -> 42. The breakdown is worth having before anyone trims it. Timing arms: go 2.02, csharp 1.09, csharp_csproj 3.22, dart 0.41, ruby 2.90, kotlin 0.85, php 3.46, java 1.57, cobol 0.09, swift 0.46, rust 0.85, python 1.22, javascript 3.23, typescript 2.72, vue 2.89, c 0.86, cpp 0.91 (28.7 s, from 39.8 s: repsFor() accounts for all of it, and every second of it comes from the six languages whose cheapest cell is 20-28 ms); heap arms 3.43 s for SEVENTEEN languages, from 2.06 s for eight (every registered language is measured now; the nine added cost 1.37 s, of which kotlin alone is 0.57 s \u2014 see _heap_bound_note), and 2.1 s came from 3.0 s for seven when flattening retired the warm-up pass; module load 3.9 s. --check pays one import that report mode does not: the inventory arm loads pipeline/registry.ts, which drags in every registered scope resolver and its providers. Measured in isolation with the bench's own static imports already resident, that import costs 6.3-6.5 s on one box and 9.3-10.0 s on another \u2014 i.e. it consumes almost the whole repsFor win, which is why --check did not get faster. It is loaded dynamically at the point of use rather than at the top of the file, so report mode does not pay it and both modes take their measurements in the same module state. IT WAS WEIGHED AND KEPT, on the number that decides it: the benchmarks job is not CI's critical path. On the last green run of main it took 9 m 23 s against 12 m 58 s for the sharded coverage job that gates the merge, so ~4 m 40 s of slack sits above this bench and those seconds buy zero merge latency. Moving the arm to a vitest file would move the registry load ONTO the critical path, and would weaken it as well: this reconciles LANG_REGISTRY's SupportedLanguages values, which are what the five dispatcher branches key off, whereas a test that cannot import measure.mjs can only reconcile this file's arm NAMES plus a hand-written rule for de-aliasing csharp_csproj. The contract test import-target-index-reuse.contract.test.ts already covers the ADAPTER-boundary contract for every registered resolver; this arm covers a different claim, that the BENCH covers the pipeline. The ts family is still the largest single block of the timing phase (8.8 s) \u2014 its cost is suffixResolve probing ~39 extensions per path part on a miss, which is the real resolver and cannot be tuned away from the bench side. IF IT HAS TO SHRINK, drop collide and collide_large for typescript and vue and nothing else: -3.9 s, and it is the only cut that removes near-duplicate work rather than coverage, because all three run the same resolveTsTarget over the same buildSuffixIndex and javascript keeps the collide arm that covers their shared collision axis. Do NOT reach for REPS_MAX: it is 15 because depth_ratio tripped its own budget about 1 run in 20 at 5 and once at 7, and lowering it would re-open that for the eleven languages whose cheapest cell is sub-5 ms \u2014 which is where every recorded trip happened. The six languages it was safe to lower have already been lowered, per language and from a measurement, by repsFor(). ---- THE FIFTH ARGUMENT (context) AND THE TWO ARMS IT MOVED ---- resolveOne now makes run.ts's five-argument call for the two hooks that declare a fifth parameter, so php and python time the legs behind it. Nothing else moved: the other fifteen arms are handed no context and build no ParsedFile[] at all, and over five runs their five ms numbers and four ratios sit exactly where they did. Both languages' ten fingerprints, resolved counts and distinct_outcomes are IDENTICAL \u2014 the leg AGREES with the cascade on this corpus, which is the whole reason the context arm had to be added rather than leaving the fingerprint to notice. PHP: small_ms 27.762 -> 35.125 (+26.5%) and collide_ms 29.407 -> 36.182 (+23.0%), which is filesByDirectory plus, on every import that resolves, a candidate gather over the resolved file's directory and a localDefs filter; the ms ceilings keep PHP's own 4.21x and 4.26x multipliers (117 -> 148, 125 -> 154). depth_ratio 1.144 -> 1.283 and the 1.9 budget is UNCHANGED, which makes it 1.48x measured rather than 1.66x: directoryAliases emits one entry per path segment, so filesByDirectory is O(files x depth) and the depth arm is the only one that can see it \u2014 that budget got TIGHTER relative to its measurement, not looser, and 1.48x sits inside the 1.37-1.75x band the other sixteen carry. Its heap reading rises 37576816 -> 49574008 (+31.9%) for the same structure, and the reading is the MEMO rather than the workspace it indexes: newPass allocates the ParsedFile objects before retainedPassBytes takes its baseline sample, so they sit outside the delta. PYTHON, WHOSE FIGURES ARE THE LEAST SETTLED THING IN THIS FILE AND ARE RECORDED IN TWO SNAPSHOTS BECAUSE OF IT. A named import is the only spelling that reads context.parsedFiles, and it costs up to three entries into the resolver per import (package probe, exports check, submodule probe) where the synthetic namespace spelling this arm used to pass costs one. Against the resolver as it stood when the call shape changed that read small_ms 1.76 -> 5.751 and collide_ms 1.929 -> 5.894, ~3.1x. Against the resolver a few commits later \u2014 which stopped re-running the whole tail after a null package probe, a double-probe this bench could not previously see because the namespace spelling never entered that branch \u2014 the same arms read 4.404 and 4.505. The ceilings are 18 and 19, chosen to clear BOTH: 4.09x and 4.22x of the current numbers, 3.13x and 3.22x of the higher ones, so neither state is red. Retighten toward 4x once that resolver settles. ITS DEPTH ARM WAS DILUTED AND THE BUDGET IS RETIGHTENED TO MATCH, which is the one thing here worth arguing about: the added work is depth-FLAT, so depth_ratio FALLS 1.872 -> 1.478 while the absolute cost more than doubles, and 2.6 against 1.478 would be 1.76x \u2014 far looser than the 1.39x #2913 chose deliberately to lock its own fix in. 2.1 restores that multiplier (1.42x). THE TWO MUTATION SCORES #2913 RECORDED (3.123 for reverting the per-directory memo, 2.734 for reverting the nested-name rejection) WERE TAKEN AGAINST THE OLD CALL SHAPE AND HAVE NOT BEEN RE-TAKEN. Modelled forward, with the depth-quadratic term reappearing in every resolver entry so its absolute contribution scales with the entry count, they land near 2.8 and 2.4 \u2014 both above 2.1, and the second BELOW 2.6, which is the arithmetic that decided the budget. Re-run the two mutations before trusting the lock-in claim above. python's heap reading is unchanged (10543152 recorded; 10529848-10544616 across eight runs) because its probe misses before the branch that reads parsedFiles \u2014 see _blind_spot for why no probe can reach that memo. Every figure in this section is the MAXIMUM over its snapshot's runs (five, then three), with peak-to-peak 1.031-1.058 on php and 1.019-1.081 on python, taken on a box that was NOT idle and with another change landing in python's resolver mid-measurement. Re-take them serially before merging.", + "_arms_note": "Five timing arms, one memory arm and one deterministic arm elsewhere, because none of them gates alone. scaling_ratio (t_large/t_small)/(1600/400) catches cost growing with FILE COUNT \u2014 the #2877-#2880, #2901, #2902 and #2908 regressions themselves; every one of those legs was Theta(files) per import, so a revert scores ~4 here by construction. depth_ratio (t_deep/t_small at a FIXED file count, ~6x the path components) catches cost growing with path DEPTH, which scaling_ratio divides out and structurally cannot see; buildSuffixIndex (C#, Ruby, PHP, Java) emits one entry per component, while Kotlin's declared-package index is depth-free while Go, Dart and COBOL, whose indexes are depth-free, sit at ~1.0. csharp's depth_budget has now been retightened twice for the same reason, and the second time it did lock the win in. It was 5 against a then-measured 3.318; #2903 made buildSuffixIndex's dirMap lazy and it became 3.5 against 2.31, with the file stating plainly that 3.5 did NOT lock that win in because a revert to an eager dirMap scores 3.318 and passes. Extending the laziness to the two SUFFIX maps drops it again, to 1.438 (java likewise 2.214 -> 1.402), because the deep arm has ~6x the path components and an O(files x depth) build of a map the no-csproj leg never reads is exactly the cost that scales with depth. Both are now 2.2, which is this file's 1.5x convention against measurements whose own peak-to-peak over 4 runs is 1.04x and 1.07x \u2014 and 2.2 DOES lock it in: an eager rebuild scores 2.3+ and fails. The other fifteen depth budgets sit at 1.37-1.75x measured and are unchanged. collide_scaling_ratio is the same measurement on a SHARED-LEAF layout (svcN/internal, SrcN/Models, com/example/model in every service, a repeated mod0.dart/mod0.rb/Mod0.cpy basename) carrying an identical file, import and resolved count: the small/large/deep arms mint one directory name per index, so every index bucket in them holds exactly ONE entry (measured: max last-segment bucket 1 and max matching directories 1 for go and csharp at 400 and 1600 files; max basename bucket 1 for dart and ruby), and bucket cardinality is the only non-constant term the new indexes have. On the shared-leaf shape go, csharp, dart and java legitimately score 2.1-3.9 because the bucket grows with the file count BY CONSTRUCTION \u2014 this is a limit on the SCOPE of the \"independent of corpus size\" claim, not a regression (the indexed code is still faster there than the pre-change full scan); their collide budgets say so honestly instead of pretending 1.8. Ruby, Kotlin, PHP and COBOL answer from keyed maps and are collision-immune, so they keep the linear 1.8 budget and that immunity is the assertion. csharp_csproj is the one arm that runs the other way: its shared leaf collapses dirsByLastSegment to the single key Models, so the slash-free sweep (see CSPROJ_CONFIGS) is CHEAPER on the collide layout than on the unique one and its expensive scale arm is large, not collide_large. Its 1.8 collide budget is therefore the linear one, and the arm that carries its real cost is the unique one. The collide arm is also the only arm that reaches filesDirectlyInPkgDir's dirCount > 1 merge (go: 388 multi-directory calls at 400 files, up to 9 directories; 1517 at 1600 files, up to 34) and the only one that reaches COBOL's copybook-over-source tier tie-break, which needs one bookname to name two files. small_ms_ceiling and collide_ms_ceiling are ABSOLUTE (~4x the measured arm), because a constant-factor regression that grows both scale arms equally passes every ratio. The five arms added here use 4.2x, the middle of the 3.7-4.6x the original five already carry; the two COBOL arms use ~5x, the multiplier dart's sub-1 ms arm has always carried, because a fixed scheduler hiccup is a larger fraction of a smaller number \u2014 measured over 8 runs they sat at 0.25-0.37 ms and 0.18-0.30 ms, and the pre-#2908 two-scans-per-COPY implementation costs ~300 ms on the same arm, so 2.0 and 1.5 still separate fixed from broken by two orders of magnitude. NOISE, measured rather than assumed: depth_ratio divides two sub-3 ms numbers (Dart's are sub-1 ms) and is by far the noisiest arm here, so it set N for the whole file. fastest() is a min-of-N estimator, so N is the knob. Over 22 --check runs on an idle box, peak-to-peak: at N=5 go ran 0.757-1.748 (2.31x) and tripped its own 1.6 budget about 1 run in 20; at N=7 (the kotlin-import-target setting) Dart still ran 0.678-2.043 (3.01x) and tripped once; at N=15 (bench/cfg, bench/schema-pairs, bench/callable-value-flow) every language collapsed to a 1.13-1.26x swing with 22/22 passing. The budgets were NOT widened; the estimator was fixed instead, which is why the headroom above is real rather than granted. N IS NOW PER LANGUAGE, and that is a refinement of the same finding rather than a retreat from it. The overshoot of min-of-K against min-of-15 is a function of the CELL's absolute duration, not of the language: replayed against two independent runs' full sample sets, the worst overshoots at K=7 land on swift.small (0.43 ms, 31.8%) and dart.collide (1.5 ms, 37.6%), while every cell at or above 10 ms overshoots by at most 6.3%. So repsFor() keeps 15 while a language's cheapest arm is under 5 ms and otherwise spends ~150 ms per cell, floored at 7 \u2014 15 for go, csharp, dart, kotlin, java, cobol, swift, rust, python, c and cpp (every language the flakiness above was ever about, cheapest arm 0.19-3.2 ms) and 7-8 for csharp_csproj, ruby, php, javascript, typescript and vue (cheapest arm 20-28 ms). Per LANGUAGE, not per cell, so all five arms of a language share one estimator and the four ratios stay comparisons of like with like. The replay passed all 85 cells on all five gates at 0.4-0.7 of budget and saved 12.8 s and 12.4 s of a 46 s run; min-of-7 also reads slightly HIGHER than min-of-15, so the ceilings get marginally more sensitive rather than less. Confirmed on 4 fresh runs with the adaptive estimator live: every small arm inside 1.12x peak-to-peak and every collide arm inside 1.07x, with the six 7-8 rep languages at 1.008-1.071 \u2014 no worse than the 11 that kept 15. The chosen N is reported per language as `reps`. heap_ceiling_bytes bounds the retained per-pass import index, the only arm here that can see memory: buildSuffixIndex emits maps at O(files x depth), the profile package-dir-index.ts cites #2649 to avoid for itself, and csharp, ruby, php and java all retained NOTHING across imports at BASE (C#'s no-csproj leg and PHP's and Java's every leg re-scanned the raw Set; Ruby rebuilt and discarded a suffix index per require). It is measured at 8000 and 32000 files at HEAP_PAD depth rather than at the timing arms' sizes, because the finding is an ABSOLUTE footprint at repository scale. THE ARM NOW READS WHAT THE LANGUAGE READS, and that change is the whole reason this file was re-baselined. Four of these arms used to call getWorkspaceFileIndex(set) directly and then read index.all.length, which asks no suffix question at all \u2014 harmless only while buildSuffixIndex built both maps eagerly. The moment they went lazy the direct call built NO map, csharp, ruby, php and java each reported 0 B at 32000 files, and 0 B is under every ceiling: --check printed PASS over four gates that had silently become ceilings over nothing, which is precisely the failure this file's own header warns about for rust and cobol. Every arm now resolves a real MISSING import through the real resolver (HEAP_PROBE_TARGET, asserted to miss), so the maps it forces are the maps production forces, and a resolver that starts asking a new question moves the number without anyone editing the bench. That makes the READ PATTERN the dominant term, and the eight numbers say so: java 34958600 B and csharp 29862200 B ask index.get and never getInsensitive; php 37579888 B asks getInsensitive and never get, plus its own first-proper-suffix map; ruby 41025360 B and javascript 26745296 B read get(s) || getInsensitive(s) and pay for both, the second DERIVED from the first; and csharp_csproj 73705944 B additionally asks getFilesInDir. csharp_csproj IS NOW GATED, reversing the earlier decision that it would be 'a ceiling on a duplicate': at +20.8% of the C# index it was one, and at 2.47x of it \u2014 same corpus, same getWorkspaceFileIndex, three maps instead of one \u2014 it is the witness that the read pattern is the footprint. The old RESIDUAL note is superseded by that number: a dirMap-sized addition is no longer +18%, and a consumer that asks all three questions blows csharp's ceiling by 1.64x rather than sliding under it. A SECOND MEASUREMENT BIAS was removed at the same time and it moved every figure here, so do not read these against the old ones as if only the read pattern changed. buildFiles mints paths with template literals, which V8 keeps as ropes; the first traversal that slices one flattens it, allocating the flat string and dropping the rope's pieces, so a build measured over an unflattened corpus reports the index MINUS that net release \u2014 11% low, uniformly. bytes_small was read over a corpus a discarded warm-up pass had already flattened and bytes_large over a fresh one, so every ratio read ~0.85-0.89 for structures that are exactly linear in the file count. measureHeap now flattens each corpus before measuring it; all eight ratios read 0.998-1.017, and the warm-up pass is gone because with the corpus flat a language's first and second reads agree to within 0.3%. python's figure rises from 7624992 to 10362976 for this reason and not because anything regressed, and then to 10543152 (+1.7%) because #2913's nestedDirNames set is retained for the pass, and then FALLS to 6360936 (-39.7%) for a reason worth knowing: byBasename holds roughly one bucket per file, and building each with `[]` followed by `push` made V8 grow the backing store to its 16-slot minimum, so every single-file bucket retained 15 empty pointer slots. Constructing the one-element buckets directly (`set(base, [entry])`) is byte-identical in contents and 3.9 MiB smaller at 32000 paths \u2014 37% of what this arm used to read was empty array slots \u2014 the ancestorsByDir memo itself is NOT in this reading, because python's probe target misses at the nested-name rejection and never reaches the walk, so this arm does not bound that memo; measured separately with a probe that does reach it, a 32000-file corpus with every file in its own 10-deep directory retains ~19 MB, which would clear this ceiling, so repointing python's heap probe at a walking spelling means re-recording the ceiling in the same change, and c is unchanged at 10018816 because its basename map does not slice paths. Its ceiling is 1.5x the measured arm, and the DIFFERENCE FROM THE 4x TIMING CONVENTION IS DELIBERATE \u2014 do not harmonise it back. 4x exists because runner contention dominates a wall-clock number; this one has essentially no measurement noise (across 4 runs the widest spread was 0.11% on python, 0.03% on csharp_csproj and 0.00% \u2014 identical to the byte \u2014 on ruby, php, java, javascript and c, and the same holds across separate processes), so 4x would throw away almost all of the gate's power and sail straight past the regression this arm exists to catch. 1.5x still tolerates ~50% of cross-platform and Node-version drift, far more than a Node major bump plausibly moves heapUsed accounting; it catches a duplicated index (+100%) or a second exactMap-sized suffix map (+~85%). heap_floor_fraction is the arm the 0 B incident proved was missing. A ceiling can only say 'not too big'; nothing said 'still measuring something', which is why four dead arms passed. The floor is 0.5 x each language's RECORDED READING (heap_reading_bytes), which is half the measured size and says so. It used to be 0.33 x the CEILING, described the same way \u2014 true only while every ceiling stayed at exactly 1.5x its reading, a convention this file states and nothing enforces, so re-tuning one ceiling upward would have loosened that language's floor by the same factor in the one direction a floor exists to watch. The two forms agree to within 0.8% for all eight today, so this is a correction of derivation, not of strength. It sits ~400x above the readings' own reproducibility and far below any collapse. A genuine 2x memory WIN trips it too, and that is intended: like a fingerprint move, it must be explained and re-baselined rather than absorbed. COBOL is left out for the opposite reason: its index is two Map, O(files) with no depth term, and at 32000 files its retained delta does not clear the noise of the measurement itself. heap_ratio_budget, the linear-growth check across the 4x file-count gap, is the orthogonal arm: it sees per-file and per-depth growth but not a constant factor. ---- THE EIGHT LANGUAGES ADDED LAST (swift, rust, python, javascript, typescript, vue, c, cpp) ---- They carry the SAME five arms and the same gates; what differs is which arm can actually fail for each, because each resolver has a different cost axis, and the budgets below say so instead of copying a number across. Every figure quoted is the MAXIMUM over 5 full runs on an idle box, and the peak-to-peak of every one of these arms stayed inside 1.10x over those runs \u2014 tighter than the 1.13-1.26x the original nine record, because none of these arms divides two sub-1 ms numbers the way dart depth_ratio does. depth_budget is ~1.5x measured throughout: swift 2.3 (1.487), rust 2.1 (1.377), javascript 2.1 (1.376), typescript 2.1 (1.381), vue 2.3 (1.563), c 3.0 (1.990), cpp 3.0 (1.999). PYTHON WAS 11 AGAINST 7.389 AND IS NOW 2.6 AGAINST 1.872, because #2913 fixed the resolver rather than the budget. Its INDEX was always depth-free; hasRepoCandidate and resolveAbsoluteFromFiles each rebuilt one ancestor prefix per directory component of the importer on EVERY import, and the index's own dirPrefixes build inserted one entry per component per file, so the resolver was quadratic in path depth where every other language here is linear or flat. The prefixes are a pure function of the importer's DIRECTORY, so they are now memoized per directory inside getPythonFileIndex (ancestorsByDir), the leading segment is rejected up front against a set of nested directory names, the module and package buckets are consulted before the walk rather than inside it, and the dirPrefixes build stops at the first ancestor already stored. All five fingerprints are byte-identical, so it is a hoist. The budget is 2.2, and BOTH numbers behind it were re-measured on a quiet box AFTER the context leg below started being measured, because that change moved the arm: the work it adds is depth-FLAT, so python's absolute cost more than doubled while depth_ratio FELL to 1.405-1.563 over 5 serial runs (peak-to-peak 1.11x). A budget carried over from before that change would have been slack against a smaller ratio. 2.2 is 1.41x the measured maximum, inside the 1.37-1.75x band the other fifteen sit in, and it LOCKS THE WIN IN: reverting the per-directory ancestor memo alone scores 2.524 and reverting the nested-name rejection alone scores 2.553, both measured under the current call shape, so each fails at 2.2 with 13% to spare. Do not read those two figures as the pre-#2913 cost \u2014 7.239 was that, and the gap closed because the bare-import tier stopped walking at all (see below). The other two parts of the fix are not gated by this arm and are not meant to be: reverting the bucket prune or the dirPrefixes early break lands under any budget this arm's noise supports, so they are gated deterministically instead, by the prefix-parity and package-probe arms of test/unit/scope-resolution/python/python-importer-ancestors.test.ts and python-import-target-parity.test.ts, which go red on exactly those two mutations. A timing budget catches what it can measure; the counts catch the rest. THE BARE-IMPORT TIER (`import os`, single segment, no dot) was a separate O(depth) walk in import-resolvers/python.ts that this bench cannot see at all, because every python arm here spells its imports with a dot and returns at the `pathLike.includes('/')` guard before reaching it. It ran TWICE per `from x import y` \u2014 the package probe's recursion re-ran the whole tail on identical inputs \u2014 and is now one memoized chain plus an O(1) proof-of-absence against the index's basename buckets: 12/24/72 Set probes at depth 1/4/16 became a flat 2, and 11.615 us/import at 18 path components became 0.740. Gated by probe COUNT in test/unit/scope-resolution/python/python-import-probe-count.test.ts, not here. collide_scaling_budget splits three ways. Three languages scan a bucket that grows with the corpus and get their measured value x1.5: swift 4.9 (3.279 \u2014 its bucket is the module file list it RETURNS, and its collide arm is four modules instead of dirs of them so that bucket is fileCount/4, i.e. 100 files at 400 and 400 at 1600), c 3.8 (2.535) and cpp 4.0 (2.639, the same basename bucket its suffix fallback walks). Four answer from keyed maps and keep the linear 1.8 \u2014 python 1.097, javascript 1.083, typescript 1.053, vue 1.079 \u2014 and that immunity IS the assertion, exactly as for ruby, kotlin, php and cobol. RUST IS THE ONE ARM THAT WAS REDESIGNED RATHER THAN BUDGETED. It resolves by probing candidate paths with allFilePaths.has(...) and never searches, so its cost is O(path segments) and provably flat in the file count (1.095 scaling, 1.061 collide scaling): a shared-leaf collide arm for rust would have asserted nothing, which is worse than no arm. Its collide corpus is instead a deep module tree (src/l0/l1/l2/l3/l4/mod{d}) whose targets carry ~2x the :: segments, so the arm exercises the axis that CAN grow, its 1.8 budget asserts the flatness across file counts, and collide_ms_ceiling 19 bounds the absolute cost of the long-path probe. small_ms_ceiling and collide_ms_ceiling are ~4x measured as everywhere else: rust 10/19 (2.609/4.704), python 7/8 (1.76/1.929, retightened from 12/15 against 3.044/3.771 by #2913), javascript 85/89 (21.254/22.145), typescript 85/86 (21.250/21.464), vue 81/93 (20.164/23.227), c 7/11 (1.620/2.850), cpp 7/12 (1.581/3.009). Swift takes ~5x (2 against 0.421 and 4 against 0.821) \u2014 the multiplier dart and cobol already carry, because a fixed scheduler hiccup is a larger fraction of a sub-1 ms number. ONE CAVEAT ON THE THREE ts-FAMILY MS NUMBERS, stated because nothing else in this file would reveal it: resolveTsTarget carries a per-pass resolveCache keyed currentFile::importPath, which no other resolver here has, and ~10% of this corpus is repeat pairs. Their us/import is therefore a slight underestimate of a cold resolve. It is left in rather than defeated because it is what the real pipeline does, and it is identical across all three so the arms stay comparable. HEAP for the eight: rust, swift, typescript, vue, cpp and cobol are still NOT gated, all of them measured before being left out. rust builds no index on this hook (16 B at 8000 files, 0 B at 32000); swift holds one pointer per file-times-segment and mints no strings, reading 0.98 MB at 8000 files against 0.29 MB at 32000 \u2014 a 4x larger corpus reading 3x SMALLER, which is what a measurement below its own noise floor looks like, and the same reading cobol gives (0.54 MB then 0 B); typescript and vue duplicate javascript through the same builder over the same-shaped corpus, and cpp duplicates c (10021320 against 10016960, 0.04% apart). Those four duplications are the ONLY exclusions that still rest on 'it would be a duplicate', and they are duplicates of a builder AND of a read pattern, which is the pairing csharp_csproj failed once the read pattern started to matter \u2014 if any of the four ever diverges in what it ASKS the index, it earns an arm the same way csharp_csproj just did. All eight gated arms are read the same way now (retainedPassBytes, one real import), so unlike before they are directly comparable to one another. WALL CLOCK \u2014 ~33-35 s in report mode, down from ~46 s, and ~44-45 s for --check, which is essentially UNCHANGED from ~46 s. Only report mode got faster; do not read the pair as 46 -> 42. The breakdown is worth having before anyone trims it. Timing arms: go 2.02, csharp 1.09, csharp_csproj 3.22, dart 0.41, ruby 2.90, kotlin 0.85, php 3.46, java 1.57, cobol 0.09, swift 0.46, rust 0.85, python 1.22, javascript 3.23, typescript 2.72, vue 2.89, c 0.86, cpp 0.91 (28.7 s, from 39.8 s: repsFor() accounts for all of it, and every second of it comes from the six languages whose cheapest cell is 20-28 ms); heap arms 3.43 s for SEVENTEEN languages, from 2.06 s for eight (every registered language is measured now; the nine added cost 1.37 s, of which kotlin alone is 0.57 s \u2014 see _heap_bound_note), and 2.1 s came from 3.0 s for seven when flattening retired the warm-up pass; module load 3.9 s. --check pays one import that report mode does not: the inventory arm loads pipeline/registry.ts, which drags in every registered scope resolver and its providers. Measured in isolation with the bench's own static imports already resident, that import costs 6.3-6.5 s on one box and 9.3-10.0 s on another \u2014 i.e. it consumes almost the whole repsFor win, which is why --check did not get faster. It is loaded dynamically at the point of use rather than at the top of the file, so report mode does not pay it and both modes take their measurements in the same module state. IT WAS WEIGHED AND KEPT, on the number that decides it: the benchmarks job is not CI's critical path. On the last green run of main it took 9 m 23 s against 12 m 58 s for the sharded coverage job that gates the merge, so ~4 m 40 s of slack sits above this bench and those seconds buy zero merge latency. Moving the arm to a vitest file would move the registry load ONTO the critical path, and would weaken it as well: this reconciles LANG_REGISTRY's SupportedLanguages values, which are what the five dispatcher branches key off, whereas a test that cannot import measure.mjs can only reconcile this file's arm NAMES plus a hand-written rule for de-aliasing csharp_csproj. The contract test import-target-index-reuse.contract.test.ts already covers the ADAPTER-boundary contract for every registered resolver; this arm covers a different claim, that the BENCH covers the pipeline. The ts family is still the largest single block of the timing phase (8.8 s) \u2014 its cost is suffixResolve probing ~39 extensions per path part on a miss, which is the real resolver and cannot be tuned away from the bench side. IF IT HAS TO SHRINK, drop collide and collide_large for typescript and vue and nothing else: -3.9 s, and it is the only cut that removes near-duplicate work rather than coverage, because all three run the same resolveTsTarget over the same buildSuffixIndex and javascript keeps the collide arm that covers their shared collision axis. Do NOT reach for REPS_MAX: it is 15 because depth_ratio tripped its own budget about 1 run in 20 at 5 and once at 7, and lowering it would re-open that for the eleven languages whose cheapest cell is sub-5 ms \u2014 which is where every recorded trip happened. The six languages it was safe to lower have already been lowered, per language and from a measurement, by repsFor(). ---- THE FIFTH ARGUMENT (context) AND THE TWO ARMS IT MOVED ---- resolveOne now makes run.ts's five-argument call for the two hooks that declare a fifth parameter, so php and python time the legs behind it. Nothing else moved: the other fifteen arms are handed no context and build no ParsedFile[] at all, and over five runs their five ms numbers and four ratios sit exactly where they did. Both languages' ten fingerprints, resolved counts and distinct_outcomes are IDENTICAL \u2014 the leg AGREES with the cascade on this corpus, which is the whole reason the context arm had to be added rather than leaving the fingerprint to notice. PHP now runs its sole timing, depth, collision and heap workloads with Composer's PSR-4 config. The canonical recording is small_ms 12.515, collide_ms 13.684, scaling_ratio 1.077, collide_scaling_ratio 0.994, depth_ratio 1.536 and 13407592 retained bytes. Its ceilings are 55/60 ms, 2.4 depth and 20200000 bytes, preserving normal cross-run headroom without splitting PHP into benchmark identities. PYTHON, WHOSE FIGURES ARE THE LEAST SETTLED THING IN THIS FILE AND ARE RECORDED IN TWO SNAPSHOTS BECAUSE OF IT. A named import is the only spelling that reads context.parsedFiles, and it costs up to three entries into the resolver per import (package probe, exports check, submodule probe) where the synthetic namespace spelling this arm used to pass costs one. Against the resolver as it stood when the call shape changed that read small_ms 1.76 -> 5.751 and collide_ms 1.929 -> 5.894, ~3.1x. Against the resolver a few commits later \u2014 which stopped re-running the whole tail after a null package probe, a double-probe this bench could not previously see because the namespace spelling never entered that branch \u2014 the same arms read 4.404 and 4.505. The ceilings are 18 and 19, chosen to clear BOTH: 4.09x and 4.22x of the current numbers, 3.13x and 3.22x of the higher ones, so neither state is red. Retighten toward 4x once that resolver settles. ITS DEPTH ARM WAS DILUTED AND THE BUDGET IS RETIGHTENED TO MATCH, which is the one thing here worth arguing about: the added work is depth-FLAT, so depth_ratio FALLS 1.872 -> 1.478 while the absolute cost more than doubles, and 2.6 against 1.478 would be 1.76x \u2014 far looser than the 1.39x #2913 chose deliberately to lock its own fix in. 2.1 restores that multiplier (1.42x). THE TWO MUTATION SCORES #2913 RECORDED (3.123 for reverting the per-directory memo, 2.734 for reverting the nested-name rejection) WERE TAKEN AGAINST THE OLD CALL SHAPE AND HAVE NOT BEEN RE-TAKEN. Modelled forward, with the depth-quadratic term reappearing in every resolver entry so its absolute contribution scales with the entry count, they land near 2.8 and 2.4 \u2014 both above 2.1, and the second BELOW 2.6, which is the arithmetic that decided the budget. Re-run the two mutations before trusting the lock-in claim above. python's heap reading is unchanged (10543152 recorded; 10529848-10544616 across eight runs) because its probe misses before the branch that reads parsedFiles \u2014 see _blind_spot for why no probe can reach that memo. Every figure in this section is the MAXIMUM over its snapshot's runs (five, then three), with peak-to-peak 1.031-1.058 on php and 1.019-1.081 on python, taken on a box that was NOT idle and with another change landing in python's resolver mid-measurement. Re-take them serially before merging.", "_triage": "Every ratio and ms ceiling here is a TIMING signal \u2014 re-run on an idle machine before investigating; runner contention dominates. depth_ratio is the noisiest of them by a wide margin (it divides two sub-3 ms numbers, and Dart's are sub-1 ms): if exactly one arm fails and it is that one, suspect the machine first. N is 15 for every language whose cheapest arm is under 5 ms, rather than this bench's original 5, specifically to hold that arm's peak-to-peak swing under 1.26x \u2014 see _arms_note for the measured distributions and for why the six languages that drop to 7-8 are the ones where cell size makes it safe \u2014 so a depth_ratio failure that REPRODUCES is a real signal, not noise. Each language's chosen N is printed as `reps`; read it before blaming the estimator. The fingerprint, shape and heap arms are the opposite: deterministic (over 4 runs the heap arm's widest spread was 0.11% on python and 0.00% on java, javascript and c), a re-run never changes them, and they must never be wished away. TWO heap failures mean the arm STOPPED MEASURING rather than that memory grew, and both are deterministic: a heap floor failure says the probe no longer forces the index it used to (this is how four arms read 0 B when buildSuffixIndex went lazy, and 0 B passes every ceiling), and a `heap probe ... resolved` throw says a probe target that must MISS now hits, so the reading is a materialized answer and the legs past it were never reached. A heap BOUND failure is deterministic in the same way and means one specific thing: a language excluded from the budgeted tier has grown a structure, or started asking its index a question it did not ask when the exclusion was recorded \u2014 never a timing signal, never a re-run, and never fixed by raising the bound without saying what grew. The context arm is deterministic too, and a failure there means one specific thing rather than a range of them: run.ts's fifth argument is not reaching that resolver from this bench, or the leg behind it stopped running. Never a timing signal, never a re-run. TIGHTENED IN #2881, because the measurements they bound got faster and a budget left alone while its reading falls is a gate loosening without anyone deciding to. Each new value holds the headroom the old one expressed over the old reading, computed from `_measured` on both sides: kotlin depth 3.4 -> 2.8 (reading 2.219 -> 1.813), go depth 1.6 -> 1.4 (1.169 -> 0.999), csharp depth 2.2 -> 2.0 (1.438 -> 1.279), java depth 2.2 -> 2.1 (1.402 -> 1.354), kotlin collide_scaling 1.8 -> 1.65 (1.179 -> 1.081), go collide_scaling 5.5 -> 5.1 (3.763 -> 3.465). The ABSOLUTE ms ceilings were deliberately NOT tightened by the same reasoning: they carry runner-contention headroom rather than measurement headroom, and a ratio is runner-speed-invariant where a millisecond is not.", "_floor": "Measured against the pre-change implementations on THIS corpus at 150/600 files: go 3.36, csharp 4.10, dart 3.32, ruby 3.87. The issues report 4.00 / 3.43 / 4.05 on their own corpora; those are DIFFERENT numbers from different repositories and are not reproduced here \u2014 what they and these share is that both independently land in the quadratic band, well clear of the ~1.0 a linear result gives. Note also that this floor was taken at 150/600 while the gate runs at 400/1600, so it is a lower bound on what the pre-change code would score today. Kotlin's own bench measured its pre-index floor at 3.737. The four resolvers added later were NOT re-floored on this corpus, and the reason is that they do not need to be: every one of their pre-change legs walked the whole file set per import (PHP one findIndex per path part per extension, Java one scan per stripped prefix, COBOL two full scans per COPY, C# csproj one normalizedFileList pass per import per matching config), so their scaling_ratio is ~4 by construction rather than by measurement. Their per-import costs were measured on their own issue corpora instead: PHP 96.40 ms -> 0.036 ms, Java 8.05 ms -> 0.62 ms, COBOL 3879 us -> 10.5 us, C# csproj 1103 us -> 7.6 us. The 1.8 budget sits well above the linear result and well below every one of those. The eight languages added last were NOT floored either, and for a different reason again: they are not fixes, so there is no pre-change implementation to floor against. Their scaling budgets are the global linear 1.8 and the point of the arms is to hold the current numbers (measured 1.01-1.13) rather than to separate a fix from a break. The one exception is javascript, which IS a fix and does have a floor: 6448.9 us per import at 2000 files and 25972.6 us at 8000 \u2014 4.12x the per-import cost for 4x the files, i.e. O(imports x files) \u2014 against 28.5 / 27.4 us with the index PR #2911 gave it, and 25.0 / 27.0 us for TypeScript over the identical corpus.", "_rebaselined_2910_java_declared_packages": "#2910 replaces Java path-suffix fallback with declared-package resolution. The benchmark now restores package capture side channels, threads parsedFiles through javaScopeResolver, proves the context leg with a positive path/package-mismatch probe, and models the collide arm as one package declared across service paths. External imports now remain unresolved; local exact and wildcard imports preserve the 1153/4681 workload. Java's index is package/type maps rather than suffix maps: bytes_large 34958600 -> 3676984, with its floor and ceiling re-recorded together. Depth and collision scaling budgets tighten to the shared linear 1.8 gate.", @@ -34,7 +35,7 @@ "dart": 1.6, "ruby": 2.2, "kotlin": 1.8, - "php": 1.9, + "php": 1.8, "java": 1.8, "cobol": 1.6, "swift": 2.3, @@ -53,7 +54,7 @@ "dart": 3, "ruby": 77, "kotlin": 12, - "php": 148, + "php": 55, "java": 17, "cobol": 2, "swift": 2, @@ -72,7 +73,7 @@ "dart": 6, "ruby": 95, "kotlin": 12, - "php": 154, + "php": 60, "java": 26, "cobol": 1.5, "swift": 4, @@ -92,7 +93,7 @@ "csharp": 44900000, "csharp_csproj": 110600000, "ruby": 61600000, - "php": 74400000, + "php": 59410824, "java": 5600000, "python": 9541404, "c": 15000000 @@ -107,7 +108,7 @@ "csharp": 29869080, "csharp_csproj": 73703384, "ruby": 41020808, - "php": 49574008, + "php": 39607216, "java": 3676984, "python": 6360936, "c": 10018816 @@ -439,44 +440,47 @@ "small": { "files": 400, "imports": 3200, - "resolved": 1153, - "distinct_outcomes": 2867, - "fingerprint": "3bb31eb4cd444b240e56b151007004f2f810bb5ee3f111b7b57738ea17c819b2" + "resolved": 1152, + "distinct_outcomes": 2871, + "fingerprint": "0e9b0839544137054dcc5a9fcc9c6972fee954c2b8780905d79201556a7e4315" }, "large": { "files": 1600, "imports": 12800, - "resolved": 4681, + "resolved": 4680, "distinct_outcomes": 11517, - "fingerprint": "1c313a83acf55ec58994fc55016754488ae2d352aefaeb84a2e3ecbb928b3479" + "fingerprint": "f69730d7df13cd12b59344d596d4918a718c6eda62d4179b296b5f8174af7d88" }, "deep": { "files": 400, "imports": 3200, - "resolved": 1153, - "distinct_outcomes": 2867, - "fingerprint": "94bdf5cb27b7a1bb0d24e2ba0157ba71dcf61ec726059dd5a0462377a1d0180b" + "resolved": 1152, + "distinct_outcomes": 2871, + "fingerprint": "ded2c1504ff813c596b74093f9352c25b358ad1e67c78e61dd028b57ef05ae61" }, "collide": { "files": 400, "imports": 3200, - "resolved": 1153, - "distinct_outcomes": 2695, - "fingerprint": "61038746f1386bfc747784e7ce6bc52522bc4585259668e22e29f93291b0b3a5" + "resolved": 1152, + "distinct_outcomes": 2871, + "fingerprint": "76c89603524105061b0a9032587702c5ff1d59d8233527799b0515f1f926960e" }, "collide_large": { "files": 1600, "imports": 12800, - "resolved": 4681, - "distinct_outcomes": 10845, - "fingerprint": "c41d254ce8703339576e5642f67dfef81c97445c75db184bb65dc26b4d4715ef" + "resolved": 4680, + "distinct_outcomes": 11517, + "fingerprint": "e88e95736fd8a0f9b27fcb363136c582fe94bcf0885e41efbb4307c367f97f50" }, - "fingerprint": "1c313a83acf55ec58994fc55016754488ae2d352aefaeb84a2e3ecbb928b3479", + "fingerprint": "f69730d7df13cd12b59344d596d4918a718c6eda62d4179b296b5f8174af7d88", "heap": { "files_small": 8000, "files_large": 32000, "path_segments": 14, - "probe": "Vendor0\\Ghost\\Missing" + "probe": "App\\HeapGhost0\\AbsentHeapProbe", + "resolution_config": "App=d0/d1/d2/d3/d4/d5/d6/d7/src/App", + "external_probe": "Vendor0\\Ghost\\Missing", + "external_result": "" }, "context": { "target": "App\\Ns0\\Dup", @@ -484,11 +488,11 @@ "without_context": "src/App/Ns0/Dup.php" }, "_measured": { - "collide_ms": 35.91, - "collide_scaling_ratio": 1.068, - "depth_ratio": 1.268, - "scaling_ratio": 1.079, - "small_ms": 34.023 + "collide_ms": 10.581, + "collide_scaling_ratio": 1.026, + "depth_ratio": 1.158, + "scaling_ratio": 1.05, + "small_ms": 10.511 } }, "java": { @@ -1012,7 +1016,7 @@ } } }, - "_blind_spot": "MEASURED, so nobody has to rediscover it: a full workspace scan reintroduced on 1-in-32 imports passes EVERY arm here \u2014 dart scored 1.458 scaling and 1.736 ms against the 1.8 budget and 4 ms ceiling of an earlier revision. At 1-in-8 the scaling arm catches it (2.414). The gate that NARROWS this is not a timing gate at all: test/unit/scope-resolution/import-target-index-parity.test.ts counts iterations of the file-set Set and reads 14 instead of 1 for that same 1-in-32 mutation, deterministically and for all five languages. It does NOT close it. The counter watches the Set, and the resolvers no longer read the Set \u2014 they read materialized copies of the same file list: WorkspaceFileIndex.normalized and .all (C#, Ruby), Dart's byBasename buckets, and PackageDirIndex.filesByDir (Go, C#). A 1-in-32 scan over any of those three touches the Set zero extra times, so it passes the parity test AND passes --check. Closing it would take an iteration counter on the materialized arrays themselves. Read the two gates together; tightening these ceilings toward the noise floor to chase that case would only buy flaky CI. CONFIRMED THE HARD WAY by PR #2911: JavaScript resolution was scanning ImportPassCache.normalizedFileList on every import \u2014 a materialized array, not the Set \u2014 at 25972 us per import at 8000 files, and no instrument on the #2901-#2909 branch could see it. It took a differential parity test over 211200 old-vs-new pairs to find. The arms added here would have caught THAT one on absolute ms (85 ms budget against a 20 ms arm; the unindexed resolver costs ~83000 ms on the same corpus), which is the argument for gating every registered language rather than only the ones a PR happens to touch. THE SECOND BLIND SPOT IS CLOSED, and this records what closing it changed. This harness used to call the inner resolvers with the NO-CONTEXT shape: run.ts calls provider.resolveImportTarget with five arguments, the fifth being { parsedFiles, parsedImport }, and resolveOne supplied three. resolveOne now makes the production call, newPass mints the ParsedFile[] FIRST and derives the path set from it exactly as run.ts does, and both legs behind the argument run on every import of their arms \u2014 PHP's named/alias function-or-const leg over filesByDirectory(context.parsedFiles), whose memo defeated measures 197.0 us -> 9976.2 us per import (50.6x), and Python's from-import submodule-precedence branch, the only spelling that reads context.parsedFiles at all. Fifteen of the seventeen arms cannot observe a context (their hooks declare three or four parameters) and are handed none, so their numbers did not move; which two CAN is now reconciled against SCOPE_RESOLVERS' hook arity rather than asserted in prose. NOTHING ELSE IN THIS FILE COULD HAVE GATED IT, which is why the context arm exists: on this corpus the leg AGREES with the cascade for every import, so all ten of PHP's and Python's fingerprints, their resolved counts and their distinct_outcomes are unchanged; a dropped context makes the timing arms FASTER and no arm here has a lower bound on ms; and the heap floor (0.5 x 49573840 = 24.8 MB) still passes the 37576816 B a no-context PHP pass reads. The arm is one import per language resolved through resolveOne twice, with and without the pass's parsedFiles, whose two answers must DIFFER and must both match what is recorded. WHAT REMAINS UNMEASURED, narrowed rather than deleted: Python's parsedFileByPath memo is exercised by the five timing arms and cannot be reached by the heap arm at all, because retainedPassBytes requires a probe that MISSES while every path that builds that memo returns a non-null packageTarget \u2014 so no ceiling bounds that Map (one pointer per parsed file, O(files), no depth term) and the contract test's count gate is what holds it to one build per pass. PHP's leg is measured with NO composer.json, so namespaceDirectories only ever returns the directory of an already-resolved file and the PSR-4 mapping branch stays unreached, exactly as csharp cannot reach the csproj leg; closing that is a second PHP arm on the csharp_csproj precedent, not a parameter. And the const tail of PHP's leg is a different ANSWER at the same cost \u2014 it runs the identical candidate gather and localDefs filter and diverges in the last two lines \u2014 so it is gated by count in test/unit/scope-resolution/import-target-index-reuse.contract.test.ts, which stays the gate to read alongside this file.", + "_blind_spot": "MEASURED, so nobody has to rediscover it: a full workspace scan reintroduced on 1-in-32 imports passes EVERY arm here \u2014 dart scored 1.458 scaling and 1.736 ms against the 1.8 budget and 4 ms ceiling of an earlier revision. At 1-in-8 the scaling arm catches it (2.414). The gate that NARROWS this is not a timing gate at all: test/unit/scope-resolution/import-target-index-parity.test.ts counts iterations of the file-set Set and reads 14 instead of 1 for that same 1-in-32 mutation, deterministically and for all five languages. It does NOT close it. The counter watches the Set, and the resolvers no longer read the Set \u2014 they read materialized copies of the same file list: WorkspaceFileIndex.normalized and .all (C#, Ruby), Dart's byBasename buckets, and PackageDirIndex.filesByDir (Go, C#). A 1-in-32 scan over any of those three touches the Set zero extra times, so it passes the parity test AND passes --check. Closing it would take an iteration counter on the materialized arrays themselves. Read the two gates together; tightening these ceilings toward the noise floor to chase that case would only buy flaky CI. CONFIRMED THE HARD WAY by PR #2911: JavaScript resolution was scanning ImportPassCache.normalizedFileList on every import \u2014 a materialized array, not the Set \u2014 at 25972 us per import at 8000 files, and no instrument on the #2901-#2909 branch could see it. It took a differential parity test over 211200 old-vs-new pairs to find. The arms added here would have caught THAT one on absolute ms (85 ms budget against a 20 ms arm; the unindexed resolver costs ~83000 ms on the same corpus), which is the argument for gating every registered language rather than only the ones a PR happens to touch. THE SECOND BLIND SPOT IS CLOSED, and this records what closing it changed. This harness used to call the inner resolvers with the NO-CONTEXT shape: run.ts calls provider.resolveImportTarget with five arguments, the fifth being { parsedFiles, parsedImport }, and resolveOne supplied three. resolveOne now makes the production call, newPass mints the ParsedFile[] FIRST and derives the path set from it exactly as run.ts does, and both legs behind the argument run on every import of their arms \u2014 PHP's named/alias function-or-const leg over filesByDirectory(context.parsedFiles), whose memo defeated measures 197.0 us -> 9976.2 us per import (50.6x), and Python's from-import submodule-precedence branch, the only spelling that reads context.parsedFiles at all. Fifteen of the seventeen arms cannot observe a context (their hooks declare three or four parameters) and are handed none, so their numbers did not move; which two CAN is now reconciled against SCOPE_RESOLVERS' hook arity rather than asserted in prose. NOTHING ELSE IN THIS FILE COULD HAVE GATED IT, which is why the context arm exists: fingerprints and shape can remain unchanged while dropping context only makes timing faster. The deterministic context arm is therefore the guard for this wiring. The arm is one import per language resolved through resolveOne twice, with and without the pass's parsedFiles, whose two answers must DIFFER and must both match what is recorded. WHAT REMAINS UNMEASURED, narrowed rather than deleted: Python's parsedFileByPath memo is exercised by the five timing arms and cannot be reached by the heap arm at all, because retainedPassBytes requires a probe that MISSES while every path that builds that memo returns a non-null packageTarget \u2014 so no ceiling bounds that Map (one pointer per parsed file, O(files), no depth term) and the contract test's count gate is what holds it to one build per pass. PHP's sole arm carries a representative Composer PSR-4 map, so mapped hits and authoritative misses exercise that production branch directly. And the const tail of PHP's leg is a different ANSWER at the same cost \u2014 it runs the identical candidate gather and localDefs filter and diverges in the last two lines \u2014 so it is gated by count in test/unit/scope-resolution/import-target-index-reuse.contract.test.ts, which stays the gate to read alongside this file.", "_depth_budget_note_2953": "javascript/typescript/vue moved from 2.0-2.1 to ~2.2 in #2953 and their budgets were raised to 2.6, which is a real shift with an understood cause rather than a loosened guard. Declared resolution never walks path components, so the deep arm's uniform d0/../d15/ prefix reaches these resolvers as the tsconfig baseUrl (see tsBaseUrlFor in measure.mjs) and every candidate string carries it: resolveFile probes ~11 extensions plus their /index forms, and hashing a 60-character path costs more than hashing a 12-character one. The growth is linear in path LENGTH and independent of file COUNT, which is what the ratio exists to bound - a resolver that started walking the corpus again would move scaling_ratio, not just this. Measured over three runs on a loaded box: js 2.109/2.257/2.240, ts 2.129/2.222/2.467, vue 2.116/2.151/2.102.", "_heap_bound_note_2953": "javascript, typescript and vue moved from heap_reading_bytes/heap_ceiling_bytes to heap_bound_bytes in #2953. They retained 26745296 B (js, ts) and 28884016 B (vue) at 32000 files for a per-pass SuffixIndex over the whole file list; they now build no per-pass structure at all and read 0-16 B, because declared resolution derives nothing from the file set. That is a real saving rather than an arm that stopped measuring - the distinction this floor exists to make - and the evidence it is real is that the resolver fingerprints did NOT move: the same corpus resolves to the same targets, once the config it always implied is passed explicitly. The 1048576 B bound is rust's, chosen the same way: far above a 16 B reading, far below the index whose return it must catch." } diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index eb6b72bcf..c7ea7cc0e 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -399,12 +399,9 @@ * per parsed file, O(files) with no depth term, and the count gate in * import-target-index-reuse.contract.test.ts is what holds it to one build * per pass; - * - PHP's leg is measured with NO composer.json — `resolutionConfig` is - * undefined here, as it always has been — so `namespaceDirectories` only - * ever returns the directory of an already-resolved file and the PSR-4 - * mapping branch stays unreached, exactly as `csharp` cannot reach the - * csproj leg. Closing that is a second PHP arm on the `csharp_csproj` - * precedent, not a parameter; + * - PHP runs with the Composer PSR-4 configuration every production project + * supplies. Configured hits and unmatched dependency misses share one + * workload, so the Composer gate cannot become an unmeasured fast path; * - the `const` tail of PHP's leg (`candidateFiles.length === 1`) is a * different ANSWER, not a different cost: `function` runs the identical * candidate gather and `localDefs` filter and diverges only in the last two @@ -550,13 +547,10 @@ const HEAP_LARGE = 32000; const HEAP_PAD = 8; /** The languages whose retained per-pass index carries a BUDGET — a ceiling, a * floor derived from `heap_reading_bytes`, and the linear-growth ratio arm. - * All eight are measured the same way as the other nine (`retainedPassBytes`, - * one real import through the real resolver); what this list decides is which - * GATE a reading gets, not whether it is taken. The first five reach the shared - * `WorkspaceFileIndex` and retained NOTHING at BASE; `csharp_csproj` is the - * same corpus through the same index under the csproj context, and it is here - * rather than excluded as a duplicate because after #2903 its READ PATTERN, - * not its corpus, decides the number. + * All arms are measured through `retainedPassBytes`, one real import through + * the real resolver; this list decides which GATE a reading gets, not whether + * it is taken. The configured C# arm stays here because its read pattern + * reaches retained structures that the unconfigured arm cannot observe. * * The remaining three are `HEAP_BOUNDED`, DERIVED from this list rather than * written beside it, and they carry an upper bound and NO floor. That asymmetry @@ -565,7 +559,7 @@ const HEAP_PAD = 8; * would gate the noise. rust reads 16 B at both scales; swift's ratio is 0.888 * and cobol's 1.082, both outside the linearity every budgeted arm shows, so a * floor and a ratio arm would be measuring the measurement. See the MEMORY - * section of the header for what re-measuring all seventeen found. */ + * section of the header for what re-measuring the full inventory found. */ const HEAP_BUDGETED = [ 'csharp', 'csharp_csproj', @@ -601,7 +595,7 @@ const HEAP_BUDGETED = [ /** * The arms handed the fifth `context` argument — `{ parsedFiles, parsedImport }` - * — because their registered hook DECLARES it. Four of seventeen, and the + * — because their registered hook DECLARES it. Four of seventeen arms, and the * inventory arm at the foot of this file reconciles that claim against * `SCOPE_RESOLVERS` in both directions rather than trusting this line. * @@ -714,6 +708,15 @@ const joinBase = (baseUrl, rest) => (baseUrl === '' ? rest : `${baseUrl}/${rest} */ const tsBaseUrlFor = (pad) => pad === 0 ? '' : Array.from({ length: pad }, (_, n) => `d${n}`).join('/'); +const phpComposerConfigFor = (pad) => ({ + psr4: new Map([['App', joinBase(tsBaseUrlFor(pad), 'src/App')]]), + authoritativePsr4: new Set(['App']), +}); +const renderPhpComposerConfig = (config) => + [...config.psr4] + .map(([namespace, directory]) => `${namespace || ''}=${directory || ''}`) + .sort() + .join(';'); /** Keyed by LAYOUT name, so there is no `csharp_csproj` row: `buildFiles` * aliases that arm to `csharp` before this table is read. */ const EXTENSION = { @@ -919,7 +922,7 @@ function collideDir(lang, d, i) { `mod${d}/src/main/kotlin/com/example/models/inner/com/example/models` : `mod${d}/src/main/kotlin/com/example/models`; } - if (lang === 'php') return `svc${d}/src/Models`; + if (lang === 'php') return `src/App/Svc${d}/Models`; if (lang === 'java') { return d % 7 === 0 ? `svc${d}/src/main/java/com/example/model/inner/model` @@ -1035,6 +1038,12 @@ function buildFiles(lang, fileCount, pad, shape) { : ext; files.push(`${prefix}${dir}/${stem}${suffix}`); } + // One real suffix decoy makes the PHP external gate observable: with the + // gate, Vendor0 stays unresolved; without it, suffix fallback resolves this + // path and the exact fingerprint/external-probe result changes. + if (layout === 'php' && files.length > 0) { + files[files.length - 1] = `${prefix}legacy/Vendor0/Ghost/Missing.php`; + } return files; } @@ -1145,9 +1154,8 @@ function kotlinBenchmarkPackage(filePath) { * The owner segment is the file's own directory name (`Ns7`, `Models`, `pkg7`), * which is stable across the `small`, `deep` and `collide` arms — so the `deep` * arm differs from `small` in path DEPTH alone, exactly as it does for the path - * set. That matters here: `directoryAliases` emits one entry per path segment, - * so `filesByDirectory` is O(files × depth) and the depth arm is the only one - * that can see it. + * set. `filesByDirectory` is exact and linear in the file count; the shared + * suffix index remains the path-depth-sensitive structure this arm measures. */ function buildParsedFiles(lang, files) { const parsedFiles = []; @@ -1247,21 +1255,18 @@ function uniqueTarget(lang, { local, r, d, j, dirs }) { : `com.ghost${(r >>> 4) % 97}.deep.Missing`; } if (lang === 'php') { - // Backslash-separated, the way a `use` statement is actually written; the - // resolver normalizes them. No composer.json is threaded (the adapter's - // `resolutionConfig` is left undefined), so every one of these lands on - // `suffixResolve` — the leg that ran one `findIndex` over every file per - // path part per extension, ~50 of them, and measured 96.40 ms per import at - // 20k files before #2901. - return local - ? `App\\Ns${d}\\File${j}` - : (r >>> 3) % 2 === 0 - ? [ - 'Psr\\Log\\LoggerInterface', - 'Symfony\\Component\\Console\\Command', - 'Doctrine\\ORM\\EntityManager', - ][(r >>> 4) % 3] - : `Vendor${(r >>> 4) % 97}\\Ghost\\Missing`; + if (local) { + const namespace = d % 7 === 0 ? `Ns${d}\\Sub\\Ns${d}` : `Ns${d}`; + const leadingSeparator = (r >>> 3) % 4 === 0 ? '\\' : ''; + return `${leadingSeparator}App\\${namespace}\\File${j}`; + } + return (r >>> 3) % 2 === 0 + ? [ + 'Psr\\Log\\LoggerInterface', + 'Symfony\\Component\\Console\\Command', + 'Doctrine\\ORM\\EntityManager', + ][(r >>> 4) % 3] + : `Vendor${(r >>> 4) % 97}\\Ghost\\Missing`; } if (lang === 'java') { // Java has NO in-repo-namespace gate (#2910 is filed for it), so a JDK @@ -1451,21 +1456,17 @@ function collideTarget(lang, { local, r, d, j, dirs }) { : `com.ghost${(r >>> 4) % 97}.deep.Missing`; } if (lang === 'php') { - // `Models\Mod{n}` is carried by every service, so the segment-suffix key it - // resolves through holds one entry no matter how many files exist: PHP - // answers from keyed maps and is collision-IMMUNE, which is what this arm - // asserts. The local spelling still always resolves, as it does on the - // unique layout — PHP's cascade strips leading segments, so even the - // nested-same-name slice is reachable by a shorter suffix. - return local - ? `App\\Models\\Mod${Math.floor(j / dirs)}` - : (r >>> 3) % 2 === 0 - ? [ - 'Psr\\Log\\LoggerInterface', - 'Symfony\\Component\\Console\\Command', - 'Doctrine\\ORM\\EntityManager', - ][(r >>> 4) % 3] - : `Vendor${(r >>> 4) % 97}\\Ghost\\Missing`; + if (local) { + const leadingSeparator = (r >>> 3) % 4 === 0 ? '\\' : ''; + return `${leadingSeparator}App\\Svc${j % dirs}\\Models\\Mod${Math.floor(j / dirs)}`; + } + return (r >>> 3) % 2 === 0 + ? [ + 'Psr\\Log\\LoggerInterface', + 'Symfony\\Component\\Console\\Command', + 'Doctrine\\ORM\\EntityManager', + ][(r >>> 4) % 3] + : `Vendor${(r >>> 4) % 97}\\Ghost\\Missing`; } if (lang === 'java') { // Every file declares the same package despite living under different @@ -1607,6 +1608,9 @@ function buildRepo(lang, fileCount, pad = 0, shape = 'unique') { imports.push([from, mintTarget(lang, { local, r, d, j, dirs })]); } } + if (lang === 'php' && imports.length > 0) { + imports[0] = [files[0], 'Vendor0\\Ghost\\Missing']; + } return { files, imports }; } @@ -1667,7 +1671,7 @@ function newPass(lang, files, pad = 0) { restoreBenchmarkSideChannels(lang, parsedFiles); return { allFilePaths: new Set(parsedFiles.map((f) => f.filePath)), - config: undefined, + config: lang === 'php' ? phpComposerConfigFor(pad) : undefined, parsedFiles, }; } @@ -2039,7 +2043,9 @@ const HEAP_PROBE_TARGET = { // (`getFilesInDir`) before answering null — the three-map read pattern. csharp_csproj: 'App.Missing0', ruby: 'gem0/missing/thing', - php: 'Vendor0\\Ghost\\Missing', + // A mapped-but-missing class forces the Composer mapping and suffix-index + // read paths. The separate external probe below keeps the fast gate visible. + php: 'App\\HeapGhost0\\AbsentHeapProbe', java: 'com.google.common.vendor0.Missing', javascript: 'vendor0/lib/missing', python: 'vendor0.deep.missing', @@ -2107,11 +2113,24 @@ function measureHeap(lang) { GC(); GC(); const probe = HEAP_PROBE_TARGET[lang]; - const read = (files) => retainedPassBytes(lang, files, probe); + const read = (files) => retainedPassBytes(lang, files, probe, lang === 'php' ? HEAP_PAD : 0); const small = flatten(buildFiles(lang, HEAP_SMALL, HEAP_PAD, 'unique')); const bytesSmall = read(small); const large = flatten(buildFiles(lang, HEAP_LARGE, HEAP_PAD, 'unique')); const bytesLarge = read(large); + const phpGateShape = + lang === 'php' + ? (() => { + const externalProbe = 'Vendor0\\Ghost\\Missing'; + const config = phpComposerConfigFor(HEAP_PAD); + const pass = newPass(lang, large, HEAP_PAD); + return { + resolution_config: renderPhpComposerConfig(config), + external_probe: externalProbe, + external_result: renderResolved(resolveOne(lang, large[0], externalProbe, pass)), + }; + })() + : {}; return { files_small: HEAP_SMALL, files_large: HEAP_LARGE, @@ -2121,6 +2140,7 @@ function measureHeap(lang) { bytes_large: bytesLarge, mib_large: Number((bytesLarge / 1024 / 1024).toFixed(2)), ratio: Number((bytesLarge / bytesSmall / (HEAP_LARGE / HEAP_SMALL)).toFixed(3)), + ...phpGateShape, }; } @@ -2213,10 +2233,11 @@ const CONTEXT_PROBE = { function measureContext(lang) { const { from, target, parsedFiles } = CONTEXT_PROBE[lang]; const allFilePaths = new Set(parsedFiles.map((f) => f.filePath)); + const config = lang === 'php' ? phpComposerConfigFor(0) : undefined; const answer = (files) => { restoreBenchmarkSideChannels(lang, files ?? []); return renderResolved( - resolveOne(lang, from, target, { allFilePaths, config: undefined, parsedFiles: files }), + resolveOne(lang, from, target, { allFilePaths, config, parsedFiles: files }), ); }; return { @@ -2249,11 +2270,11 @@ if (CHECK && GC === null) { /** * Every arm, and the registered language each one exercises. * - * This used to be a hand-written list of seventeen strings under a comment + * This used to be a hand-written list of language strings under a comment * claiming it was "every language in `SCOPE_RESOLVERS`" — a claim nothing in * the file could check, because the file never imported the registry. Adding a * resolver to `pipeline/registry.ts` is two lines, neither of which is this - * one, so a seventeenth registered language would have shipped ungated and + * one, so a newly registered language would have shipped ungated and * printed PASS. That is not a hypothetical failure mode: JavaScript reached * `suffixResolve` with no index at all and measured 25 972 µs per import at * 8000 files (PR #2911) for exactly as long as nothing gated it. @@ -2265,10 +2286,9 @@ if (CHECK && GC === null) { * uses ten files away, and the same "one row per language" table * `bench/cfg/measure.mjs` keeps. * - * The mapping is many-to-one on purpose: `csharp` and `csharp_csproj` are two - * arms over one registered resolver, differing only in whether `csharpConfigs` - * is supplied, because the no-csproj arm returns before it can reach the leg - * #2902 indexed. + * The mapping is many-to-one only for C#: the configured arm reaches the + * csproj branch that the default arm cannot observe. PHP's sole arm carries + * its production Composer configuration directly. */ const LANG_REGISTRY = { go: SupportedLanguages.Go, @@ -2457,7 +2477,7 @@ const SCALE_SHAPE = { 'one of them alone moves nothing in the others.', }; /** The same, for the heap arm — the four inputs that decide what it measures. - * Asserted for all seventeen, budgeted tier and bounded tier alike, and it is + * Asserted for all seventeen arms, budgeted tier and bounded tier alike, and it is * the bounded tier that needs it most: a bound is a single comparison, so a * probe swapped for one that reaches less is a bound over a smaller workload * and there is no floor beside it to notice. @@ -2474,6 +2494,13 @@ const HEAP_SHAPE = { 'ceiling, floor, bound and ratio passing over an arm that changed workload. Deterministic: ' + 'a re-run will not change it.', }; +const PHP_HEAP_SHAPE = { + fields: [...HEAP_SHAPE.fields, 'resolution_config', 'external_probe', 'external_result'], + why: + HEAP_SHAPE.why + + ' PHP also pins the Composer mapping and a suffix-matchable external decoy so the mapped ' + + 'index path and the external fast gate remain separate observable arms.', +}; /** The same, for the `context` arm. All three fields are exact strings, not * bounds: this arm has no measurement noise at all — it resolves one import * two ways over a three-file corpus — so anything less than equality would be @@ -2496,7 +2523,7 @@ const CONTEXT_SHAPE = { * a fifth parameter. */ const armShapes = (lang) => [ ...SCALES.map((scale) => [scale, SCALE_SHAPE]), - ['heap', HEAP_SHAPE], + ['heap', lang === 'php' ? PHP_HEAP_SHAPE : HEAP_SHAPE], ...(CONTEXT_LANGS.includes(lang) ? [['context', CONTEXT_SHAPE]] : []), ]; diff --git a/gitnexus/bench/java-lombok-synthesis/baselines.json b/gitnexus/bench/java-lombok-synthesis/baselines.json new file mode 100644 index 000000000..9e14f8f3c --- /dev/null +++ b/gitnexus/bench/java-lombok-synthesis/baselines.json @@ -0,0 +1,8 @@ +{ + "_comment": "Baselines for bench/java-lombok-synthesis/measure.mjs --check (#2885). fingerprint is sha256 over synthetic Method node ids on the lombok_large corpus (800 @Data entities × 4 fields × 2 accessors = 6400 methods). no_lombok arm must emit 0 methods. Budgets are timing gates with CI headroom.", + "fingerprint": "b935d6894d32de7594d5887bb62af6ade2b66b19d6846700a05ef3baf1ed1eb1", + "scaling_budget": 1.6, + "_scaling_note": "(t_large/t_small)/(800/250) on the lombok arm. Measured ~1.01.", + "widening_overhead_budget": 2.5, + "_widening_overhead_note": "lombok_large_ms / no_lombok_large_ms using an unannotated, shape-equivalent four-field control. Measured about 1.24; budget guards against a pathological feature-arm regression." +} diff --git a/gitnexus/bench/java-lombok-synthesis/measure.mjs b/gitnexus/bench/java-lombok-synthesis/measure.mjs new file mode 100644 index 000000000..512af1e12 --- /dev/null +++ b/gitnexus/bench/java-lombok-synthesis/measure.mjs @@ -0,0 +1,124 @@ +/** + * Build-free throughput + identity bench for Java Lombok accessor synthesis. + * + * Arms: + * - no_lombok: unannotated fields (shape-equivalent control) — synthesizer no-ops + * - lombok_heavy: @Data classes (feature path) + * + * Times synthesizeLombokAccessors over N separate files (not one giant buffer). + * + * Usage: + * node --import tsx bench/java-lombok-synthesis/measure.mjs + * node --import tsx bench/java-lombok-synthesis/measure.mjs --check + */ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { synthesizeLombokAccessors } from '../../src/core/ingestion/languages/java/lombok-synthesizer.ts'; +import { + fingerprintIds, + minSample, + runBaselineCheck, + runMethodCountCheck, +} from '../lib/identity-guard.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const SMALL = 250; +const LARGE = 800; +const REPS = 15; +const WARMUP = 5; + +function entitySource(i, mode) { + if (mode === 'lombok') { + return `import lombok.Data; +@Data +public class Entity${i} { + private String id; + private String name; + private boolean active; + private Long amount; +} +`; + } + return `public class Entity${i} { + private String id; + private String name; + private boolean active; + private Long amount; +} +`; +} + +function ownerMap(tree, filePath) { + const map = new Map(); + const walk = (node) => { + if (node.type === 'class_declaration') { + const name = node.childForFieldName('name')?.text; + if (name) map.set(node.id, `Class:${filePath}:${name}`); + } + for (const c of node.children) walk(c); + }; + walk(tree.rootNode); + return map; +} + +function prepare(mode, fileCount) { + const files = []; + for (let i = 0; i < fileCount; i++) { + const parser = new Parser(); + parser.setLanguage(Java); + const filePath = `bench/${mode}/Entity${i}.java`; + const tree = parser.parse(entitySource(i, mode)); + files.push({ tree, filePath, owners: ownerMap(tree, filePath) }); + } + return files; +} + +function runAll(files) { + const nodes = []; + for (const f of files) { + const result = synthesizeLombokAccessors(f.tree, f.filePath, f.owners); + for (const n of result.nodes) nodes.push(n.id); + } + return nodes; +} + +function measure(mode, fileCount) { + const files = prepare(mode, fileCount); + const { last, ms } = minSample(() => runAll(files), WARMUP, REPS); + return { + files: fileCount, + ms, + methods: last.length, + fingerprint: fingerprintIds(last), + }; +} + +const report = { + no_lombok_small: measure('bare', SMALL), + no_lombok_large: measure('bare', LARGE), + lombok_small: measure('lombok', SMALL), + lombok_large: measure('lombok', LARGE), +}; +report.scaling_ratio = Number( + (report.lombok_large.ms / report.lombok_small.ms / (LARGE / SMALL)).toFixed(3), +); +report.widening_overhead = Number( + (report.lombok_large.ms / Math.max(report.no_lombok_large.ms, 0.001)).toFixed(3), +); +report.fingerprint = report.lombok_large.fingerprint; + +runMethodCountCheck(report, { + no_lombok_large: 0, + lombok_large: 6400, +}); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +runBaselineCheck(report, BASELINE_PATH); diff --git a/gitnexus/bench/java-wildcard-route-constants/baselines.json b/gitnexus/bench/java-wildcard-route-constants/baselines.json new file mode 100644 index 000000000..961d9a883 --- /dev/null +++ b/gitnexus/bench/java-wildcard-route-constants/baselines.json @@ -0,0 +1,8 @@ +{ + "_comment": "Baselines for bench/java-wildcard-route-constants/measure.mjs --check (#3110). fingerprint is sha256 over 800 materialized route bindings from 800 constant files and must match the named-import control. The benchmark builds the constant import index once per repo pass, matching ingestion and group wiring.", + "fingerprint": "8114e613e93ce0ef6220b810850888592e0e822fe04bf2eb5d8fc4ec3dbba5ef", + "scaling_budget": 1.6, + "_scaling_note": "(t_large/t_small)/(800/250) while both constant files and wildcard importers scale. Measured about 1.14 with the suffix index; repeated candidate scans are quadratic.", + "absolute_ms_budget": 10, + "_absolute_ms_note": "Wildcard materialization for 800 controllers. Measured about 1.1 ms; the generous ceiling catches gross regressions without treating the near-zero named-import control as a stable ratio denominator." +} diff --git a/gitnexus/bench/java-wildcard-route-constants/measure.mjs b/gitnexus/bench/java-wildcard-route-constants/measure.mjs new file mode 100644 index 000000000..50eda340c --- /dev/null +++ b/gitnexus/bench/java-wildcard-route-constants/measure.mjs @@ -0,0 +1,158 @@ +/** + * Build-free throughput + identity benchmark for Java wildcard-static route constants. + * + * Arms: + * - named: explicit `import static ...ApiPaths.ROUTE_n` control + * - wildcard: `import static ...ApiPaths.*` feature path + * + * Parsing is prepared outside the timer. The measured path mirrors ingestion: + * build the constant-key index once, materialize pending wildcard imports, then + * read the resulting binding. Route folding itself has separate integration + * coverage and an older per-fold index cost shared by both arms. + * + * Usage: + * node --import tsx bench/java-wildcard-route-constants/measure.mjs + * node --import tsx bench/java-wildcard-route-constants/measure.mjs --check + */ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { + extractJavaModuleConstants, + prepareJavaRouteConstants, +} from '../../src/core/ingestion/route-extractors/java-const-resolver.ts'; +import { + fingerprintIds, + minSampleFresh, + runBaselineCheck, + runCountCheck, + runFingerprintParityCheck, +} from '../lib/route-constant-guard.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); +const SMALL = 250; +const LARGE = 800; +const REPS = 15; +const WARMUP = 5; + +const parser = new Parser(); +parser.setLanguage(Java); + +function constantsSource(i) { + return `package bench.constants; +public final class ApiPaths${i} { + public static final String ROUTE = "/api/routes/${i}"; +} +`; +} + +function controllerSource(i, mode) { + const fqn = `bench.constants.ApiPaths${i}`; + const imported = mode === 'wildcard' ? `import static ${fqn}.*;` : `import static ${fqn}.ROUTE;`; + return `package bench.web; +${imported} +class Controller${i} {} +`; +} + +function cloneConstants(mc) { + return { + literals: new Map(mc.literals), + exprs: new Map(mc.exprs), + imports: new Map(mc.imports), + wildcardImports: mc.wildcardImports ? [...mc.wildcardImports] : undefined, + unfoldableDeclarations: new Set(mc.unfoldableDeclarations ?? []), + }; +} + +function prepare(mode, fileCount) { + const constants = []; + const controllers = []; + for (let i = 0; i < fileCount; i++) { + constants.push({ + key: `bench/constants/ApiPaths${i}.java`, + constants: extractJavaModuleConstants(parser.parse(constantsSource(i))), + }); + controllers.push({ + key: `bench/web/Controller${i}.java`, + route: 'ROUTE', + constants: extractJavaModuleConstants(parser.parse(controllerSource(i, mode))), + }); + } + return { constants, controllers }; +} + +function instantiate(prepared) { + const repo = new Map(); + for (const constant of prepared.constants) { + repo.set(constant.key, cloneConstants(constant.constants)); + } + const controllers = []; + for (const controller of prepared.controllers) { + repo.set(controller.key, cloneConstants(controller.constants)); + controllers.push({ key: controller.key, route: controller.route }); + } + return { repo, controllers }; +} + +function runAll(instance) { + const { repo, controllers } = instance; + prepareJavaRouteConstants(repo); + const bindings = []; + for (const controller of controllers) { + const mc = repo.get(controller.key); + const binding = mc.imports.get(controller.route); + if (binding) { + bindings.push( + `${controller.key}:${controller.route}:${binding.module}:${binding.originalName}`, + ); + } + } + return bindings; +} + +function measure(mode, fileCount) { + const prepared = prepare(mode, fileCount); + // Expansion mutates each importing file's `imports` map. Give every timed + // sample a fresh repo, but build those clones outside the timer. + const { last, ms } = minSampleFresh(() => instantiate(prepared), runAll, WARMUP, REPS); + return { + files: fileCount, + ms, + bindings: last.length, + fingerprint: fingerprintIds(last), + }; +} + +const report = { + named_small: measure('named', SMALL), + named_large: measure('named', LARGE), + wildcard_small: measure('wildcard', SMALL), + wildcard_large: measure('wildcard', LARGE), +}; +report.scaling_ratio = Number( + (report.wildcard_large.ms / report.wildcard_small.ms / (LARGE / SMALL)).toFixed(3), +); +report.overhead_us_per_binding = Number( + ( + ((report.wildcard_large.ms - report.named_large.ms) * 1000) / + report.wildcard_large.bindings + ).toFixed(3), +); +report.absolute_ms = report.wildcard_large.ms; +report.fingerprint = report.wildcard_large.fingerprint; + +runCountCheck(report, 'bindings', { + named_large: LARGE, + wildcard_large: LARGE, +}); +runFingerprintParityCheck(report, 'named_large', 'wildcard_large'); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +runBaselineCheck(report, BASELINE_PATH); diff --git a/gitnexus/bench/kotlin-jvm-accessors/baselines.json b/gitnexus/bench/kotlin-jvm-accessors/baselines.json new file mode 100644 index 000000000..6ef877719 --- /dev/null +++ b/gitnexus/bench/kotlin-jvm-accessors/baselines.json @@ -0,0 +1,8 @@ +{ + "_comment": "Baselines for bench/kotlin-jvm-accessors/measure.mjs --check (#2885). fingerprint is sha256 over synthetic Method node ids on the data_large corpus (800 data classes × 4 vars × 2 accessors = 6400 methods). no_props arm uses @JvmField so kotlinc and the synthesizer emit 0 accessor methods. Budgets are timing gates with CI headroom.", + "fingerprint": "18e4f295a437a747c486699e8ec5d310d9bde54437d9a96356a1b1bf8442b0ef", + "scaling_budget": 1.6, + "_scaling_note": "(t_large/t_small)/(800/250) on the data-class arm. Measured ~1.02.", + "widening_overhead_budget": 2.5, + "_widening_overhead_note": "data_large_ms / no_props_large_ms. The @JvmField control preserves four property declarations without accessors; budget guards against a pathological synthesis-arm regression." +} diff --git a/gitnexus/bench/kotlin-jvm-accessors/measure.mjs b/gitnexus/bench/kotlin-jvm-accessors/measure.mjs new file mode 100644 index 000000000..0edbfb1e4 --- /dev/null +++ b/gitnexus/bench/kotlin-jvm-accessors/measure.mjs @@ -0,0 +1,121 @@ +/** + * Build-free throughput + identity bench for Kotlin JVM accessor synthesis. + * + * Arms: + * - no_props: @JvmField properties with no JVM accessors (control) + * - data_class: data class constructor properties (feature path) + * + * Usage: + * node --import tsx bench/kotlin-jvm-accessors/measure.mjs + * node --import tsx bench/kotlin-jvm-accessors/measure.mjs --check + */ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Parser from 'tree-sitter'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { getLanguageGrammar } from '../../src/core/tree-sitter/parser-loader.ts'; +import { synthesizeLombokAccessors } from '../../src/core/ingestion/languages/kotlin/lombok-synthesizer.ts'; +import { + fingerprintIds, + minSample, + runBaselineCheck, + runMethodCountCheck, +} from '../lib/identity-guard.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const SMALL = 250; +const LARGE = 800; +const REPS = 15; +const WARMUP = 5; + +function entitySource(i, mode) { + if (mode === 'data') { + return `data class Entity${i}(var id: String, var name: String, var active: Boolean, var amount: Long) +`; + } + // @JvmField suppresses accessors in kotlinc and in the synthesizer while + // retaining the same four property declarations as the feature arm. + return `class Entity${i} { + @JvmField var id: String = "" + @JvmField var name: String = "" + @JvmField var active: Boolean = false + @JvmField var amount: Long = 0 +} +`; +} + +function ownerMap(tree, filePath) { + const map = new Map(); + const walk = (node) => { + if (node.type === 'class_declaration' || node.type === 'object_declaration') { + const name = + node.childForFieldName('name')?.text ?? + node.namedChildren.find((c) => c.type === 'type_identifier')?.text; + if (name) map.set(node.id, `Class:${filePath}:${name}`); + } + for (const c of node.children) walk(c); + }; + walk(tree.rootNode); + return map; +} + +function prepare(mode, fileCount) { + const files = []; + const lang = getLanguageGrammar(SupportedLanguages.Kotlin); + for (let i = 0; i < fileCount; i++) { + const parser = new Parser(); + parser.setLanguage(lang); + const filePath = `bench/${mode}/Entity${i}.kt`; + const tree = parser.parse(entitySource(i, mode)); + files.push({ tree, filePath, owners: ownerMap(tree, filePath), parser }); + } + return files; +} + +function runAll(files) { + const nodes = []; + for (const f of files) { + const result = synthesizeLombokAccessors(f.tree, f.filePath, f.owners); + for (const n of result.nodes) nodes.push(n.id); + } + return nodes; +} + +function measure(mode, fileCount) { + const files = prepare(mode, fileCount); + const { last, ms } = minSample(() => runAll(files), WARMUP, REPS); + return { + files: fileCount, + ms, + methods: last.length, + fingerprint: fingerprintIds(last), + }; +} + +const report = { + no_props_small: measure('hand', SMALL), + no_props_large: measure('hand', LARGE), + data_small: measure('data', SMALL), + data_large: measure('data', LARGE), +}; +report.scaling_ratio = Number( + (report.data_large.ms / report.data_small.ms / (LARGE / SMALL)).toFixed(3), +); +report.widening_overhead = Number( + (report.data_large.ms / Math.max(report.no_props_large.ms, 0.001)).toFixed(3), +); +report.fingerprint = report.data_large.fingerprint; + +runMethodCountCheck(report, { + no_props_large: 0, + data_large: 6400, +}); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +runBaselineCheck(report, BASELINE_PATH); diff --git a/gitnexus/bench/kotlin-star-route-constants/baselines.json b/gitnexus/bench/kotlin-star-route-constants/baselines.json new file mode 100644 index 000000000..a68d3ba9f --- /dev/null +++ b/gitnexus/bench/kotlin-star-route-constants/baselines.json @@ -0,0 +1,10 @@ +{ + "_comment": "Baselines for bench/kotlin-star-route-constants/measure.mjs --check (#3110). fingerprint is sha256 over 800 folded route facts from 800 constant files and must match the explicit-import control. The feature arm resolves package-star names through one prepared KotlinConstantIndex.", + "fingerprint": "881101236c511d73d3894d3c9bd2e4a166e3437329149dd99fd16c256435482e", + "scaling_budget": 1.6, + "_scaling_note": "(t_large/t_small)/(800/250) while both constant files and importing controllers scale. Measured about 1.07-1.14.", + "widening_overhead_budget": 2.5, + "_widening_overhead_note": "star_large_ms / named_large_ms. Measured below 1.0; budget guards against a pathological star-lookup regression.", + "absolute_ms_budget": 5, + "_absolute_ms_note": "Package-star folding for 800 controllers. Measured below 0.6 ms; budget includes substantial CI headroom." +} diff --git a/gitnexus/bench/kotlin-star-route-constants/measure.mjs b/gitnexus/bench/kotlin-star-route-constants/measure.mjs new file mode 100644 index 000000000..299accdb8 --- /dev/null +++ b/gitnexus/bench/kotlin-star-route-constants/measure.mjs @@ -0,0 +1,156 @@ +/** + * Build-free throughput + identity benchmark for Kotlin package-star route constants. + * + * Arms: + * - named: explicit `import bench.constants.ROUTE_n` control + * - star: `import bench.constants.*` feature path + * + * Parsing is prepared outside the timer. The measured path mirrors the Kotlin + * group plugin: overlay one importing controller on the prepared constant + * index, then fold its route. + * + * Usage: + * node --import tsx bench/kotlin-star-route-constants/measure.mjs + * node --import tsx bench/kotlin-star-route-constants/measure.mjs --check + */ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Parser from 'tree-sitter'; +import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.ts'; +import { + buildKotlinConstantIndex, + extractKotlinModuleConstants, + foldKotlinOperands, + overlayKotlinConstantIndex, +} from '../../src/core/ingestion/route-extractors/kotlin-const-resolver.ts'; +import { + fingerprintIds, + minSampleFresh, + runBaselineCheck, + runCountCheck, + runFingerprintParityCheck, +} from '../lib/route-constant-guard.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); +const SMALL = 250; +const LARGE = 800; +const REPS = 15; +const WARMUP = 5; + +const parser = new Parser(); +parser.setLanguage(requireVendoredGrammar('tree-sitter-kotlin')); + +function constantsSource(i) { + return `package bench.constants +const val ROUTE_${i} = "/api/routes/${i}" +`; +} + +function controllerSource(i, mode) { + const route = `ROUTE_${i}`; + const imported = mode === 'star' ? 'import bench.constants.*' : `import bench.constants.${route}`; + return `package bench.web +${imported} +class Controller${i} +`; +} + +function cloneConstants(mc) { + return { + literals: new Map(mc.literals), + exprs: new Map(mc.exprs), + imports: new Map(mc.imports), + wildcardImports: mc.wildcardImports ? [...mc.wildcardImports] : undefined, + packageName: mc.packageName, + unfoldableDeclarations: new Set(mc.unfoldableDeclarations), + topLevelDeclarations: new Set(mc.topLevelDeclarations), + }; +} + +function prepare(mode, fileCount) { + const constants = []; + const controllers = []; + for (let i = 0; i < fileCount; i++) { + constants.push({ + key: `bench/constants/ApiPaths${i}.kt`, + constants: extractKotlinModuleConstants(parser.parse(constantsSource(i))), + }); + controllers.push({ + key: `bench/web/Controller${i}.kt`, + route: `ROUTE_${i}`, + constants: extractKotlinModuleConstants(parser.parse(controllerSource(i, mode))), + }); + } + return { constants, controllers }; +} + +function instantiate(prepared) { + const baseRepo = new Map(); + for (const constant of prepared.constants) { + baseRepo.set(constant.key, cloneConstants(constant.constants)); + } + const controllers = prepared.controllers.map((controller) => ({ + key: controller.key, + route: controller.route, + constants: cloneConstants(controller.constants), + })); + return { baseRepo, controllers }; +} + +function runAll(instance) { + const { baseRepo, controllers } = instance; + const baseIndex = buildKotlinConstantIndex(baseRepo); + const routes = []; + for (const controller of controllers) { + const index = overlayKotlinConstantIndex(baseIndex, controller.key, controller.constants); + const route = foldKotlinOperands( + controller.key, + [{ kind: 'ref', name: controller.route }], + index.repo, + [], + index, + ); + if (route !== null) routes.push(`${controller.key}:${route}`); + } + return routes; +} + +function measure(mode, fileCount) { + const prepared = prepare(mode, fileCount); + const { last, ms } = minSampleFresh(() => instantiate(prepared), runAll, WARMUP, REPS); + return { + files: fileCount, + ms, + routes: last.length, + fingerprint: fingerprintIds(last), + }; +} + +const report = { + named_small: measure('named', SMALL), + named_large: measure('named', LARGE), + star_small: measure('star', SMALL), + star_large: measure('star', LARGE), +}; +report.scaling_ratio = Number( + (report.star_large.ms / report.star_small.ms / (LARGE / SMALL)).toFixed(3), +); +report.widening_overhead = Number( + (report.star_large.ms / Math.max(report.named_large.ms, 0.001)).toFixed(3), +); +report.absolute_ms = report.star_large.ms; +report.fingerprint = report.star_large.fingerprint; + +runCountCheck(report, 'routes', { + named_large: LARGE, + star_large: LARGE, +}); +runFingerprintParityCheck(report, 'named_large', 'star_large'); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +runBaselineCheck(report, BASELINE_PATH); diff --git a/gitnexus/bench/lib/identity-guard.mjs b/gitnexus/bench/lib/identity-guard.mjs new file mode 100644 index 000000000..b73a1a241 --- /dev/null +++ b/gitnexus/bench/lib/identity-guard.mjs @@ -0,0 +1,62 @@ +/** + * Shared fingerprint + --check for JVM accessor synthesis benches. + */ +import fs from 'node:fs'; +import crypto from 'node:crypto'; + +export function fingerprintIds(ids) { + return crypto + .createHash('sha256') + .update([...ids].sort().join('\n')) + .digest('hex'); +} + +export function minSample(run, warmup, reps) { + for (let w = 0; w < warmup; w++) run(); + const samples = []; + let last; + for (let r = 0; r < reps; r++) { + const t0 = performance.now(); + last = run(); + samples.push(performance.now() - t0); + } + return { last, ms: Math.min(...samples) }; +} + +export function runMethodCountCheck(report, expectedCounts) { + const errors = []; + for (const [arm, expected] of Object.entries(expectedCounts)) { + const actual = report[arm]?.methods; + if (actual !== expected) { + errors.push(`${arm}.methods ${String(actual)} != ${expected}`); + } + } + if (errors.length) { + console.error(JSON.stringify({ report, errors }, null, 2)); + process.exit(1); + } +} + +export function runBaselineCheck(report, baselinePath) { + const baseline = JSON.parse(fs.readFileSync(baselinePath, 'utf-8')); + const errors = []; + if (report.fingerprint !== baseline.fingerprint) { + errors.push(`fingerprint drift: ${report.fingerprint} != ${baseline.fingerprint}`); + } + if (report.scaling_ratio > baseline.scaling_budget) { + errors.push(`scaling_ratio ${report.scaling_ratio} > ${baseline.scaling_budget}`); + } + if ( + baseline.widening_overhead_budget !== undefined && + report.widening_overhead > baseline.widening_overhead_budget + ) { + errors.push( + `widening_overhead ${report.widening_overhead} > ${baseline.widening_overhead_budget}`, + ); + } + if (errors.length) { + console.error(JSON.stringify({ report, errors }, null, 2)); + process.exit(1); + } + console.log(JSON.stringify({ ok: true, report }, null, 2)); +} diff --git a/gitnexus/bench/lib/route-constant-guard.mjs b/gitnexus/bench/lib/route-constant-guard.mjs new file mode 100644 index 000000000..cae8b95fc --- /dev/null +++ b/gitnexus/bench/lib/route-constant-guard.mjs @@ -0,0 +1,77 @@ +/** Shared fingerprint + --check helpers for route-constant benchmarks. */ +import fs from 'node:fs'; +import crypto from 'node:crypto'; + +export function fingerprintIds(ids) { + return crypto + .createHash('sha256') + .update([...ids].sort().join('\n')) + .digest('hex'); +} + +/** Min sample for mutating benchmarks that need fresh state per repetition. */ +export function minSampleFresh(create, run, warmup, reps) { + for (let w = 0; w < warmup; w++) run(create()); + const samples = []; + let last; + for (let r = 0; r < reps; r++) { + const state = create(); + const t0 = performance.now(); + last = run(state); + samples.push(performance.now() - t0); + } + return { last, ms: Math.min(...samples) }; +} + +export function runCountCheck(report, field, expectedCounts) { + const errors = []; + for (const [arm, expected] of Object.entries(expectedCounts)) { + const actual = report[arm]?.[field]; + if (actual !== expected) { + errors.push(`${arm}.${field} ${String(actual)} != ${expected}`); + } + } + failIfNeeded(report, errors); +} + +export function runFingerprintParityCheck(report, leftArm, rightArm) { + const left = report[leftArm]?.fingerprint; + const right = report[rightArm]?.fingerprint; + failIfNeeded( + report, + left === right ? [] : [`${leftArm}.fingerprint ${left} != ${rightArm}.fingerprint ${right}`], + ); +} + +export function runBaselineCheck(report, baselinePath) { + const baseline = JSON.parse(fs.readFileSync(baselinePath, 'utf-8')); + const errors = []; + if (report.fingerprint !== baseline.fingerprint) { + errors.push(`fingerprint drift: ${report.fingerprint} != ${baseline.fingerprint}`); + } + if (report.scaling_ratio > baseline.scaling_budget) { + errors.push(`scaling_ratio ${report.scaling_ratio} > ${baseline.scaling_budget}`); + } + if ( + baseline.absolute_ms_budget !== undefined && + report.absolute_ms > baseline.absolute_ms_budget + ) { + errors.push(`absolute_ms ${report.absolute_ms} > ${baseline.absolute_ms_budget}`); + } + if ( + baseline.widening_overhead_budget !== undefined && + report.widening_overhead > baseline.widening_overhead_budget + ) { + errors.push( + `widening_overhead ${report.widening_overhead} > ${baseline.widening_overhead_budget}`, + ); + } + failIfNeeded(report, errors); + console.log(JSON.stringify({ ok: true, report }, null, 2)); +} + +function failIfNeeded(report, errors) { + if (errors.length === 0) return; + console.error(JSON.stringify({ report, errors }, null, 2)); + process.exit(1); +} diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index b5b609ba7..272d081c0 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -212,8 +212,10 @@ "_rebaselined_blind_spots_2856": "#2856 blind-spots series: the JS/TS SCOPE queries gained capture rules, so fingerprint drift is expected and additive. Verified before re-baselining by diffing the capture-name sets in both scope queries against origin/main: TypeScript gained exactly @reference.read.identifier (A2 bare-identifier reads in value positions) and @reference.type (R2-2 type references, so a declared contract stops reporting incoming:{}); JavaScript gained exactly @reference.read.identifier, @reference.read.destructured (R2-1c) and @reference.write.property-key (R2-1b record-construction writes). NOTHING was removed on either side \u2014 the delta is a pure superset, which is the check that no existing capture moved. capture_groups_small/large are unchanged (4503/14403) because those measure the SYNTHETIC scaling source, which this branch does not touch; only the fixture-corpus count moves. capture_groups_fp 2097 -> 2338 and fixture_count 146 -> 151 from 21 new lang-resolution fixtures. Scaling stayed linear and inside budget: typescript 1.116 < 1.5, javascript 1.010 < 1.5. Prior typescript ed92588e0fc7b28b3a0174339ac378b4dd85965fe007db1208dea97a65ce0571 -> f66a3e6f1e096431e7046505129a627deaa00ca0de5bc846b080591b397248f7; prior javascript 806f70ad3cce5fc849f6d06a08ace8a95f92a1ea84a2418fddabb1eef5846594 -> 2026993b81b873839dd2ef8797d9c14d9c48516b2b57b05ac17d8d43f2f4eba3." }, "kotlin": { - "fingerprint": "973d702510002dda76166e017c5eca90cae37a139f5a512877b7a1b04ad19dc5", + "fingerprint": "a9d3f0db7547ff47856159debf15d2a6f427efca97a6af27b4004174ed432132", "scaling_budget": 1.5, + "_rebaselined_interface_abstract_2885": "#2885: Kotlin interface property accessors stay in the capture set (groups still 5753/18403 and capture_groups_fp 2563) but Method isAbstract is now true for body-less interface properties, which changes accessor-plan identity in the fixture digest. Prior 82ae5e1f750580383344d4c84c400a290474528cd502be4af8cd56705819a683 -> aeafc7a87402c933786ef582b7c98683b1822b78fa909e605cb97552867fa0d5; CI scaling 0.838 < 1.5.", + "_rebaselined_jvm_property_accessors_2885": "#2885: Kotlin val/var properties now emit JVM getter/setter scope and declaration captures, including data-class constructor properties and custom accessors. Synthetic scaling counts move 4753/15203 -> 5753/18403; fixture-corpus groups move 2367 -> 2563. Accessor declaration sidecars use the canonical @declaration.qualified_name key, preserve same-name owner identity, follow JvmAbi is-prefix naming, and suppress @JvmName-renamed accessors until their custom names are modeled. Prior f98e7e936afbce0e99588285cfc603bf945fd58c5de45271860509a5d90eb832 -> 82ae5e1f750580383344d4c84c400a290474528cd502be4af8cd56705819a683; scaling 0.869 < 1.5.", "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior bddba25d5a88152bbbee8d70e82c944b5302accb4b625df782adb1d4f7a7ac12 -> e856951c2a779163d555dadc8e1bf59304a86caed78ac1f450d9caa2b50f63d1; scaling 1.090 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: Kotlin callable-reference flow facts with invocation-result suppression. Prior 4900431791f2b9280009deb2b82659c26ead8aa6fb8731190a7c505dec5a9041 -> bddba25d5a88152bbbee8d70e82c944b5302accb4b625df782adb1d4f7a7ac12; scaling 0.880 < 1.5.", "_added": "#1951: bench coverage added (was ungated); scale source heritage-bearing (: Base()); js/kotlin O(n^2) findNodeAtRange-per-match fixed to threaded captured node, now linear.", @@ -226,11 +228,12 @@ "_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior d3c4d2fa0d82d248a2299cfc888b067187ad1faf2c87a97f93c6ed835eefc3f1 -> c1f0cc9058ab11b7cd6fc8b440deb6db2b2f530f2eb21178923e68a3d0796c4b.", "_rebaselined_2766_await_subscript_emission": "#2766: extractMixedChain now walks THROUGH await and subscript nodes and peels transparent wrappers at loop entry, so sites whose receiver is `repos[0]` or `(await f())` mint a receiver chain where they previously minted none. EMISSION CHANGE: more sites carry `@reference.receiver-chain`; no existing chain changed shape. Only go and kotlin drifted of 15 \u2014 the two whose fixture corpora contain such receivers. Prior c1f0cc9058ab11b7cd6fc8b440deb6db2b2f530f2eb21178923e68a3d0796c4b -> efd5dbf80ffcd3bab2834d1010f6fe2b239dcc5d58229938dea9cff8d0f380f2.", "_rebaselined_2960_declared_package_fixture": "#2960 adds four Kotlin declared-package import-resolution fixture files. This is fixture-corpus growth only: fixture_count 137 -> 141 and capture_groups_fp 2334 -> 2367; the synthetic capture counts remain 4753/15203, no Kotlin scope-capture query or implementation changed, and package resolution runs after capture. Other language fingerprints matched their baselines in the same CI run. Prior a184f8ff0ae40d246db855b63f7ff26bda3afac03e5f4c76e4593c7e2cefce54 -> f98e7e936afbce0e99588285cfc603bf945fd58c5de45271860509a5d90eb832.", - "capture_groups_small": 4753, - "capture_groups_large": 15203, - "capture_groups_fp": 2367, + "capture_groups_small": 5753, + "capture_groups_large": 18403, + "capture_groups_fp": 2563, "fixture_count": 141, "_rebaselined_1432_member_call_callee_name": "#1432 (Zig): the shared callable-flow reader no longer names a callee by simple name for a MEMBER call (`@callable-flow.direct-callee-name` requires a direct designator: `f(x)`, `ns.f(x)`), and a member call is a field-stored-callable invoke only when a MEMBER store (`o.f = handler`) or a declared callable-typed field is visible - a same-named plain binding no longer gates it. CAPTURE-EMISSION CHANGE, not fixture growth (fixture_count unchanged). Only drift: `users.map { it.name }.forEach { name -> println(name) }` (kotlin-lambda-scopes/App.kt) loses `direct-callee-name|forEach` (member call). capture_groups_fp 2334 (unchanged). Prior a184f8ff0ae40d246db855b63f7ff26bda3afac03e5f4c76e4593c7e2cefce54 -> 5a181af0dbc9451937da0964c40d3f3f9820914ca429d873bb5c812b5e2b9284.", - "_rebaselined_1432_rebase_onto_2960": "#1432 rebase onto main @ aac7515d: the kotlin fingerprint is a COMBINATION of two independent changes, so neither side of the merge conflict was correct on its own and resolving it by picking a side would have committed a fingerprint no run can reproduce. main's #2960 added four declared-package fixture files (fixture_count 137 -> 141, capture_groups_fp 2334 -> 2367); this branch's `_rebaselined_1432_member_call_callee_name` drops `direct-callee-name|forEach` from one member call. Recomputed under both: capture_groups_fp 2367 and fixture_count 141 match main's committed counts EXACTLY (this branch's change is emission-only and moves no count), capture_groups_small/large stay 4753/15203 (the SYNTHETIC scaling source, which neither change touches), scaling 1.003 < 1.5, and the other 14 languages report ok against their committed baselines in the same run - which is the check that the rebase replayed nothing else into the capture stream. Attribution is exact rather than inferred: moving test/fixtures/lang-resolution/kotlin-import-package-evidence aside and re-running returns kotlin to 5a181af0dbc9451937da0964c40d3f3f9820914ca429d873bb5c812b5e2b9284 byte-for-byte with fixture_count back at 137 and capture_groups_fp back at 2334 - this branch's pre-rebase value - so the whole delta is #2960's corpus growth layered on top, with nothing else moving. Prior (this branch, pre-rebase) 5a181af0dbc9451937da0964c40d3f3f9820914ca429d873bb5c812b5e2b9284 and (main) f98e7e936afbce0e99588285cfc603bf945fd58c5de45271860509a5d90eb832 -> 973d702510002dda76166e017c5eca90cae37a139f5a512877b7a1b04ad19dc5." + "_rebaselined_1432_rebase_onto_2960": "#1432 rebase onto main @ aac7515d: the kotlin fingerprint is a COMBINATION of two independent changes, so neither side of the merge conflict was correct on its own and resolving it by picking a side would have committed a fingerprint no run can reproduce. main's #2960 added four declared-package fixture files (fixture_count 137 -> 141, capture_groups_fp 2334 -> 2367); this branch's `_rebaselined_1432_member_call_callee_name` drops `direct-callee-name|forEach` from one member call. Recomputed under both: capture_groups_fp 2367 and fixture_count 141 match main's committed counts EXACTLY (this branch's change is emission-only and moves no count), capture_groups_small/large stay 4753/15203 (the SYNTHETIC scaling source, which neither change touches), scaling 1.003 < 1.5, and the other 14 languages report ok against their committed baselines in the same run - which is the check that the rebase replayed nothing else into the capture stream. Attribution is exact rather than inferred: moving test/fixtures/lang-resolution/kotlin-import-package-evidence aside and re-running returns kotlin to 5a181af0dbc9451937da0964c40d3f3f9820914ca429d873bb5c812b5e2b9284 byte-for-byte with fixture_count back at 137 and capture_groups_fp back at 2334 - this branch's pre-rebase value - so the whole delta is #2960's corpus growth layered on top, with nothing else moving. Prior (this branch, pre-rebase) 5a181af0dbc9451937da0964c40d3f3f9820914ca429d873bb5c812b5e2b9284 and (main) f98e7e936afbce0e99588285cfc603bf945fd58c5de45271860509a5d90eb832 -> 973d702510002dda76166e017c5eca90cae37a139f5a512877b7a1b04ad19dc5.", + "_rebaselined_1432_merge_main_2885": "#1432 merge of main @ 212e007a: the kotlin fingerprint is again a COMBINATION of two independent changes \u2014 main's #2885 JVM property accessors / interface-abstract (4753/15203 -> 5753/18403, capture_groups_fp 2367 -> 2563) and this branch's `_rebaselined_1432_member_call_callee_name` (drops `direct-callee-name|forEach` from one member call). Neither side's value reproduces under the merged tree. Recomputed under both: counts 5753/18403/2563 and fixture_count 141 match main's committed counts EXACTLY (this branch's change is emission-only), scaling 1.061 < 1.5, and csharp / cpp / typescript measure byte-for-byte at this branch's committed values (main did not touch them since the merge-base) while the other 11 languages report ok \u2014 the check that the merge replayed nothing else into the capture stream. Prior (main) aeafc7a87402c933786ef582b7c98683b1822b78fa909e605cb97552867fa0d5 and (this branch) 973d702510002dda76166e017c5eca90cae37a139f5a512877b7a1b04ad19dc5 -> a9d3f0db7547ff47856159debf15d2a6f427efca97a6af27b4004174ed432132." } } diff --git a/gitnexus/bench/spring-config-bindings/baselines.json b/gitnexus/bench/spring-config-bindings/baselines.json new file mode 100644 index 000000000..97f881dd4 --- /dev/null +++ b/gitnexus/bench/spring-config-bindings/baselines.json @@ -0,0 +1,8 @@ +{ + "_comment": "Baselines for bench/spring-config-bindings/measure.mjs --check (#2412). fingerprint is sha256 over position-free Kotlin config-consumer fact ids on the wildcard_large corpus (800 files × 2 @Value properties + 1 @ConfigurationProperties class = 2400 facts). Both arms must fingerprint identically: the wildcard arm adds a sibling nested type named `Value`, which must not suppress the imported Spring annotation. Budgets are timing gates with CI headroom.", + "fingerprint": "34776f883427479befbeb3c09eaae2260ba778e769bff195044d3cb8f5ad9889", + "scaling_budget": 1.6, + "_scaling_note": "(t_large/t_small)/(800/250) on the wildcard arm. Measured ~0.99.", + "widening_overhead_budget": 1.8, + "_widening_overhead_note": "wildcard_large_ms / exact_large_ms. The exact-import control resolves each annotation from imports.exact before any shadow check, so this isolates the wildcard path's per-annotation lexical shadow walk. Measured ~1.09; budget guards against a per-annotation rescan of the file's declarations." +} diff --git a/gitnexus/bench/spring-config-bindings/measure.mjs b/gitnexus/bench/spring-config-bindings/measure.mjs new file mode 100644 index 000000000..94ac94bf2 --- /dev/null +++ b/gitnexus/bench/spring-config-bindings/measure.mjs @@ -0,0 +1,164 @@ +/** + * Build-free throughput + identity bench for Kotlin Spring config-consumer + * capture (#2412). + * + * Arms (identical corpora except the import style): + * - exact: explicit `import ...annotation.Value` control, which resolves the + * annotation from `imports.exact` before any shadow check runs + * - wildcard: `import ...annotation.*` feature path, where every simple-name + * annotation pays the lexical local-type shadow walk. Each file also + * declares a sibling nested type named `Value` that must NOT suppress the + * Spring annotation — the file-wide-shadow regression fixed on this branch. + * + * Parsing is prepared outside the timer; the measured path is the capture + * function the Kotlin worker calls on its own AST. + * + * Usage: + * node --import tsx bench/spring-config-bindings/measure.mjs + * node --import tsx bench/spring-config-bindings/measure.mjs --check + */ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Parser from 'tree-sitter'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { getLanguageGrammar } from '../../src/core/tree-sitter/parser-loader.ts'; +import { captureKotlinSpringConfigConsumerFacts } from '../../src/core/ingestion/languages/kotlin/spring-config-bindings.ts'; +import { fingerprintIds, minSample, runBaselineCheck } from '../lib/identity-guard.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const SMALL = 250; +const LARGE = 800; +const REPS = 15; +const WARMUP = 5; +/** Two @Value properties plus one @ConfigurationProperties class per file. */ +const FACTS_PER_FILE = 3; + +function consumerSource(i, mode) { + const imports = + mode === 'wildcard' + ? `import org.springframework.beans.factory.annotation.* +import org.springframework.boot.context.properties.*` + : `import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.context.properties.ConfigurationProperties`; + + return `package bench.config +${imports} + +class Shadowing${i} { + class Value +} + +@ConfigurationProperties(prefix = "svc.${i}") +class Props${i} { + var endpoint: String? = null +} + +class Consumer${i} { + @Value("\\\${app.key${i}}") + var timeout: Int = 0 + + @Value("\\\${app.other${i}:5}") + var other: String? = null + + fun decoy() {} +} +`; +} + +/** Position-free fact identity, so both arms are directly comparable. */ +function factId(fact) { + const consumer = fact.consumer; + return consumer.kind === 'value' + ? `value|${consumer.fieldName}|${[...consumer.keys].sort().join(',')}` + : `configuration-properties|${consumer.className}|${consumer.prefix}`; +} + +function prepare(mode, fileCount) { + const files = []; + const lang = getLanguageGrammar(SupportedLanguages.Kotlin); + for (let i = 0; i < fileCount; i++) { + const parser = new Parser(); + parser.setLanguage(lang); + const filePath = `bench/${mode}/Consumer${i}.kt`; + files.push({ tree: parser.parse(consumerSource(i, mode)), filePath, parser }); + } + return files; +} + +function runAll(files) { + const ids = []; + for (const f of files) { + for (const fact of captureKotlinSpringConfigConsumerFacts(f.tree.rootNode, f.filePath)) { + ids.push(factId(fact)); + } + } + return ids; +} + +function measure(mode, fileCount) { + const files = prepare(mode, fileCount); + const { last, ms } = minSample(() => runAll(files), WARMUP, REPS); + return { + files: fileCount, + ms, + facts: last.length, + fingerprint: fingerprintIds(last), + }; +} + +function failIfNeeded(current, errors) { + if (errors.length === 0) return; + console.error(JSON.stringify({ report: current, errors }, null, 2)); + process.exit(1); +} + +function runFactCountCheck(current, expectedCounts) { + const errors = []; + for (const [arm, expected] of Object.entries(expectedCounts)) { + const actual = current[arm]?.facts; + if (actual !== expected) errors.push(`${arm}.facts ${String(actual)} != ${expected}`); + } + failIfNeeded(current, errors); +} + +/** + * A wildcard import plus a sibling `Value` declaration must capture exactly the + * facts the explicit-import control captures. + */ +function runFingerprintParityCheck(current, leftArm, rightArm) { + const left = current[leftArm]?.fingerprint; + const right = current[rightArm]?.fingerprint; + failIfNeeded( + current, + left === right ? [] : [`${leftArm}.fingerprint ${left} != ${rightArm}.fingerprint ${right}`], + ); +} + +const report = { + exact_small: measure('exact', SMALL), + exact_large: measure('exact', LARGE), + wildcard_small: measure('wildcard', SMALL), + wildcard_large: measure('wildcard', LARGE), +}; +report.scaling_ratio = Number( + (report.wildcard_large.ms / report.wildcard_small.ms / (LARGE / SMALL)).toFixed(3), +); +report.widening_overhead = Number( + (report.wildcard_large.ms / Math.max(report.exact_large.ms, 0.001)).toFixed(3), +); +report.fingerprint = report.wildcard_large.fingerprint; + +runFactCountCheck(report, { + exact_large: LARGE * FACTS_PER_FILE, + wildcard_large: LARGE * FACTS_PER_FILE, +}); +runFingerprintParityCheck(report, 'exact_large', 'wildcard_large'); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +runBaselineCheck(report, BASELINE_PATH); diff --git a/gitnexus/bench/v8-sidecar/measure.mjs b/gitnexus/bench/v8-sidecar/measure.mjs new file mode 100644 index 000000000..68867b9ba --- /dev/null +++ b/gitnexus/bench/v8-sidecar/measure.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node +/** + * Optional V8 sidecar warm-load bench (#3089). + * + * Not part of `npm test`. Measures repeated warm loads of the `.v8` ParsedFile + * shards already on disk through the production loader. Replay of identical + * shards is throughput-only — it is not unique-object scale. + * + * Copies the store into a temporary workspace first. The source cache is + * never mutated. + * + * Usage (from gitnexus/): + * node --expose-gc --import tsx bench/v8-sidecar/measure.mjs + */ +import { cp, mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { loadParsedFilesForPaths } from '../../src/storage/parsedfile-store.ts'; +import { inspectV8Cache } from '../../src/storage/v8-sidecar.ts'; + +const srcStorage = process.argv[2]; +if (!srcStorage) { + console.error('usage: node --expose-gc --import tsx bench/v8-sidecar/measure.mjs '); + process.exit(2); +} + +const srcStoreDir = path.join(srcStorage, 'parsedfile-store'); +const benchRoot = await mkdtemp(path.join(tmpdir(), 'gnx-v8-bench-')); +const storeDir = path.join(benchRoot, 'parsedfile-store'); +const PATH_SOURCE_SHARDS = 8; +const RUNS = 3; + +try { + await cp(srcStoreDir, storeDir, { recursive: true }); + + const names = (await readdir(storeDir)) + .filter((f) => f.endsWith('.v8') && !f.includes('.v8.')) + .sort(); + if (names.length === 0) { + throw new Error( + `no .v8 ParsedFile shards in ${srcStoreDir} — run an analyze that populates the store first`, + ); + } + + const want = new Set(); + let sourceShards = 0; + for (const name of names) { + const inspected = await inspectV8Cache(path.join(storeDir, name)); + if (!inspected) continue; + sourceShards++; + for (const filePath of inspected.paths) want.add(filePath); + if (sourceShards >= PATH_SOURCE_SHARDS) break; + } + if (want.size === 0) { + throw new Error( + `no file paths readable from ${names.length} shard(s) in ${srcStoreDir} — shards may be from another Node/V8 runtime, so re-analyze with this runtime`, + ); + } + + const rss = () => Math.round(process.memoryUsage().rss / 1024 / 1024); + const heap = () => Math.round(process.memoryUsage().heapUsed / 1024 / 1024); + + const run = async (label) => { + if (typeof globalThis.gc === 'function') globalThis.gc(); + const t0 = performance.now(); + const loaded = await loadParsedFilesForPaths(benchRoot, want); + const ms = Math.round(performance.now() - t0); + if (loaded.size !== want.size) { + throw new Error(`incomplete V8 load: requested ${want.size} paths but loaded ${loaded.size}`); + } + if (typeof globalThis.gc === 'function') globalThis.gc(); + console.log( + JSON.stringify({ + label, + shards: names.length, + wantPaths: want.size, + files: loaded.size, + ms, + rssMiB: rss(), + heapUsedMiB: heap(), + }), + ); + }; + + for (let i = 1; i <= RUNS; i++) { + await run(`v8-load-${i}`); + } +} finally { + await rm(benchRoot, { recursive: true, force: true }); +} diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index cac9af79d..555e0c747 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,12 +1,12 @@ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { @@ -14,15 +14,18 @@ "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", "busboy": "^1.6.0", + "chokidar": "^5.0.0", "cli-progress": "^3.12.0", "commander": "^15.0.0", "cors": "^2.8.5", "express": "^5.2.1", "express-rate-limit": "^8.4.1", + "fast-xml-parser": "^5.11.1", "glob": "^13.0.6", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", + "graphql": "^17.0.2", "ignore": "^7.0.5", "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", @@ -826,9 +829,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -845,9 +845,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -864,9 +861,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -883,9 +877,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -902,9 +893,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -921,9 +909,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -940,9 +925,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -959,9 +941,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -978,9 +957,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1003,9 +979,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1028,9 +1001,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1053,9 +1023,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1078,9 +1045,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1103,9 +1067,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1128,9 +1089,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1153,9 +1111,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1444,6 +1399,18 @@ } } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@oxc-project/types": { "version": "0.144.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.144.0.tgz", @@ -1619,9 +1586,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1639,9 +1603,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1659,9 +1620,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1679,9 +1637,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1699,9 +1654,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1719,9 +1671,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1960,9 +1909,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "version": "26.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.3.0.tgz", + "integrity": "sha512-L3fgrnchriRC2ExBflb8j4uZZURHZfQsmQeyVzhjcHW4kkwVyo8/0h1B2MVzMTrYUJYu6G7EWs14hW/L9putqw==", "devOptional": true, "license": "MIT", "dependencies": { @@ -2238,6 +2187,18 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/apache-arrow": { "version": "21.1.0", "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.1.0.tgz", @@ -2467,6 +2428,21 @@ "url": "https://github.com/chalk/chalk-template?sponsor=1" } }, + "node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -3091,6 +3067,45 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", + "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3377,6 +3392,15 @@ "graphology-types": ">=0.23.0" } }, + "node_modules/graphql": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-17.0.2.tgz", + "integrity": "sha512-FRWbddMxfkjiB7z+aQDWIR+E34xo9I8c9mtK2RPv8PmMzKRvrdsreHL/Ui/TmwHJfhHChEtsFPyMHKI+xuarQQ==", + "license": "MIT", + "engines": { + "node": "^22.0.0 || ^24.0.0 || ^25.0.0 || >=26.0.0" + } + }, "node_modules/guid-typescript": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", @@ -3542,6 +3566,18 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/isexe": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", @@ -3616,9 +3652,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.3.0.tgz", - "integrity": "sha512-muutsYr+e2+d3rTgUGslq5rxbBlUy3cJ61IsHag2QNDQV+7zXWjkUpmALIajhrlLlrgRUiymj6U3zUr/TMK84Q==", + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.4.0.tgz", + "integrity": "sha512-jE7vUJIebKzYQI5xu4co5CRBDlDEYnHrdzsxs4O2giCz4v2SbVMYKpmt1D9L38OKQAeCWmrOTRiCV93u0UkaJA==", "funding": [ { "type": "github", @@ -3831,9 +3867,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -3855,9 +3888,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -3879,9 +3909,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -3903,9 +3930,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4335,15 +4359,15 @@ } }, "node_modules/onnxruntime-common": { - "version": "1.27.0", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.27.0.tgz", - "integrity": "sha512-3KxL5wIVqa8Ex08jxSzncm9CMgw8CjOFyOQ7SxvG9o0cVLlhTNKXyIQuTbtX4tGPJEf73OER2xrjt4HJSBL4ow==", + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.29.0.tgz", + "integrity": "sha512-/F63/e2VJoaVXGGNu6S5QH7jivBThGO95OzAVXXQ8hTta/b1QxI8udHa6cI3+3mAb5WWIIaMMwfZw01oivjJ1g==", "license": "MIT" }, "node_modules/onnxruntime-node": { - "version": "1.27.0", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.27.0.tgz", - "integrity": "sha512-QEzGwrvNBgv4uPVdnbHsOGG4G6T96mdlcFI8aAKPjMU8wOPpVocPXb6k3QGkaZagVTv2G9Bnnbo6Z3JdXr1fQw==", + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.29.0.tgz", + "integrity": "sha512-WjiVVB72riILz8HbYvxvmjKyE/WmkYoSfKY++axo5jAR609HQg8MwiG/HhShpTcJfmmAdzxxmB+MMST3A+SiPA==", "hasInstallScript": true, "license": "MIT", "optional": true, @@ -4353,9 +4377,9 @@ "linux" ], "dependencies": { - "adm-zip": "^0.5.16", + "adm-zip": "^0.6.0", "global-agent": "^4.1.3", - "onnxruntime-common": "1.27.0" + "onnxruntime-common": "1.29.0" } }, "node_modules/onnxruntime-web": { @@ -4407,6 +4431,21 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -4594,9 +4633,9 @@ "license": "MIT" }, "node_modules/protobufjs": { - "version": "7.6.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", - "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", "hasInstallScript": true, "license": "BSD-3-Clause", "optional": true, @@ -4700,6 +4739,19 @@ "rc": "cli.js" } }, + "node_modules/readdirp": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", + "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -5140,6 +5192,21 @@ "node": ">=0.10.0" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -5825,6 +5892,21 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index af3d3812f..d99734523 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.6.9", + "version": "1.6.10", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", @@ -60,15 +60,18 @@ "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", "busboy": "^1.6.0", + "chokidar": "^5.0.0", "cli-progress": "^3.12.0", "commander": "^15.0.0", "cors": "^2.8.5", "express": "^5.2.1", "express-rate-limit": "^8.4.1", + "fast-xml-parser": "^5.11.1", "glob": "^13.0.6", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", + "graphql": "^17.0.2", "ignore": "^7.0.5", "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts index 1a026a5e0..2c52dd86b 100644 --- a/gitnexus/scripts/cross-platform-shard.ts +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -3,7 +3,7 @@ * * WHY THIS EXISTS. `run-cross-platform.ts` used to hand vitest the whole file * list plus `--shard=i/n`, and vitest partitions by file COUNT. Runtime on this - * suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 361 s + * suite is wildly uneven — measured on the Windows runner, `cli-e2e` is 621 s * and `worker-pool` 221 s, while most files are under a second — so a * count-split routinely put several of the heaviest suites on one shard. That * is #2449, and this file's sibling header has documented the symptom ("the @@ -44,7 +44,9 @@ * partition depend on the very machine load it is trying to protect against. */ export const WINDOWS_WEIGHTS_SEC: Readonly> = { - 'test/integration/cli-e2e.test.ts': 361, + // Re-measured after the analyze --watch e2e landed in #3072. The previous + // 361 s entry undercharged this suite and left shard 1 close to the watchdog. + 'test/integration/cli-e2e.test.ts': 621, 'test/integration/worker-pool.test.ts': 222, 'test/unit/incremental-vector-extension-ordering.test.ts': 87, // ESTIMATE, not a measurement (#2841): this suite drives more full @@ -65,6 +67,16 @@ export const WINDOWS_WEIGHTS_SEC: Readonly> = { 'test/integration/antigravity-hook-e2e.test.ts': 7, 'test/unit/index-lock.test.ts': 5, 'test/unit/setup.test.ts': 5, + // ESTIMATE, not a measurement. This file asserts almost nothing; it READS — + // one 4893-file pass over every tracked text file, plus an 830-file pass over + // `src/`. Measured at 2.3 s and 0.3 s per pass on a virtualised and a local + // Linux filesystem respectively, so the cost is entirely per-file open + // latency, which is the term Windows inflates most (NTFS plus Defender on + // every read). Scaled from the slower Linux figure to keep the split + // conservative rather than let the 8 s PER_FILE_OVERHEAD floor under-charge + // a file that touches more paths than anything else here. Replace with a real + // figure after the first green Windows matrix run. + 'test/unit/source-control-bytes.test.ts': 15, }; /** diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 1994abf6b..44e45fbbb 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -90,6 +90,7 @@ const PLATFORM_LOGIC = [ 'test/unit/ignore-service.test.ts', 'test/unit/group/bridge-db.test.ts', 'test/unit/group/bridge-db-edge.test.ts', + 'test/unit/group/fs-utils.test.ts', 'test/unit/onnxruntime-node-resolver.test.ts', // Windows cmd.exe arg-quoting + compose-and-spawn for the npm install (#2372): // the quoting rules and win32 single-string spawn shape are OS-sensitive, so @@ -113,6 +114,9 @@ const PLATFORM_LOGIC = [ // POSIX and Windows — the fail-closed path-claim semantics must hold on the // real windows-latest path implementation (#2419/#2420). 'test/unit/server-api-repo-resolution.test.ts', + // #3073: cwd-based repository selection canonicalizes real paths, compares + // platform separators/case, and rejects nested Git-boundary fallthrough. + 'test/unit/calltool-dispatch.test.ts', // The index write-lock (#2658) selects its backend by process.platform — the // OS socket lock (Windows named pipe / Linux abstract socket) vs the file // fallback — and its socket-backend describe block is gated to linux/win32. @@ -140,6 +144,7 @@ const LBUG_NATIVE = [ // opens them through the pool adapter (native addon + bridge file locking). // Windows is skipped in-file (describeReopen) due to the bridge reopen lock. 'test/integration/group/cross-trace-e2e.test.ts', + 'test/integration/group/graphql-resolve-symbol.test.ts', 'test/integration/local-backend.test.ts', 'test/integration/local-backend-calltool.test.ts', 'test/integration/search-core.test.ts', @@ -208,6 +213,18 @@ const SPAWN_CLI = [ // exposed a file-backend double-admit race here (#2658 review); the reclaim is // now judgment-verified so a live holder is never displaced. 'test/integration/analyze-index-lock-concurrency.test.ts', + // The per-group sync lock (R9), same class of guarantee one level up: real + // child processes contend for one group's lock while this process runs a real + // `syncGroup`, and the CLI case spawns the real command. Everything that + // varies here is platform-owned — which backend `selectBackend()` picks + // (Windows named pipe / Linux abstract socket / macOS file lock), kernel + // auto-release on SIGKILL vs. the file backend's pid-liveness reclaim, and + // `mkdir` over an occupied path. The fail-closed cases pin + // GITNEXUS_INDEX_LOCK_BACKEND=file so the filesystem branch is exercised on + // every OS rather than only where it is the default; no case is skipped on + // any platform, because a skipped case turns "a sync that cannot be protected + // does not run" into a claim that holds on Ubuntu only. + 'test/integration/group/group-sync-lock-concurrency.test.ts', // The three `dist/` module-load closure guards, all built on the shared // child-process probe in `test/helpers/module-load-probe.ts`. That probe IS // the platform-varying part: it spawns `process.execPath` in array form, @@ -220,7 +237,7 @@ const SPAWN_CLI = [ // Cheap: measured on the Windows runner at 448 ms, 53 ms and sub-second. An // earlier attempt to register them still turned the matrix red — not from // their own cost, but because vitest sharded by file COUNT, so inserting any - // file re-partitioned the list and happened to cluster `cli-e2e` (361 s) with + // file re-partitioned the list and happened to cluster `cli-e2e` (621 s) with // `cli-limit-e2e` (75 s) on one shard. The split is weight-aware now // (`scripts/cross-platform-shard.ts`), so a cheap file can no longer move a // heavy one. @@ -259,8 +276,31 @@ const NATIVE_ADDON_SMOKE = [ // platforms (CRLF, symlinks, permissions, temp dirs) const FILESYSTEM = [ 'test/integration/filesystem-walker.test.ts', + 'test/integration/watch-filesystem.test.ts', 'test/integration/markdown-processor-crlf.test.ts', 'test/integration/ignore-and-skip-e2e.test.ts', + // Pins that the bridge pairing verdict is measured before the database is + // opened. The property it protects is about mtime behavior across OS and + // filesystem, and the alternative — really opening the bridge — cannot run on + // Windows at all (in-process write→read reopen of the same bridge.lbug is a + // documented limitation). Running it on every platform is the whole point: + // Windows is where an unverified assumption about mtime would hurt most. + 'test/unit/group/bridge-pairing-precedes-open.test.ts', + // The raw-control-byte guard reads every tracked text file `git ls-files` + // reports — 4893 of them — and decides membership from the git path, which is + // always `/`-separated no matter what the host separator is. Both halves of + // that are platform-varying: the collector basename-matches with + // `path.posix.basename` against `git ls-files -z` output while the reads go + // through `path.join`, so on Windows the same string is consumed under two + // separator conventions in one pass, and only a real windows-latest run + // proves they agree. It is also the file-count-heaviest read loop in the + // suite, so it is where a per-file filesystem cost (NTFS + Defender, or + // macOS's slower stat path) would show up first. No case is skipped on any + // platform: a guard that only holds on Ubuntu is not a guard on the file + // whose NUL it exists to catch. Budget: the heaviest single case is one + // 4893-file pass — 2.3 s on a slow virtualised filesystem, 0.34 s on a local + // disk — against a 30 s testTimeout. + 'test/unit/source-control-bytes.test.ts', ]; const ALL_CROSS_PLATFORM = [ diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index 853d44860..be02d92fd 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -21,13 +21,20 @@ Run from the project root. This parses all source files, builds the knowledge gr | Flag | Effect | | -------------- | ---------------------------------------------------------------- | +| `--watch` | Keep a Git repository index current with serialized refreshes | +| `--debounce ` | Watch quiet period before refresh (default: 300 ms) | | `--force` | Force full re-index even if up to date | | `--embeddings` | Enable embedding generation for semantic search (off by default) | | `--drop-embeddings` | Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` preserves them. | | `--pdg` | Build the program-dependence layers used by `explain` and `pdg_query` (taint, CDG, and REACHING_DEF). | +| `--spring-actuator ` | Import opt-in Spring Boot Actuator mappings, beans, conditions, configprops, and env snapshots. Forces a full rebuild; unsupported with `--watch`. | **When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale. In Claude Code, a PostToolUse hook detects staleness after `git commit` and `git merge` and notifies the agent to run `analyze` — the hook does not run analyze itself, to avoid blocking the agent for up to 120s and risking KuzuDB corruption on timeout. +For Spring runtime enrichment, pass a JSON bundle, one endpoint JSON file, or a directory containing endpoint files. Route evidence is authoritative only when `runtimeConfirmed === true`; `runtimeSource` records provenance and may also accompany `handler-conflict`. Env/configprops values are never persisted. + +Use `node .gitnexus/run.cjs analyze --watch` for a long-lived local Git repository. It performs an initial analysis, queues scanner-admitted file changes, and retries intact failed batches with bounded backoff. Watch refreshes update only the graph: they skip AGENTS.md / CLAUDE.md injection and standard skill installation, so run a one-shot `analyze` when those generated files need updating. Watch rejects one-shot or context-output flags including `--force`, embedding flags, `--skills`, `--default-branch`, `--skip-agents-md`, `--skip-skills`, `--no-stats`, `--self-commit`, `--index-only`, and `--skip-git`. It never pulls remotes. Scheduled remote clone/pull is a different command: `gitnexus auto-sync`. Bare `gitnexus watch` is reserved and does not start either job. Running MCP and `serve` processes periodically check for a published replacement and reopen it without a restart. MCP checks are throttled to once every five seconds, so a tool call before the next check can briefly use the previous index. + ### status — Check index freshness ```bash @@ -55,15 +62,19 @@ Deletes the `.gitnexus/` directory and unregisters the repo from the global regi node .gitnexus/run.cjs wiki ``` -Generates repository documentation from the knowledge graph using an LLM. Requires an API key (saved to `~/.gitnexus/config.json` on first use). +Generates repository documentation from the knowledge graph using an LLM. HTTP providers require an API key (saved to `~/.gitnexus/config.json` on first use). Local CLI providers (`--provider cursor|claude|codex|opencode|grok`) use your existing CLI login. | Flag | Effect | | ------------------- | ----------------------------------------- | -| `--force` | Force full regeneration | +| `--force` | Force full regeneration, also required to re-generate an existing wiki in a different language | +| `--provider ` | LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax). Local CLIs (`cursor`, `claude`, `codex`, `opencode`, `grok`) use your existing CLI login and skip `--api-key`. | | `--model ` | LLM model (default: MiniMax-M3) | | `--base-url ` | LLM API base URL | | `--api-key ` | LLM API key | | `--concurrency ` | Parallel LLM calls (default: 3) | +| `--timeout ` | LLM request timeout in seconds (default: disabled) | +| `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | ### list — Show all indexed repos @@ -82,5 +93,5 @@ Lists all repositories registered in `~/.gitnexus/registry.json`. The MCP `list_ ## Troubleshooting - **"Not inside a git repository"**: Run from a directory inside a git repo -- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server +- **Index is stale after re-analyzing**: Wait for the next MCP tool call to reopen the published index; this normally takes no more than five seconds - **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding diff --git a/gitnexus/skills/gitnexus-impact-analysis.md b/gitnexus/skills/gitnexus-impact-analysis.md index 4fb73f3e6..85d90c90d 100644 --- a/gitnexus/skills/gitnexus-impact-analysis.md +++ b/gitnexus/skills/gitnexus-impact-analysis.md @@ -93,6 +93,15 @@ dispatch, cross-language calls), so few-callers ⇒ LOW does **not** apply. The result carries a `riskNote` saying so. Confirm with a text search before treating the symbol as safe to change or delete. +`risk` is the edit gate: warn on HIGH/CRITICAL and stop on UNKNOWN until the +uncertainty is resolved. Within single-repo mode, compare File and symbol +targets with local `riskSharedAxes` (direct/total only). Within group mode, +compare only group results: their `riskSharedAxes` overlays resolved +cross-repo crossings on that local value. Never use either field to waive the +edit gate. Check `riskScale.unusedAxes` before comparing kinds: MCP File walks +omit process/module axes, while web Graph-RAG expands File targets to in-file +symbols before enrichment. + ## Tools **impact** — the primary tool for symbol blast radius. If MCP is unavailable, use `node .gitnexus/run.cjs impact --direction upstream --repo .` instead: diff --git a/gitnexus/src/cli/ai-context.ts b/gitnexus/src/cli/ai-context.ts index 258aeb46c..c7b3a934f 100644 --- a/gitnexus/src/cli/ai-context.ts +++ b/gitnexus/src/cli/ai-context.ts @@ -11,6 +11,7 @@ import path from 'path'; import { fileURLToPath } from 'url'; import { type GeneratedSkillInfo } from './generated-skill.js'; import { STANDARD_SKILL_CATALOG } from './standard-skills.js'; +import { isEnoent } from './editor-targets.js'; import { logger } from '../core/logger.js'; // ESM equivalent of __dirname @@ -42,6 +43,8 @@ export interface AIContextOptions { * "no PDG layer" note, so advertising it on a non-`--pdg` index is noise. */ hasPdg?: boolean; + /** Whether this index includes opt-in Spring Actuator runtime evidence. */ + hasSpringActuator?: boolean; } const GITNEXUS_START_MARKER = ''; @@ -136,6 +139,8 @@ export interface GitNexusContentOptions { * line below — false (default) omits it, so a non-pdg index doesn't advertise * a tool that only returns a "no PDG layer" note. */ hasPdg?: boolean; + /** Whether Route nodes may carry Spring Actuator runtime evidence. */ + hasSpringActuator?: boolean; } export function generateGitNexusContent( @@ -151,6 +156,7 @@ export function generateGitNexusContent( runnerPath = '.gitnexus/run.cjs', defaultBranch = 'main', hasPdg = false, + hasSpringActuator = false, } = opts; const generatedRows = generatedSkills && generatedSkills.length > 0 @@ -218,16 +224,19 @@ This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${s ## Always Do -- **MUST run impact analysis before editing.** Use \`impact({target: "symbolName", direction: "upstream"})\` (MCP) or \`${runner} impact "symbolName" --direction upstream --repo .\` (CLI fallback); report callers, processes, and risk. Never substitute grep for graph analysis.${ +- **MUST run impact before editing.** Use \`impact({target: "symbolName", direction: "upstream"})\` or \`${runner} impact "symbolName" --direction upstream --repo .\`; report callers, processes, and risk. Never substitute grep for graph analysis.${ hasPdg ? ` For unified PDG impact, add \`mode: "pdg"\` with optional \`line: \` — it returns statement-level \`affectedStatements\` over CDG + REACHING_DEF and inter-procedural symbols in \`interproceduralByDepth\`/\`byDepth\`; no-layer/degraded PDG results are UNKNOWN-risk notes (\`--pdg\` layer). CLI equivalent: \`${runner} impact "symbolName" --direction upstream --mode pdg --line --repo .\`.` : '' } - **MUST analyze graph changes before committing.** Use \`detect_changes({scope: "all"})\` (MCP) or \`${runner} detect-changes --scope all --repo .\` (CLI fallback). \`partial: true\` or \`truncated: true\` is not a clean check — a zero means unseen, not unaffected; re-run it. For regression review: \`detect_changes({scope: "compare", base_ref: ${JSON.stringify(markdownSafeBranch(defaultBranch))}})\` or \`${runner} detect-changes --scope compare --base-ref ${JSON.stringify(markdownSafeBranch(defaultBranch))} --repo .\`. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- MUST warn on HIGH/CRITICAL \`risk\` pre-edit; never use \`riskSharedAxes\` to waive a HIGH/CRITICAL \`risk\` warning. Compare File/symbol: MCP File omits axes; Graph-RAG expands File. - **MUST treat \`risk: UNKNOWN\` as unresolved, not as low.** An empty caller set is not evidence the symbol is unused — it can also mean the callers are not resolvable by the index (plain-object property access, dynamic dispatch, cross-language calls). \`impact\` pairs \`UNKNOWN\` with a \`riskNote\` saying so. Confirm with a text search before treating the symbol as safe to change or delete; do not proceed on the strength of a zero. -- When exploring unfamiliar code, use \`query({search_query: "concept"})\` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use \`context({name: "symbolName"})\`. +- **MUST use \`query({search_query: "concept"})\` for concepts/flows, \`context({name: "symbolName"})\` for a named symbol, or \`impact\` for blast radius, on read-only callers, dependencies, imports, or execution flow.** Graph first; text search only for empty/\`UNKNOWN\`/literals.${ + hasSpringActuator + ? '\n- Spring Actuator runtime evidence is enabled. A Route is authoritative only when `runtimeConfirmed === true`; `runtimeSource` is provenance and may also describe conflicts. Snapshot values are never persisted.' + : '' + } - For security review, \`explain({target: "fileOrSymbol"})\` lists taint findings (source→sink flows; needs \`analyze --pdg\`).${ hasPdg ? `\n- For control/data dependence, \`pdg_query({mode: "controls", target: "fileOrSymbol"})\` answers "under what condition does X run?" (CDG, incl. guard clauses) and \`pdg_query({mode: "flows", target, variable})\` traces "where does variable Y flow?" (REACHING_DEF). \`--pdg\` layer.` @@ -432,17 +441,84 @@ export async function shouldMirrorSkillsToAgents(repoPath: string): Promise { + try { + return await fs.readFile(filePath, 'utf-8'); + } catch (err) { + if (isEnoent(err)) return null; + throw err; + } +} + +function skillBytesDiverge(existing: string | null, bundled: string): boolean { + return existing !== null && existing !== bundled; +} + +/** Write bundled skill bytes unless an existing file already differs. */ +async function writeSkillUnlessDivergent(filePath: string, content: string): Promise { + const existing = await readUtf8IfPresent(filePath); + if (skillBytesDiverge(existing, content)) { + logger.warn(`Preserved customized skill ${filePath}; ${SKILL_PRESERVE_HINT}.`); + return true; + } + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, 'utf-8'); + return false; +} + +async function inspectLegacySkillDir( + legacyDir: string, +): Promise<{ nestedExisting: string | null; hasSiblings: boolean } | null> { + let entries: string[]; + try { + entries = await fs.readdir(legacyDir); + } catch (err) { + if (isEnoent(err)) return null; + throw err; + } + const nestedExisting = entries.includes('SKILL.md') + ? await fs.readFile(path.join(legacyDir, 'SKILL.md'), 'utf-8') + : null; + return { + nestedExisting, + hasSiblings: entries.some((entry) => entry !== 'SKILL.md'), + }; +} + +function formatSkillInstallLine( + prefix: string, + total: number, + preserved: number, + allWrittenSuffix: string, + partialSuffix: string, +): string { + if (preserved > 0) { + return `${prefix} (${total - preserved} written, ${preserved} ${partialSuffix})`; + } + return `${prefix} (${total} ${allWrittenSuffix})`; +} + /** * Install GitNexus skills as direct children of .claude/skills/ * Works natively with Claude Code, Cursor, and GitHub Copilot. * Mirrored to .agents/skills/ when .agents/ exists. */ -async function installSkills( - repoPath: string, -): Promise<{ skills: string[]; agentsMirror: boolean }> { +async function installSkills(repoPath: string): Promise<{ + skills: string[]; + agentsMirror: boolean; + claudePreserved: number; + agentsPreserved: number; + legacyPreserved: number; +}> { const skillsDir = path.join(repoPath, '.claude', 'skills'); const legacySkillsDir = path.join(skillsDir, 'gitnexus'); const installedSkills: string[] = []; + let claudePreserved = 0; + let agentsPreserved = 0; + let legacyPreserved = 0; const agentsMirror = await shouldMirrorSkillsToAgents(repoPath); for (const skill of STANDARD_SKILL_CATALOG.filter( @@ -452,9 +528,6 @@ async function installSkills( const skillPath = path.join(skillDir, 'SKILL.md'); try { - // Create skill directory - await fs.mkdir(skillDir, { recursive: true }); - // Try to read from package skills directory const packageSkillPath = path.join(__dirname, '..', '..', 'skills', `${skill.name}.md`); let skillContent: string; @@ -476,14 +549,13 @@ Use GitNexus tools to accomplish this task. `; } - await fs.writeFile(skillPath, skillContent, 'utf-8'); + if (await writeSkillUnlessDivergent(skillPath, skillContent)) claudePreserved += 1; // Mirror to .agents/skills/ for agents that read repo-local skills if (agentsMirror) { try { - const agentsSkillDir = path.join(repoPath, '.agents', 'skills', skill.name); - await fs.mkdir(agentsSkillDir, { recursive: true }); - await fs.writeFile(path.join(agentsSkillDir, 'SKILL.md'), skillContent, 'utf-8'); + const agentsSkillPath = path.join(repoPath, '.agents', 'skills', skill.name, 'SKILL.md'); + if (await writeSkillUnlessDivergent(agentsSkillPath, skillContent)) agentsPreserved += 1; } catch (err) { logger.warn({ err }, `Warning: Could not mirror skill ${skill.name} to .agents/skills:`); } @@ -495,7 +567,20 @@ Use GitNexus tools to accomplish this task. // deep. Remove only the child owned by this installer; unknown siblings // under the legacy grouping directory may be user-authored and survive. try { - await fs.rm(path.join(legacySkillsDir, skill.name), { recursive: true, force: true }); + const legacyDir = path.join(legacySkillsDir, skill.name); + const nestedSkill = path.join(legacyDir, 'SKILL.md'); + const leftover = await inspectLegacySkillDir(legacyDir); + if (leftover !== null && skillBytesDiverge(leftover.nestedExisting, skillContent)) { + logger.warn(`Preserved customized skill ${nestedSkill}; ${SKILL_PRESERVE_HINT}.`); + legacyPreserved += 1; + } else if (leftover?.hasSiblings) { + logger.warn( + `Preserved legacy skill directory ${legacyDir} because it contains operator-owned files.`, + ); + legacyPreserved += 1; + } else if (leftover !== null) { + await fs.rm(legacyDir, { recursive: true, force: true }); + } } catch (err) { logger.warn({ err }, `Warning: Could not remove legacy skill ${skill.name}:`); } @@ -505,7 +590,13 @@ Use GitNexus tools to accomplish this task. } } - return { skills: installedSkills, agentsMirror }; + return { + skills: installedSkills, + agentsMirror, + claudePreserved, + agentsPreserved, + legacyPreserved, + }; } /** @@ -551,6 +642,7 @@ export async function generateAIContextFiles( runnerPath, defaultBranch: options?.defaultBranch ?? 'main', hasPdg: options?.hasPdg ?? false, + hasSpringActuator: options?.hasSpringActuator ?? false, }); const createdFiles: string[] = []; @@ -583,12 +675,37 @@ export async function generateAIContextFiles( // Install standard skills directly under .claude/skills/ (unless --skip-skills) if (!options?.skipSkills) { - const { skills: installedSkills, agentsMirror } = await installSkills(repoPath); + const { + skills: installedSkills, + agentsMirror, + claudePreserved, + agentsPreserved, + legacyPreserved, + } = await installSkills(repoPath); if (installedSkills.length > 0) { - createdFiles.push(`.claude/skills/gitnexus-*/ (${installedSkills.length} skills)`); + createdFiles.push( + formatSkillInstallLine( + '.claude/skills/gitnexus-*/', + installedSkills.length, + claudePreserved, + 'skills', + 'preserved', + ), + ); if (agentsMirror) { createdFiles.push( - `.agents/skills/gitnexus-*/ (${installedSkills.length} skills mirrored for .agents)`, + formatSkillInstallLine( + '.agents/skills/gitnexus-*/', + installedSkills.length, + agentsPreserved, + 'skills mirrored for .agents', + 'preserved for .agents', + ), + ); + } + if (legacyPreserved > 0) { + createdFiles.push( + `.claude/skills/gitnexus// (legacy directories preserved: ${legacyPreserved})`, ); } } diff --git a/gitnexus/src/cli/analyze-config.ts b/gitnexus/src/cli/analyze-config.ts index 6e1afc7bb..3d040fac1 100644 --- a/gitnexus/src/cli/analyze-config.ts +++ b/gitnexus/src/cli/analyze-config.ts @@ -30,6 +30,7 @@ import fs from 'node:fs'; import path from 'node:path'; +import { readRepoControlFile } from '../config/repo-control-file.js'; import type { AnalyzeOptions } from './analyze-options.js'; export const GITNEXUS_RC_FILENAME = '.gitnexusrc'; @@ -59,7 +60,8 @@ type ValueKind = | 'string-array' | 'numeric-string' | 'embeddings' - | 'branch'; + | 'branch' + | 'path'; interface KeySpec { /** The `AnalyzeOptions` field this config key normalizes into. */ @@ -107,6 +109,9 @@ const KEY_SPECS: Record = { // built-in convention set, is otherwise invisible to route_map consumers. // Listing it here adds it to the cross-file consumer scan. fetchWrappers: { target: 'fetchWrappers', kind: 'string-array' }, + // Explicit local Actuator snapshot input (#2418). The path itself is safe in + // project config; payload contents are never copied into the graph wholesale. + springActuator: { target: 'springActuator', kind: 'path' }, // Auth token AND dims are intentionally CLI/env-only — no embeddingAuthToken // or embeddingDims key here: // - the token keeps secrets out of a committed .gitnexusrc; @@ -225,6 +230,17 @@ const normalizeValue = (kind: ValueKind, value: unknown, key: string): unknown = throw new GitNexusRcError(`${source} must be a string branch name.`); } return validateBranchName(value, source); + case 'path': { + if (typeof value !== 'string') { + throw new GitNexusRcError(`${source} must be a file or directory path.`); + } + const trimmed = value.trim(); + if (!trimmed) { + throw new GitNexusRcError(`${source} must not be empty.`); + } + assertNoHiddenChars(trimmed, source); + return trimmed; + } case 'string': { if (typeof value !== 'string') { throw new GitNexusRcError(`${source} must be a string.`); @@ -370,7 +386,6 @@ const normalizeLevel = ( */ export function loadAnalyzeConfig(repoRoot: string): Partial | undefined { const filePath = path.join(repoRoot, GITNEXUS_RC_FILENAME); - let raw: string; try { raw = fs.readFileSync(filePath, 'utf-8'); @@ -379,6 +394,25 @@ export function loadAnalyzeConfig(repoRoot: string): Partial | u throw new GitNexusRcError(`Could not read ${GITNEXUS_RC_FILENAME}: ${(err as Error).message}`); } + return parseAnalyzeConfig(raw); +} + +/** Load `.gitnexusrc` through the strict bounded reader used by watch mode. */ +export async function loadAnalyzeConfigStrict( + repoRoot: string, +): Promise | undefined> { + let raw: string | null; + try { + raw = await readRepoControlFile(repoRoot, GITNEXUS_RC_FILENAME); + } catch (err) { + throw new GitNexusRcError(`Could not read ${GITNEXUS_RC_FILENAME}: ${(err as Error).message}`); + } + return raw === null ? undefined : parseAnalyzeConfig(raw); +} + +function parseAnalyzeConfig(rawInput: string): Partial { + let raw = rawInput; + // Strip a leading UTF-8 BOM: Node's 'utf-8' decode keeps it, and JSON.parse // then fails with a confusing "Unexpected token" on an otherwise-valid file // (#1996 tri-review). Only one leading BOM is stripped; in-string control diff --git a/gitnexus/src/cli/analyze-options.ts b/gitnexus/src/cli/analyze-options.ts index c3d1b3e8f..a749589f1 100644 --- a/gitnexus/src/cli/analyze-options.ts +++ b/gitnexus/src/cli/analyze-options.ts @@ -15,6 +15,10 @@ * import cycle. `analyze.ts` re-exports the type for existing importers. */ export interface AnalyzeOptions { + /** Keep this repository current with serialized incremental refreshes. */ + watch?: boolean; + /** Watch quiet period in milliseconds. */ + debounce?: string; force?: boolean; repairFts?: boolean; /** @@ -120,6 +124,11 @@ export interface AnalyzeOptions { * outside the built-in convention still produces `route_map` consumers. */ fetchWrappers?: string[]; + /** + * Explicit local Spring Boot Actuator snapshot input (#2418). Accepts a JSON + * bundle or a directory containing endpoint JSON files. Disabled by default. + */ + springActuator?: string; /** OpenAI-compatible embeddings base URL (incl. /v1). Overrides GITNEXUS_EMBEDDING_URL. */ embeddingBaseUrl?: string; /** Embedding model name. Overrides GITNEXUS_EMBEDDING_MODEL. */ diff --git a/gitnexus/src/cli/analyze-watch.ts b/gitnexus/src/cli/analyze-watch.ts new file mode 100644 index 000000000..2e4744bc5 --- /dev/null +++ b/gitnexus/src/cli/analyze-watch.ts @@ -0,0 +1,504 @@ +/** Local incremental watch (`gitnexus analyze --watch`). Remote auto-sync lives in `auto-sync.ts`. */ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { watch, type FSWatcher } from 'chokidar'; +import { createWatchIgnorePredicate } from '../config/ignore-service.js'; +import { + analyzeFailureMayHaveMutatedLiveIndex, + runFullAnalysis, + type AnalyzeOptions as CoreAnalyzeOptions, + type AnalyzeResult, +} from '../core/run-analyze.js'; +import { getGitRoot, hasGitDir } from '../storage/git.js'; +import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; +import { GITNEXUS_DIR } from '../storage/repo-meta.js'; +import { + loadAnalyzeConfigStrict, + mergeAnalyzeOptions, + validateBranchName, +} from './analyze-config.js'; +import type { AnalyzeOptions } from './analyze-options.js'; +import { ensureHeap } from './analyze.js'; +import { cliError, cliInfo, cliWarn } from './cli-message.js'; +import { + WATCH_FULL_REFRESH_PATH, + WatchRefreshQueue, + type WatchRefreshError, +} from './watch-queue.js'; + +const DEFAULT_DEBOUNCE_MS = 300; +const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_FILE_SIZE_KB = 32 * 1024; +const TRANSIENT_WATCH_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'ENOTDIR', 'EPERM']); + +export type WatchCliOptions = AnalyzeOptions; + +function posixWatchPath(filePath: string): string { + return filePath.replace(/\\/g, '/').replace(/^\.\/+/, ''); +} + +export function isRelevantWatchPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath); + return ( + normalized.length > 0 && + normalized !== '.' && + !normalized.startsWith('../') && + !path.posix.isAbsolute(normalized) && + !path.win32.isAbsolute(filePath) + ); +} + +function isIgnoreControlPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath); + return normalized === '.gitignore' || normalized === '.gitnexusignore'; +} + +function isConfigControlPath(filePath: string): boolean { + return posixWatchPath(filePath) === '.gitnexusrc'; +} + +function isAnalyzerOwnedWatchPath(filePath: string): boolean { + const normalized = posixWatchPath(filePath).replace(/\/+$/, ''); + return normalized === GITNEXUS_DIR || normalized.startsWith(`${GITNEXUS_DIR}/`); +} + +function repoRelativeWatchPath(repoPath: string, candidate: string): string | null { + const relative = path.relative(repoPath, candidate).replace(/\\/g, '/'); + if (!relative || relative.startsWith('../') || path.isAbsolute(relative)) return null; + return relative; +} + +export interface WatchEnvironmentBaseline { + readonly maxFileSize: string | undefined; + readonly workerTimeout: string | undefined; + readonly verbose: string | undefined; +} + +function setEnvironment(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +function positiveInteger( + value: string | undefined, + flag: string, + maximum?: number, +): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) + throw new Error(`${flag} must be a positive integer`); + if (maximum !== undefined && parsed > maximum) { + throw new Error(`${flag} must not exceed ${maximum}`); + } + return parsed; +} + +export async function resolveWatchOptions( + repoPath: string, + cli: WatchCliOptions, + baseline: WatchEnvironmentBaseline, + reportIgnoredConfig: (names: readonly string[]) => void = () => {}, +): Promise { + const config = (await loadAnalyzeConfigStrict(repoPath)) ?? {}; + const merged = mergeAnalyzeOptions(cli, config); + const unsupported = [ + ['--force', cli.force], + ['--repair-fts', cli.repairFts], + ['--embeddings', cli.embeddings], + ['--drop-embeddings', cli.dropEmbeddings], + ['--skills', cli.skills], + ['--default-branch', cli.defaultBranch], + ['--skip-agents-md', cli.skipAgentsMd], + ['--skip-skills', cli.skipSkills], + ['--no-stats', cli.stats === false], + ['--self-commit', cli.selfCommit], + ['--index-only', cli.indexOnly], + ['--skip-git', cli.skipGit], + ['--spring-actuator', cli.springActuator], + ['walCheckpointThreshold', cli.walCheckpointThreshold], + ['embeddingThreads', cli.embeddingThreads], + ['embeddingBatchSize', cli.embeddingBatchSize], + ['embeddingSubBatchSize', cli.embeddingSubBatchSize], + ['embeddingDevice', cli.embeddingDevice], + ['embeddingBaseUrl', cli.embeddingBaseUrl], + ['embeddingModel', cli.embeddingModel], + ['--embedding-auth-token', cli.embeddingAuthToken], + ['--embedding-dims', cli.embeddingDims], + ].filter(([, value]) => value !== undefined && value !== false); + if (unsupported.length > 0) { + throw new Error( + `analyze --watch does not support ${unsupported.map(([name]) => name).join(', ')}`, + ); + } + reportIgnoredConfig( + [ + ['embeddings', config.embeddings], + ['dropEmbeddings', config.dropEmbeddings], + ['defaultBranch', config.defaultBranch], + ['skipAgentsMd', config.skipAgentsMd !== undefined], + ['skipSkills', config.skipSkills !== undefined], + ['stats', config.stats !== undefined], + ['springActuator', config.springActuator], + ['walCheckpointThreshold', config.walCheckpointThreshold], + ['embeddingThreads', config.embeddingThreads], + ['embeddingBatchSize', config.embeddingBatchSize], + ['embeddingSubBatchSize', config.embeddingSubBatchSize], + ['embeddingDevice', config.embeddingDevice], + ['embeddingBaseUrl', config.embeddingBaseUrl], + ['embeddingModel', config.embeddingModel], + ] + .filter(([, value]) => value !== undefined && value !== false) + .map(([name]) => String(name)), + ); + const branch = + merged.branch === undefined ? undefined : validateBranchName(merged.branch, '--branch'); + const workerPoolSize = positiveInteger(merged.workers, '--workers'); + const workerTimeoutSeconds = positiveInteger(merged.workerTimeout, 'workerTimeout'); + const maxFileSize = positiveInteger(merged.maxFileSize, 'maxFileSize', MAX_FILE_SIZE_KB); + + setEnvironment( + 'GITNEXUS_MAX_FILE_SIZE', + maxFileSize === undefined ? baseline.maxFileSize : String(maxFileSize), + ); + if (workerTimeoutSeconds !== undefined) { + process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS = String(workerTimeoutSeconds * 1000); + } else { + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baseline.workerTimeout); + } + setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose); + + return { + pdg: merged.pdg, + branch, + registryName: merged.name, + allowDuplicateName: merged.allowDuplicateName, + workerPoolSize, + fetchWrappers: merged.fetchWrappers, + skipAgentsMd: true, + skipSkills: true, + noStats: true, + atomicIncremental: process.platform !== 'win32', + }; +} + +function refreshSummary( + result: AnalyzeResult, + observedPaths: readonly string[], + durationMs: number, + lastSuccessfulRefreshAt: string, +): string { + const measured = result.incrementalStats; + const changed = measured?.changedFiles ?? (result.alreadyUpToDate ? 0 : observedPaths.length); + const reparsed = + measured?.reparsedFiles ?? + (typeof result.pipelineResult?.reparsedFileCount === 'number' + ? result.pipelineResult.reparsedFileCount + : 0); + const dependents = measured?.affectedDependents ?? 0; + const mode = measured?.writeMode ?? (result.alreadyUpToDate ? 'no-op' : 'full'); + return ( + `Refresh complete: ${changed} changed, ${reparsed} re-parsed, ` + + `${dependents} affected dependent(s), ${durationMs}ms, ${mode}; ` + + `last success ${lastSuccessfulRefreshAt}` + ); +} + +async function waitUntilReady(watcher: FSWatcher): Promise { + await new Promise((resolve, reject) => { + const ready = () => { + watcher.off('error', failed); + resolve(); + }; + const failed = (error: unknown) => { + watcher.off('ready', ready); + reject(error); + }; + watcher.once('ready', ready); + watcher.once('error', failed); + }); +} + +export interface WatchFileLoop { + readonly waitForIdle: () => Promise; + readonly close: () => Promise; +} + +class WatchControlReloadError extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = 'WatchControlReloadError'; + } +} + +export function shouldStopAfterWatchRefreshFailure( + error: unknown, + paths: readonly string[], +): boolean { + return ( + paths.length > 0 && + !(error instanceof WatchControlReloadError) && + analyzeFailureMayHaveMutatedLiveIndex(error) + ); +} + +/** Start the real filesystem watcher with bounded, serialized refreshes. */ +export async function startWatchFileLoop( + repoPath: string, + debounceMs: number, + refresh: (paths: readonly string[]) => Promise, + onError: WatchRefreshError, + onWatcherError: (error: unknown) => void = (error) => onError(error, []), +): Promise { + let ignorePath = await createWatchIgnorePredicate(repoPath); + let ignoreControlValid = true; + const queue = new WatchRefreshQueue( + async (paths) => { + if (paths.some(isIgnoreControlPath) || !ignoreControlValid) { + const retryingInvalidControls = !ignoreControlValid; + try { + ignorePath = await createWatchIgnorePredicate(repoPath); + ignoreControlValid = true; + watcher.add(repoPath); + } catch (error) { + ignoreControlValid = false; + throw new WatchControlReloadError( + retryingInvalidControls + ? new Error( + 'Ignore controls remain invalid; fix them before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + await refresh(paths); + }, + onError, + debounceMs, + { + maxWaitMs: Math.max(2_000, debounceMs * 10), + maxPendingPaths: 1_000, + holdEventsUntilInitialRefresh: true, + isPriorityPath: (filePath) => isIgnoreControlPath(filePath) || isConfigControlPath(filePath), + }, + ); + + const watcher: FSWatcher = watch(repoPath, { + ignoreInitial: true, + atomic: true, + followSymlinks: false, + awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 20 }, + ignored: (candidate, stats) => { + const relative = repoRelativeWatchPath(repoPath, candidate); + if (relative !== null && isAnalyzerOwnedWatchPath(relative)) return true; + if (relative !== null && (isIgnoreControlPath(relative) || isConfigControlPath(relative))) { + return false; + } + return ignorePath(candidate, stats?.isDirectory() ?? false); + }, + }); + watcher.on('all', (event, changedPath) => { + if (event !== 'add' && event !== 'change' && event !== 'unlink') return; + const relative = repoRelativeWatchPath(repoPath, changedPath); + if (relative && isRelevantWatchPath(relative) && !isAnalyzerOwnedWatchPath(relative)) { + queue.enqueue(relative); + } + }); + watcher.on('error', (error) => { + // Chokidar can surface a transient EPERM on Windows while an ignored + // analyzer-owned path is replaced. Re-arm the root and force one bounded + // catch-up refresh so a missed event cannot leave the graph stale. Other + // watcher errors may mean coverage was lost and remain fatal. + if (TRANSIENT_WATCH_ERROR_CODES.has((error as NodeJS.ErrnoException).code ?? '')) { + watcher.add(repoPath); + queue.enqueue(WATCH_FULL_REFRESH_PATH); + return; + } + onWatcherError(error); + }); + + try { + await waitUntilReady(watcher); + await queue.runInitial(); + } catch (error) { + await watcher.close(); + await queue.close(); + throw error; + } + + return { + waitForIdle: () => queue.waitForIdle(), + close: async () => { + await watcher.close(); + await queue.close(); + }, + }; +} + +export async function watchCommandWithRunnerIdentity( + runnerIdentityAtBootstrap: AnalyzerRunnerIdentity, + inputPath?: string, + cliOptions: WatchCliOptions = {}, +): Promise { + if (await ensureHeap({ cleanForwardedTermination: true })) return; + + const requestedRepoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (requestedRepoPath === null || !hasGitDir(requestedRepoPath)) { + cliError(' gitnexus analyze --watch requires a Git repository.'); + process.exitCode = 1; + return; + } + const repoPath = await fs.realpath(requestedRepoPath); + const baselineEnvironment: WatchEnvironmentBaseline = { + maxFileSize: process.env.GITNEXUS_MAX_FILE_SIZE, + workerTimeout: process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS, + verbose: process.env.GITNEXUS_VERBOSE, + }; + try { + let ignoredConfigSignature: string | undefined; + const reportIgnoredConfig = (names: readonly string[]) => { + const signature = [...names].sort().join(','); + if (signature === ignoredConfigSignature) return; + ignoredConfigSignature = signature; + if (names.length > 0) { + cliWarn(`Watch mode ignores unsupported .gitnexusrc settings: ${names.join(', ')}.`); + } + }; + let debounceMs: number; + let analyzeOptions: CoreAnalyzeOptions; + try { + debounceMs = + positiveInteger( + cliOptions.debounce ?? String(DEFAULT_DEBOUNCE_MS), + '--debounce', + MAX_TIMER_DELAY_MS, + ) ?? DEFAULT_DEBOUNCE_MS; + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + } catch (error) { + cliError(` ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + return; + } + + let stopWatching!: () => void; + const stopped = new Promise((resolve) => { + stopWatching = resolve; + }); + const stop = () => stopWatching(); + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + try { + let loop: WatchFileLoop; + let fatalRefreshError: unknown; + let configControlValid = true; + let lastSuccessfulRefreshAt: string | undefined; + try { + loop = await startWatchFileLoop( + repoPath, + debounceMs, + async (paths) => { + if (paths.some(isConfigControlPath) || !configControlValid) { + const retryingInvalidConfig = !configControlValid; + try { + analyzeOptions = await resolveWatchOptions( + repoPath, + cliOptions, + baselineEnvironment, + reportIgnoredConfig, + ); + configControlValid = true; + } catch (error) { + configControlValid = false; + throw new WatchControlReloadError( + retryingInvalidConfig + ? new Error( + 'Configuration remains invalid; fix it before indexing more changes.', + { + cause: error, + }, + ) + : error, + ); + } + } + const startedAt = Date.now(); + const result = await runFullAnalysis( + repoPath, + analyzeOptions, + { + onProgress: () => {}, + onLog: + process.env.GITNEXUS_VERBOSE === '1' + ? (message) => cliInfo(` ${message}`) + : undefined, + }, + runnerIdentityAtBootstrap, + ); + lastSuccessfulRefreshAt = new Date().toISOString(); + if (paths.length === 0) { + cliInfo( + result.alreadyUpToDate + ? `Watching ${repoPath}; index is up to date.` + : `Watching ${repoPath}; initial index ready in ${Date.now() - startedAt}ms.`, + ); + } else { + cliInfo( + refreshSummary(result, paths, Date.now() - startedAt, lastSuccessfulRefreshAt), + ); + } + }, + (error, paths) => { + const detail = paths.length > 0 ? ` (${paths.length} queued path(s))` : ''; + if (shouldStopAfterWatchRefreshFailure(error, paths)) { + fatalRefreshError = error; + cliError( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping because the live index may have been updated in place.', + ); + stopWatching(); + return; + } + const lastSuccess = lastSuccessfulRefreshAt ?? 'none yet'; + cliWarn( + `Refresh failed${detail}: ${error instanceof Error ? error.message : String(error)}. ` + + `Retry scheduled; last success ${lastSuccess}.`, + ); + }, + (error) => { + fatalRefreshError = error; + cliError( + `Watcher failed: ${error instanceof Error ? error.message : String(error)}. ` + + 'Watch mode is stopping.', + ); + stopWatching(); + }, + ); + } catch (error) { + cliError( + ` Unable to start watcher: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + return; + } + + await stopped; + await loop.close(); + if (fatalRefreshError !== undefined) process.exitCode = 1; + } finally { + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); + } + } finally { + setEnvironment('GITNEXUS_MAX_FILE_SIZE', baselineEnvironment.maxFileSize); + setEnvironment('GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', baselineEnvironment.workerTimeout); + setEnvironment('GITNEXUS_VERBOSE', baselineEnvironment.verbose); + } +} diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index e2208a3c8..beb553e01 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -363,6 +363,7 @@ interface RespawnExit { stdout?: string; stderr?: string; message?: string; + forwardedSignal?: NodeJS.Signals; } const appendOutputTail = (tail: string, chunk: unknown): string => { @@ -395,17 +396,28 @@ const runRespawnedAnalyze = ( let stdout = ''; let stderr = ''; let settled = false; - const finish = (exit: RespawnExit): void => { - if (settled) return; - settled = true; - resolve(exit); - }; - + let forwardedSignal: NodeJS.Signals | undefined; const child = spawn(process.execPath, [...args], { stdio: ['inherit', 'pipe', 'pipe'], windowsHide: true, env, }); + const forwardSignal = (signal: NodeJS.Signals): void => { + forwardedSignal ??= signal; + if (child.exitCode === null && child.signalCode === null) child.kill(signal); + }; + const forwardSigint = () => forwardSignal('SIGINT'); + const forwardSigterm = () => forwardSignal('SIGTERM'); + const finish = (exit: RespawnExit): void => { + if (settled) return; + settled = true; + process.removeListener('SIGINT', forwardSigint); + process.removeListener('SIGTERM', forwardSigterm); + resolve({ ...exit, forwardedSignal }); + }; + + process.once('SIGINT', forwardSigint); + process.once('SIGTERM', forwardSigterm); child.stdout?.on('data', (chunk) => { stdout = appendOutputTail(stdout, chunk); @@ -548,7 +560,16 @@ export function parseMaxOldSpaceMb(nodeOptions: string): number | null { * tooling), not a deliberate per-run choice: warn and respawn with the * auto cap. Pre-#2649 this returned early and large repos then OOM'd on * whatever heap the environment happened to specify. */ -async function ensureHeap(): Promise { +export function forwardedSignalExitCode(signal: NodeJS.Signals, cleanTermination: boolean): number { + if (cleanTermination) return 0; + if (signal === 'SIGINT') return 130; + if (signal === 'SIGTERM') return 143; + return 1; +} + +export async function ensureHeap( + options: { cleanForwardedTermination?: boolean } = {}, +): Promise { // Explicit opt-out disables auto-sizing ENTIRELY — both the ambient-pin // override and the default v8-limit respawn — and is honored SILENTLY: // the operator already made the call, and stderr-sensitive consumers @@ -590,6 +611,13 @@ async function ensureHeap(): Promise { }; if (shouldBridgeRespawnProgressTty()) childEnv[RESPAWN_PROGRESS_ENV] = '1'; const childExit = await runRespawnedAnalyze(childArgs, childEnv); + if (childExit.forwardedSignal !== undefined) { + process.exitCode = forwardedSignalExitCode( + childExit.forwardedSignal, + options.cleanForwardedTermination === true, + ); + return true; + } if (childExit.status !== 0 || childExit.signal) { if (childProcessLikelyOom(childExit)) { cliError( @@ -640,6 +668,8 @@ const ANALYZE_CLI_ENV_KEYS = [ 'GITNEXUS_EMBEDDING_SUB_BATCH_SIZE', 'GITNEXUS_EMBEDDING_DEVICE', 'GITNEXUS_ANALYZE_PROGRESS_ACTIVE', + 'GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS', + 'GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO', 'GITNEXUS_EMBEDDING_URL', 'GITNEXUS_EMBEDDING_MODEL', 'GITNEXUS_EMBEDDING_API_KEY', @@ -740,6 +770,19 @@ export const analyzeCommandWithRunnerIdentity = async ( options?: AnalyzeOptions, ): Promise => analyzeCommand(inputPath, options, runnerIdentityAtBootstrap); +export async function analyzeOrWatchCommandWithRunnerIdentity( + runnerIdentityAtBootstrap: AnalyzerRunnerIdentity, + inputPath?: string, + options: AnalyzeOptions = {}, +): Promise { + if (options.watch) { + const { watchCommandWithRunnerIdentity } = await import('./analyze-watch.js'); + await watchCommandWithRunnerIdentity(runnerIdentityAtBootstrap, inputPath, options); + return; + } + await analyzeCommandWithRunnerIdentity(runnerIdentityAtBootstrap, inputPath, options); +} + const analyzeCommandImpl = async ( inputPath?: string, cliOptions?: AnalyzeOptions, @@ -1331,6 +1374,7 @@ const analyzeCommandImpl = async ( // Extra fetch-wrapper names from `.gitnexusrc` (#1589/#1852 residual); // forwarded to the routes phase consumer scan. fetchWrappers: options.fetchWrappers, + springActuatorPath: options.springActuator, // The CLI always process.exit()s after this returns (success path at the // end of analyzeCommandImpl, error/interrupt paths via process.exit too), // so the finalize close skips the native conn/db close — it can double-free @@ -1395,6 +1439,9 @@ const analyzeCommandImpl = async ( console.error = origError; bar.stop(); console.log(' Already up to date\n'); + if (runOptions.registryName) { + console.log(` Registry name: ${result.repoName}\n`); + } if (baseRefRefreshed.length > 0) { console.log( ` Updated base_ref to "${resolvedDefaultBranch}" in ${baseRefRefreshed.join(', ')}\n`, @@ -1486,6 +1533,7 @@ const analyzeCommandImpl = async ( // exercised on the `--skills` path by analyze-no-stats-bridge.test.ts. noStats: options.stats === false, hasPdg: options.pdg === true, + hasSpringActuator: options.springActuator !== undefined, }, ); } diff --git a/gitnexus/src/cli/auto-sync.ts b/gitnexus/src/cli/auto-sync.ts new file mode 100644 index 000000000..880deb5d5 --- /dev/null +++ b/gitnexus/src/cli/auto-sync.ts @@ -0,0 +1,125 @@ +/** Remote auto-sync CLI (`gitnexus auto-sync`). Local incremental watch lives in `analyze-watch.ts`. */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + getAutoSyncConfigPath, + getAutoSyncMutexPath, + readAutoSyncWatchStatus, + resetAutoSyncState, + startAutoSyncWatch, + stopAutoSyncWatch, + type WatchStatusRecord, +} from '../core/auto-sync/index.js'; + +export async function autoSyncCommand(action = 'start'): Promise { + if (action === 'init') { + await initWatchConfig(); + return; + } + if (action === 'reset') { + if (!(await resetAutoSyncState())) { + process.stderr.write( + `[auto-sync] Cannot reset analysis state while the watch mutex is held. Confirm no watch process is running, then remove ${getAutoSyncMutexPath()}.\n`, + ); + process.exitCode = 1; + return; + } + process.stdout.write('[auto-sync] Reset analysis state.\n'); + return; + } + if (action === 'status') { + printStatus(await readAutoSyncWatchStatus()); + return; + } + if (action === 'stop') { + if ((await stopAutoSyncWatch()) !== 'stopped') process.exitCode = 1; + return; + } + if (action === 'restart') { + const result = await stopAutoSyncWatch(); + if (result === 'refused' || result === 'timeout') { + process.exitCode = 1; + return; + } + await startWatchProcess(); + return; + } + if (action !== 'start') { + process.stderr.write(`[auto-sync] Unknown auto-sync action: ${action}\n`); + process.exitCode = 1; + return; + } + await startWatchProcess(); +} + +async function startWatchProcess(): Promise { + const handle = await startAutoSyncWatch(); + if (!handle) { + process.exitCode = 1; + return; + } + + const stop = () => { + void handle.stop().then( + () => { + process.stderr.write('[auto-sync] Watch stopped.\n'); + process.exit(0); + }, + (error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[auto-sync] Failed to stop watch: ${message}\n`); + process.exit(1); + }, + ); + }; + process.once('SIGINT', stop); + process.once('SIGTERM', stop); +} + +function printStatus(status: WatchStatusRecord): void { + const parts = [`state=${status.state}`]; + if (status.pid) parts.push(`pid=${status.pid}`); + if (status.configPath) parts.push(`config=${status.configPath}`); + if (status.message) parts.push(`message=${status.message}`); + parts.push(`updated_at=${status.updatedAt}`); + process.stdout.write(`${parts.join(' ')}\n`); +} + +async function initWatchConfig(): Promise { + const configPath = getAutoSyncConfigPath(); + try { + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile( + configPath, + defaultSyncConfig(path.resolve(path.dirname(configPath), 'repos')), + { + flag: 'wx', + }, + ); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'EEXIST') { + process.stderr.write(`[auto-sync] Config already exists: ${configPath}\n`); + process.exitCode = 1; + return; + } + throw err; + } + process.stdout.write(`[auto-sync] Created ${configPath}\n`); +} + +function defaultSyncConfig(localPath: string): string { + return [ + 'sync_interval_minutes: 10', + 'max_concurrency: 1', + 'repo_git_timeout: 10s', + 'analyze_timeout: 5m', + 'analyze_failure_threshold: 3', + 'projects:', + ` - local_path: ${localPath}`, + ' branches: [master, main]', + ' overwrite_local_changes: false', + ' remote_urls:', + ' - git@github.com:owner/repo.git', + '', + ].join('\n'); +} diff --git a/gitnexus/src/cli/detect-changes-format.ts b/gitnexus/src/cli/detect-changes-format.ts index 98ecffa3e..9e91cc330 100644 --- a/gitnexus/src/cli/detect-changes-format.ts +++ b/gitnexus/src/cli/detect-changes-format.ts @@ -6,6 +6,7 @@ type DetectChangesSummary = { changed_count?: number; affected_count?: number; risk_level?: string; + message?: string; }; type ChangedSymbol = { @@ -55,6 +56,29 @@ export function formatDetectChangesResult(result: unknown): string { ); if ((summary.changed_count ?? 0) === 0) { + // Parse-fail payloads set `partial` and an honest `message` (#2915/#3131). + // Production *clean* trees also set English `message: 'No changes detected.'` + // — that must go through `t('tool.detectChanges.noChanges')` or zh-CN never + // fires. Only pass the backend string through on a degraded/parse-fail run. + if ( + payload.partial && + typeof summary.message === 'string' && + summary.message.trim().length > 0 + ) { + return [...notes, summary.message.trim()].join('\n'); + } + // Confirmed no-overlap: files parsed, mapping succeeded, zero symbols. + // `queryDegraded` is `partial: true` with the same counts and no message — + // do not call that a confirmed mapping (#3131 honesty). + if (!payload.partial && (summary.changed_files ?? 0) > 0) { + return [ + ...notes, + t('tool.detectChanges.noOverlappingSymbols', { files: summary.changed_files }), + ].join('\n'); + } + if (payload.partial) { + return notes.join('\n'); + } return [...notes, t('tool.detectChanges.noChanges')].join('\n'); } diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index caf19bc03..b2f656c39 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -302,6 +302,20 @@ function formatTruncationSuffix(result: { return label ? ` (by ${label})` : ''; } +function pushCallgraphRiskLines(lines: string[], result: any): void { + if (result.risk) { + lines.push(`Risk: ${result.risk}`); + } + if (result.riskNote) { + lines.push(String(result.riskNote)); + } + if (result.riskScale?.comparableAcrossKinds === false && result.riskSharedAxes) { + lines.push( + `Shared-axes risk: ${result.riskSharedAxes} (process/module axes are unavailable — compare File vs symbol only; do not use this to waive a HIGH/CRITICAL risk warning)`, + ); + } +} + export function formatImpactResult(result: any): string { if (result.error) { const suggestion = result.suggestion ? `\nSuggestion: ${result.suggestion}` : ''; @@ -567,14 +581,21 @@ export function formatImpactResult(result: any): string { // #1858 — "isolated" is a confident claim. If an interface / indirection // boundary is on the path, the true count is a lower bound, not zero; // callers binding via DI / dynamic dispatch were not traced. Say so instead. + const lines: string[] = []; if (result.epistemic === 'lower-bound') { - const lines = [ + lines.push( `${target?.name || '?'}: no direct ${direction} dependencies traced, but this is a LOWER BOUND — unresolved indirection on the path (actual impact may be higher):`, - ]; + ); for (const b of result.boundaries || []) lines.push(` • ${b}`); - return lines.join('\n'); + } else if (direction === 'upstream') { + lines.push( + `${target?.name || '?'}: No ${direction} callers resolved. This is not evidence the symbol is unused or isolated.`, + ); + } else { + lines.push(`${target?.name || '?'}: No ${direction} dependencies found.`); } - return `${target?.name || '?'}: No ${direction} dependencies found. This symbol appears isolated.`; + pushCallgraphRiskLines(lines, result); + return lines.join('\n'); } const lines: string[] = []; @@ -594,6 +615,7 @@ export function formatImpactResult(result: any): string { ); for (const b of result.boundaries || []) lines.push(` • ${b}`); } + pushCallgraphRiskLines(lines, result); lines.push(''); const depthLabels: Record = { diff --git a/gitnexus/src/cli/group.ts b/gitnexus/src/cli/group.ts index 3111f69d0..abc13fa2b 100644 --- a/gitnexus/src/cli/group.ts +++ b/gitnexus/src/cli/group.ts @@ -1,6 +1,8 @@ // gitnexus/src/cli/group.ts import { createRequire } from 'node:module'; import type { Command } from 'commander'; +import type { RegistryWriteOutcome } from '../core/group/sync.js'; +import type { MatchType } from '../core/group/types.js'; import { logger } from '../core/logger.js'; const _require = createRequire(import.meta.url); @@ -120,16 +122,43 @@ export function registerGroupCommands(program: Command): void { indexStale: boolean; contractsStale: boolean; missing: boolean; + /** + * Optional here on purpose: a payload produced before the split + * carries no such key, and an absent one must degrade to the + * label this command has always printed rather than to the new + * one — an unrecorded cause is not evidence of a cause. + */ + unresolvable?: boolean; + unresolvableReason?: string; commitsBehind?: number; } >; missingRepos?: string[]; + unreadableRepos?: string[]; + suppressedMatchStages?: string[]; }; console.log(' Repo index / contracts staleness:'); for (const [repoPath, row] of Object.entries(st.repos || {})) { if (row.missing) { - console.log(` ${repoPath.padEnd(25)} MISSING (not in registry or unreadable)`); + // Two different facts with two different remedies: a repo the + // registry never heard of is fixed by indexing it, while an entry + // the resolver choked on is fixed by repairing the registry. + // Printing "no entry in the registry" for the second one states a + // cause that was never measured, and points at the wrong repair. + if (row.unresolvable) { + // The reason can be multi-line — an ambiguous registry names + // every colliding clone. Fold it onto this row's line rather + // than truncating it: those paths are what the operator acts on, + // and a table row that swallows half its own explanation is the + // failure this label exists to stop. + const why = (row.unresolvableReason ?? 'the registry entry could not be resolved') + .replace(/\s+/g, ' ') + .trim(); + console.log(` ${repoPath.padEnd(25)} UNRESOLVABLE (${why})`); + continue; + } + console.log(` ${repoPath.padEnd(25)} MISSING (no entry in the registry)`); continue; } const idx = row.indexStale @@ -138,9 +167,41 @@ export function registerGroupCommands(program: Command): void { const ctr = row.contractsStale ? ' CONTRACTS_STALE' : ''; console.log(` ${repoPath.padEnd(25)} ${idx}${ctr}`); } + // `undefined` and `[]` are different answers here: a registry written + // before this was tracked has no opinion, while an empty array is a + // measurement. Printing nothing for both would let an unmeasured sync + // read as evidence that every index opened cleanly. + // + // `undefined` covers two ways of not knowing — the field is absent, or + // it held something that was not a list of repo paths and `getStatus` + // declined to guess. Naming only the first would make a corrupt + // registry read as a merely old one, which is the same shape of wrong + // answer this command exists to stop giving. + const unreadable = st.unreadableRepos; + if (unreadable === undefined) { + console.log( + `\n Last sync unreadable repos: not recorded` + + `\n (the registry predates this field, or its value could not be read)` + + `\n Re-run \`gitnexus group sync\` to record it.`, + ); + } else if (unreadable.length > 0) { + console.log(`\n Last sync unreadable repos: ${unreadable.join(', ')}`); + } if ((st.missingRepos || []).length > 0) { console.log(`\n Last sync missing repos: ${st.missingRepos!.join(', ')}`); } + // Only the populated case prints. Absent means a registry that predates + // the field, and empty is the ordinary clean sync — neither is worth a + // line, whereas a narrowed registry changes how every later answer + // should be read. + const skippedStages = st.suppressedMatchStages ?? []; + if (skippedStages.length > 0) { + console.log( + `\n Last sync skipped matching stages: ${skippedStages.join(', ')}` + + `\n Cross-links those stages would have found are absent by request.` + + `\n Re-run \`gitnexus group sync\` without --exact-only for the complete set.`, + ); + } } finally { await backend.dispose().catch(() => {}); } @@ -149,39 +210,137 @@ export function registerGroupCommands(program: Command): void { group .command('sync ') .description('Sync Contract Registry — extract contracts and build cross-links') - .option('--skip-embeddings', 'Exact + BM25 only (no embedding fallback)') - .option('--exact-only', 'Exact match only') - .option('--allow-stale', 'Skip stale index warnings') - .option('--verbose', 'Show each cross-link detail') + .option( + '--exact-only', + 'Skip wildcard service matching; cross-link on exact contract-id match only (manifest links still apply)', + ) + .option('--verbose', 'Show additional sync diagnostics') .option('--json', 'JSON output') .action(async (name: string, opts: Record) => { const { getGroupDir, getDefaultGitnexusDir } = await import('../core/group/storage.js'); const { loadGroupConfig } = await import('../core/group/config-parser.js'); - const { syncGroup } = await import('../core/group/sync.js'); + const { syncGroup, formatGroupSyncAmbiguousError } = await import('../core/group/sync.js'); + const { GroupSyncLockError } = await import('../core/group/group-lock.js'); + const { RegistryAmbiguousTargetError } = await import('../storage/repo-manager.js'); const groupDir = getGroupDir(getDefaultGitnexusDir(), name); const config = await loadGroupConfig(groupDir); console.log(`Syncing group "${name}" (${Object.keys(config.repos).length} repos)...\n`); - const result = await syncGroup(config, { - groupDir, - allowStale: Boolean(opts.allowStale), - verbose: Boolean(opts.verbose), - skipEmbeddings: Boolean(opts.skipEmbeddings), - exactOnly: Boolean(opts.exactOnly), - }); + let result: Awaited>; + try { + result = await syncGroup(config, { + groupDir, + verbose: Boolean(opts.verbose), + exactOnly: Boolean(opts.exactOnly), + }); + } catch (err) { + if (err instanceof RegistryAmbiguousTargetError) { + logger.error(`⚠️ Did not sync group "${name}": ${formatGroupSyncAmbiguousError(err)}`); + process.exitCode = 1; + return; + } + // A sync that could not take the group's lock did NOT run and wrote + // nothing (R9 fails closed). That is an operator-actionable outcome, not + // a crash, so report it as a failed command rather than letting it + // surface as an unhandled rejection with a stack trace — commander's + // async actions have no error handler, so an uncaught throw here would + // print exactly that. + if (!(err instanceof GroupSyncLockError)) throw err; + logger.error(`⚠️ Did not sync group "${name}": ${err.message}`); + process.exitCode = 1; + return; + } if (opts.json) { console.log(JSON.stringify(result, null, 2)); } else { - console.log(`\nMatching cascade:`); - const exactLinks = result.crossLinks.filter((l) => l.matchType === 'exact'); - console.log(` exact: ${exactLinks.length} cross-links (confidence 1.0)`); - console.log(` unmatched: ${result.unmatched.length} contracts`); - console.log( - `\nWrote contracts.json (${result.contracts.length} contracts, ${result.crossLinks.length} cross-links)`, - ); + // Repos we could not read are the most likely explanation for a small + // or empty contract count, so they are reported before the counts — + // otherwise a run that read nothing looks exactly like a clean run. + if (result.unreadableRepos.length > 0) { + // No "re-run with GITNEXUS_LOG_LEVEL=warn" hint: the default level is + // `info`, and pino emits `warn` (40) at `info` (30), so the reason was + // already printed by this same run — raising the level to `warn` would + // only suppress the surrounding `info` output. + console.log( + `\n ⚠️ Could not extract contracts from: ${result.unreadableRepos.join(', ')}` + + `\n None of their contracts are included in this sync (the warning above says why),` + + `\n or check \`gitnexus doctor\` in the affected repo.`, + ); + } + if (result.missingRepos.length > 0) { + console.log( + `\n ⚠️ Not found in the registry: ${result.missingRepos.join(', ')}` + + `\n Index them with \`gitnexus analyze\`, or remove them from group.yaml.`, + ); + } + // Every stage that produced a link, not just `exact`. This used to print + // `Matching cascade:` and then count `exact` alone, while the `Wrote + // contracts.json (…)` line below reports `result.crossLinks.length` — + // which also includes `manifest` and `wildcard` links. For any group with + // those, the two numbers disagreed with nothing on screen explaining why. + // Summing the stages here makes them reconcile by construction. + console.log(`\nMatching:`); + // Exhaustive by construction, same idiom as OUTCOME_LINE below: adding a + // MatchType fails the build here instead of silently going uncounted and + // reopening the very mismatch this replaced. Every stage prints even at + // zero — a stage that is absent reads as "did not apply", not "found none". + const STAGE_COUNTS: Record = { + exact: 0, + manifest: 0, + wildcard: 0, + }; + for (const link of result.crossLinks) STAGE_COUNTS[link.matchType] += 1; + // A stage the sync was told to skip is reported as skipped, not as a + // zero count. The two are different facts — "ran, matched nothing" and + // "never ran" — and printing both as `0` is the same conflation this + // block replaced. Driven by what the sync did (`suppressedMatchStages`) + // rather than by what the caller asked for, so it stays correct on the + // outcomes where the run ended without writing a registry. + for (const stage of Object.keys(STAGE_COUNTS) as MatchType[]) { + const count = STAGE_COUNTS[stage]; + const label = `${stage}:`.padEnd(10); + if (result.suppressedMatchStages.includes(stage)) { + console.log(` ${label} skipped (--exact-only)`); + continue; + } + const confidence = stage === 'exact' ? ' (confidence 1.0)' : ''; + console.log(` ${label} ${count} cross-links${confidence}`); + } + console.log(` ${'unmatched:'.padEnd(10)} ${result.unmatched.length} contracts`); + // Driven by what actually happened to the file. This line used to be + // unconditional, so a run that deliberately preserved the previous + // registry still announced `Wrote contracts.json (0 contracts, 0 + // cross-links)` — a confident false statement about persisted state, on + // the exact path this command exists to make legible. + // Exhaustive by construction: a `Record` keyed on the union means a + // new outcome fails the build here instead of printing nothing, which + // is what previously pushed a distinct state into `preserved` and made + // this summary false on one of the two branches it then covered. + const OUTCOME_LINE: Record = { + written: + `\nWrote contracts.json (${result.contracts.length} contracts, ` + + `${result.crossLinks.length} cross-links)`, + preserved: + `\nKept the previous contracts.json — no repo in this group could be read.` + + `\n Its contracts and cross-links are unchanged; only the unreadable/missing` + + `\n repo lists were refreshed to describe THIS run. Fix the repos above and re-run.`, + superseded: + `\nDid NOT touch contracts.json — no repo in this group could be read, and another` + + `\n sync replaced the file while this one waited for the group lock. That sync's` + + `\n result stands and this run's repo lists were NOT recorded: they describe a` + + `\n group state older than what is on disk. Fix the repos above and re-run.`, + 'no-prior-registry': + `\nDid NOT write contracts.json — no repo in this group could be read,` + + `\n and there is no previous contracts.json to fall back on. Fix the repos` + + `\n above and re-run.`, + // Nothing to say: the caller asked for no write. + 'not-attempted': null, + }; + const line = OUTCOME_LINE[result.registryOutcome]; + if (line) console.log(line); } }); @@ -281,11 +440,28 @@ export function registerGroupCommands(program: Command): void { // repos — reporting it as crossings understates a fan-out cap the // same way #2787's totals did. const dropped = (raw as { truncatedRepos?: string[] })?.truncatedRepos ?? []; - console.log( - dropped.length > 0 - ? ` risk is a LOWER BOUND — fan-out stopped early; crossings to ${dropped.length} repo(s) not traversed: ${dropped.join(', ')}` - : ' risk is a LOWER BOUND — the local impact walk did not complete (every bridge crossing was traversed)', - ); + const reason = (raw as { truncationReason?: string })?.truncationReason; + // Keyed on the REASON, not on which incidental fact happens to be + // non-empty. `truncatedRepos` is populated for a structural gap too + // — the bridge's incomplete repos are unioned into it even when ZERO + // crossings were attempted — so branching on its length first + // reported "fan-out stopped early" for a run where nothing stopped + // early, and omitted the only remedy that works. Same false-cause + // shape the contract listing was just re-gated for, one command over. + const floorReason = (): string => { + if (reason === 'suppressed-stage') { + return 'the last sync skipped a matching stage (--exact-only); re-run `gitnexus group sync` without it for the complete graph'; + } + if (reason === 'incomplete-sync') { + return dropped.length > 0 + ? `the last sync could not account for ${dropped.join(', ')}; their contracts are absent from every query against this bridge — re-run \`gitnexus group sync\`` + : 'the last sync could not say which repos it read — re-run `gitnexus group sync`'; + } + return dropped.length > 0 + ? `fan-out stopped early; crossings to ${dropped.length} repo(s) not traversed: ${dropped.join(', ')}` + : 'the local impact walk did not complete (every bridge crossing was traversed)'; + }; + console.log(` risk is a LOWER BOUND — ${floorReason()}`); } } } finally { @@ -370,7 +546,15 @@ export function registerGroupCommands(program: Command): void { return; } - const { contracts, crossLinks } = raw as { + const { + contracts, + crossLinks, + truncated, + unreadableRepos, + missingRepos, + suppressedMatchStages, + truncationReason, + } = raw as { contracts: Array<{ role: string; contractId: string; @@ -384,10 +568,21 @@ export function registerGroupCommands(program: Command): void { confidence: number; contractId: string; }>; + truncated?: boolean; + suppressedMatchStages?: string[]; + truncationReason?: string; + unreadableRepos?: string[]; + missingRepos?: string[]; }; if (opts.json) { - console.log(JSON.stringify({ contracts, crossLinks }, null, 2)); + // The whole payload, not a re-serialized subset. Destructuring the two + // fields this command happens to print and rebuilding an object from + // them dropped everything else the service returned — which is how the + // completeness fields were invisible here while the MCP tool carried + // them. Printing `raw` means a field added to the service reaches + // `--json` without a matching edit in this file. + console.log(JSON.stringify(raw, null, 2)); } else { console.log(`Contracts (${contracts.length}):`); for (const c of contracts) { @@ -399,6 +594,39 @@ export function registerGroupCommands(program: Command): void { ` ${l.from.repo} -> ${l.to.repo} [${l.matchType}, conf=${l.confidence}] ${l.contractId}`, ); } + // Separate from `truncated` below, and deliberately so: that one means + // the sync could not read something and the remedy is to fix the repo. + // This one means the sync was ASKED to skip a stage, and the remedy is + // to re-run without the flag. A listing narrowed on purpose is still + // narrowed, and without this the human view showed nothing at all. + if (suppressedMatchStages && suppressedMatchStages.length > 0) { + console.log( + `\n⚠️ This listing is a lower bound: the last sync skipped ${suppressedMatchStages.join(', ')} matching` + + `\n (--exact-only), so cross-links that stage would have found are absent.` + + `\n Re-run \`gitnexus group sync\` without --exact-only for the complete set.`, + ); + } + // Gated on the REASON, not just the flag. A suppressed stage sets + // `truncated` with both repo lists empty, which sent this block down + // its else-branch and printed "the last sync did not record which + // repos it could read" — a false statement, with the wrong remedy, + // about a sync that recorded them fine. The suppressed-stage warning + // above already said the true thing. When a repo gap co-occurs the + // reason is 'incomplete-sync' (the repo side takes precedence in + // `crossRepoCompleteness`), so this block still runs for it. + if (truncated && truncationReason !== 'suppressed-stage') { + // Counts above are a floor, not a census. Name the repos when the + // registry recorded them, and say so plainly when it did not — a + // listing that cannot say what it is missing is still incomplete. + const absent = [...(unreadableRepos ?? []), ...(missingRepos ?? [])]; + console.log( + absent.length > 0 + ? `\n⚠️ This listing is incomplete: the last sync could not account for ${absent.join(', ')}.` + + `\n Contracts from those repos are absent, so the counts above are a lower bound.` + : `\n⚠️ This listing is incomplete: the last sync did not record which repos it could` + + `\n read, so the counts above are a lower bound. Re-run group sync.`, + ); + } } } finally { await backend.dispose().catch(() => {}); diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index 58f28d11a..283e99832 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -13,6 +13,8 @@ const COMMAND_DESCRIPTION_KEYS = { '': 'help.description.root', setup: 'help.command.setup.description', uninstall: 'help.command.uninstall.description', + watch: 'help.command.watch.description', + 'auto-sync': 'help.command.autoSync.description', analyze: 'help.command.analyze.description', index: 'help.command.index.description', serve: 'help.command.serve.description', @@ -72,6 +74,8 @@ const OPTION_DESCRIPTION_KEYS = { 'analyze|--embedding-batch-size ': 'help.option.analyze.embeddingBatchSize', 'analyze|--embedding-sub-batch-size ': 'help.option.analyze.embeddingSubBatchSize', 'analyze|--embedding-device ': 'help.option.analyze.embeddingDevice', + 'analyze|--watch': 'help.option.analyze.watch', + 'analyze|--debounce ': 'help.option.analyze.debounce', 'index|-f, --force': 'help.option.index.force', 'index|--allow-non-git': 'help.option.index.allowNonGit', 'mcp|--http': 'help.option.mcp.http', @@ -155,9 +159,7 @@ const OPTION_DESCRIPTION_KEYS = { 'embeddings install|--cuda': 'help.option.embeddings.install.cuda', 'embeddings install|--force': 'help.option.embeddings.install.force', 'group create|--force': 'help.option.group.create.force', - 'group sync|--skip-embeddings': 'help.option.group.sync.skipEmbeddings', 'group sync|--exact-only': 'help.option.group.sync.exactOnly', - 'group sync|--allow-stale': 'help.option.group.sync.allowStale', 'group sync|--verbose': 'help.option.group.sync.verbose', 'group sync|--json': 'help.option.json', 'group impact|--target ': 'help.option.group.impact.target', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index aaaf442cb..58904c48f 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -33,6 +33,17 @@ export const en = { 'status.workspaceIndexLabel': "Workspace index: last analyzed on '{{primary}}' (re-run gitnexus analyze to follow the current branch)", 'status.status': 'Status', + 'status.indexContentCurrent': 'Index content: matches all {{count}} covered file(s)', + 'status.indexContentDrifted': + 'Index content: {{changed}} changed, {{added}} added, {{deleted}} deleted', + 'status.indexContentMore': ' ...and {{count}} more {{label}}', + 'status.indexContentUnmeasurable': + 'Index content: not comparable ({{reason}}); fell back to the working-tree check', + 'status.indexContentScanFailed': + 'Index content: coverage scan failed; treating the index as stale', + 'status.driftChanged': 'changed', + 'status.driftAdded': 'added', + 'status.driftDeleted': 'deleted', 'status.upToDate': '✅ up-to-date', 'status.stale': '⚠️ stale (re-run gitnexus analyze)', 'clean.deleteAll': 'This will delete GitNexus indexes for {{count}} repo(s):', @@ -65,6 +76,8 @@ export const en = { 'tool.warn.unknownKind': "--kind '{{kind}}' is not a known symbol kind (e.g. Function, Class, Method); it will not narrow the result.", 'tool.detectChanges.noChanges': 'No changes detected.', + 'tool.detectChanges.noOverlappingSymbols': + 'Diff touched {{files}} file(s) but no indexed symbols overlap those hunks — not a clean tree.', 'tool.detectChanges.partial': 'PARTIAL RESULT: a graph query failed, so changed symbols may be missing. Do not read this as a clean pre-commit check.', 'tool.detectChanges.truncated': @@ -132,6 +145,16 @@ export const en = { 'One-time setup: configure MCP for Cursor, Claude Code, Antigravity, OpenCode, CodeBuddy, Qoder, Codex', 'help.command.uninstall.description': 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', + 'help.command.autoSync.description': + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + 'help.autoSync.details': + '\nActions: init, start (default), restart, stop, status, reset\nConfiguration: GITNEXUS_HOME/watch_config.yml\nRuntime files: GITNEXUS_HOME/watch/watch.pid, watch.mutex, watch.owner.json, watch.status.json, auto-sync-state.json\nRecovery: mutexes with verified dead owners are reclaimed automatically; invalid or legacy mutexes fail closed and require manual removal after confirming no watch process is running.\nWrites: GITNEXUS_HOME/watch/project_commit_info.txt\nRemote URLs: only SSH URLs on github.com, gitlab.com, and gitee.com are allowed.\nRuns once immediately, then repeats on sync_interval_minutes.', + 'help.command.watch.description': + 'Ambiguous: use `analyze --watch` for local files, or `auto-sync` for scheduled remotes', + 'help.watch.details': + '\n`gitnexus watch` does not start a watcher.\n Local working-tree incremental index: gitnexus analyze --watch\n Scheduled remote clone/pull + analyze: gitnexus auto-sync start\n', + 'error.watch.ambiguous': + '`gitnexus watch` is ambiguous.\n Local working-tree incremental index: gitnexus analyze --watch\n Scheduled remote clone/pull + analyze: gitnexus auto-sync start\n', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', @@ -190,7 +213,7 @@ export const en = { 'help.option.analyze.skills': 'Generate repo-specific skill files from detected communities (no-op when --index-only is also set).', 'help.option.analyze.skipAgentsMd': - 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md', + 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md. Does not skip standard skills in .claude/skills or .agents/skills; use --skip-skills for those. Community skills from --skills are unaffected.', 'help.option.analyze.noStats': 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md', 'help.option.analyze.selfCommit': 'Auto-commit AGENTS.md/CLAUDE.md changes after analyze (opt-in, off by default). Scoped to only those two files (never `git add -A`); no-ops if neither exists, neither changed, or the repo has no git identity configured.', @@ -217,6 +240,8 @@ export const en = { 'help.option.analyze.embeddingBatchSize': 'Number of nodes per embedding batch', 'help.option.analyze.embeddingSubBatchSize': 'Number of chunks per embedding model call', 'help.option.analyze.embeddingDevice': 'Embedding device: auto, cpu, dml, cuda, or wasm', + 'help.option.analyze.watch': 'Keep the index current with serialized incremental refreshes', + 'help.option.analyze.debounce': 'Watch quiet period before refreshing (milliseconds)', 'help.option.index.force': 'Register even if index metadata is missing (stats will be empty)', 'help.option.index.allowNonGit': 'Allow registering folders that are not Git repositories', 'help.option.port': 'Port number', @@ -225,7 +250,7 @@ export const en = { 'help.option.mcp.host': 'HTTP bind address (only with --http). Default: 127.0.0.1 (loopback). Use 0.0.0.0 to expose to all interfaces.', 'help.option.mcp.authToken': - 'Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.', + "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.", 'help.option.force.confirmation': 'Skip confirmation prompt', 'help.option.uninstall.force': 'Apply the changes (default is a dry-run preview)', 'help.option.clean.all': 'Clean all indexed repos', @@ -234,7 +259,7 @@ export const en = { 'Clean parked LadybugDB recovery sidecars (missing-shadow WAL quarantines and dirty-recovery parks)', 'help.option.wiki.force': 'Force full regeneration even if up to date', 'help.option.wiki.provider': - 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: minimax)', + 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax)', 'help.option.wiki.model': 'LLM model or deployment name (default: MiniMax-M3)', 'help.option.wiki.baseUrl': 'LLM API base URL. Azure v1: https://{resource}.openai.azure.com/openai/v1', @@ -293,10 +318,9 @@ export const en = { 'help.option.embeddings.install.force': 'Install into the runtime prefix even when the stack already resolves', 'help.option.group.create.force': 'Overwrite existing group', - 'help.option.group.sync.skipEmbeddings': 'Exact + BM25 only (no embedding fallback)', - 'help.option.group.sync.exactOnly': 'Exact match only', - 'help.option.group.sync.allowStale': 'Skip stale index warnings', - 'help.option.group.sync.verbose': 'Show each cross-link detail', + 'help.option.group.sync.exactOnly': + 'Skip wildcard service matching; cross-link on exact contract-id match only (manifest links still apply)', + 'help.option.group.sync.verbose': 'Show additional sync diagnostics', 'help.option.status.json': 'Emit machine-readable index and analyzer provenance', 'help.option.json': 'JSON output', 'help.option.group.impact.target': 'Symbol or file name to analyze', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 0ed1c7f1e..de9249cc4 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -37,6 +37,15 @@ export const zhCN = { 'status.workspaceIndexLabel': "工作区索引：最近在 '{{primary}}' 分支上分析（重新运行 gitnexus analyze 以跟随当前分支）", 'status.status': '状态', + 'status.indexContentCurrent': '索引内容：与覆盖的全部 {{count}} 个文件一致', + 'status.indexContentDrifted': + '索引内容：{{changed}} 个已修改，{{added}} 个新增，{{deleted}} 个已删除', + 'status.indexContentMore': ' ……另有 {{count}} 个 {{label}}', + 'status.indexContentUnmeasurable': '索引内容：无法比对（{{reason}}），已回退到工作区检查', + 'status.indexContentScanFailed': '索引内容：覆盖扫描失败，按过期处理', + 'status.driftChanged': '已修改', + 'status.driftAdded': '新增', + 'status.driftDeleted': '已删除', 'status.upToDate': '✅ 已是最新', 'status.stale': '⚠️ 已过期（重新运行 gitnexus analyze）', 'clean.deleteAll': '将删除 {{count}} 个仓库的 GitNexus 索引：', @@ -69,6 +78,8 @@ export const zhCN = { 'tool.warn.unknownKind': "--kind '{{kind}}' 不是已知的符号类型（如 Function、Class、Method），不会用于缩小结果范围。", 'tool.detectChanges.noChanges': '未检测到变更。', + 'tool.detectChanges.noOverlappingSymbols': + 'diff 触及 {{files}} 个文件，但没有索引符号与这些 hunk 重叠 — 并非干净工作区。', 'tool.detectChanges.partial': '结果不完整：图查询失败，可能遗漏已变更符号。请勿将其视为通过的提交前检查。', 'tool.detectChanges.truncated': @@ -133,6 +144,16 @@ export const zhCN = { '一次性设置：为 Cursor、Claude Code、Antigravity、OpenCode、CodeBuddy、Qoder、Codex 配置 MCP', 'help.command.uninstall.description': '撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', + 'help.command.autoSync.description': + '控制基于 GITNEXUS_HOME/watch_config.yml 的定时 clone/pull 和分析', + 'help.autoSync.details': + '\n操作：init、start（默认）、restart、stop、status、reset\n配置：GITNEXUS_HOME/watch_config.yml\n运行时文件：GITNEXUS_HOME/watch/watch.pid、watch.mutex、watch.owner.json、watch.status.json、auto-sync-state.json\n恢复：已验证 owner 退出的 mutex 会自动回收；无效或旧版 mutex 会安全拒绝，确认没有 watch 进程运行后再手动删除。\n写入：GITNEXUS_HOME/watch/project_commit_info.txt\n远程地址：仅允许 github.com、gitlab.com 和 gitee.com 上的 SSH 地址。\n启动后立即运行一次，之后按 sync_interval_minutes 重复。', + 'help.command.watch.description': + '含义不明确：本地文件请用 `analyze --watch`，定时远程同步请用 `auto-sync`', + 'help.watch.details': + '\n`gitnexus watch` 不会启动监视器。\n 本地工作区增量索引：gitnexus analyze --watch\n 定时远程 clone/pull 并分析：gitnexus auto-sync start\n', + 'error.watch.ambiguous': + '`gitnexus watch` 含义不明确。\n 本地工作区增量索引：gitnexus analyze --watch\n 定时远程 clone/pull 并分析：gitnexus auto-sync start\n', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', @@ -179,7 +200,8 @@ export const zhCN = { '重建时删除现有嵌入。默认情况下，未传 `--embeddings` 的 `analyze` 会保留索引中已有嵌入。', 'help.option.analyze.skills': '根据检测到的社区生成仓库专属 skill 文件（同时设置 --index-only 时无效）。', - 'help.option.analyze.skipAgentsMd': '跳过更新 AGENTS.md 和 CLAUDE.md 中的 gitnexus 区块', + 'help.option.analyze.skipAgentsMd': + '跳过更新 AGENTS.md 和 CLAUDE.md 中的 gitnexus 区块。不会跳过 .claude/skills 或 .agents/skills 下的标准 skill；如需跳过那些请使用 --skip-skills。--skills 生成的社区 skill 不受影响。', 'help.option.analyze.noStats': '从 AGENTS.md 和 CLAUDE.md 中省略易变的文件/符号计数', 'help.option.analyze.selfCommit': '在 analyze 后自动提交 AGENTS.md/CLAUDE.md 的变更（默认关闭，需显式开启）。仅限这两个文件（绝不使用 `git add -A`）；若两者均不存在、均未变更，或仓库未配置 git 身份，则不执行任何操作。', @@ -203,6 +225,8 @@ export const zhCN = { 'help.option.analyze.embeddingBatchSize': '每个嵌入批次的节点数', 'help.option.analyze.embeddingSubBatchSize': '每次嵌入模型调用的分块数', 'help.option.analyze.embeddingDevice': '嵌入设备：auto、cpu、dml、cuda 或 wasm', + 'help.option.analyze.watch': '监视本地源文件变更并串行执行增量刷新', + 'help.option.analyze.debounce': '刷新前的静默等待时间（毫秒）', 'help.option.index.force': '即使缺少索引元数据也注册（统计为空）', 'help.option.index.allowNonGit': '允许注册非 Git 仓库文件夹', 'help.option.port': '端口号', @@ -211,7 +235,7 @@ export const zhCN = { 'help.option.mcp.host': 'HTTP 绑定地址（仅与 --http 搭配使用）。默认：127.0.0.1（回环）。使用 0.0.0.0 向所有接口开放。', 'help.option.mcp.authToken': - '要求 Authorization 头携带此 Bearer Token（仅与 --http 搭配使用）；也可通过 GITNEXUS_MCP_AUTH_TOKEN 环境变量设置。非回环绑定（--host 0.0.0.0/::）时必填，否则拒绝启动。', + '要求 Authorization 头携带此 Bearer Token（仅与 --http 搭配使用）；也可通过 GITNEXUS_MCP_AUTH_TOKEN 环境变量设置，该变量同时为 gitnexus serve 的 /api/mcp 路由启用 MCP Bearer 认证。非回环绑定（--host 0.0.0.0/::）时必填，否则拒绝启动。', 'help.option.force.confirmation': '跳过确认提示', 'help.option.uninstall.force': '应用更改（默认仅为预演预览）', 'help.option.clean.all': '清理所有已索引仓库', @@ -220,7 +244,7 @@ export const zhCN = { '清理已暂存的 LadybugDB 恢复 sidecar（missing-shadow WAL 隔离文件与 dirty-recovery 暂存文件）', 'help.option.wiki.force': '即使已是最新也强制完整重新生成', 'help.option.wiki.provider': - 'LLM 提供商：minimax、openai、openrouter、azure、custom、cursor、claude、codex 或 opencode（默认：minimax）', + 'LLM 提供商：minimax、openai、openrouter、azure、custom、cursor、claude、codex、opencode 或 grok（默认：minimax）', 'help.option.wiki.model': 'LLM 模型或 deployment 名称（默认：MiniMax-M3）', 'help.option.wiki.baseUrl': 'LLM API base URL。Azure v1：https://{resource}.openai.azure.com/openai/v1', @@ -273,10 +297,9 @@ export const zhCN = { '同时下载 CUDA GPU 二进制文件（运行 onnxruntime-node 的 NuGet postinstall；代理后请设置 GLOBAL_AGENT_HTTPS_PROXY）', 'help.option.embeddings.install.force': '即使嵌入组件已可解析，也强制安装到运行时目录', 'help.option.group.create.force': '覆盖现有仓库组', - 'help.option.group.sync.skipEmbeddings': '仅使用 exact + BM25（不使用嵌入回退）', - 'help.option.group.sync.exactOnly': '仅精确匹配', - 'help.option.group.sync.allowStale': '跳过过期索引警告', - 'help.option.group.sync.verbose': '显示每条跨仓库链接详情', + 'help.option.group.sync.exactOnly': + '跳过通配符服务匹配，仅按契约 ID 精确匹配建立跨仓链接（清单声明的链接仍然生效）', + 'help.option.group.sync.verbose': '显示额外的同步诊断信息', 'help.option.status.json': '输出机器可读的索引和分析器来源信息', 'help.option.json': 'JSON 输出', 'help.option.group.impact.target': '要分析的符号或文件名', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 1ccf75c2f..d48bc65c9 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -45,6 +45,22 @@ program .option('-f, --force', 'Apply the changes (default is a dry-run preview)') .action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand')); +program + .command('auto-sync [action]') + .description( + 'Control scheduled repository clone/pull and analysis from GITNEXUS_HOME/watch_config.yml', + ) + .addHelpText('after', () => t('help.autoSync.details')) + .action(createLazyAction(() => import('./auto-sync.js'), 'autoSyncCommand')); + +program + .command('watch [action]') + .description( + 'Ambiguous: use `analyze --watch` for local files, or `auto-sync` for scheduled remotes', + ) + .addHelpText('after', () => t('help.watch.details')) + .action(createLazyAction(() => import('./watch.js'), 'watchAmbiguousCommand')); + // Baseline of GITNEXUS_EMBEDDING_DIMS captured by the analyze preAction hook // before it overwrites the var, so the postAction hook can restore it. The // analyzeCommand env snapshot is taken AFTER this hook runs, so it cannot undo @@ -57,6 +73,8 @@ let dimsEnvCaptured = false; program .command('analyze [path]') .description('Index a repository (full analysis)') + .option('--watch', 'Keep the index current with serialized incremental refreshes') + .option('--debounce ', 'Watch quiet period before refreshing (default: 300 milliseconds)') .option('-f, --force', 'Force full re-index even if up to date') .option('--repair-fts', 'Repair/rebuild search FTS indexes without full re-analysis') .option( @@ -74,7 +92,10 @@ program 'Generate repo-specific skill files from detected communities ' + '(no-op when --index-only is also set).', ) - .option('--skip-agents-md', 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md') + .option( + '--skip-agents-md', + 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md. Does not skip standard skills in .claude/skills or .agents/skills; use --skip-skills for those. Community skills from --skills are unaffected.', + ) .option( '--pdg', 'Build the control-flow-graph / PDG substrate (BasicBlock nodes + CFG edges) ' + @@ -137,6 +158,11 @@ program '--workers ', 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', ) + .option( + '--spring-actuator ', + 'Import local Spring Boot Actuator JSON snapshots (mappings, beans, conditions, ' + + 'configprops, env). Explicit opt-in; disabled by default.', + ) .option('--embedding-threads ', 'Limit local ONNX embedding CPU threads') .option('--embedding-batch-size ', 'Number of nodes per embedding batch') .option('--embedding-sub-batch-size ', 'Number of chunks per embedding model call') @@ -162,6 +188,11 @@ program ) .addHelpText('after', () => t('help.analyze.environment')) .hook('preAction', (thisCommand: Command) => { + const analyzeOpts = thisCommand.opts(); + if (analyzeOpts['debounce'] !== undefined && analyzeOpts['watch'] !== true) { + process.stderr.write('\n --debounce requires --watch\n\n'); + process.exit(1); + } // ONLY GITNEXUS_EMBEDDING_DIMS must be set here: schema.ts reads it at // module-load time during the lazy import('./analyze.js') below (via the // static chain analyze.ts → run-analyze.ts → schema.ts), so deferring to @@ -169,7 +200,7 @@ program // lazily at runtime (readConfig), so analyzeCommandImpl is their sole // setter — keeping them out of this hook means they fall under the impl's // env snapshot/restore and don't leak across in-process invocations. - const dimsOpt = thisCommand.opts()['embeddingDims']; + const dimsOpt = analyzeOpts['embeddingDims']; if (dimsOpt !== undefined) { // Validate + normalize BEFORE writing the env var: schema.ts throws on a // bad value at module-load, which — on the synchronous program.parse() @@ -202,7 +233,7 @@ program createAnalyzerLbugLazyAction( () => import('../core/analyzer-identity.js'), () => import('./analyze.js'), - 'analyzeCommandWithRunnerIdentity', + 'analyzeOrWatchCommandWithRunnerIdentity', import.meta.url, ), ); @@ -238,7 +269,7 @@ program ) .option( '--auth-token ', - 'Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.', + "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.", ) .action(createLbugLazyAction(() => import('./mcp.js'), 'mcpCommand')); @@ -303,7 +334,7 @@ program .option('-f, --force', 'Force full regeneration even if up to date') .option( '--provider ', - 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: minimax)', + 'LLM provider: minimax, openai, openrouter, azure, custom, cursor, claude, codex, opencode, or grok (default: minimax)', ) .option('--model ', 'LLM model or deployment name (default: MiniMax-M3)') .option( diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index 003e548a9..6dd0f9b85 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -1090,14 +1090,30 @@ async function installSkillsTo(targetDir: string): Promise { const skillDir = path.join(targetDir, skillName); try { - if (source.isDirectory) { - const dirSource = path.join(skillsRoot, skillName); - await copyDirRecursive(dirSource, skillDir); - } else { - const flatSource = path.join(skillsRoot, `${skillName}.md`); - const content = await fs.readFile(flatSource, 'utf-8'); + const sourceSkillPath = source.isDirectory + ? path.join(skillsRoot, skillName, 'SKILL.md') + : path.join(skillsRoot, `${skillName}.md`); + const destinationSkillPath = path.join(skillDir, 'SKILL.md'); + const [sourceSkillContent, destinationSkillContent] = await Promise.all([ + fs.readFile(sourceSkillPath, 'utf-8'), + fs.readFile(destinationSkillPath, 'utf-8').catch((err) => { + if (!isEnoent(err)) throw err; + return null; + }), + ]); + + const preserved = + destinationSkillContent !== null && destinationSkillContent !== sourceSkillContent; + if (preserved && !source.isDirectory) { + console.log( + `[gitnexus] preserved customized skill ${destinationSkillPath}; ` + + 'delete the file and rerun setup to refresh it.', + ); + } else if (source.isDirectory) { + await copyDirRecursive(path.join(skillsRoot, skillName), skillDir); + } else if (!preserved) { await fs.mkdir(skillDir, { recursive: true }); - await fs.writeFile(path.join(skillDir, 'SKILL.md'), content, 'utf-8'); + await fs.writeFile(destinationSkillPath, sourceSkillContent, 'utf-8'); } // A directory superseded by a shipped rename is warned about, never @@ -1113,7 +1129,7 @@ async function installSkillsTo(targetDir: string): Promise { ); } } - installed.push(skillName); + if (!preserved) installed.push(skillName); } catch { // Source skill not found — skip } @@ -1133,9 +1149,23 @@ async function copyDirRecursive(src: string, dest: string): Promise { const destPath = path.join(dest, entry.name); if (entry.isDirectory()) { await copyDirRecursive(srcPath, destPath); - } else { - await fs.copyFile(srcPath, destPath); + continue; } + const [srcBuf, destBuf] = await Promise.all([ + fs.readFile(srcPath), + fs.readFile(destPath).catch((err) => { + if (!isEnoent(err)) throw err; + return null; + }), + ]); + if (destBuf !== null && !destBuf.equals(srcBuf)) { + console.log( + `[gitnexus] preserved customized skill ${destPath}; ` + + 'delete the file and rerun setup to refresh it.', + ); + continue; + } + await fs.writeFile(destPath, srcBuf); } } diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index 09eb415d3..e0ee08bcc 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -18,8 +18,69 @@ import { resolveAnalyzerRunnerIdentity, } from '../core/analyzer-identity.js'; import { getIndexIncompleteReasons } from '../core/index-freshness.js'; +import { detectIndexContentDrift, type IndexContentDrift } from '../core/index-content-drift.js'; import { t } from './i18n/index.js'; +/** How many drifted paths the report names before summarizing the rest. */ +const DRIFT_SAMPLE_LIMIT = 10; + +/** + * Machine-readable form of the per-file comparison. `'not-checked'` is its own + * value rather than a silent omission: it says the index was already stale on + * metadata alone, so the scan was skipped, which is not the same claim as a + * scan that ran and found nothing. + */ +const describeContentDrift = (drift: IndexContentDrift | undefined) => { + if (!drift) return { status: 'not-checked' as const }; + if (drift.kind === 'current') { + return { status: 'current' as const, coveredFiles: drift.coveredFileCount }; + } + if (drift.kind === 'unmeasurable') { + return { status: 'unmeasurable' as const, reason: drift.reason }; + } + return { + status: 'drifted' as const, + counts: { + changed: drift.changed.length, + added: drift.added.length, + deleted: drift.deleted.length, + }, + changed: drift.changed.slice(0, DRIFT_SAMPLE_LIMIT), + added: drift.added.slice(0, DRIFT_SAMPLE_LIMIT), + deleted: drift.deleted.slice(0, DRIFT_SAMPLE_LIMIT), + truncated: { + changed: drift.changed.length > DRIFT_SAMPLE_LIMIT, + added: drift.added.length > DRIFT_SAMPLE_LIMIT, + deleted: drift.deleted.length > DRIFT_SAMPLE_LIMIT, + }, + }; +}; + +/** Escape control characters in repo-relative paths before printing. */ +const formatDriftPath = (rel: string): string => + /[\u0000-\u001f\u007f]/.test(rel) ? JSON.stringify(rel) : rel; +const printDriftDetail = (drift: Extract): void => { + console.log( + t('status.indexContentDrifted', { + changed: drift.changed.length, + added: drift.added.length, + deleted: drift.deleted.length, + }), + ); + const labelled: [string, readonly string[]][] = [ + [t('status.driftChanged'), drift.changed], + [t('status.driftAdded'), drift.added], + [t('status.driftDeleted'), drift.deleted], + ]; + for (const [label, paths] of labelled) { + for (const p of paths.slice(0, DRIFT_SAMPLE_LIMIT)) { + console.log(` ${label}: ${formatDriftPath(p)}`); + } + const remaining = paths.length - DRIFT_SAMPLE_LIMIT; + if (remaining > 0) console.log(t('status.indexContentMore', { count: remaining, label })); + } +}; + export interface StatusOptions { json?: boolean; } @@ -85,14 +146,36 @@ export const statusCommand = async (options: StatusOptions = {}) => { currentRunnerIdentity, ); const incompleteReasons = getIndexIncompleteReasons(activeMeta); - // A matching HEAD is not enough: `analyze` re-indexes a dirty working tree, - // so a repo with uncommitted source changes is stale even at the same commit. - // Skip the check for non-git folders (currentCommit === '') to match analyze. - const isUpToDate = + const metadataIsCurrent = currentCommit === activeMeta.lastCommit && runnerIdentityIsCurrent && - incompleteReasons.length === 0 && - (currentCommit === '' || !isWorkingTreeDirty(repo.repoPath)); + incompleteReasons.length === 0; + + // A matching HEAD is not enough: `analyze` re-indexes changed content at the + // same commit, so the files the index covers must still be compared against + // disk. Only worth the scan once the cheap metadata checks agree, and skipped + // for non-git folders (currentCommit === '') to match analyze. + const contentDrift: IndexContentDrift | undefined = + metadataIsCurrent && currentCommit !== '' + ? await detectIndexContentDrift( + repo.repoPath, + activeMeta.fileHashes, + activeMeta.indexCoverage, + ) + : undefined; + + // The repo-wide dirty flag survives only as the fallback for metadata written + // before `fileHashes` existed. Where the per-file comparison can run it + // decides, so a file the index does not cover no longer pins a byte-current + // index to a "stale" verdict that `analyze` is powerless to clear (#3077). + const contentIsCurrent = + contentDrift === undefined || + contentDrift.kind === 'current' || + (contentDrift.kind === 'unmeasurable' && + contentDrift.reason === 'no-file-hashes' && + !isWorkingTreeDirty(repo.repoPath)); + + const isUpToDate = metadataIsCurrent && contentIsCurrent; if (options.json) { console.log( JSON.stringify({ @@ -111,6 +194,7 @@ export const statusCommand = async (options: StatusOptions = {}) => { commit: currentCommit, runnerIdentity: currentRunnerIdentity, }, + contentDrift: describeContentDrift(contentDrift), status: isUpToDate ? 'up-to-date' : 'stale', }), ); @@ -137,5 +221,16 @@ export const statusCommand = async (options: StatusOptions = {}) => { console.log(`Index incomplete reasons: ${JSON.stringify(incompleteReasons)}`); } console.log(`${t('status.currentRunnerIdentity')}: ${JSON.stringify(currentRunnerIdentity)}`); + if (contentDrift?.kind === 'current') { + console.log(t('status.indexContentCurrent', { count: contentDrift.coveredFileCount })); + } else if (contentDrift?.kind === 'drifted') { + printDriftDetail(contentDrift); + } else if (contentDrift?.kind === 'unmeasurable') { + if (contentDrift.reason === 'scan-failed') { + console.log(t('status.indexContentScanFailed')); + } else if (!isUpToDate) { + console.log(t('status.indexContentUnmeasurable', { reason: contentDrift.reason })); + } + } console.log(`${t('status.status')}: ${isUpToDate ? t('status.upToDate') : t('status.stale')}`); }; diff --git a/gitnexus/src/cli/watch-queue.ts b/gitnexus/src/cli/watch-queue.ts new file mode 100644 index 000000000..3f701ef50 --- /dev/null +++ b/gitnexus/src/cli/watch-queue.ts @@ -0,0 +1,184 @@ +export type WatchRefresh = (paths: readonly string[]) => Promise; +export type WatchRefreshError = (error: unknown, paths: readonly string[]) => void; + +export const WATCH_FULL_REFRESH_PATH = '*'; + +export interface WatchRefreshQueueOptions { + readonly maxWaitMs?: number; + readonly maxPendingPaths?: number; + readonly retryBaseDelayMs?: number; + readonly retryMaxDelayMs?: number; + readonly holdEventsUntilInitialRefresh?: boolean; + readonly isPriorityPath?: (filePath: string) => boolean; +} + +/** Debounces filesystem events and guarantees that refreshes never overlap. */ +export class WatchRefreshQueue { + private readonly pending = new Set(); + private readonly idleWaiters = new Set<() => void>(); + private timer: ReturnType | undefined; + private active: Promise | undefined; + private closed = false; + private initialPending = false; + private firstPendingAt: number | undefined; + private overflowed = false; + private consecutiveFailures = 0; + private retryNotBefore: number | undefined; + + constructor( + private readonly refresh: WatchRefresh, + private readonly onError: WatchRefreshError, + private readonly debounceMs: number, + private readonly options: WatchRefreshQueueOptions = {}, + ) { + this.initialPending = options.holdEventsUntilInitialRefresh === true; + } + + enqueue(filePath: string): void { + if (this.closed) return; + this.addPendingPath(filePath); + this.firstPendingAt ??= Date.now(); + if (!this.initialPending && this.active === undefined) this.schedule(); + } + + private addPendingPath(filePath: string): void { + const maxPendingPaths = this.options.maxPendingPaths ?? 1_000; + const priority = this.options.isPriorityPath?.(filePath) === true; + if (this.pending.has(filePath)) { + // A duplicate does not increase memory use or imply that paths were dropped. + } else if (this.pending.size < maxPendingPaths) { + this.pending.add(filePath); + } else { + this.overflowed = true; + if (priority) { + const evictable = [...this.pending].find( + (pendingPath) => this.options.isPriorityPath?.(pendingPath) !== true, + ); + if (evictable !== undefined) { + this.pending.delete(evictable); + this.pending.add(filePath); + } + } + } + } + + /** Run the initial refresh while still queueing events that arrive during it. */ + async runInitial(): Promise { + if (this.closed) return; + if (this.active !== undefined) throw new Error('Watch refresh is already running'); + try { + await this.runBatch([], true); + } finally { + this.initialPending = false; + if (!this.closed && this.hasPendingWork()) this.schedule(); + else this.resolveIdleWaiters(); + } + } + + async waitForIdle(): Promise { + if (this.isIdle()) return; + await new Promise((resolve) => this.idleWaiters.add(resolve)); + } + + async close(): Promise { + this.closed = true; + if (this.timer !== undefined) clearTimeout(this.timer); + this.timer = undefined; + this.pending.clear(); + this.firstPendingAt = undefined; + this.overflowed = false; + this.consecutiveFailures = 0; + this.retryNotBefore = undefined; + // A refresh rejection is already surfaced through `onError` (or through + // runInitial). Closing from that handler can race the runBatch `finally`, + // so consume the same rejection here instead of reporting it twice. + await this.active?.catch(() => {}); + this.resolveIdleWaiters(); + } + + private schedule(retryDelayMs?: number): void { + if (this.timer !== undefined) clearTimeout(this.timer); + const maxWaitMs = this.options.maxWaitMs ?? Math.max(this.debounceMs, 2_000); + const now = Date.now(); + if (retryDelayMs !== undefined) this.retryNotBefore = now + retryDelayMs; + const elapsed = this.firstPendingAt === undefined ? 0 : now - this.firstPendingAt; + const debounced = Math.max(0, Math.min(this.debounceMs, maxWaitMs - elapsed)); + // An event arriving mid-backoff merges into the pending batch but must not + // pull the retry earlier than the deadline the backoff already committed to. + const delay = + retryDelayMs ?? + (this.retryNotBefore === undefined + ? debounced + : Math.max(debounced, this.retryNotBefore - now)); + this.timer = setTimeout(() => { + this.timer = undefined; + void this.drain(); + }, delay); + } + + private async drain(): Promise { + if (this.closed || this.active !== undefined || !this.hasPendingWork()) return; + const paths = [ + ...(this.overflowed ? [WATCH_FULL_REFRESH_PATH] : []), + ...[...this.pending].sort(), + ]; + this.pending.clear(); + this.firstPendingAt = undefined; + this.overflowed = false; + this.retryNotBefore = undefined; + await this.runBatch(paths, false); + } + + private async runBatch(paths: readonly string[], propagateError: boolean): Promise { + let work: Promise; + try { + work = this.refresh(paths); + } catch (error) { + work = Promise.reject(error); + } + this.active = work; + let retryDelayMs: number | undefined; + try { + await work; + this.consecutiveFailures = 0; + } catch (error) { + if (propagateError) throw error; + try { + await this.onError(error, paths); + } catch { + // Refresh failures are already handled here; a reporter must not + // reject the detached drain promise and become an unhandled rejection. + } + if (!this.closed) { + if (paths.includes(WATCH_FULL_REFRESH_PATH)) this.overflowed = true; + for (const filePath of paths) { + if (filePath !== WATCH_FULL_REFRESH_PATH) this.addPendingPath(filePath); + } + this.firstPendingAt = Date.now(); + this.consecutiveFailures++; + const base = this.options.retryBaseDelayMs ?? Math.max(250, this.debounceMs); + const maximum = this.options.retryMaxDelayMs ?? 30_000; + retryDelayMs = Math.min(maximum, base * 2 ** (this.consecutiveFailures - 1)); + } + } finally { + if (this.active === work) this.active = undefined; + if (!this.closed && !this.initialPending && this.hasPendingWork()) + this.schedule(retryDelayMs); + else this.resolveIdleWaiters(); + } + } + + private hasPendingWork(): boolean { + return this.overflowed || this.pending.size > 0; + } + + private isIdle(): boolean { + return this.active === undefined && this.timer === undefined && !this.hasPendingWork(); + } + + private resolveIdleWaiters(): void { + if (!this.isIdle() && !this.closed) return; + for (const resolve of this.idleWaiters) resolve(); + this.idleWaiters.clear(); + } +} diff --git a/gitnexus/src/cli/watch.ts b/gitnexus/src/cli/watch.ts new file mode 100644 index 000000000..2a779b51a --- /dev/null +++ b/gitnexus/src/cli/watch.ts @@ -0,0 +1,7 @@ +/** Reserved CLI verb: never starts either watch product. */ +import { t } from './i18n/index.js'; + +export async function watchAmbiguousCommand(_action?: string): Promise { + process.stderr.write(t('error.watch.ambiguous')); + process.exitCode = 1; +} diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index d65d130a7..007451ef5 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -25,6 +25,7 @@ import { type LLMProvider, } from '../core/wiki/llm-client.js'; import { detectCursorCLI } from '../core/wiki/cursor-client.js'; +import { detectGrokCLI } from '../core/wiki/grok-client.js'; import { detectLocalCLI } from '../core/wiki/local-cli-client.js'; import { logger } from '../core/logger.js'; @@ -65,20 +66,22 @@ function parsePositiveIntegerOption( function isLocalProvider( provider: LLMProvider | undefined, -): provider is 'cursor' | 'claude' | 'codex' | 'opencode' { +): provider is 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok' { return ( provider === 'cursor' || provider === 'claude' || provider === 'codex' || - provider === 'opencode' + provider === 'opencode' || + provider === 'grok' ); } -function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode') { +function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok') { if (provider === 'cursor') return 'cursorModel'; if (provider === 'claude') return 'claudeModel'; if (provider === 'codex') return 'codexModel'; if (provider === 'opencode') return 'opencodeModel'; + if (provider === 'grok') return 'grokModel'; throw new Error(`Unsupported local provider: ${provider satisfies never}`); } @@ -287,7 +290,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) if (!llmConfig.apiKey && !isLocalProvider(llmConfig.provider)) { console.log(' Error: No LLM API key found.'); console.log(' Set MINIMAX_API_KEY, GITNEXUS_API_KEY, or OPENAI_API_KEY,'); - console.log(' or pass --api-key , or use --provider cursor|claude|codex|opencode.\n'); + console.log( + ' or pass --api-key , or use --provider cursor|claude|codex|opencode|grok.\n', + ); process.exitCode = 1; return; } @@ -301,9 +306,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) const hasClaude = detectLocalCLI('claude'); const hasCodex = detectLocalCLI('codex'); const hasOpenCode = detectLocalCLI('opencode'); + const hasGrok = detectGrokCLI(); const localChoices: Array<{ choice: string; - provider: 'cursor' | 'claude' | 'codex' | 'opencode'; + provider: 'cursor' | 'claude' | 'codex' | 'opencode' | 'grok'; }> = []; // Provider selection @@ -346,6 +352,14 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) }); console.log(` [${choice}] OpenCode CLI (local, uses your OpenCode login/config)`); } + if (hasGrok) { + const choice = String(nextChoice++); + localChoices.push({ + choice, + provider: 'grok', + }); + console.log(` [${choice}] Grok CLI (local, uses your Grok Build login)`); + } console.log(''); const maxChoice = String(nextChoice - 1); diff --git a/gitnexus/src/config/ignore-service.ts b/gitnexus/src/config/ignore-service.ts index 163998a8e..ac2dc7704 100644 --- a/gitnexus/src/config/ignore-service.ts +++ b/gitnexus/src/config/ignore-service.ts @@ -1,7 +1,9 @@ import ignore, { type Ignore } from 'ignore'; +import { existsSync } from 'fs'; import fs from 'fs/promises'; import nodePath from 'path'; import type { Path } from 'path-scurry'; +import { readRepoControlFile } from './repo-control-file.js'; import { logger } from '../core/logger.js'; import { getCoreExcludesFilePath, getGitInfoExcludePath } from '../storage/git.js'; @@ -31,12 +33,15 @@ const DEFAULT_IGNORE_LIST = new Set([ // 'packages' removed - commonly used for monorepo source code (lerna, pnpm, yarn workspaces) 'venv', '.venv', - 'env', '.env', + // Bare `env/` can be application source or a Python virtual environment. + // Path-aware rules below prune it at the root and wherever pyvenv.cfg marks + // a virtual environment, while preserving ordinary nested source folders. '__pycache__', '.pytest_cache', '.mypy_cache', 'site-packages', + 'dist-packages', '.tox', 'eggs', '.eggs', @@ -54,13 +59,33 @@ const DEFAULT_IGNORE_LIST = new Set([ 'obj', 'target', // Java/Rust '.next', + // `.next` is Next.js's build CACHE; `_next` is the EMITTED output, and the two + // are different directories. A Capacitor/Cordova shell copies the emitted + // bundle to `/app/src/main/assets/public/_next/static/…`, where none + // of the path segments hit this list — so a mobile-wrapped Next.js app had its + // shipped bundle indexed as source, and every Route node it produced pointed at + // a webpack chunk rather than code anyone wrote (#3007). + // + // The name is deliberately unanchored. No `/_next` form matches a + // root-level `_next/static/…`, which is the shape the reported repo has, so + // anchoring it would miss the case it was added for. The accepted cost is a + // hand-written directory literally named `_next`; recover one with a bare + // `!_next/` line in `.gitnexusignore`. + '_next', '.nuxt', '.output', '.vercel', '.netlify', '.serverless', '_build', - 'public/build', + // `'public/build'` used to sit here. This set is tested one path SEGMENT at a + // time, and `isHardcodedIgnoredDirectory(name)` takes a bare directory name, + // so a slash-containing member could never match either — it was inert. Its + // paths were never unignored though: bare `'build'` above already prunes + // `public/build/**`, so removing the entry changes no behavior (#3007). + // `test/unit/ignore-build-output.test.ts` keeps the next slash-bearing entry + // in this set — or in IGNORED_FILES, ROOT_ARTIFACT_DIRECTORIES or + // IGNORED_EXTENSIONS — from dying the same way. '.parcel-cache', '.turbo', '.svelte-kit', @@ -86,11 +111,11 @@ const DEFAULT_IGNORE_LIST = new Set([ // Generated/Compiled '.generated', - 'generated', 'auto-generated', + // Bare `generated/` can contain tracked source-of-truth code. Build output + // remains covered by .gitignore/.gitnexusignore and the unambiguous names. 'monaco-workers', // Monaco editor web-worker bundles generated for browser runtime '.terraform', - '.serverless', // Documentation (optional - might want to keep) // 'docs', @@ -106,6 +131,14 @@ const DEFAULT_IGNORE_LIST = new Set([ '__snapshots__', ]); +// Ambiguous names that conventionally denote generated artifacts only at the +// repository root. Nested directories with these names are frequently source +// modules (for example apps/web/src/env or packages/api/generated). +const ROOT_ARTIFACT_DIRECTORIES = new Set(['env', 'generated']); + +const isRootArtifactDirectory = (relativePath: string, name: string): boolean => + !relativePath.includes('/') && ROOT_ARTIFACT_DIRECTORIES.has(name); + const IGNORED_EXTENSIONS = new Set([ // Images '.png', @@ -290,6 +323,10 @@ export const shouldIgnorePath = (filePath: string): boolean => { const fileName = parts[parts.length - 1]; const fileNameLower = fileName.toLowerCase(); + if (parts.length > 0 && isRootArtifactDirectory(parts[0], parts[0])) { + return true; + } + // Laravel compiles Blade templates into generated PHP cache files under // storage/framework/views. Source templates live in resources/views and are // handled separately; compiled cache should not become source-of-truth. Keep @@ -329,10 +366,8 @@ export const shouldIgnorePath = (filePath: string): boolean => { if ( fileNameLower.includes('.bundle.') || fileNameLower.includes('.chunk.') || - fileNameLower.includes('.generated.') || - fileNameLower.endsWith('.d.ts') + fileNameLower.includes('.generated.') ) { - // TypeScript declaration files return true; } @@ -344,6 +379,20 @@ export const isHardcodedIgnoredDirectory = (name: string): boolean => { return DEFAULT_IGNORE_LIST.has(name); }; +/** Apply directory ignore rules that depend on repository-relative depth. */ +export const isHardcodedIgnoredDirectoryAtPath = ( + repoRoot: string, + directoryPath: string, +): boolean => { + const name = nodePath.basename(directoryPath); + if (isHardcodedIgnoredDirectory(name)) return true; + + const relative = nodePath.relative(repoRoot, directoryPath).replace(/\\/g, '/'); + if (isRootArtifactDirectory(relative, name)) return true; + + return name === 'env' && existsSync(nodePath.join(directoryPath, 'pyvenv.cfg')); +}; + /** * Load .gitignore and .gitnexusignore rules from the repo root. * Returns an `ignore` instance with all patterns, or null if no files found. @@ -353,6 +402,8 @@ export interface IgnoreOptions { noGitignore?: boolean; /** Skip core.excludesFile and $GIT_COMMON_DIR/info/exclude. Defaults to GITNEXUS_NO_GLOBAL_IGNORE env var. */ noGlobalIgnore?: boolean; + /** Fail repository-control reloads closed so long-lived watchers keep their prior predicate. */ + strictRepoControlFiles?: boolean; } export const loadIgnoreRules = async ( @@ -394,20 +445,56 @@ export const loadIgnoreRules = async ( for (const filename of filenames) { try { - const content = await fs.readFile(nodePath.join(repoPath, filename), 'utf-8'); + const content = options?.strictRepoControlFiles + ? await readRepoControlFile(repoPath, filename) + : await fs.readFile(nodePath.join(repoPath, filename), 'utf-8'); + if (content === null) continue; ig.add(content); hasRules = true; } catch (err: unknown) { const code = (err as NodeJS.ErrnoException).code; - if (code !== 'ENOENT') { - logger.warn(` Warning: could not read ${filename}: ${(err as Error).message}`); - } + if (!options?.strictRepoControlFiles && code === 'ENOENT') continue; + if (options?.strictRepoControlFiles) throw err; + logger.warn(` Warning: could not read ${filename}: ${(err as Error).message}`); } } return hasRules ? ig : null; }; +/** + * Build a synchronous predicate for long-lived filesystem watchers. + * + * Unlike {@link createIgnoreFilter}, callers pass ordinary absolute or + * repository-relative paths instead of path-scurry `Path` objects. The rule + * precedence deliberately mirrors the scanner: explicit negations win over + * hardcoded defaults unless a more-specific rule re-ignores the path. + */ +export const createWatchIgnorePredicate = async ( + repoPath: string, + options?: IgnoreOptions, +): Promise<(candidatePath: string, isDirectory?: boolean) => boolean> => { + const ig = await loadIgnoreRules(repoPath, { ...options, strictRepoControlFiles: true }); + const repoRoot = nodePath.resolve(repoPath); + + return (candidatePath: string, isDirectory = false): boolean => { + const absolute = nodePath.isAbsolute(candidatePath) + ? nodePath.resolve(candidatePath) + : nodePath.resolve(repoRoot, candidatePath); + const rel = nodePath.relative(repoRoot, absolute).replace(/\\/g, '/'); + if (!rel) return false; + if (rel === '..' || rel.startsWith('../') || nodePath.isAbsolute(rel)) return true; + + if (ig && hasExplicitUnignore(ig, rel) && !ig.ignores(isDirectory ? `${rel}/` : rel)) { + return false; + } + + if (ig && ig.ignores(isDirectory ? `${rel}/` : rel)) return true; + if (isDirectory && isHardcodedIgnoredDirectoryAtPath(repoRoot, absolute)) return true; + return shouldIgnorePath(rel); + }; +}; + /** * Walk ancestor segments of `rel` and check whether `.gitnexusignore` * (or `.gitignore`) contains an explicit `!pattern` negation that @@ -496,8 +583,10 @@ export const createIgnoreFilter = async (repoPath: string, options?: IgnoreOptio // last-match-wins: `!__tests__/` + `__tests__/generated/` still // blocks descent into `__tests__/generated/`. if (ig && rel && hasExplicitUnignore(ig, rel) && !ig.ignores(rel + '/')) return false; - // Hardcoded list: block descent into well-known noise directories. - if (DEFAULT_IGNORE_LIST.has(p.name)) return true; + // Hardcoded and path-aware rules prune whole trees before glob walks them. + if (rel && isHardcodedIgnoredDirectoryAtPath(repoPath, nodePath.join(repoPath, rel))) { + return true; + } // Check against .gitignore / .gitnexusignore patterns. // Since childrenIgnored is only called for directories, always test with // a trailing slash. This ensures directory-only negation patterns (e.g. diff --git a/gitnexus/src/config/repo-control-file.ts b/gitnexus/src/config/repo-control-file.ts new file mode 100644 index 000000000..13e08adb0 --- /dev/null +++ b/gitnexus/src/config/repo-control-file.ts @@ -0,0 +1,117 @@ +import fs from 'node:fs'; +import * as path from 'node:path'; + +export const MAX_REPO_CONTROL_FILE_BYTES = 1024 * 1024; + +/** Read a bounded, regular control file owned by the repository root. */ +export async function readRepoControlFile( + repoRoot: string, + filename: string, +): Promise { + const requestedRoot = path.resolve(repoRoot); + const requested = path.resolve(requestedRoot, filename); + const relative = path.relative(requestedRoot, requested); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`${filename} resolves outside the repository root`); + } + + try { + const canonicalRoot = fs.realpathSync(requestedRoot); + const beforeOpen = fs.lstatSync(requested); + if (beforeOpen.isSymbolicLink()) throw new Error(`${filename} must not be a symbolic link`); + if (!beforeOpen.isFile()) throw new Error(`${filename} must be a regular file`); + if (beforeOpen.nlink !== 1) throw new Error(`${filename} must not be a hard link`); + if (beforeOpen.size > MAX_REPO_CONTROL_FILE_BYTES) { + throw new Error(`${filename} exceeds ${MAX_REPO_CONTROL_FILE_BYTES} bytes`); + } + return await new Promise((resolve, reject) => { + const stream = fs.createReadStream(requested, { + flags: 'r', + start: 0, + end: MAX_REPO_CONTROL_FILE_BYTES, + autoClose: true, + }); + const chunks: Buffer[] = []; + let totalBytes = 0; + let validated = false; + let settled = false; + + const finish = (value: string): void => { + if (settled) return; + settled = true; + resolve(value); + }; + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + reject(error); + }; + + stream.pause(); + stream.once('open', (fd) => { + try { + const opened = fs.fstatSync(fd); + if (!opened.isFile()) throw new Error(`${filename} must be a regular file`); + if (opened.nlink !== 1) throw new Error(`${filename} must not be a hard link`); + if (opened.size > MAX_REPO_CONTROL_FILE_BYTES) { + throw new Error(`${filename} exceeds ${MAX_REPO_CONTROL_FILE_BYTES} bytes`); + } + + const entry = fs.lstatSync(requested); + if (entry.isSymbolicLink()) throw new Error(`${filename} must not be a symbolic link`); + if ( + !entry.isFile() || + entry.nlink !== 1 || + entry.dev !== opened.dev || + entry.ino !== opened.ino + ) { + throw new Error(`${filename} moved or was replaced while being opened`); + } + const canonicalFile = fs.realpathSync(requested); + const canonicalRelative = path.relative(canonicalRoot, canonicalFile); + if (canonicalRelative.startsWith('..') || path.isAbsolute(canonicalRelative)) { + throw new Error(`${filename} resolves outside the repository root`); + } + const canonical = fs.statSync(canonicalFile); + if ( + canonical.nlink !== 1 || + canonical.dev !== opened.dev || + canonical.ino !== opened.ino + ) { + throw new Error(`${filename} moved or was replaced while being opened`); + } + + validated = true; + stream.resume(); + } catch (error) { + fail(error); + stream.destroy(); + } + }); + stream.on('data', (chunk: Buffer | string) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += bytes.length; + if (totalBytes > MAX_REPO_CONTROL_FILE_BYTES) { + fail(new Error(`${filename} exceeds ${MAX_REPO_CONTROL_FILE_BYTES} bytes`)); + stream.destroy(); + return; + } + chunks.push(bytes); + }); + stream.once('end', () => { + if (!validated) { + fail(new Error(`${filename} could not be validated`)); + return; + } + finish(Buffer.concat(chunks, totalBytes).toString('utf8')); + }); + stream.once('error', fail); + stream.once('close', () => { + if (!settled) fail(new Error(`${filename} closed before it could be read`)); + }); + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} diff --git a/gitnexus/src/core/analyzer-identity.ts b/gitnexus/src/core/analyzer-identity.ts index 6bc80848f..628f27716 100644 --- a/gitnexus/src/core/analyzer-identity.ts +++ b/gitnexus/src/core/analyzer-identity.ts @@ -15,6 +15,7 @@ */ import { + accessSync, closeSync, constants as fsConstants, existsSync, @@ -38,6 +39,7 @@ import { spawnSync } from 'node:child_process'; import { isDeepStrictEqual } from 'node:util'; import os from 'node:os'; import path from 'node:path'; +import { parseTruthyEnv } from './ingestion/utils/env.js'; import { fileURLToPath } from 'node:url'; import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js'; @@ -2335,8 +2337,30 @@ function snapshotCacheGuardDirect(request: CacheGuardRequest): CacheGuardResult } } -function snapshotCacheGuards(requests: CacheGuardRequest[]): CacheGuardResult[] { +function installTreeUnwritable(packageRoot: string, buildRoot: string): boolean { + for (const dir of [packageRoot, buildRoot]) { + try { + accessSync(dir, fsConstants.W_OK); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'EACCES' || code === 'EROFS') return true; + } + } + return false; +} + +function snapshotCacheGuards( + requests: CacheGuardRequest[], + packageRoot: string, + buildRoot: string, +): CacheGuardResult[] { if (requests.length < 128) return requests.map(snapshotCacheGuardDirect); + if ( + parseTruthyEnv(process.env.GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS) || + installTreeUnwritable(packageRoot, buildRoot) + ) { + return requests.map(snapshotCacheGuardDirect); + } try { const probe = spawnSync( process.execPath, @@ -2468,7 +2492,7 @@ function validateIdentityCache( return { mode, absolutePath }; }); options.onCacheValidationPass?.({ guardCount: requests.length }); - const actual = snapshotCacheGuards(requests); + const actual = snapshotCacheGuards(requests, cache.packageRoot, cache.buildRoot); const mismatch = actual.findIndex( (result, index) => !isDeepStrictEqual(result, entries[index][1]), ); diff --git a/gitnexus/src/core/auto-sync/analysis-worker-launch.ts b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts new file mode 100644 index 000000000..d57a251a6 --- /dev/null +++ b/gitnexus/src/core/auto-sync/analysis-worker-launch.ts @@ -0,0 +1,210 @@ +import { fork, type ChildProcess } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import type { AnalyzeOptions, AnalyzeResult } from '../run-analyze.js'; +import type { WorkerMessage } from '../../server/analyze-worker-protocol.js'; +import { autoHeapCapMb } from '../ingestion/utils/effective-ram.js'; + +const _require = createRequire(import.meta.url); +export type AutoSyncAnalysisRunner = ( + repoPath: string, + options: AnalyzeOptions, + timeoutMs: number, + signal?: AbortSignal, + onCancellationRequested?: () => void, + concurrency?: number, +) => Promise>; + +interface AnalysisWorker extends Pick { + stdout?: Pick | null; + stderr?: Pick | null; + unref?: () => void; + channel?: { unref(): void } | null; +} + +/** + * How long the parent keeps waiting after asking a worker to cancel. + * + * Must stay below `stopAutoSyncWatch`'s process-exit budget, or a worker wedged + * past its safe point still turns `watch stop` into a timeout. + */ +const AUTO_SYNC_CANCEL_GRACE_MS = 5_000; + +export interface AutoSyncAnalysisLaunchDeps { + forkWorker: (workerPath: string, execArgv: string[]) => AnalysisWorker; + setTimeoutFn: typeof setTimeout; + clearTimeoutFn: typeof clearTimeout; + cancelGraceMs: number; +} + +const DEFAULT_DEPS: AutoSyncAnalysisLaunchDeps = { + forkWorker: (workerPath, execArgv) => + fork(workerPath, [], { + execArgv, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }), + setTimeoutFn: setTimeout, + clearTimeoutFn: clearTimeout, + cancelGraceMs: AUTO_SYNC_CANCEL_GRACE_MS, +}; + +/** + * Per-worker V8 heap cap for one tick. + * + * `autoHeapCapMb()` is a whole-machine figure, so handing it to every fork + * over-commits memory by the parallelism factor. Admission already bounds + * parallelism to `floor(availableMemoryGB / 2)`, so dividing here keeps the sum + * of worker heaps inside the machine budget while leaving `max_concurrency` + * free to mean what it says. The two rules compose to a ~1.5GB per-worker floor. + */ +export function resolveWorkerHeapMb(concurrency = 1): number { + const slots = Number.isFinite(concurrency) && concurrency >= 1 ? Math.floor(concurrency) : 1; + return Math.max(1, Math.min(8192, Math.floor(autoHeapCapMb() / slots))); +} + +export function createAutoSyncAnalysisRunner( + overrides: Partial = {}, +): AutoSyncAnalysisRunner { + const deps = { ...DEFAULT_DEPS, ...overrides }; + return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency) => + new Promise>((resolve, reject) => { + if (signal?.aborted) { + reject(new Error('Analysis cancelled.')); + return; + } + const callerPath = fileURLToPath(import.meta.url); + const isDev = callerPath.endsWith('.ts'); + const workerPath = path.join( + path.dirname(callerPath), + '../../server', + isDev ? 'analyze-worker.ts' : 'analyze-worker.js', + ); + if (!existsSync(workerPath)) { + reject(new Error(`Auto-sync analyze worker is missing: ${workerPath}`)); + return; + } + const workerHeapMb = resolveWorkerHeapMb(concurrency); + const execArgv = isDev + ? [ + '--import', + pathToFileURL(_require.resolve('tsx/esm')).href, + `--max-old-space-size=${workerHeapMb}`, + ] + : [`--max-old-space-size=${workerHeapMb}`]; + const child = deps.forkWorker(workerPath, execArgv); + child.stdout?.resume(); + child.stderr?.resume(); + + let terminalOutcome: WorkerMessage | undefined; + let terminationError: Error | undefined; + let settled = false; + let graceTimer: ReturnType | undefined; + const cleanup = () => { + deps.clearTimeoutFn(timeout); + deps.clearTimeoutFn(graceTimer); + signal?.removeEventListener('abort', onAbort); + }; + // Stop the parent owning a worker it has given up waiting for. An + // established IPC channel keeps this event loop alive even after unref, + // so both handles have to go. Never a kill: the child may be inside + // native work and is left to reach its own safe point. + const releaseChild = () => { + child.channel?.unref?.(); + child.unref?.(); + }; + const settle = (error?: Error, result?: Pick) => { + if (settled) return; + settled = true; + cleanup(); + if (error) reject(error); + else resolve(result!); + }; + const requestCancellation = (error: Error) => { + if (settled || terminationError) return; + terminationError = error; + deps.clearTimeoutFn(timeout); + onCancellationRequested?.(); + // IPC has the same semantics on macOS and Windows. The worker exits only + // after reaching a JS-visible safe point; this parent keeps ownership until then. + try { + child.send({ type: 'cancel' }); + } catch { + // A closed IPC channel still has an exit/error path. Do not force-kill a + // worker that may be inside native code. + } + // Bounded wait. A worker stuck past its safe point would otherwise leave + // this promise pending forever, wedging `activeRun` so `stop()` — and the + // `watch stop` waiting on this process to exit — can never finish. Settle + // the parent's wait and drop the IPC channel's hold on this event loop; + // an established channel keeps the parent alive even after unref. The + // child is deliberately left running rather than killed mid-write. + graceTimer = deps.setTimeoutFn(() => { + if (settled) return; + releaseChild(); + settle( + new Error( + `${error.message} The analyze worker did not exit within ${deps.cancelGraceMs}ms; ` + + 'it was left running so its native work is not interrupted.', + ), + ); + }, deps.cancelGraceMs); + }; + const timeout = deps.setTimeoutFn( + () => requestCancellation(new Error(`Analysis timed out after ${timeoutMs}ms.`)), + timeoutMs, + ); + const onAbort = () => requestCancellation(new Error('Analysis cancelled.')); + signal?.addEventListener('abort', onAbort, { once: true }); + + child.on('message', (message: WorkerMessage) => { + // Once timeout/cancellation requested shutdown, its reason owns the + // result. A terminal IPC can already be queued behind cancellation. + if (message.type === 'progress' || terminalOutcome || terminationError) return; + terminalOutcome = message; + deps.clearTimeoutFn(timeout); + }); + child.on('error', (error) => { + const workerError = new Error(`Auto-sync analyze worker error: ${error.message}`); + requestCancellation(workerError); + // This settles immediately rather than waiting out the grace, so the + // grace timer that would otherwise have released the child is cleared + // by cleanup(). Release it here instead — an errored channel does not + // mean the worker stopped. + releaseChild(); + settle(workerError); + }); + child.on('exit', (code, childSignal) => { + if (settled) return; + if (terminationError) { + settle(terminationError); + return; + } + if (terminalOutcome?.type === 'complete') { + settle(undefined, { stats: terminalOutcome.result.stats }); + return; + } + if (terminalOutcome?.type === 'error') { + settle(new Error(terminalOutcome.message)); + return; + } + settle( + new Error( + `Auto-sync analyze worker exited before completion (${childSignal ?? code ?? 'unknown'}).`, + ), + ); + }); + try { + child.send({ type: 'start', repoPath, options }); + } catch (error) { + const startError = new Error( + `Failed to start auto-sync analyze worker: ${(error as Error).message}`, + ); + requestCancellation(startError); + settle(startError); + } + }); +} + +export const runAutoSyncAnalysis = createAutoSyncAnalysisRunner(); diff --git a/gitnexus/src/core/auto-sync/config.ts b/gitnexus/src/core/auto-sync/config.ts new file mode 100644 index 000000000..fa91aa8b4 --- /dev/null +++ b/gitnexus/src/core/auto-sync/config.ts @@ -0,0 +1,367 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { normalizeConfiguredCloneRoot } from './path-security.js'; + +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +export const AUTO_SYNC_CONFIG_FILE = 'watch_config.yml'; +const GROUP_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +const MIN_SYNC_INTERVAL_MINUTES = 5; +const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_SYNC_INTERVAL_MINUTES = Math.floor(MAX_TIMER_DELAY_MS / 60_000); +const DEFAULT_REPO_GIT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_CONCURRENCY = 1; +export const DEFAULT_ANALYZE_FAILURE_THRESHOLD = 3; +const MIN_ANALYZE_FAILURE_THRESHOLD = 2; +const ALLOWED_REMOTE_HOSTS = new Set(['github.com', 'gitlab.com', 'gitee.com']); + +/** + * A single clone/pull must fit inside one sync interval and inside an hour. + * This is also the guard for the unit slip the bare-number rule invites: + * `repo_git_timeout: 600000` means 600000 SECONDS (~7 days), which clears the + * Node timer ceiling and would silently disable the timeout. + */ +const MAX_REPO_GIT_TIMEOUT_MS = 3_600_000; + +// Mirrors REPO_NAME_PATTERN in server/git-clone.ts. Deliberately duplicated +// rather than imported: git-clone.ts already imports from this module, so the +// reverse edge would be a cycle. +const REMOTE_REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/; +// Same charset for a namespace segment: GitLab subgroups allow exactly these, +// and excluding separators is what stops a segment smuggling in traversal. +const REMOTE_PATH_SEGMENT_PATTERN = REMOTE_REPO_NAME_PATTERN; + +export interface AutoSyncProjectConfig { + localPath: string; + groupName?: string; + overwriteLocalChanges: boolean; + branches: string[]; + remoteUrls: string[]; +} + +export interface AutoSyncConfig { + configPath: string; + syncIntervalMinutes: number; + repoGitTimeoutMs: number; + analyzeTimeoutMs: number; + maxConcurrency: number; + analyzeFailureThreshold: number; + projects: AutoSyncProjectConfig[]; +} + +export type AutoSyncConfigLoadResult = + | { ok: true; config: AutoSyncConfig } + | { ok: false; reason: 'missing' | 'unreadable' | 'invalid'; message: string }; + +export function getAutoSyncConfigPath(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, AUTO_SYNC_CONFIG_FILE); +} + +export function parseBranchCandidates(branchValue: unknown): string[] { + const rawItems = Array.isArray(branchValue) + ? branchValue.flatMap((item) => String(item).split(',')) + : String(branchValue ?? '').split(','); + const branches: string[] = []; + const seen = new Set(); + for (const item of rawItems) { + const branch = item.trim(); + if (!branch || seen.has(branch)) continue; + seen.add(branch); + branches.push(branch); + } + return branches; +} + +export async function loadAutoSyncConfig( + configPath = getAutoSyncConfigPath(), +): Promise { + let content: string; + try { + content = await fs.readFile(configPath, 'utf-8'); + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return { + ok: false, + reason: 'missing', + message: `[auto-sync] Missing config file: ${configPath}. Auto sync is skipped.`, + }; + } + return { + ok: false, + reason: 'unreadable', + message: `[auto-sync] Unable to read config file: ${configPath}. Auto sync is skipped.`, + }; + } + + try { + return { ok: true, config: parseAutoSyncConfig(content, configPath) }; + } catch (err: unknown) { + return { + ok: false, + reason: 'invalid', + message: `[auto-sync] Invalid watch_config.yml: ${(err as Error).message}. Auto sync is skipped.`, + }; + } +} + +export function parseAutoSyncConfig(content: string, configPath: string): AutoSyncConfig { + const raw = yaml.load(content, { schema: yaml.JSON_SCHEMA }) as Record; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('expected a YAML object'); + } + + const errors: string[] = []; + const interval = Number(raw.sync_interval_minutes); + if (!Number.isInteger(interval) || interval <= 0) { + errors.push('sync_interval_minutes must be a positive integer'); + } else if (interval < MIN_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must be at least ${MIN_SYNC_INTERVAL_MINUTES}`); + } else if (interval > MAX_SYNC_INTERVAL_MINUTES) { + errors.push(`sync_interval_minutes must not exceed ${MAX_SYNC_INTERVAL_MINUTES}`); + } + + // YAML booleans survive JSON_SCHEMA (`true`/`false`). `Number(true) === 1` + // would otherwise pass the integer check and silently mean concurrency 1. + let maxConcurrency = DEFAULT_MAX_CONCURRENCY; + if (raw.max_concurrency !== undefined) { + if (typeof raw.max_concurrency !== 'number' || !Number.isInteger(raw.max_concurrency)) { + errors.push('max_concurrency must be a positive integer'); + } else if (raw.max_concurrency <= 0) { + errors.push('max_concurrency must be a positive integer'); + } else { + maxConcurrency = raw.max_concurrency; + } + } + + const repoGitTimeoutMs = + raw.repo_git_timeout === undefined + ? DEFAULT_REPO_GIT_TIMEOUT_MS + : parseDurationMs(raw.repo_git_timeout); + const maxRepoGitTimeoutMs = + Number.isInteger(interval) && + interval >= MIN_SYNC_INTERVAL_MINUTES && + interval <= MAX_SYNC_INTERVAL_MINUTES + ? Math.min(interval * 60_000, MAX_REPO_GIT_TIMEOUT_MS) + : undefined; + if (!Number.isInteger(repoGitTimeoutMs) || repoGitTimeoutMs <= 0) { + errors.push('repo_git_timeout must be a positive duration such as 10s'); + } else if (repoGitTimeoutMs > MAX_TIMER_DELAY_MS) { + errors.push(`repo_git_timeout must not exceed ${MAX_TIMER_DELAY_MS}ms`); + } else if (maxRepoGitTimeoutMs !== undefined && repoGitTimeoutMs > maxRepoGitTimeoutMs) { + errors.push( + `repo_git_timeout must not exceed ${maxRepoGitTimeoutMs}ms (the lesser of 1h and ` + + `sync_interval_minutes); a bare number is interpreted as seconds, so use an explicit ` + + `unit such as 600000ms or 10m`, + ); + } + + const maxAnalyzeTimeoutMs = + Number.isInteger(interval) && + interval >= MIN_SYNC_INTERVAL_MINUTES && + interval <= MAX_SYNC_INTERVAL_MINUTES + ? interval * 30_000 + : undefined; + const analyzeTimeoutMs = + raw.analyze_timeout === undefined + ? (maxAnalyzeTimeoutMs ?? 0) + : parseDurationMs(raw.analyze_timeout); + if (!Number.isInteger(analyzeTimeoutMs) || analyzeTimeoutMs <= 0) { + errors.push('analyze_timeout must be a positive duration such as 30m'); + } else if (maxAnalyzeTimeoutMs !== undefined && analyzeTimeoutMs > maxAnalyzeTimeoutMs) { + errors.push( + `analyze_timeout must not exceed half of sync_interval_minutes (${maxAnalyzeTimeoutMs / 60_000}m)`, + ); + } + + const analyzeFailureThreshold = + raw.analyze_failure_threshold === undefined + ? DEFAULT_ANALYZE_FAILURE_THRESHOLD + : Number(raw.analyze_failure_threshold); + if ( + !Number.isInteger(analyzeFailureThreshold) || + analyzeFailureThreshold < MIN_ANALYZE_FAILURE_THRESHOLD + ) { + errors.push(`analyze_failure_threshold must be an integer >= ${MIN_ANALYZE_FAILURE_THRESHOLD}`); + } + + const rawProjects = raw.projects; + if (!Array.isArray(rawProjects) || rawProjects.length === 0) { + errors.push('projects must contain at least one project'); + } + + const projects: AutoSyncProjectConfig[] = []; + if (Array.isArray(rawProjects)) { + rawProjects.forEach((projectValue, index) => { + const project = projectValue as Record; + if (!project || typeof project !== 'object' || Array.isArray(project)) { + errors.push(`projects[${index}] must be an object`); + return; + } + + const localPath = typeof project.local_path === 'string' ? project.local_path.trim() : ''; + if (!localPath) { + errors.push(`projects[${index}].local_path is required`); + } else { + try { + normalizeConfiguredCloneRoot(localPath); + } catch (err: unknown) { + errors.push(`projects[${index}].local_path ${(err as Error).message}`); + } + } + + const remoteUrls = Array.isArray(project.remote_urls) + ? project.remote_urls.map((url) => String(url).trim()).filter(Boolean) + : []; + if (remoteUrls.length === 0) { + errors.push(`projects[${index}].remote_urls must contain at least one URL`); + } + for (let urlIndex = 0; urlIndex < remoteUrls.length; urlIndex += 1) { + try { + validateAutoSyncRemoteUrl(remoteUrls[urlIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].remote_urls[${urlIndex}] ${(err as Error).message}`); + } + } + + if (project.branch !== undefined && project.branches !== undefined) { + errors.push(`projects[${index}] must not set both branch and branches`); + } + const branches = parseBranchCandidates( + project.branches !== undefined ? project.branches : project.branch, + ); + if (branches.length === 0) errors.push(`projects[${index}].branches is required`); + for (let branchIndex = 0; branchIndex < branches.length; branchIndex += 1) { + try { + validateAutoSyncBranchName(branches[branchIndex]); + } catch (err: unknown) { + errors.push(`projects[${index}].branches[${branchIndex}] ${(err as Error).message}`); + } + } + + const groupName = + typeof project.group_name === 'string' && project.group_name.trim() + ? project.group_name.trim() + : undefined; + if (groupName && !GROUP_NAME_PATTERN.test(groupName)) { + errors.push(`projects[${index}].group_name is invalid`); + } + + const overwriteLocalChanges = + project.overwrite_local_changes === undefined ? false : project.overwrite_local_changes; + if (typeof overwriteLocalChanges !== 'boolean') { + errors.push(`projects[${index}].overwrite_local_changes must be a boolean`); + } + + if (localPath && remoteUrls.length > 0 && branches.length > 0) { + projects.push({ + localPath, + groupName, + overwriteLocalChanges: overwriteLocalChanges === true, + branches, + remoteUrls, + }); + } + }); + } + + if (errors.length > 0) throw new Error(errors.join('; ')); + return { + configPath, + syncIntervalMinutes: interval, + repoGitTimeoutMs, + analyzeTimeoutMs, + maxConcurrency, + analyzeFailureThreshold, + projects, + }; +} + +export function validateAutoSyncRemoteUrl(remoteUrl: string): void { + const trimmed = remoteUrl.trim(); + if (trimmed.includes('?') || trimmed.includes('#')) { + throw new Error('must not include query strings or fragments'); + } + const match = /^git@([^:\s/]+):([^\s]+)$/.exec(trimmed); + if (!match) { + throw new Error('must use an SSH URL on github.com, gitlab.com, or gitee.com'); + } + const host = match[1].toLowerCase(); + const repoPath = match[2]; + if (!ALLOWED_REMOTE_HOSTS.has(host)) { + throw new Error('host must be one of github.com, gitlab.com, or gitee.com'); + } + const pathParts = repoPath.split('/'); + // Every segment becomes a directory component: the namespace segments build + // the clone path and the last one names the repo. So each is held to the same + // charset, which is what keeps a separator out of a segment — on Windows + // `..\..\outside` is traversal even though the segment is not literally `..`, + // and testing the raw string for `..` instead would reject an ordinary + // `foo..bar`. Traversal is a whole segment; a separator is a character. + const namespaceParts = pathParts.slice(0, -1); + if ( + repoPath.startsWith('/') || + pathParts.length < 2 || + pathParts.some((part) => !part || part === '.' || part === '..') || + namespaceParts.some((part) => !REMOTE_PATH_SEGMENT_PATTERN.test(part)) + ) { + throw new Error('path must include owner/repo without traversal'); + } + // The final segment becomes the on-disk clone directory via `extractRepoName`, + // whose name rules are stricter than the path check above: a backslash — or + // anything outside `[A-Za-z0-9._-]` — passes here and then throws once per + // tick inside the sync loop instead of at config load. These rules are a + // strict superset, so anything accepted here is accepted there. + const lastSegment = pathParts[pathParts.length - 1]; + const repoName = /\.git$/i.test(lastSegment) ? lastSegment.slice(0, -4) : lastSegment; + if ( + !repoName || + repoName === '.' || + repoName === '..' || + repoName === 'unknown' || + repoName.startsWith('-') || + !REMOTE_REPO_NAME_PATTERN.test(repoName) + ) { + throw new Error( + 'repository name must use only letters, digits, ".", "_", or "-" and must not be "unknown"', + ); + } +} + +export function validateAutoSyncBranchName(branch: string): void { + if (!branch.trim()) throw new Error('must not be empty'); + if (/[\s\0-\x1f\x7f]/.test(branch)) + throw new Error('must not contain whitespace or control characters'); + if (/[~^:?*[\\]/.test(branch)) throw new Error('contains characters not allowed in a git ref'); + if (branch.startsWith('-')) throw new Error('must not start with "-"'); + if (branch.startsWith('/')) throw new Error('must not start with "/"'); + if (branch.includes('..')) throw new Error('must not contain ".."'); + if (branch.includes('`')) throw new Error('must not contain backticks'); + if (branch.endsWith('/') || branch.endsWith('.')) throw new Error('must not end with "/" or "."'); + if (branch.includes('//')) throw new Error('must not contain consecutive slashes'); + if (branch.includes('@{')) throw new Error('must not contain "@{"'); + if ( + branch + .split('/') + .some( + (component) => + component.startsWith('.') || component.endsWith('.') || component.endsWith('.lock'), + ) + ) + throw new Error('must not contain hidden, trailing-dot, or .lock path components'); +} + +export function parseDurationMs(value: unknown): number { + if (typeof value === 'number') return value * 1_000; + const raw = String(value ?? '').trim(); + const match = /^(\d+)(ms|s|m)?$/.exec(raw); + if (!match) return Number.NaN; + const amount = Number(match[1]); + const unit = match[2] ?? 's'; + if (unit === 'ms') return amount; + if (unit === 's') return amount * 1_000; + return amount * 60_000; +} diff --git a/gitnexus/src/core/auto-sync/index.ts b/gitnexus/src/core/auto-sync/index.ts new file mode 100644 index 000000000..b02948779 --- /dev/null +++ b/gitnexus/src/core/auto-sync/index.ts @@ -0,0 +1,57 @@ +export { + AUTO_SYNC_CONFIG_FILE, + getAutoSyncConfigPath, + loadAutoSyncConfig, + parseAutoSyncConfig, + parseBranchCandidates, + parseDurationMs, + validateAutoSyncBranchName, + validateAutoSyncRemoteUrl, + type AutoSyncConfig, + type AutoSyncConfigLoadResult, + type AutoSyncProjectConfig, +} from './config.js'; +export { + buildStateKey, + getAutoSyncMutexPath, + getAutoSyncWatchDir, + getAutoSyncStatePath, + getProjectCommitInfoPath, + loadAutoSyncState, + resetAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + writeProjectCommitInfo, + type AutoSyncAnalyzeStatus, + type AutoSyncCommitState, + type AutoSyncCommitStateEntry, + type ProjectCommitInfoEntry, +} from './state.js'; +export { extractRepoNameFromRemoteUrl } from './repo.js'; +export { + normalizeConfiguredCloneRoot, + quarantineAutoSyncPartial, + resolveConfiguredCloneRoot, + type AutoSyncCloneRoot, +} from './path-security.js'; +export { + addRepoToGroup, + getAutoSyncRepoIdentity, + getConfiguredRepoPath, + resolveActualConcurrency, + runAutoSyncOnce, + syncGroupByName, + type AutoSyncLogger, + type AutoSyncRunDeps, + type AutoSyncRunResult, +} from './runner.js'; +export { + getAutoSyncWatchPaths, + readAutoSyncWatchStatus, + startAutoSyncWatch, + stopAutoSyncWatch, + type AutoSyncStartHandle, + type AutoSyncWatchStopResult, + type AutoSyncWatchPaths, + type WatchStatusRecord, +} from './starter.js'; diff --git a/gitnexus/src/core/auto-sync/path-security.ts b/gitnexus/src/core/auto-sync/path-security.ts new file mode 100644 index 000000000..ed9d9201b --- /dev/null +++ b/gitnexus/src/core/auto-sync/path-security.ts @@ -0,0 +1,286 @@ +import fs from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { getAutoSyncWatchDir } from './state.js'; + +const WINDOWS_DANGEROUS_ROOTS = + process.platform === 'win32' + ? [ + process.env.SystemRoot, + process.env.ProgramData, + process.env.ProgramFiles, + process.env['ProgramFiles(x86)'], + ].filter((entry): entry is string => Boolean(entry)) + : []; + +const DANGEROUS_ROOTS = new Set( + [ + '/', + os.homedir(), + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ...WINDOWS_DANGEROUS_ROOTS, + ].map((entry) => path.resolve(entry)), +); + +const DANGEROUS_PARENT_ROOTS = new Set( + [ + os.tmpdir(), + '/bin', + '/boot', + '/dev', + '/etc', + '/lib', + '/lib64', + '/opt', + '/proc', + '/private/tmp', + '/private/var', + '/root', + '/sbin', + '/sys', + '/tmp', + '/usr', + '/var', + ...WINDOWS_DANGEROUS_ROOTS, + ].map((entry) => path.resolve(entry)), +); + +const QUARANTINE_RETENTION_DAYS = 14; +const QUARANTINE_MAX_ENTRIES_PER_REPO = 5; + +// `auto-sync----` — see quarantineAutoSyncPartial. +// The UUID is the only fixed-shape field, so it anchors the grouping key, and +// everything after it is the basename (`[A-Za-z0-9._-]` by construction). +const QUARANTINE_ENTRY_PATTERN = + /^auto-sync-.+-\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-(.+)$/i; + +export interface AutoSyncCloneRoot { + root: string; + quarantineRoot: string; + quarantineRetentionDays: number; +} + +export async function resolveConfiguredCloneRoot(localPath: string): Promise { + const root = normalizeConfiguredCloneRoot(localPath); + assertNotDangerousRoot(root); + await assertNoSymlinkPath(root); + await fs.mkdir(root, { recursive: true }); + await assertDirectoryOwnerAndPermissions(root); + const realRoot = await fs.realpath(root); + assertContainedOrSame( + root, + realRoot, + 'Configured clone root realpath escaped its normalized path', + ); + assertNotDangerousRoot(realRoot); + assertNotGitNexusInternalRoot(realRoot); + const quarantineRoot = path.join(getAutoSyncWatchDir(), 'quarantine'); + await pruneQuarantineEntries(quarantineRoot); + + return { + root: realRoot, + quarantineRoot, + quarantineRetentionDays: QUARANTINE_RETENTION_DAYS, + }; +} + +export function normalizeConfiguredCloneRoot(localPath: string): string { + const value = localPath.trim(); + if (!value) throw new Error('local_path is required'); + if (!path.isAbsolute(value)) throw new Error('local_path must be an absolute path'); + if (value.split(path.sep).includes('..')) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + const resolved = path.resolve(value); + if (resolved !== path.normalize(value)) { + throw new Error('local_path must be normalized and must not contain traversal segments'); + } + return resolved; +} + +export async function quarantineAutoSyncPartial( + targetDir: string, + quarantineRoot: string, +): Promise { + await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o700 }); + const base = path.basename(targetDir); + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const destination = path.join( + quarantineRoot, + `auto-sync-${stamp}-${process.pid}-${randomUUID()}-${base}`, + ); + try { + await fs.rename(targetDir, destination); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'EXDEV') throw err; + await fs.cp(targetDir, destination, { recursive: true }); + await fs.rm(targetDir, { recursive: true, force: true }); + } + await fs.writeFile( + `${destination}.README.txt`, + [ + 'GitNexus auto-sync isolated a partial or unsafe clone result.', + `Created at: ${new Date().toISOString()}`, + `Original path: ${targetDir}`, + `Retention: keep for ${QUARANTINE_RETENTION_DAYS} days unless an operator reviews and removes it earlier.`, + 'Cleanup: verify the original path and remote before manual deletion.', + '', + ].join('\n'), + 'utf-8', + ); + return destination; +} + +async function pruneQuarantineEntries(quarantineRoot: string): Promise { + const cutoff = Date.now() - QUARANTINE_RETENTION_DAYS * 24 * 60 * 60 * 1_000; + // readdir and stat both resolve through a link, so a symlinked quarantine + // root would age-sweep and delete entries somewhere else entirely. + const rootStat = await fs.lstat(quarantineRoot).catch(() => undefined); + if (rootStat?.isSymbolicLink()) { + throw new Error(`Refusing symlinked auto-sync quarantine root: ${quarantineRoot}`); + } + let entries; + try { + entries = await fs.readdir(quarantineRoot); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return; + throw err; + } + const survivors = ( + await Promise.all( + entries + .filter((entry) => entry.startsWith('auto-sync-')) + .map(async (entry) => { + const entryPath = path.join(quarantineRoot, entry); + const stat = await fs.stat(entryPath).catch(() => undefined); + if (stat && stat.mtimeMs < cutoff) { + await fs.rm(entryPath, { recursive: true, force: true }); + return undefined; + } + return entry; + }), + ) + ).filter((entry): entry is string => entry !== undefined); + + // Age alone never bounds a repo that fails on every tick: one partial clone + // per tick stays inside the retention window forever. Keep the newest few per + // repo. Entries that do not match the generated naming scheme (operator + // notes, names from another version) are left to the age sweep alone. + const byRepo = new Map(); + for (const entry of survivors) { + if (entry.endsWith('.README.txt')) continue; + const repo = QUARANTINE_ENTRY_PATTERN.exec(entry)?.[1]; + if (!repo) continue; + const group = byRepo.get(repo) ?? []; + group.push(entry); + byRepo.set(repo, group); + } + await Promise.all( + [...byRepo.values()].flatMap((group) => + group + // The timestamp is the leading fixed-width field, so a descending + // string sort is newest-first. + .sort((a, b) => (a < b ? 1 : a > b ? -1 : 0)) + .slice(QUARANTINE_MAX_ENTRIES_PER_REPO) + .map(async (entry) => { + await fs.rm(path.join(quarantineRoot, entry), { recursive: true, force: true }); + await fs.rm(path.join(quarantineRoot, `${entry}.README.txt`), { force: true }); + }), + ), + ); +} + +function assertNotDangerousRoot(root: string): void { + if (root === path.resolve(getGlobalDir(), 'repos')) return; + if (DANGEROUS_ROOTS.has(root)) throw new Error(`Refusing unsafe auto-sync clone root: ${root}`); + for (const dangerousRoot of DANGEROUS_PARENT_ROOTS) { + const rel = path.relative(dangerousRoot, root); + if (rel && !rel.startsWith('..') && !path.isAbsolute(rel)) { + throw new Error(`Refusing unsafe auto-sync clone root under ${dangerousRoot}: ${root}`); + } + } + if (path.parse(root).root === root) + throw new Error(`Refusing filesystem root as clone root: ${root}`); +} + +function assertNotGitNexusInternalRoot(root: string): void { + const gitnexusDir = path.resolve(getGlobalDir()); + const blocked = [ + path.join(gitnexusDir, 'groups'), + path.join(gitnexusDir, 'indexes'), + path.join(gitnexusDir, 'quarantine'), + path.join(getAutoSyncWatchDir(gitnexusDir), 'quarantine'), + ]; + for (const blockedRoot of blocked) { + const rel = path.relative(blockedRoot, root); + if (!rel || (!rel.startsWith('..') && !path.isAbsolute(rel))) { + throw new Error(`Refusing GitNexus internal directory as auto-sync clone root: ${root}`); + } + } +} + +async function assertNoSymlinkPath(root: string): Promise { + const parsed = path.parse(root); + let current = parsed.root; + const parts = root.slice(parsed.root.length).split(path.sep).filter(Boolean); + for (const part of parts) { + current = path.join(current, part); + let stat; + try { + stat = await fs.lstat(current); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; + throw err; + } + if (stat.isSymbolicLink()) + throw new Error(`Refusing symlink in auto-sync clone root path: ${current}`); + } +} + +export async function assertDirectoryOwnerAndPermissions(root: string): Promise { + const stat = await fs.stat(root); + if (!stat.isDirectory()) throw new Error(`auto-sync clone root is not a directory: ${root}`); + // POSIX uid/mode have no meaning on Windows, and this runs on every tick for + // every project, so throwing here failed 100% of repos forever while `watch + // status` still read `running`. Skip the ownership assertions rather than the + // whole feature: the caller's other guards — dangerous-root rejection + // (including the Windows system roots), symlink refusal, realpath containment + // and the GitNexus-internal-root check — all still apply, and managed git runs + // with `core.hooksPath` pinned to the null device. + if (process.platform === 'win32') return; + if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { + throw new Error(`auto-sync clone root is owned by uid ${stat.uid}, not current process uid`); + } + const mode = stat.mode & 0o777; + const groupWritable = (mode & 0o020) !== 0; + const worldWritable = (mode & 0o002) !== 0; + if (worldWritable) { + throw new Error(`Refusing world-writable auto-sync clone root: ${root}`); + } + if (groupWritable) { + throw new Error(`Refusing group-writable auto-sync clone root: ${root}`); + } +} + +function assertContainedOrSame(root: string, child: string, message: string): void { + const rel = path.relative(root, child); + if (rel.startsWith('..') || path.isAbsolute(rel)) throw new Error(message); +} diff --git a/gitnexus/src/core/auto-sync/repo.ts b/gitnexus/src/core/auto-sync/repo.ts new file mode 100644 index 000000000..ed8e1731d --- /dev/null +++ b/gitnexus/src/core/auto-sync/repo.ts @@ -0,0 +1,7 @@ +import { extractRepoName } from '../../server/git-clone.js'; +import { validateAutoSyncRemoteUrl } from './config.js'; + +export function extractRepoNameFromRemoteUrl(remoteUrl: string): string { + validateAutoSyncRemoteUrl(remoteUrl); + return extractRepoName(remoteUrl); +} diff --git a/gitnexus/src/core/auto-sync/runner.ts b/gitnexus/src/core/auto-sync/runner.ts new file mode 100644 index 000000000..ed5f7afd7 --- /dev/null +++ b/gitnexus/src/core/auto-sync/runner.ts @@ -0,0 +1,561 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { loadGroupConfig } from '../group/config-parser.js'; +import { getDefaultGitnexusDir, getGroupDir } from '../group/storage.js'; +import { syncGroup } from '../group/sync.js'; +import { registerRepo, resolveBranchPlacement, type RepoMeta } from '../../storage/repo-manager.js'; +import { extractRepoNameFromRemoteUrl } from './repo.js'; +import { cloneOrPull, runGit } from '../../server/git-clone.js'; +import { resolveConfiguredCloneRoot } from './path-security.js'; +import { + buildStateKey, + loadAutoSyncState, + saveAutoSyncState, + shouldAnalyzeCommit, + writeProjectCommitInfo, + type AutoSyncAnalyzeStatus, + type AutoSyncCommitStateEntry, + type ProjectCommitInfoEntry, +} from './state.js'; +import type { AutoSyncConfig, AutoSyncProjectConfig } from './config.js'; +import { validateAutoSyncRemoteUrl } from './config.js'; +import { runAutoSyncAnalysis, type AutoSyncAnalysisRunner } from './analysis-worker-launch.js'; + +export interface AutoSyncLogger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export interface AutoSyncRunDeps { + cloneOrPull: typeof cloneOrPull; + getCurrentBranch: (repoPath: string, timeoutMs: number) => Promise; + getCurrentCommit: (repoPath: string, timeoutMs: number) => Promise; + runAnalysis: AutoSyncAnalysisRunner; + registerRepo: typeof registerRepo; + resolveBranchPlacement: typeof resolveBranchPlacement; + loadState: typeof loadAutoSyncState; + saveState: typeof saveAutoSyncState; + writeCommitInfo: typeof writeProjectCommitInfo; + addRepoToGroup: typeof addRepoToGroup; + syncGroupByName: typeof syncGroupByName; + resolveCloneRoot: typeof resolveConfiguredCloneRoot; + getAvailableMemoryGB: () => number; +} + +export interface AutoSyncRunResult { + synced: number; + analyzed: number; + skippedAnalysis: number; + failed: number; +} + +const _require = createRequire(import.meta.url); +const yaml = _require('js-yaml') as typeof import('js-yaml'); + +const DEFAULT_LOGGER: AutoSyncLogger = { + info: (message) => process.stderr.write(`${message}\n`), + warn: (message) => process.stderr.write(`${message}\n`), + error: (message) => process.stderr.write(`${message}\n`), +}; + +const DEFAULT_DEPS: AutoSyncRunDeps = { + cloneOrPull, + getCurrentBranch: async (repoPath, timeoutMs) => { + const branch = (await runGit(['branch', '--show-current'], repoPath, { timeoutMs })).trim(); + return branch || undefined; + }, + getCurrentCommit: async (repoPath, timeoutMs) => + (await runGit(['rev-parse', 'HEAD'], repoPath, { timeoutMs })).trim(), + runAnalysis: runAutoSyncAnalysis, + registerRepo, + resolveBranchPlacement, + loadState: loadAutoSyncState, + saveState: saveAutoSyncState, + writeCommitInfo: writeProjectCommitInfo, + addRepoToGroup, + syncGroupByName, + resolveCloneRoot: resolveConfiguredCloneRoot, + getAvailableMemoryGB: () => Math.floor(process.availableMemory?.() ?? 0) / 1024 / 1024 / 1024, +}; + +export async function runAutoSyncOnce( + config: AutoSyncConfig, + options: { + deps?: Partial; + logger?: AutoSyncLogger; + now?: () => Date; + signal?: AbortSignal; + onAnalysisCancellationRequested?: () => void; + } = {}, +): Promise { + const deps = { ...DEFAULT_DEPS, ...options.deps }; + const logger = options.logger ?? DEFAULT_LOGGER; + const now = options.now ?? (() => new Date()); + throwIfAborted(options.signal); + const state = await deps.loadState(); + throwIfAborted(options.signal); + const groupsToSync = new Set(); + const groupStateKeys = new Map(); + const result: AutoSyncRunResult = { synced: 0, analyzed: 0, skippedAnalysis: 0, failed: 0 }; + const commitInfoEntries: ProjectCommitInfoEntry[] = []; + const actualConcurrency = resolveActualConcurrency( + config.maxConcurrency, + deps.getAvailableMemoryGB(), + ); + logger.info( + `[auto-sync] Starting sync loop with max_concurrency=${actualConcurrency} analyze_failure_threshold=${config.analyzeFailureThreshold}.`, + ); + + const workItems = await buildWorkItems(config, deps); + // What will actually run at once. One repo means one worker, so the common + // single-project case still hands that worker the whole machine budget. + const analysisParallelism = Math.max(1, Math.min(actualConcurrency, workItems.length)); + const repoResults = await mapWithConcurrency( + workItems, + actualConcurrency, + options.signal, + async (item) => { + const lastSyncTime = now().toISOString(); + try { + throwIfAborted(options.signal); + if (!item.cloneRoot || !item.repoName || !item.targetDir) { + throw new Error(item.error ?? 'Invalid auto-sync work item'); + } + const repoName = item.repoName; + const targetDir = item.targetDir; + const syncResult = await syncFirstAvailableBranch({ + item, + repoName, + targetDir, + timeoutMs: config.repoGitTimeoutMs, + deps, + logger, + }); + throwIfAborted(options.signal); + if (syncResult.ok === false) { + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}; no configured branch could be pulled: ${syncResult.message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir, + branch: item.project.branches[0], + status: syncResult.status, + analyzeConsecutiveFailures: 0, + lastSyncTime, + }; + } + + const currentBranch = syncResult.branch; + + const currentCommit = await deps.getCurrentCommit(targetDir, config.repoGitTimeoutMs); + const stateKey = buildStateKey(targetDir, currentBranch); + const previous = state[stateKey]; + let analyzeStatus: AutoSyncAnalyzeStatus = 'skipped'; + let analyzedCommitId = previous?.analyzedCommitId; + let analyzeConsecutiveFailures = previous?.analyzeConsecutiveFailures ?? 0; + let lastAnalyzeError = previous?.lastAnalyzeError; + const groupSyncPending = previous?.groupSyncPending === true; + let stats: RepoMeta['stats'] | undefined; + + if (previous && previous.codeCommitId !== currentCommit) { + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } + + if (analyzeConsecutiveFailures >= config.analyzeFailureThreshold) { + analyzeStatus = 'threshold_skipped'; + logger.error( + `[auto-sync] Skip analysis for ${targetDir}; analyze consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold} reached threshold. Fix the repository or clear auto-sync state before retrying.`, + ); + } else if ( + shouldAnalyzeCommit({ + currentCommit, + previousAnalyzedCommit: previous?.analyzedCommitId, + previousStatus: previous?.lastAnalyzeStatus, + }) + ) { + try { + const analysis = await deps.runAnalysis( + targetDir, + { branch: currentBranch, skipAgentsMd: true, skipSkills: true }, + config.analyzeTimeoutMs, + options.signal, + options.onAnalysisCancellationRequested, + analysisParallelism, + ); + throwIfAborted(options.signal); + stats = analysis.stats; + analyzeStatus = 'success'; + analyzedCommitId = currentCommit; + analyzeConsecutiveFailures = 0; + lastAnalyzeError = undefined; + } catch (err: unknown) { + if (options.signal?.aborted) throw err; + analyzeStatus = 'failed'; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = shortErrorMessage(err); + logger.error( + `[auto-sync] Analysis failed for ${targetDir}; consecutive failures ${analyzeConsecutiveFailures}/${config.analyzeFailureThreshold}: ${lastAnalyzeError}`, + ); + } + } else { + logger.info(`[auto-sync] Skip analysis for ${targetDir}; commit unchanged.`); + } + throwIfAborted(options.signal); + + return { + kind: 'synced' as const, + project: item.project, + repoName, + remoteUrl: item.remoteUrl, + targetDir, + branch: currentBranch, + currentCommit, + analyzedCommitId, + analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + groupSyncPending, + stats, + stateKey, + lastSyncTime, + }; + } catch (err: unknown) { + if (options.signal?.aborted) throw err; + logger.error( + `[auto-sync] Repository sync failed for ${item.remoteUrl}: ${(err as Error).message}`, + ); + return { + kind: 'failed' as const, + project: item.project, + remoteUrl: item.remoteUrl, + targetDir: item.targetDir ?? '', + status: 'sync_failed' as const, + lastSyncTime, + }; + } + }, + ); + + for (const repoResult of repoResults) { + if (repoResult.kind === 'failed') { + result.failed += 1; + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + status: repoResult.status, + lastSyncTime: repoResult.lastSyncTime, + }); + continue; + } + + result.synced += 1; + let analyzeStatus = repoResult.analyzeStatus; + let analyzeConsecutiveFailures = repoResult.analyzeConsecutiveFailures; + let lastAnalyzeError = repoResult.lastAnalyzeError; + let analyzedCommitId = repoResult.analyzedCommitId; + if (analyzeStatus === 'success') { + const meta: RepoMeta = { + repoPath: repoResult.targetDir, + lastCommit: repoResult.currentCommit, + indexedAt: repoResult.lastSyncTime, + stats: repoResult.stats!, + branch: repoResult.branch, + remoteUrl: repoResult.remoteUrl, + }; + try { + // Reproduce the placement the analyze worker already made. Registering + // without a branch always takes the primary/flat arm, which relabels a + // pinned branch entry with whatever this tick happened to sync — visible + // on the documented branch-fallback path. + const placement = await deps.resolveBranchPlacement( + repoResult.targetDir, + repoResult.branch, + ); + await deps.registerRepo(repoResult.targetDir, meta, { + name: getAutoSyncRepoIdentity(repoResult.remoteUrl), + // Omitted rather than passed as undefined, so a primary index is + // registered with the same option shape it had before this branch. + ...(placement.branch ? { branch: placement.branch } : {}), + }); + result.analyzed += 1; + } catch (err: unknown) { + analyzeStatus = 'failed'; + analyzedCommitId = undefined; + analyzeConsecutiveFailures += 1; + lastAnalyzeError = `Repository registration failed: ${shortErrorMessage(err)}`; + result.failed += 1; + logger.error(`[auto-sync] ${lastAnalyzeError}`); + } + } else if (analyzeStatus === 'failed') { + result.failed += 1; + } else { + result.skippedAnalysis += 1; + } + + const stateEntry: AutoSyncCommitStateEntry = { + codeCommitId: repoResult.currentCommit, + analyzedCommitId, + lastAnalyzeStatus: analyzeStatus, + analyzeConsecutiveFailures, + lastAnalyzeError, + groupSyncPending: repoResult.groupSyncPending, + lastSyncTime: repoResult.lastSyncTime, + }; + state[repoResult.stateKey] = stateEntry; + + commitInfoEntries.push({ + remoteUrl: repoResult.remoteUrl, + localPath: repoResult.targetDir, + branch: repoResult.branch, + codeCommitId: repoResult.currentCommit, + analyzedCommitId, + status: analyzeStatus, + analyzeConsecutiveFailures, + analyzeFailureThreshold: config.analyzeFailureThreshold, + lastAnalyzeError, + lastSyncTime: repoResult.lastSyncTime, + }); + + if (repoResult.project.groupName) { + let groupMembershipOk = false; + let membershipAdded = false; + try { + membershipAdded = await deps.addRepoToGroup( + repoResult.project, + getAutoSyncRepoIdentity(repoResult.remoteUrl), + getAutoSyncRepoIdentity(repoResult.remoteUrl), + ); + groupMembershipOk = true; + } catch (err: unknown) { + result.failed += 1; + logger.error( + `[auto-sync] Group update failed for ${repoResult.project.groupName}: ${(err as Error).message}`, + ); + } + if ( + groupMembershipOk && + (analyzeStatus === 'success' || + (membershipAdded && analyzeStatus === 'skipped') || + (analyzeStatus === 'skipped' && repoResult.groupSyncPending)) + ) { + const groupName = repoResult.project.groupName; + groupsToSync.add(groupName); + const keys = groupStateKeys.get(groupName) ?? []; + keys.push(repoResult.stateKey); + groupStateKeys.set(groupName, keys); + } + } + } + + await deps.saveState(state); + await deps.writeCommitInfo(commitInfoEntries); + let groupStateChanged = false; + for (const groupName of groupsToSync) { + try { + await deps.syncGroupByName(groupName); + for (const stateKey of groupStateKeys.get(groupName) ?? []) { + if (state[stateKey].groupSyncPending) { + state[stateKey].groupSyncPending = false; + groupStateChanged = true; + } + } + } catch (err: unknown) { + result.failed += 1; + for (const stateKey of groupStateKeys.get(groupName) ?? []) { + if (!state[stateKey].groupSyncPending) { + state[stateKey].groupSyncPending = true; + groupStateChanged = true; + } + } + logger.error(`[auto-sync] Group sync failed for ${groupName}: ${(err as Error).message}`); + } + } + if (groupStateChanged) await deps.saveState(state); + return result; +} + +function shortErrorMessage(err: unknown): string { + const message = err instanceof Error ? err.message : String(err); + return message.replace(/\s+/g, ' ').slice(0, 240); +} + +export function getConfiguredRepoPath( + project: Pick, + repoName: string, + remoteUrl?: string, +): string { + if (!remoteUrl) return path.resolve(project.localPath, repoName); + const identity = getAutoSyncRepoIdentity(remoteUrl); + return path.resolve(project.localPath, ...identity.split('/').slice(0, -1), repoName); +} + +export async function addRepoToGroup( + project: Pick, + groupPath: string, + registryName = groupPath, +): Promise { + if (!project.groupName) return false; + const groupDir = getGroupDir(getDefaultGitnexusDir(), project.groupName); + const config = await loadGroupConfig(groupDir); + if (config.repos[groupPath] === registryName) return false; + if (config.repos[groupPath] !== undefined) { + throw new Error(`group path ${groupPath} is already mapped to ${config.repos[groupPath]}`); + } + config.repos[groupPath] = registryName; + await writeGroupConfigAtomic(path.join(groupDir, 'group.yaml'), config); + return true; +} + +export function getAutoSyncRepoIdentity(remoteUrl: string): string { + validateAutoSyncRemoteUrl(remoteUrl); + const [, host, remotePath] = /^git@([^:\s/]+):([^\s]+)$/.exec(remoteUrl.trim())!; + return `${host.toLowerCase()}/${remotePath.replace(/\.git$/i, '')}`; +} + +export async function syncGroupByName(groupName: string): Promise { + const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName); + const config = await loadGroupConfig(groupDir); + await syncGroup(config, { groupDir }); +} + +async function writeGroupConfigAtomic(filePath: string, config: unknown): Promise { + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, yaml.dump(config), 'utf-8'); + await fs.rename(tmpPath, filePath); +} + +export function resolveActualConcurrency(configured: number, availableMemoryGB: number): number { + const memoryLimit = Math.max(1, Math.floor(availableMemoryGB / 2)); + return Math.max(1, Math.min(configured, memoryLimit)); +} + +async function buildWorkItems( + config: AutoSyncConfig, + deps: AutoSyncRunDeps, +): Promise { + const items: AutoSyncWorkItem[] = []; + const targetOwners = new Map(); + for (const project of config.projects) { + let cloneRoot: AutoSyncWorkItem['cloneRoot']; + try { + cloneRoot = await deps.resolveCloneRoot(project.localPath); + } catch (err: unknown) { + for (const remoteUrl of project.remoteUrls) { + items.push({ project, remoteUrl, error: shortErrorMessage(err) }); + } + continue; + } + for (const remoteUrl of project.remoteUrls) { + try { + const repoName = extractRepoNameFromRemoteUrl(remoteUrl); + const targetDir = getConfiguredRepoPath({ localPath: cloneRoot.root }, repoName, remoteUrl); + const previous = targetOwners.get(targetDir); + if (previous !== undefined) { + throw new Error( + `Duplicate auto-sync targetDir ${targetDir} for ${previous} and ${remoteUrl}`, + ); + } + targetOwners.set(targetDir, remoteUrl); + items.push({ project, remoteUrl, cloneRoot, repoName, targetDir }); + } catch (err: unknown) { + items.push({ project, remoteUrl, error: shortErrorMessage(err) }); + } + } + } + return items; +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + signal: AbortSignal | undefined, + worker: (item: T) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let nextIndex = 0; + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (nextIndex < items.length) { + throwIfAborted(signal); + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await worker(items[currentIndex]); + throwIfAborted(signal); + } + }); + // Settle every runner before surfacing a failure. Promise.all rejects on the + // first error while siblings are still inside a clone or waiting on an + // analyze fork, and the caller treats that rejection as "the run is over" — + // it releases the watch mutex and exits, orphaning those children. Each + // runner already refuses new work at the abort check above, so waiting here + // costs nothing on the cancel path. + const settlements = await Promise.allSettled(runners); + const failure = settlements.find((s) => s.status === 'rejected'); + if (failure) throw (failure as PromiseRejectedResult).reason; + return results; +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new Error('Auto-sync run cancelled.'); +} + +interface AutoSyncWorkItem { + project: AutoSyncProjectConfig; + remoteUrl: string; + cloneRoot?: Awaited>; + repoName?: string; + targetDir?: string; + error?: string; +} + +async function syncFirstAvailableBranch(input: { + item: AutoSyncWorkItem; + repoName: string; + targetDir: string; + timeoutMs: number; + deps: AutoSyncRunDeps; + logger: AutoSyncLogger; +}): Promise< + | { ok: true; branch: string } + | { ok: false; status: 'branch_unavailable' | 'sync_timeout'; message: string } +> { + const failures: string[] = []; + let sawTimeout = false; + for (const branch of input.item.project.branches) { + try { + await input.deps.cloneOrPull(input.item.remoteUrl, input.targetDir, undefined, { + allowedCloneRoot: input.item.cloneRoot!.root, + expectedRepoName: input.repoName, + quarantineRoot: input.item.cloneRoot!.quarantineRoot, + allowAutoSyncSsh: true, + timeoutMs: input.timeoutMs, + branch, + overwriteLocalChanges: input.item.project.overwriteLocalChanges, + }); + const currentBranch = await input.deps.getCurrentBranch(input.targetDir, input.timeoutMs); + if (currentBranch === branch) return { ok: true, branch }; + failures.push(`${branch}: checked out ${currentBranch ?? ''}`); + input.logger.warn( + `[auto-sync] Branch ${branch} for ${input.item.remoteUrl} synced but current branch is ${currentBranch ?? ''}; trying next branch.`, + ); + } catch (err: unknown) { + const message = (err as Error).message; + if (message.includes('timed out')) sawTimeout = true; + failures.push(`${branch}: ${message}`); + input.logger.warn( + `[auto-sync] Branch ${branch} unavailable for ${input.item.remoteUrl}: ${message}`, + ); + } + } + return { + ok: false, + status: sawTimeout ? 'sync_timeout' : 'branch_unavailable', + message: failures.join('; '), + }; +} diff --git a/gitnexus/src/core/auto-sync/starter.ts b/gitnexus/src/core/auto-sync/starter.ts new file mode 100644 index 000000000..624e092ec --- /dev/null +++ b/gitnexus/src/core/auto-sync/starter.ts @@ -0,0 +1,643 @@ +import fs from 'node:fs/promises'; +import crypto from 'node:crypto'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { acquireFileLock, FileLockBusyError } from '../../storage/file-lock.js'; +import { getGlobalDir } from '../../storage/repo-manager.js'; +import { isProcessAlive, readProcessStartTime } from '../../utils/process-identity.js'; +import { loadAutoSyncConfig } from './config.js'; +import { runAutoSyncOnce } from './runner.js'; +import { getAutoSyncMutexPath, getAutoSyncWatchDir } from './state.js'; + +export interface AutoSyncStartHandle { + stop(): Promise; +} + +export type WatchStatusState = + | 'running' + | 'cancelling' + | 'stopping' + | 'stopped' + | 'stale' + | 'error'; +export type AutoSyncWatchStopResult = 'stopped' | 'not_running' | 'refused' | 'timeout'; + +export interface WatchStatusRecord { + state: WatchStatusState; + pid?: number; + ownerId?: string; + configPath?: string; + message?: string; + updatedAt: string; +} + +export interface WatchOwnerRecord { + pid: number; + ownerId: string; + processStartTime: string; + createdAt: string; +} + +interface WatchStopRequestRecord { + pid: number; + ownerId: string; + processStartTime: string; + requestedAt: string; +} + +const WATCH_STOP_POLL_MS = 250; + +export interface AutoSyncWatchPaths { + pidPath: string; + mutexPath: string; + ownerPath: string; + statusPath: string; +} + +export interface AutoSyncWatchControlDeps { + isProcessAlive(pid: number): boolean; + readProcessCommand(pid: number): string | undefined; + readProcessStartTime(pid: number): string | undefined; + sleep(ms: number): Promise; +} + +export function getAutoSyncWatchPaths(gitnexusDir = getGlobalDir()): AutoSyncWatchPaths { + const watchDir = getAutoSyncWatchDir(gitnexusDir); + return { + pidPath: path.join(watchDir, 'watch.pid'), + mutexPath: getAutoSyncMutexPath(gitnexusDir), + ownerPath: path.join(watchDir, 'watch.owner.json'), + statusPath: path.join(watchDir, 'watch.status.json'), + }; +} + +export async function startAutoSyncWatch( + options: { + setIntervalFn?: typeof setInterval; + clearIntervalFn?: typeof clearInterval; + runOnce?: typeof runAutoSyncOnce; + stderr?: Pick; + keepAlive?: boolean; + paths?: AutoSyncWatchPaths; + deps?: Partial; + } = {}, +): Promise { + const stderr = options.stderr ?? process.stderr; + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const ownerId = crypto.randomUUID(); + const processStartTime = deps.readProcessStartTime(process.pid); + if (!processStartTime) { + stderr.write('[auto-sync] Unable to verify the watch process start time.\n'); + return null; + } + await fs.mkdir(path.dirname(paths.pidPath), { recursive: true }); + const releaseLock = await acquireWatchLock(paths, deps, stderr, processStartTime); + if (!releaseLock) return null; + + try { + await writeWatchOwner(paths, { + pid: process.pid, + ownerId, + processStartTime, + createdAt: new Date().toISOString(), + }); + await writeAtomicText(paths.pidPath, `${process.pid}\n`); + + const loaded = await loadAutoSyncConfig(); + if (loaded.ok === false) { + stderr.write(`${loaded.message}\n`); + await writeWatchStatus(paths, { + state: 'error', + pid: process.pid, + ownerId, + message: loaded.message, + updatedAt: new Date().toISOString(), + }); + await cleanupWatchFiles(paths, ownerId, releaseLock); + return null; + } + await writeWatchStatus(paths, { + state: 'running', + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + updatedAt: new Date().toISOString(), + }); + + const runOnce = options.runOnce ?? runAutoSyncOnce; + const setIntervalFn = options.setIntervalFn ?? setInterval; + const clearIntervalFn = options.clearIntervalFn ?? clearInterval; + let activeRun: Promise | undefined; + let activeAbortController: AbortController | undefined; + let stopping = false; + let statusWrite = Promise.resolve(); + const updateStatus = (state: WatchStatusState, message?: string) => { + const write = statusWrite.then(() => + writeWatchStatus(paths, { + state, + pid: process.pid, + ownerId, + configPath: loaded.config.configPath, + message, + updatedAt: new Date().toISOString(), + }), + ); + statusWrite = write.catch(() => {}); + return write; + }; + const reportStatusWriteFailure = (error: unknown) => { + stderr.write(`[auto-sync] Failed to publish watch status: ${(error as Error).message}\n`); + }; + const runSafely = () => { + if (stopping) return; + if (activeRun) { + stderr.write('[auto-sync] Previous run is still active; skipping overlapping run.\n'); + return; + } + const startedAt = new Date(); + stderr.write(`[auto-sync] Watch loop started at ${startedAt.toISOString()}.\n`); + const abortController = new AbortController(); + const run = runOnce(loaded.config, { + signal: abortController.signal, + onAnalysisCancellationRequested: () => { + if (!stopping) { + void updateStatus( + 'cancelling', + 'Analysis cancellation requested; waiting for the worker to reach a safe shutdown point.', + ).catch(reportStatusWriteFailure); + } + }, + }) + .then((result) => { + stderr.write( + `[auto-sync] Watch loop finished: synced=${result.synced} analyzed=${result.analyzed} skipped=${result.skippedAnalysis} failed=${result.failed}.\n`, + ); + }) + .catch((err: unknown) => { + stderr.write(`[auto-sync] Scheduled run failed: ${(err as Error).message}\n`); + stderr.write('[auto-sync] Watch loop finished: failed.\n'); + }) + .finally(async () => { + if (activeRun === run) { + activeRun = undefined; + activeAbortController = undefined; + } + if (!stopping) { + await updateStatus('running').catch(reportStatusWriteFailure); + } + }); + activeRun = run; + activeAbortController = abortController; + }; + + let stopPromise: Promise | undefined; + const stop = () => + (stopPromise ??= (async () => { + stopping = true; + clearIntervalFn(timer); + clearIntervalFn(controlTimer); + activeAbortController?.abort(); + try { + await updateStatus('stopping'); + await activeRun?.catch(() => {}); + await updateStatus('stopped'); + } finally { + await cleanupWatchFiles(paths, ownerId, releaseLock); + } + })()); + const checkStopRequest = async () => { + const request = await readStopRequest(stopRequestPath(paths, ownerId)); + if ( + request?.pid === process.pid && + request.ownerId === ownerId && + request.processStartTime === processStartTime + ) { + void stop().catch((error: unknown) => { + stderr.write(`[auto-sync] Failed to stop watch: ${(error as Error).message}\n`); + }); + } + }; + + runSafely(); + const controlTimer = setIntervalFn(() => void checkStopRequest(), WATCH_STOP_POLL_MS); + const timer = setIntervalFn(runSafely, loaded.config.syncIntervalMinutes * 60_000); + if (options.keepAlive === false) { + controlTimer.unref?.(); + timer.unref?.(); + } + return { stop }; + } catch (error) { + await cleanupWatchFiles(paths, ownerId, releaseLock).catch(() => {}); + throw error; + } +} + +async function acquireWatchLock( + paths: AutoSyncWatchPaths, + deps: AutoSyncWatchControlDeps, + stderr: Pick, + processStartTime: string, +): Promise<(() => Promise) | null> { + try { + return await acquireFileLock(paths.mutexPath, { + pid: process.pid, + processStartTime, + isProcessAlive: deps.isProcessAlive, + readProcessStartTime: deps.readProcessStartTime, + }); + } catch (err: unknown) { + if (!(err instanceof FileLockBusyError)) throw err; + } + + const owner = await readOwnerFile(paths.ownerPath); + if (!owner) { + stderr.write( + `[auto-sync] Watch mutex is held but owner metadata is not ready or invalid. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return null; + } + if (!deps.isProcessAlive(owner.pid)) { + stderr.write( + `[auto-sync] Watch mutex remains after owner pid ${owner.pid} exited. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return null; + } + const reason = getWatchProcessIdentityError(owner, deps); + if (reason) { + stderr.write(`[auto-sync] Refusing to trust existing watch pid ${owner.pid}; ${reason}.\n`); + return null; + } + stderr.write(`[auto-sync] Watch is already running with pid ${owner.pid}.\n`); + return null; +} + +export async function stopAutoSyncWatch( + options: { + paths?: AutoSyncWatchPaths; + stderr?: Pick; + deps?: Partial; + timeoutMs?: number; + pollMs?: number; + } = {}, +): Promise { + const stderr = options.stderr ?? process.stderr; + const paths = options.paths ?? getAutoSyncWatchPaths(); + const deps = resolveWatchDeps(options.deps); + const timeoutMs = options.timeoutMs ?? 10_000; + const pollMs = options.pollMs ?? 100; + const pid = await readPid(paths.pidPath); + if (!pid) { + const owner = await readOwnerFile(paths.ownerPath); + if (owner && deps.isProcessAlive(owner.pid)) { + stderr.write( + `[auto-sync] Watch appears to be starting with pid ${owner.pid}; pid file is not ready.\n`, + ); + return 'refused'; + } + if (owner || (await fileExists(paths.mutexPath))) { + stderr.write( + `[auto-sync] Watch ownership is stale or incomplete. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return 'refused'; + } + stderr.write('[auto-sync] Watch is not running.\n'); + return 'not_running'; + } + if (!deps.isProcessAlive(pid)) { + stderr.write( + `[auto-sync] Watch pid ${pid} is stale. Confirm no watch process is running, then remove ${paths.mutexPath}.\n`, + ); + return 'refused'; + } + + const owner = await readVerifiedWatchOwner(paths, pid, deps); + if (owner.ok === false) { + stderr.write(`[auto-sync] Refusing to stop pid ${pid}; ${owner.reason}.\n`); + return 'refused'; + } + + const currentPid = await readPid(paths.pidPath); + const currentOwner = await readVerifiedWatchOwner(paths, pid, deps); + if ( + currentPid !== pid || + currentOwner.ok === false || + currentOwner.owner.ownerId !== owner.owner.ownerId + ) { + stderr.write(`[auto-sync] Refusing to stop pid ${pid}; watch ownership changed.\n`); + return 'refused'; + } + + await writeAtomicText( + stopRequestPath(paths, owner.owner.ownerId), + `${JSON.stringify({ + pid, + ownerId: owner.owner.ownerId, + processStartTime: owner.owner.processStartTime, + requestedAt: new Date().toISOString(), + } satisfies WatchStopRequestRecord)}\n`, + ); + stderr.write(`[auto-sync] Stop requested for watch pid ${pid}.\n`); + const stopped = await waitForProcessExit(pid, { + deps, + timeoutMs, + pollMs, + processStartTime: owner.owner.processStartTime, + }); + if (!stopped) { + stderr.write(`[auto-sync] Watch pid ${pid} did not exit within ${timeoutMs}ms.\n`); + return 'timeout'; + } + return 'stopped'; +} + +export async function readAutoSyncWatchStatus( + paths = getAutoSyncWatchPaths(), + deps: Partial = {}, +): Promise { + const resolvedDeps = resolveWatchDeps(deps); + const pid = await readPid(paths.pidPath); + const stored = await readStatusFile(paths.statusPath); + const updatedAt = stored?.updatedAt ?? new Date().toISOString(); + if (pid && !resolvedDeps.isProcessAlive(pid)) { + return { + ...stored, + state: 'stale', + pid, + message: 'pid file exists but process is not running', + updatedAt, + }; + } + if (pid) { + const owner = await readVerifiedWatchOwner(paths, pid, resolvedDeps); + if (owner.ok === false) { + return { + ...stored, + state: 'error', + pid, + message: owner.reason, + updatedAt, + }; + } + if (stored?.state === 'error') { + return { + ...stored, + pid, + ownerId: owner.owner.ownerId, + updatedAt, + }; + } + return { + ...stored, + state: + stored?.state === 'cancelling' || stored?.state === 'stopping' ? stored.state : 'running', + pid, + ownerId: owner.owner.ownerId, + updatedAt, + }; + } + return stored ?? { state: 'stopped', updatedAt }; +} + +function isSafeWatchOwnerId(ownerId: string): boolean { + return ( + ownerId === path.basename(ownerId) && + !ownerId.includes('..') && + !ownerId.includes('/') && + !ownerId.includes('\\') + ); +} + +async function readOwnerFile(ownerPath: string): Promise { + try { + const raw = await fs.readFile(ownerPath, 'utf-8'); + const parsed = JSON.parse(raw) as WatchOwnerRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + isSafeWatchOwnerId(parsed.ownerId) && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime + ) { + return parsed; + } + return undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return undefined; + } +} + +async function readVerifiedWatchOwner( + paths: AutoSyncWatchPaths, + pid: number, + deps: AutoSyncWatchControlDeps, +): Promise<{ ok: true; owner: WatchOwnerRecord } | { ok: false; reason: string }> { + const [status, owner] = await Promise.all([ + readStatusFile(paths.statusPath), + readOwnerFile(paths.ownerPath), + ]); + if (!owner) return { ok: false, reason: 'watch owner is missing or invalid' }; + if (!status) return { ok: false, reason: 'watch status is missing or invalid' }; + if (owner.pid !== pid) return { ok: false, reason: 'watch owner pid does not match pid file' }; + if (status.pid !== pid) return { ok: false, reason: 'watch status pid does not match pid file' }; + if (!status.ownerId || status.ownerId !== owner.ownerId) { + return { ok: false, reason: 'watch status owner does not match watch owner' }; + } + const identityError = getWatchProcessIdentityError(owner, deps); + if (identityError) return { ok: false, reason: identityError }; + return { ok: true, owner }; +} + +function getWatchProcessIdentityError( + owner: WatchOwnerRecord, + deps: AutoSyncWatchControlDeps, +): string | undefined { + const processStartTime = deps.readProcessStartTime(owner.pid); + if (!processStartTime) return 'unable to verify process start time'; + if (processStartTime !== owner.processStartTime) return 'pid belongs to a different process'; + const command = deps.readProcessCommand(owner.pid); + if (!command) return 'unable to verify process command'; + if ( + !/(?:^|\s)(?:watch|auto-sync)(?:\s|$)/.test(command) || + !/(?:gitnexus|[\\/]cli[\\/]index\.(?:ts|[cm]?js))/.test(command) + ) { + return 'pid command is not a GitNexus auto-sync process'; + } + return undefined; +} + +async function waitForProcessExit( + pid: number, + options: { + deps: AutoSyncWatchControlDeps; + timeoutMs: number; + pollMs: number; + processStartTime?: string; + }, +): Promise { + // A bare liveness poll cannot tell "still running" from "exited, and the OS + // handed the pid to something else" — so a reused pid would keep us waiting + // on an unrelated process and then report the watch stopped once THAT exits. + // The start time identifies the process behind the number. + const isOriginalProcessAlive = () => { + if (!options.deps.isProcessAlive(pid)) return false; + if (!options.processStartTime) return true; + const startTime = options.deps.readProcessStartTime(pid); + return startTime === undefined || startTime === options.processStartTime; + }; + const deadline = Date.now() + options.timeoutMs; + while (Date.now() < deadline) { + if (!isOriginalProcessAlive()) return true; + await options.deps.sleep(options.pollMs); + } + return !isOriginalProcessAlive(); +} + +async function readPid(pidPath: string): Promise { + try { + const raw = await fs.readFile(pidPath, 'utf-8'); + const pid = Number(raw.trim()); + return Number.isInteger(pid) && pid > 0 ? pid : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw err; + } +} + +async function readStatusFile(statusPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(statusPath, 'utf-8')) as WatchStatusRecord; + return parsed && typeof parsed === 'object' ? parsed : undefined; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + return { + state: 'error', + message: `unable to read status file: ${(err as Error).message}`, + updatedAt: new Date().toISOString(), + }; + } +} + +function stopRequestPath(paths: AutoSyncWatchPaths, ownerId: string): string { + if (!isSafeWatchOwnerId(ownerId)) { + throw new Error('watch ownerId is not a safe filename component'); + } + return path.join(path.dirname(paths.pidPath), `watch.stop.${ownerId}.json`); +} + +async function readStopRequest(filePath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(filePath, 'utf-8')) as WatchStopRequestRecord; + if ( + parsed && + typeof parsed === 'object' && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.ownerId === 'string' && + parsed.ownerId && + typeof parsed.processStartTime === 'string' && + parsed.processStartTime && + typeof parsed.requestedAt === 'string' && + parsed.requestedAt + ) { + return parsed; + } + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') return undefined; + } + return undefined; +} + +async function writeWatchStatus( + paths: AutoSyncWatchPaths, + record: WatchStatusRecord, +): Promise { + await fs.mkdir(path.dirname(paths.statusPath), { recursive: true }); + const tmpPath = `${paths.statusPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, paths.statusPath); +} + +async function writeWatchOwner(paths: AutoSyncWatchPaths, record: WatchOwnerRecord): Promise { + await writeAtomicText(paths.ownerPath, `${JSON.stringify(record, null, 2)}\n`); +} + +async function cleanupWatchFiles( + paths: AutoSyncWatchPaths, + ownerId: string, + releaseLock: () => Promise, +): Promise { + try { + const owner = await readOwnerFile(paths.ownerPath); + if (owner?.ownerId === ownerId) { + if ((await readPid(paths.pidPath)) === owner.pid) await removeIfExists(paths.pidPath); + if ((await readOwnerFile(paths.ownerPath))?.ownerId === ownerId) { + await removeIfExists(paths.ownerPath); + } + await removeIfExists(stopRequestPath(paths, ownerId)); + } + } finally { + await releaseLock(); + } +} + +async function writeAtomicText(filePath: string, content: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, content, 'utf-8'); + await fs.rename(tmpPath, filePath); +} + +async function removeIfExists(filePath: string): Promise { + await fs.rm(filePath, { force: true }); +} + +async function fileExists(filePath: string): Promise { + return fs.access(filePath).then( + () => true, + () => false, + ); +} + +function resolveWatchDeps(deps: Partial = {}): AutoSyncWatchControlDeps { + return { + isProcessAlive: deps.isProcessAlive ?? isProcessAlive, + readProcessCommand: + deps.readProcessCommand ?? + ((pid) => { + try { + const command = + process.platform === 'win32' + ? execFileSync( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `(Get-CimInstance Win32_Process -Filter \"ProcessId = ${pid}\").CommandLine`, + ], + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim() + : execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + return command || undefined; + } catch { + return undefined; + } + }), + readProcessStartTime: deps.readProcessStartTime ?? readProcessStartTime, + sleep: + deps.sleep ?? + ((ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + })), + }; +} diff --git a/gitnexus/src/core/auto-sync/state.ts b/gitnexus/src/core/auto-sync/state.ts new file mode 100644 index 000000000..01f3ed2f5 --- /dev/null +++ b/gitnexus/src/core/auto-sync/state.ts @@ -0,0 +1,173 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { acquireFileLock, FileLockBusyError } from '../../storage/file-lock.js'; +import { getGlobalDir } from '../../storage/repo-manager.js'; + +export type AutoSyncAnalyzeStatus = 'success' | 'failed' | 'skipped' | 'threshold_skipped'; + +export interface AutoSyncCommitStateEntry { + codeCommitId: string; + analyzedCommitId?: string; + lastAnalyzeStatus?: AutoSyncAnalyzeStatus; + analyzeConsecutiveFailures?: number; + lastAnalyzeError?: string; + groupSyncPending?: boolean; + lastSyncTime: string; +} + +export type AutoSyncCommitState = Record; + +export function getAutoSyncWatchDir(gitnexusDir = getGlobalDir()): string { + return path.join(gitnexusDir, 'watch'); +} + +export function getAutoSyncMutexPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'watch.mutex'); +} + +export function getAutoSyncStatePath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'auto-sync-state.json'); +} + +export function getProjectCommitInfoPath(gitnexusDir = getGlobalDir()): string { + return path.join(getAutoSyncWatchDir(gitnexusDir), 'project_commit_info.txt'); +} + +export async function resetAutoSyncState(gitnexusDir = getGlobalDir()): Promise { + let releaseLock: () => Promise; + try { + releaseLock = await acquireFileLock(getAutoSyncMutexPath(gitnexusDir)); + } catch (error) { + if (error instanceof FileLockBusyError) return false; + throw error; + } + + try { + await Promise.all([ + fs.rm(getAutoSyncStatePath(gitnexusDir), { force: true }), + fs.rm(getProjectCommitInfoPath(gitnexusDir), { force: true }), + ]); + return true; + } finally { + await releaseLock(); + } +} + +export function buildStateKey(repoPath: string, branch: string): string { + return `${path.resolve(repoPath)}|${branch}`; +} + +export function shouldAnalyzeCommit(input: { + currentCommit: string; + previousAnalyzedCommit?: string; + previousStatus?: AutoSyncAnalyzeStatus; +}): boolean { + if (!input.currentCommit) return false; + if (input.previousStatus === 'failed') return true; + return input.currentCommit !== input.previousAnalyzedCommit; +} + +export async function loadAutoSyncState( + statePath = getAutoSyncStatePath(), +): Promise { + try { + const raw = await fs.readFile(statePath, 'utf-8'); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {}; + return Object.fromEntries( + Object.entries(parsed).filter((entry): entry is [string, AutoSyncCommitStateEntry] => + isAutoSyncCommitStateEntry(entry[1]), + ), + ); + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return {}; + // Corrupt JSON is genuinely unrecoverable, so rebuilding is the only move. + // An unreadable file (EACCES, EIO, EISDIR) is different: the state is + // probably intact, and returning {} here would make the tick overwrite it, + // losing every repo's analyzed commit and failure count. + if (!(err instanceof SyntaxError)) throw err; + process.stderr.write( + `[auto-sync] Ignoring corrupt state file: ${statePath}. State will be rebuilt.\n`, + ); + return {}; + } +} + +function isAutoSyncCommitStateEntry(value: unknown): value is AutoSyncCommitStateEntry { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const entry = value as Record; + return ( + typeof entry.codeCommitId === 'string' && + typeof entry.lastSyncTime === 'string' && + (entry.analyzedCommitId === undefined || typeof entry.analyzedCommitId === 'string') && + (entry.lastAnalyzeStatus === undefined || + entry.lastAnalyzeStatus === 'success' || + entry.lastAnalyzeStatus === 'failed' || + entry.lastAnalyzeStatus === 'skipped' || + entry.lastAnalyzeStatus === 'threshold_skipped') && + (entry.analyzeConsecutiveFailures === undefined || + (typeof entry.analyzeConsecutiveFailures === 'number' && + Number.isInteger(entry.analyzeConsecutiveFailures) && + entry.analyzeConsecutiveFailures >= 0)) && + (entry.lastAnalyzeError === undefined || typeof entry.lastAnalyzeError === 'string') && + (entry.groupSyncPending === undefined || typeof entry.groupSyncPending === 'boolean') + ); +} + +export async function saveAutoSyncState( + state: AutoSyncCommitState, + statePath = getAutoSyncStatePath(), +): Promise { + await fs.mkdir(path.dirname(statePath), { recursive: true }); + const tmpPath = `${statePath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8'); + await fs.rename(tmpPath, statePath); +} + +export async function writeProjectCommitInfo( + entries: ProjectCommitInfoEntry[], + infoPath = getProjectCommitInfoPath(), +): Promise { + await fs.mkdir(path.dirname(infoPath), { recursive: true }); + const lines = [ + '# GitNexus auto-sync project commit info', + `updated_at: ${new Date().toISOString()}`, + '', + ...entries.flatMap((entry) => [ + `remote: ${entry.remoteUrl}`, + `local_path: ${entry.localPath}`, + `branch: ${entry.branch ?? ''}`, + `code_commit: ${entry.codeCommitId ?? ''}`, + `analyzed_commit: ${entry.analyzedCommitId ?? ''}`, + `status: ${entry.status}`, + `analyze_consecutive_failures: ${entry.analyzeConsecutiveFailures ?? 0}`, + ...(entry.analyzeFailureThreshold === undefined + ? [] + : [`analyze_failure_threshold: ${entry.analyzeFailureThreshold}`]), + ...(entry.lastAnalyzeError ? [`last_analyze_error: ${entry.lastAnalyzeError}`] : []), + `last_sync_time: ${entry.lastSyncTime}`, + '', + ]), + ]; + const tmpPath = `${infoPath}.tmp.${process.pid}.${Date.now()}`; + await fs.writeFile(tmpPath, `${lines.join('\n')}\n`, 'utf-8'); + await fs.rename(tmpPath, infoPath); +} + +export interface ProjectCommitInfoEntry { + remoteUrl: string; + localPath: string; + branch?: string; + codeCommitId?: string; + analyzedCommitId?: string; + status: + | AutoSyncAnalyzeStatus + | 'sync_failed' + | 'branch_skipped' + | 'branch_unavailable' + | 'sync_timeout'; + analyzeConsecutiveFailures?: number; + analyzeFailureThreshold?: number; + lastAnalyzeError?: string; + lastSyncTime: string; +} diff --git a/gitnexus/src/core/group/PIPELINE.md b/gitnexus/src/core/group/PIPELINE.md index 9a8c4b972..72961cca3 100644 --- a/gitnexus/src/core/group/PIPELINE.md +++ b/gitnexus/src/core/group/PIPELINE.md @@ -16,10 +16,12 @@ flowchart TD D --> E1[TopicExtractor] D --> E2[HttpRouteExtractor] D --> E3[GrpcExtractor] + D --> E4[GraphqlExtractor] E1 --> F[ExtractedContract array
per repo] E2 --> F E3 --> F + E4 --> F B --> M[ManifestExtractor] M --> G[Manifest contracts
+ cross-links] @@ -59,14 +61,27 @@ flowchart TD **Strategy A** (graph-assisted) uses Cypher over edges already produced by the main ingestion pipeline: + - HTTP: `HANDLES_ROUTE` / `FETCHES` edges from `(File)-[]->(Route)` - topic: none (pipeline doesn't yet produce topic nodes — Strategy B only) - gRPC: none (Strategy B + proto map only) -**Strategy B** (source-scan) is 100% tree-sitter based after this PR. +**Strategy B** (source-scan) uses tree-sitter for language source and the +official GraphQL parser for `.graphql` / `.gql` operation documents. Each `*-patterns/.ts` plugin owns its grammar + S-expression queries; the top-level orchestrator imports neither. +GraphQL detection is opt-in with `detect.graphql: true`. The initial slice +recognizes imported NestJS `Query`, `Mutation`, and `Subscription` decorators on +top-level imported `Resolver` classes, plus named operation documents. Generated +object documents, static `gql` templates, and `TypedDocumentString` values are +verified against the operation and its resolved root fragments. Providers and +consumers must resolve to one exact, real graph symbol; anonymous operations, +dynamic decorator names, ambiguous generated symbols, malformed documents, +symlink escapes, and bounded-parser overflows are skipped rather than linked +approximately. `matching.exclude_links_paths` also suppresses configured GraphQL +root fields from exact cross-linking while retaining their registry entries. + ## Plugin architecture ```mermaid @@ -117,6 +132,7 @@ They use the `MATCH (n) WHERE labels(n) IN [...]` allowlist form, NOT the `MATCH (n:A|B)` disjunction — LadybugDB's parser rejects a disjunction that names a reserved keyword (e.g. `Macro`, `Union`), which is what broke the `custom` branch in #2325: + - `topic` → `labels(n) IN ['Function','Method','Class','Interface']` - `grpc`/`thrift` method → `labels(n) IN ['Function','Method']`, service → `labels(n) IN ['Class','Interface']` - `lib` → `labels(n) IN ['Module']` @@ -144,7 +160,7 @@ without coordinating through any shared state. ## Cross-repo trace (`cross-trace.ts`) A second consumer of the bridge. Where cross-impact fans a blast radius -*outward* from one symbol, cross-trace stitches a directed **path** between +_outward_ from one symbol, cross-trace stitches a directed **path** between two symbols that live in different repos: ```mermaid @@ -158,7 +174,7 @@ flowchart TD ``` It reuses the same `symbolUid` join as cross-impact, but issues its own -*pair* query (`listCrossingsBetween`) because a path needs BOTH endpoints of +_pair_ query (`listCrossingsBetween`) because a path needs BOTH endpoints of a crossing — the uid-filtered neighbor join (`resolveBridgeNeighbors`, shared with impact) returns only the far side. The crossing is clamped to one boundary (`MAX_SUPPORTED_CROSS_DEPTH`). With `pdg: true` the boundary-adjacent diff --git a/gitnexus/src/core/group/bridge-db.ts b/gitnexus/src/core/group/bridge-db.ts index 5fbe711aa..3c8c07bf2 100644 --- a/gitnexus/src/core/group/bridge-db.ts +++ b/gitnexus/src/core/group/bridge-db.ts @@ -3,14 +3,23 @@ import path from 'node:path'; import { createHash } from 'node:crypto'; import lbug from '@ladybugdb/core'; import type { LbugValue } from '@ladybugdb/core'; -import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js'; +import type { + BridgeHandle, + BridgeMeta, + StoredContract, + CrossLink, + RepoSnapshot, + MatchType, +} from './types.js'; import { BRIDGE_SCHEMA_QUERIES, BRIDGE_SCHEMA_VERSION } from './bridge-schema.js'; +import { recordedMatchStages, recordedRepoList } from './completeness.js'; import { closeLbugConnection, openLbugConnection, type LbugConnectionHandle, } from '../lbug/lbug-config.js'; import { dedupeContracts, dedupeCrossLinks } from './normalization.js'; +import { withGroupSyncLock } from './group-lock.js'; import { createLogger } from '../logger.js'; import { retryRename, writeFileAtomic } from '../../storage/fs-atomic.js'; @@ -647,15 +656,347 @@ export async function closeBridgeDb(handle: BridgeHandle): Promise { /* ------------------------------------------------------------------ */ export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise { - await writeFileAtomic(path.join(groupDir, 'meta.json'), JSON.stringify(meta, null, 2)); + // Strip the reader-only fields HERE rather than at each writer. `readBridgeMeta` + // sets both on what it returns, so any caller that reads-modifies-writes would + // round-trip them to disk — and `pairedWithDatabase` is the poisonous one: + // persisted, it tells every future reader the pair was verified when nothing + // verified it. That rule used to live in the body of the only such caller, + // which held exactly as long as there was one. There are now three writers and + // two of them read first. Enforced at the boundary, no writer can get it wrong. + const { repoListsUnreadable: _reader1, pairedWithDatabase: _reader2, ...persisted } = meta; + await writeFileAtomic(path.join(groupDir, 'meta.json'), JSON.stringify(persisted, null, 2)); } +/** + * Does `meta` still describe the `bridge.lbug` sitting next to it? + * + * `writeBridge` stamps the database's size and mtime into the metadata it + * writes, so a metadata file left over from an earlier sync cannot match a + * database that was replaced after it. Callers whose answer depends on the + * metadata being true of THIS database (cross-repo impact reads completeness + * from it) must not treat a mismatch as fact. + * + * When BOTH halves of the stamp are absent the metadata predates stamping, and + * it is judged on the write order of the two files instead — see + * {@link unstampedMetaPairsByWriteOrder}. Failing every unstamped metadata + * closed would mark all pre-existing bridges as incomplete until re-synced, + * trading a narrow window for a repo-wide regression; accepting them all hands + * back "verified" for the very window this pairing exists to catch. + * + * A stamp is a PAIR, so exactly one half present is rejected rather than waved + * through. That is not the legacy shape: something wrote a stamp and did not + * finish, which is the very condition stamping was added to detect. Joining the + * two `undefined` checks with `||` returned "verified" for precisely the shape + * that most deserves suspicion. + * + * Returns `false` when the database itself cannot be stat'd, on either path, + * since metadata describing a file that is not there describes nothing. + * + * The checks are ORDERED by how strong their evidence is, strongest first, and + * each later one is reached only because every earlier one had nothing to say. + * `provenanceUnknown` therefore comes first: a metadata file whose own writer + * says it cannot vouch for the database beside it has settled the question, and + * neither the stamp nor the write-order heuristic may overturn that. + * + * The marker is not decoration. `refreshPreservedBridgeMeta` rewrites this file + * atomically without touching the database, which leaves `meta.mtime` newer — + * the write order a paired write produces, and the one the unstamped branch + * ACCEPTS. Reading the marker after that branch (or not at all) hands back + * "verified" for a pair the same code path had just found broken. + */ +export async function bridgeMetaMatchesFile(groupDir: string, meta: BridgeMeta): Promise { + if (meta.provenanceUnknown) return false; + const stampedSize = meta.bridgeSize !== undefined; + const stampedMtime = meta.bridgeMtimeMs !== undefined; + if (!stampedSize && !stampedMtime) return unstampedMetaPairsByWriteOrder(groupDir); + if (!stampedSize || !stampedMtime) return false; + try { + const stat = await fsp.stat(path.join(groupDir, 'bridge.lbug')); + return stat.size === meta.bridgeSize && stat.mtimeMs === meta.bridgeMtimeMs; + } catch { + return false; + } +} + +/** + * Could the unstamped `meta.json` plausibly have been written by the sync that + * put this `bridge.lbug` beside it? + * + * `writeBridge` renames the database into place and writes the metadata AFTER, + * so `meta.mtime >= db.mtime` holds for any pair written together — including + * pairs written by builds from before the stamp existed, which is what makes + * this usable as back-compat rather than a repo-wide "re-sync everything". + * The only way to reach a database strictly NEWER than the metadata beside it + * is a swap whose metadata write did not land: the stale-meta-beside-a-new- + * database window, whose completeness `runGroupImpact` would otherwise spend as + * fact. + * + * This is a HEURISTIC ON WRITE ORDER, not proof of provenance. It answers "were + * these two written in the order a successful sync writes them?", and treats + * that as a proxy for "do these two belong together". It is wrong in two + * directions, and neither is theoretical: + * - FALSE ACCEPT, from a non-monotonic wall clock. `mtimeMs` is realtime, not + * monotonic, so an NTP step backwards, a VM snapshot restore or container + * clock skew between the database write and the metadata write can leave a + * genuinely mis-paired set reading as ordered. Anything that touches the + * stale metadata after a swap does the same — a restore from backup, an + * editor save, a copy that preserves only the database's times. The STAMP + * is what actually closes this; a pair that has one never reaches here. + * + * Coarse filesystem mtime granularity is NOT this hazard, despite looking + * like it: it collapses a pair written together to equal times, and equal + * is accepted, which is the correct verdict for that pair. + * + * - FALSE REJECT, from anything that rewrites the database's mtime after the + * metadata's — `cp -r`, `rsync` without `-t`, a machine move, a restore + * that replays files in directory order. An intact legacy pair is then + * demoted to a lower bound and stays there until the next successful sync + * re-stamps it; there is no other recovery, because nothing on the read + * path can distinguish it from the swap window it is imitating. + * + * This direction is the safe one — it degrades an answer to a floor rather + * than vouching for one — but it is a real, reachable cost, not a + * theoretical one, and it is NOT true that the rule can only ever demote + * pairs that were already broken. + * + * Equality counts as paired. On a filesystem with coarse mtime granularity both + * writes land in the same tick, and demanding a strictly newer metadata file + * would reject every legacy bridge there for a reason that is about the + * filesystem rather than about the bridge. + * + * A timestamp that cannot be measured is no match, the same convention the + * read-only handle cache applies to a bridge it could not stat: a comparison + * that could not be made is not a comparison that succeeded. + */ +async function unstampedMetaPairsByWriteOrder(groupDir: string): Promise { + try { + const [dbStat, metaStat] = await Promise.all([ + fsp.stat(path.join(groupDir, 'bridge.lbug')), + fsp.stat(path.join(groupDir, 'meta.json')), + ]); + return metaStat.mtimeMs >= dbStat.mtimeMs; + } catch { + return false; + } +} + +/** + * Read `meta.json`, validating the SHAPE of what it holds. + * + * The read and the parse have always been guarded — an absent or unparseable + * file answers `version: 0`, which every caller already treats as "no + * provenance". What was not guarded is a file that parses into something that + * is not this shape: `runGroupImpact` spread both repo lists directly into a + * `Set`, so a non-iterable there threw a TypeError out of the whole cross-repo + * query, from a point where the bridge lease had been taken and not yet + * released. A malformed file is a reason to answer "provenance unknown", never + * a reason to crash the question. + */ export async function readBridgeMeta(groupDir: string): Promise { + const unreadable: BridgeMeta = { version: 0, generatedAt: '', missingRepos: [] }; + let parsed: unknown; try { const content = await fsp.readFile(path.join(groupDir, 'meta.json'), 'utf-8'); - return JSON.parse(content) as BridgeMeta; + parsed = JSON.parse(content); } catch { - return { version: 0, generatedAt: '', missingRepos: [] }; + return unreadable; + } + // `JSON.parse` succeeds on `null`, `7` and `[]` too, and none of them are + // metadata. Reading `.version` off the first of those is a thrown TypeError; + // reading it off the others silently yields `undefined`, which passes the + // version gate as if the bridge had been vouched for. + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return unreadable; + + const raw = parsed as Partial; + const missingRepos = recordedRepoList(raw.missingRepos); + const unreadableRepos = recordedRepoList(raw.unreadableRepos); + // Each list is judged on its own: a file whose `unreadableRepos` is garbage + // can still carry a `missingRepos` that was genuinely measured, and throwing + // that away would turn one unknown into two. + const repoListsUnreadable = + (raw.missingRepos !== undefined && missingRepos === undefined) || + (raw.unreadableRepos !== undefined && unreadableRepos === undefined); + + const meta: BridgeMeta = { + ...raw, + // A version that is not a number cannot be compared against + // BRIDGE_SCHEMA_VERSION; `0` is this file's existing word for "provenance + // unknown", which is exactly what such a file gives us. + // `0` is this file's word for "no provenance". A version that is not a + // positive integer is not a schema version, and letting one through splits + // the four gates that read this field: `ensureBridgeReady` and + // `openBridgeDbReadOnly` both compare `> 0 && !== CURRENT` and would open + // the bridge, `bridgeExists` compares `=== 0 || === CURRENT` and would say + // it is not there, and `bridgeProvenanceUnknown` compares `=== 0` and would + // call the answer complete. Normalizing here keeps all four agreeing + // instead of teaching each one the same new case. + version: + Number.isInteger(raw.version) && (raw.version as number) > 0 ? (raw.version as number) : 0, + generatedAt: typeof raw.generatedAt === 'string' ? raw.generatedAt : '', + missingRepos: missingRepos ?? [], + }; + // Absent, not empty. `unreadableRepos` is optional and "not recorded" is a + // distinct state from "measured none", so an unusable value is dropped rather + // than carried through — `repoListsUnreadable` is what records that something + // was there and could not be read. + if (unreadableRepos) meta.unreadableRepos = unreadableRepos; + else delete meta.unreadableRepos; + // Same absent-vs-empty rule, through the one shared reader. + const suppressed = recordedMatchStages(raw.suppressedMatchStages); + if (suppressed) meta.suppressedMatchStages = suppressed; + else delete meta.suppressedMatchStages; + if (repoListsUnreadable) meta.repoListsUnreadable = true; + return meta; +} + +/* ------------------------------------------------------------------ */ +/* refreshPreservedBridgeMeta */ +/* ------------------------------------------------------------------ */ + +/** + * What a refresh did to `meta.json`. + * + * - `restamped` — the pair still matched, so the lists were refreshed + * and the stamp re-taken from the database on disk. + * - `provenance-unknown` — the pair did NOT match (or there is no database to + * match), so the lists were refreshed and the metadata + * marked as unable to vouch for the file beside it. + * - `no-bridge` — neither `meta.json` nor `bridge.lbug` exists, so + * there is no pair to keep honest and nothing written. + */ +export type PreservedBridgeMetaOutcome = 'restamped' | 'provenance-unknown' | 'no-bridge'; + +async function fileExists(filePath: string): Promise { + try { + await fsp.access(filePath); + return true; + } catch { + return false; + } +} + +/** + * Bring `meta.json`'s diagnostic lists up to date with a sync that PRESERVED + * the bridge instead of rebuilding it, without ever making the metadata claim + * more about the database than it did before. + * + * `syncGroup`'s total-failure path keeps the previous run's contracts and + * deliberately leaves `bridge.lbug` alone — the contracts that bridge holds are + * the ones being preserved. But `runGroupImpact` reads completeness from + * `meta.json`, not from `contracts.json`, so leaving the metadata alone too left + * the two files telling different stories: the registry said "this sync could + * not read svc/users" while a cross-repo query answered "complete, nothing + * depends on this" (R4/R6). + * + * The refresh is the whole difficulty. It rewrites `meta.json` atomically, so + * the file's mtime becomes now while the database's stays old — which is the + * write order a paired write produces, and precisely what + * `unstampedMetaPairsByWriteOrder` accepts. Three rules follow, and each of them + * is load-bearing: + * + * 1. Ask `bridgeMetaMatchesFile` FIRST, on the file as it stands. After the + * write the question is unanswerable, because the write is what destroys + * the evidence. + * 2. Re-stamp only when that answer was yes. Re-stamping a pair that already + * failed would MANUFACTURE the provenance the failure just denied — the + * same metadata/database mis-pairing stamping exists to prevent (KTD6). + * 3. When it was no, record `provenanceUnknown` explicitly and carry the + * existing stamp fields through verbatim. Writing "no stamp" instead is + * worse, not better: an unstamped file is judged on the two file times, + * and this write has just put them in the accepting order. + * + * Nothing here opens, reads, or writes the database. The only `stat` of it + * happens on the branch where the pair was just verified. + * + * NOT SPLIT into locked/unlocked halves the way {@link writeBridge} is, and + * deliberately. Its one caller is `syncGroup`'s preserve branch, which is + * already inside `withGroupSyncLock` — so this write is ALREADY serialized + * against every other sync of the group, and taking the lock here would be the + * second acquisition of a non-reentrant primitive that the split exists to + * avoid. An acquiring wrapper would therefore have zero production callers, + * and no test calls this function at all: it would be dead code standing in for + * a guarantee the caller already provides. If a caller outside the critical + * section ever appears, it needs the same treatment `writeBridge` got — a + * wrapper, not a lock moved down here. + */ +export async function refreshPreservedBridgeMeta( + groupDir: string, + // Deliberately NOT `suppressedMatchStages`. This path preserves an EARLIER + // sync's database, so stamping it with this run's request would claim the + // untouched bridge was built with a flag it never saw. The registry's own + // preserve write (`{ ...prior, missingRepos, unreadableRepos }`) omits it for + // exactly this reason, and the two artifacts have to agree about which run + // they describe. + diagnostics: { missingRepos: string[]; unreadableRepos: string[] }, +): Promise { + const dbPath = path.join(groupDir, 'bridge.lbug'); + const [metaOnDisk, dbOnDisk] = await Promise.all([ + fileExists(path.join(groupDir, 'meta.json')), + fileExists(dbPath), + ]); + // Nothing on either side of the pair. `readBridgeMeta` already answers + // `version: 0` — provenance unknown — for an absent file, so a file written + // here would say what the absence already says while inventing state for a + // bridge that has never existed. + if (!metaOnDisk && !dbOnDisk) return 'no-bridge'; + + const existing = await readBridgeMeta(groupDir); + const paired = await bridgeMetaMatchesFile(groupDir, existing); + + const refreshed: BridgeMeta = { ...existing, ...diagnostics }; + // NEVER PERSISTED (see `BridgeMeta`): both are things a READER computes ABOUT + // a file, and this is the first code in the repo that reads metadata and + // writes it back. The strip itself now lives in `writeBridgeMeta`, so every + // writer inherits it rather than each remembering. + + if (paired) { + const stat = await fsp.stat(dbPath).catch(() => null); + if (stat) { + refreshed.bridgeSize = stat.size; + refreshed.bridgeMtimeMs = stat.mtimeMs; + await writeBridgeMeta(groupDir, refreshed); + return 'restamped'; + } + // The database disappeared between the pairing check and this stat. There + // is nothing left to stamp, so fall through and say so rather than write a + // stamp describing a file that is gone. + } + + refreshed.provenanceUnknown = true; + await writeBridgeMeta(groupDir, refreshed); + return 'provenance-unknown'; +} + +/** + * Withdraw the bridge's claim to be complete, without touching the database. + * + * The one path this exists for: `contracts.json` committed, then the bridge + * replacement failed. The old database is still physically usable and still + * answers queries, but it now describes an EARLIER sync than the canonical + * registry beside it — so `group_contracts` can report a narrowed or advanced + * contract set while `group_impact` traverses the old graph and calls its + * answer complete. Two public surfaces, contradictory epistemic claims, from + * one sync. + * + * Setting `provenanceUnknown` is the smallest thing that makes that safe: + * `bridgeMetaMatchesFile` gives it highest precedence and refuses to vouch for + * the pair, so every cross-repo answer downgrades to a floor until a sync + * succeeds. Deliberately NOT a re-stamp — the metadata still describes the + * database it was written for, and claiming otherwise is the mis-pairing the + * preserve path is careful to avoid. Deliberately not a delete either: the + * previous graph is better than nothing as long as nobody calls it complete. + * + * Best-effort by construction. It runs inside a failure handler, so a throw + * here would replace a reported bridge failure with an unrelated one. + */ +export async function markBridgeProvenanceUnknown(groupDir: string): Promise { + try { + const existing = await readBridgeMeta(groupDir); + if (existing.version === 0) return false; + await writeBridgeMeta(groupDir, { ...existing, provenanceUnknown: true }); + return true; + } catch { + return false; } } @@ -668,6 +1009,29 @@ export interface WriteBridgeInput { crossLinks: CrossLink[]; repoSnapshots: Record; missingRepos: string[]; + /** + * Repos this sync could not extract from — see + * `ContractRegistry.unreadableRepos` for the full definition, which this + * field carries unchanged. + * + * Deliberately not restated here. The narrower wording this once had ("whose + * index could not be opened") described one of the two causes and silently + * excluded the other, an extractor that threw partway through — so the same + * field meant one thing on the registry, another on the bridge input, and a + * third on the result. One definition, referenced twice, cannot drift. + * + * Recorded in meta.json so cross-repo impact can tell "nothing depends on + * this" from "we could not look": the bridge built here is missing every + * contract those repos own. + */ + unreadableRepos?: string[]; + /** + * Matching stages the sync was asked to skip. Recorded here for the same + * reason `unreadableRepos` is: a later cross-repo query reads this bridge + * with no access to the run that built it, and a graph narrowed by request + * looks exactly like a complete one. + */ + suppressedMatchStages?: MatchType[]; } /** @@ -702,7 +1066,33 @@ function errMessage(err: unknown): string { } } -export async function writeBridge( +/** + * Rebuild `bridge.lbug` and its `meta.json`, ASSUMING THE CALLER ALREADY HOLDS + * THE GROUP SYNC LOCK for `groupDir` (R9). + * + * PRECONDITION — the group lock is held. There is exactly one production call + * site, `syncGroup` in sync.ts, and it is already inside + * `withGroupSyncLock(groupDir, …)` when it gets here. Enforced by this comment + * rather than by a type, matching `registerRepoUnlocked` / `withRegistryLock` + * in repo-manager.ts, which splits the same shape for the same reason. + * + * WHY THE SPLIT EXISTS AT ALL. The swap this function performs — old database + * aside, temp database into place, then `meta.json` written as a SECOND + * operation — is the write two concurrent syncs can interleave into a pairing + * that never existed: one sync's metadata beside the other's database. That + * needs mutual exclusion. But taking the lock HERE would be a second + * acquisition of a non-reentrant primitive inside a region that already holds + * it, and it would hang every single sync on the happy path, not some rare + * interleave. So the exclusion is the caller's, and this function only states + * the precondition. {@link writeBridge} is the acquiring wrapper for callers + * who are not already inside that region. + * + * SCOPE — writer-writer only. The reader-side promotion of a leftover + * `bridge.lbug.bak` runs on ordinary reads, outside anybody's critical section; + * `bridgeMetaMatchesFile` remains the reader's defense there and is not + * replaced by this lock. + */ +export async function writeBridgeUnlocked( groupDir: string, input: WriteBridgeInput, ): Promise { @@ -962,11 +1352,42 @@ export async function writeBridge( } await removeLbugFile(bakPath); - // 4. Write meta.json + // 4. Write the new meta.json, STAMPED WITH THE FILE IT DESCRIBES. + // + // meta.json carries the bridge's completeness, and since #3011 that is + // load-bearing: `runGroupImpact` folds `unreadableRepos ∪ missingRepos` + // into its truncation fields. The swap above and this write are two + // operations, so a sync that stops between them leaves the previous sync's + // meta beside a new database — and reading that as fact is a confidently + // wrong answer about the one thing this channel exists to make legible. + // + // Deleting the old meta before the swap would decide which way that window + // fails, but at an unacceptable price: the rename of the old database is + // wrapped in a catch that also swallows a FAILED rename (a held read-only + // handle does this on Windows), so `writeBridge` can throw with the old, + // perfectly good database still in place — and its metadata already gone, + // unrecoverably, for as long as the swap keeps failing. + // + // So destroy nothing and pair the two instead: record the size and mtime of + // the database this metadata describes, and let readers check that the pair + // still belongs together (`bridgeMetaMatchesFile`). A stale meta cannot match + // a freshly renamed database, and a sync that fails before the swap leaves a + // matching pair untouched. + const finalStat = await fsp.stat(finalPath); await writeBridgeMeta(groupDir, { version: BRIDGE_SCHEMA_VERSION, generatedAt: new Date().toISOString(), + bridgeSize: finalStat.size, + bridgeMtimeMs: finalStat.mtimeMs, missingRepos: input.missingRepos, + // Persisted whenever the caller supplied it, `[]` included: an empty list + // is the measurement "this sync accounted for every repo", and it is a + // different claim from a bridge that never recorded the field. Omitted + // only when the caller passed nothing to record. + ...(input.unreadableRepos ? { unreadableRepos: input.unreadableRepos } : {}), + ...(input.suppressedMatchStages + ? { suppressedMatchStages: input.suppressedMatchStages } + : {}), }); return report; @@ -982,6 +1403,33 @@ export async function writeBridge( } } +/** + * Rebuild `bridge.lbug` and its `meta.json` as the only writer of `groupDir`. + * + * The acquiring half of the split described on {@link writeBridgeUnlocked}: for + * callers that are NOT already inside the group's critical section, this takes + * the group sync lock around the whole swap and releases it afterwards. Two + * concurrent calls therefore run one after the other, so the `meta.json` left + * on disk is stamped for the `bridge.lbug` left on disk instead of for the + * loser's, which is the pairing the swap-plus-metadata sequence would otherwise + * let them interleave into. + * + * NOT used by `syncGroup`, and it must not be: that path already holds this + * lock, and `acquireIndexLock` is not reentrant, so routing it here would make + * every ordinary sync wait out the full `GROUP_SYNC_LOCK_TIMEOUT_MS` ceiling + * against itself. It calls {@link writeBridgeUnlocked} directly. + * + * Fails closed exactly as `withGroupSyncLock` does: if the lock cannot be + * acquired, a `GroupSyncLockError` is thrown and NOTHING is written — + * `bridge.lbug` and `meta.json` are left as they were. + */ +export async function writeBridge( + groupDir: string, + input: WriteBridgeInput, +): Promise { + return withGroupSyncLock(groupDir, () => writeBridgeUnlocked(groupDir, input)); +} + /* ------------------------------------------------------------------ */ /* openBridgeDbReadOnly */ /* ------------------------------------------------------------------ */ diff --git a/gitnexus/src/core/group/completeness.ts b/gitnexus/src/core/group/completeness.ts new file mode 100644 index 000000000..16aee0473 --- /dev/null +++ b/gitnexus/src/core/group/completeness.ts @@ -0,0 +1,169 @@ +/** + * The one computation of "is this cross-repo answer complete?" (KTD10), and the + * truncation vocabulary it speaks. + * + * A LEAF MODULE, deliberately, and that is the whole reason it exists apart from + * `cross-impact.ts`. Three surfaces need this fold — impact, trace, and the + * contract listing — but `cross-impact.ts` statically imports `bridge-db.ts`, + * and through it the native LadybugDB binding. `service.ts` therefore had to + * reach the fold through `await import('./cross-impact.js')`, which loaded that + * entire module graph on the first `group_contracts` of every process — 44-51ms + * and 8.4MB of RSS to run a `Set` union and a ternary, once per CLI invocation. + * + * Nothing here imports anything but types. Keep it that way: the moment this + * file gains a runtime import, every consumer pays for it again. + */ +import type { GroupImpactTruncationReason, MatchType } from './types.js'; + +/** + * A union rather than `Pick` so the two states are + * distinguishable by their `truncated` discriminant: a caller that folds these + * fields into its own result (see `crossRepoCompleteness`) can then read + * `truncationReason` on the truncated branch without a fallback for a value + * that cannot be absent there. + */ +export type TruncationFields = + | { truncated: false } + | { + truncated: true; + truncationReason: GroupImpactTruncationReason; + riskEpistemic: 'lower-bound'; + }; + +/** + * Build the truncation fields every `runGroupImpact` return path shares. + * + * `riskEpistemic` must follow `truncated` mechanically: it is the marker that + * tells a caller the `risk` value is a floor rather than a verdict, and + * `mergeRisk` can only under-report once a crossing is dropped. Attaching it at + * each return let two of the four paths set `truncated` without it, so a + * truncated result read as complete — deriving it in one place is what keeps + * the invariant from drifting again (#2787). + */ +export function truncationFields( + truncated: boolean, + // Only read on the truncated branch, so the not-truncated call sites omit it + // rather than passing a reason that is thrown away. + reasonIfTruncated: GroupImpactTruncationReason = 'partial', +): TruncationFields { + if (!truncated) return { truncated: false }; + return { truncated: true, truncationReason: reasonIfTruncated, riskEpistemic: 'lower-bound' }; +} + +/** + * Everything a caller needs in order to say whether a cross-repo answer is + * complete — deliberately WITHOUT naming where any of it came from. + * + * `BridgeMeta` is not in this signature, and must not be: `groupContracts` + * answers the same question from `contracts.json` (via + * `loadContractRegistryResilient`) and never opens a bridge at all, so + * `version` / `repoListsUnreadable` / `pairedWithDatabase` do not exist on that + * path. Each caller computes its own `provenanceUnknown` from whatever + * provenance IT has and passes the boolean in. + */ +export interface CrossRepoCompletenessInput { + /** + * Repos the sync could not extract from, and repos it found no entry for. + * Two independent diagnostics with one consequence — none of those repos' + * contracts are in the artifact — so they are folded into one set. + */ + unreadableRepos?: readonly string[]; + missingRepos?: readonly string[]; + /** + * Matching stages the sync was asked to skip. Absent or empty means it + * suppressed none; a populated list makes the answer a floor for a reason + * that is neither a runtime limit nor an unreadable repo. + */ + suppressedMatchStages?: readonly string[]; + /** Computed by the caller; see `bridgeProvenanceUnknown` for the bridge one. */ + provenanceUnknown: boolean; + /** + * The query's DECLARED scope, not the set of repos the walk happened to + * reach: the subgroup filter for an impact query, the two endpoint repos for + * a trace, every member for a query that names none. An incomplete repo the + * caller never asked about cannot make the caller's answer a floor, and + * marking it anyway is how the marker stops meaning anything. Passing the + * predicate in — rather than a repo list, or a subgroup — is what keeps + * narrowing a scope a call-site change. + */ + inScope: (repoPath: string) => boolean; +} + +/** The structured triple, plus the in-scope repos that produced it. */ +export type CrossRepoCompleteness = TruncationFields & { + /** + * In-scope repos absent from the artifact, deduped, in first-seen order. + * Empty on a provenance-unknown answer: nothing was measured there, and + * inventing names out of an unreadable value is not a measurement. + */ + incompleteRepos: string[]; +}; + +/** + * Read a persisted `suppressedMatchStages` list. + * + * Sibling of `recordedRepoList` and here for the same stated reason: it had + * lived in two files verbatim, so tightening one would silently leave the other. + * All-or-nothing like its sibling — a stale member (this repo has already + * retired `'bm25'` and `'embedding'`) makes the whole list unreadable rather + * than filtering down to `[]`, which on this field would mean "measured, + * nothing suppressed": a clean answer manufactured from a value we could not + * read. + */ +export function recordedMatchStages(value: unknown): MatchType[] | undefined { + if (!Array.isArray(value)) return undefined; + const known: MatchType[] = ['exact', 'manifest', 'wildcard']; + return value.every((v): v is MatchType => known.includes(v as MatchType)) ? value : undefined; +} + +/** + * The ONE computation of "is this cross-repo answer complete?" (KTD10). + * + * Three surfaces can return a partial cross-repo answer — impact, trace, and + * the contract listing — and each used to decide for itself, in its own + * vocabulary, which is how two of them ended up saying it in prose only. The + * answer is the same structured triple `GroupImpactResult` already carries, so + * an agent reading any of them learns "complete" vs "floor" the same way. + * + * `truncationFields` derives `riskEpistemic` from `truncated` mechanically, and + * is reused here rather than re-implemented for the same reason it exists: the + * marker that says "this is a floor, not a verdict" may never drift away from + * the flag that says the answer was cut short (#2787). + */ +export function crossRepoCompleteness(input: CrossRepoCompletenessInput): CrossRepoCompleteness { + const incompleteRepos = [ + ...new Set([...(input.unreadableRepos ?? []), ...(input.missingRepos ?? [])]), + ].filter((repoPath) => input.inScope(repoPath)); + // An unreadable or unaccounted repo outranks a suppressed stage: it is the + // more serious structural gap and its remedy (repair the repo, re-sync) has + // to be the one reported. A suppressed stage only decides the reason when + // the repo side is otherwise clean. + const suppressed = (input.suppressedMatchStages ?? []).length > 0; + const repoSideIncomplete = input.provenanceUnknown || incompleteRepos.length > 0; + return { + ...truncationFields( + repoSideIncomplete || suppressed, + repoSideIncomplete ? 'incomplete-sync' : 'suppressed-stage', + ), + incompleteRepos, + }; +} + +/** + * A recorded repo list is an array of strings. Anything else — a bare string, an + * object, an array of objects — is a value we could not read, which is "not + * recorded", not "none". + * + * ONE definition, deliberately. This gate is the predicate the whole + * absent-vs-empty-vs-populated distinction rests on, and it applies to the same + * two lists on both the registry and the bridge metadata. It lived in two files + * verbatim, which meant tightening it — say, to reject blank strings — would + * have fixed one surface and silently left the other. + * + * `Array.isArray` alone is not enough: only an array of strings survives + * `cli/group.ts`'s `.join(', ')` as repo paths rather than as `[object Object]`. + */ +export function recordedRepoList(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + return value.every((entry) => typeof entry === 'string') ? (value as string[]) : undefined; +} diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index 29c868171..373e5e5c4 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -1,10 +1,15 @@ import { createRequire } from 'node:module'; -import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from './types.js'; +import type { + ContractRole, + GroupConfig, + GroupManifestLink, + ManifestContractType, +} from './types.js'; const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -const VALID_CONTRACT_TYPES: ContractType[] = [ +const VALID_CONTRACT_TYPES: ManifestContractType[] = [ 'http', 'grpc', 'thrift', @@ -26,19 +31,15 @@ const VALID_ROLES: ContractRole[] = ['provider', 'consumer']; // repos that need cross-repo header tracking. const DEFAULT_DETECT = { http: true, + graphql: false, grpc: true, thrift: true, topics: true, - shared_libs: true, - embedding_fallback: true, includes: false, workspace_deps: false, }; const DEFAULT_MATCHING = { - bm25_threshold: 0.7, - embedding_threshold: 0.65, - max_candidates_per_step: 3, exclude_links_paths: [] as string[], exclude_links_param_only_paths: false, }; @@ -59,7 +60,16 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { throw new Error('repos is required in group.yaml (must be a mapping)'); } - const repos = raw.repos as Record; + const reposRaw = raw.repos as Record; + const repos: Record = {}; + for (const [memberPath, registryName] of Object.entries(reposRaw)) { + if (typeof registryName !== 'string' || registryName.trim() === '') { + throw new Error( + `repos["${memberPath}"] must be a non-empty registry name string, not ${typeof registryName}`, + ); + } + repos[memberPath] = registryName.trim(); + } const repoPaths = new Set(Object.keys(repos)); const rawLinks = (raw.links as unknown[]) || []; @@ -71,7 +81,7 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { if (!link.to || !repoPaths.has(link.to as string)) { throw new Error(`links[${i}].to "${link.to}" does not match any repo path in group`); } - if (!VALID_CONTRACT_TYPES.includes(link.type as ContractType)) { + if (!VALID_CONTRACT_TYPES.includes(link.type as ManifestContractType)) { throw new Error( `links[${i}].type "${link.type}" is invalid. Expected: ${VALID_CONTRACT_TYPES.join(', ')}`, ); @@ -89,13 +99,25 @@ export function parseGroupConfig(yamlContent: string): GroupConfig { return { from: link.from as string, to: link.to as string, - type: link.type as ContractType, + type: link.type as ManifestContractType, contract: String(link.contract), role: link.role as ContractRole, }; }); - const detect = { ...DEFAULT_DETECT, ...((raw.detect as object) || {}) }; + const rawDetect = raw.detect; + if ( + rawDetect !== undefined && + (!rawDetect || typeof rawDetect !== 'object' || Array.isArray(rawDetect)) + ) { + throw new Error('detect must be a mapping of boolean flags'); + } + for (const [key, value] of Object.entries((rawDetect as Record) || {})) { + if (key in DEFAULT_DETECT && typeof value !== 'boolean') { + throw new Error(`detect.${key} must be true or false`); + } + } + const detect = { ...DEFAULT_DETECT, ...((rawDetect as object) || {}) }; const matching = { ...DEFAULT_MATCHING, ...((raw.matching as object) || {}) }; const packages = (raw.packages as Record>) || {}; diff --git a/gitnexus/src/core/group/cross-impact.ts b/gitnexus/src/core/group/cross-impact.ts index 06c16d3fa..da9ba0de4 100644 --- a/gitnexus/src/core/group/cross-impact.ts +++ b/gitnexus/src/core/group/cross-impact.ts @@ -5,13 +5,14 @@ import fsp from 'node:fs/promises'; import path from 'node:path'; +import type { ImpactRisk } from 'gitnexus-shared'; import type { BridgeHandle, + BridgeMeta, ContractType, CrossRepoImpact, GroupConfig, GroupImpactResult, - GroupImpactTruncationReason, MatchType, OutOfScopeLink, } from './types.js'; @@ -24,12 +25,23 @@ import { } from './group-path-utils.js'; import { getGroupDir } from './storage.js'; import { + bridgeMetaMatchesFile, closeBridgeDb, getCachedBridgeReadOnly, queryBridge, readBridgeMeta, } from './bridge-db.js'; import { BRIDGE_SCHEMA_VERSION } from './bridge-schema.js'; +// Re-exported so the three surfaces keep one import site for the vocabulary, +// while the fold itself stays in a leaf module no native binding reaches. +export { + truncationFields, + crossRepoCompleteness, + type TruncationFields, + type CrossRepoCompleteness, + type CrossRepoCompletenessInput, +} from './completeness.js'; +import { truncationFields, crossRepoCompleteness } from './completeness.js'; import { compareCodeUnits } from '../../lib/utils.js'; // High limit for the local phase of group impact so collectImpactSymbolUids @@ -147,6 +159,15 @@ export function validateGroupImpactParams(params: Record): name: string; repoPath: string; target: string; + // Target selectors, same names/semantics as the single-repo impact tool + // (target_uid = zero-ambiguity lookup that wins over the name; + // file_path/kind narrow a name shared by same-named symbols). Threading + // them through HERE is what makes the MCP boundary's forwarding live — + // dropping them at this boundary silently re-broke the group-mode + // disambiguation loop once already. + target_uid?: string; + file_path?: string; + kind?: string; direction: 'upstream' | 'downstream'; maxDepth: number; crossDepth: number; @@ -161,11 +182,21 @@ export function validateGroupImpactParams(params: Record): | { ok: false; error: string } { const name = String(params.name ?? '').trim(); const repoPath = String(params.repo ?? '').trim(); - const target = String(params.target ?? '').trim(); + // Optional string, same helper shape as cross-trace's `str()`: empty/blank + // counts as absent so `target_uid: ''` degrades to the name lookup rather + // than a zero-ambiguity lookup of the empty uid. Parsed before the required + // check so UID-only callers (MCP impact schema requires `direction`, not + // `target`) are accepted. + const str = (v: unknown): string | undefined => + typeof v === 'string' && v.trim() !== '' ? v : undefined; + const targetName = String(params.target ?? '').trim(); + const target_uidEarly = str(params.target_uid); if (!name) return { ok: false, error: 'name is required' }; if (!repoPath) return { ok: false, error: 'repo is required (group repo path, e.g. app/backend)' }; - if (!target) return { ok: false, error: 'target is required' }; + if (!targetName && !target_uidEarly) + return { ok: false, error: 'target or target_uid is required' }; + const target = targetName || target_uidEarly!; if ( params.service !== undefined && params.service !== null && @@ -193,6 +224,10 @@ export function validateGroupImpactParams(params: Record): const service = normalizeServicePrefix(params.service); const subgroup = typeof params.subgroup === 'string' ? params.subgroup : undefined; + const target_uid = target_uidEarly; + const file_path = str(params.file_path); + const kind = str(params.kind); + // Clamp at the validate boundary so the downstream `deadline` (line // ~366) and `safeLocalImpact`'s `setTimeout` both see a single // bounded value. Without this, the outer deadline budgeted Phase-2 @@ -212,6 +247,9 @@ export function validateGroupImpactParams(params: Record): name, repoPath, target, + target_uid, + file_path, + kind, direction, maxDepth, crossDepth, @@ -232,6 +270,17 @@ async function resolveGroupRepo( ): Promise { const registryName = config.repos[repoPath]; if (!registryName) { + const matchingMemberPaths = Object.entries(config.repos) + .filter(([, alias]) => alias.toLowerCase() === repoPath.toLowerCase()) + .map(([memberPath]) => memberPath); + if (matchingMemberPaths.length > 0) { + return { + error: + `Unknown repo path "${repoPath}" in this group. ` + + `That value is a registry alias for member path(s): ${matchingMemberPaths.join(', ')}. ` + + `Pass the group.yaml key to --repo, not the alias.`, + }; + } return { error: `Unknown repo path "${repoPath}" in this group.` }; } try { @@ -370,7 +419,17 @@ function extractProcessNames(impact: unknown): string[] { // permanently that a PDG `risk:'UNKNOWN'` never coalesces to a confident `LOW`. // No behavior change — `'UNKNOWN'` was already handled correctly at the // `(localRisk === 'LOW' || localRisk === 'UNKNOWN')` branch below. -export function mergeRisk(localRisk: string, cross: CrossRepoImpact[]): string { +function asImpactRisk(value: unknown, fallback: ImpactRisk = 'LOW'): ImpactRisk { + return value === 'LOW' || + value === 'MEDIUM' || + value === 'HIGH' || + value === 'CRITICAL' || + value === 'UNKNOWN' + ? value + : fallback; +} + +export function mergeRisk(localRisk: ImpactRisk, cross: CrossRepoImpact[]): ImpactRisk { const traversed = cross.filter((c) => c.fanout_status !== 'not_attempted'); const highConf = traversed.some((c) => c.contract.confidence >= 0.85); if (localRisk === 'CRITICAL') return 'CRITICAL'; @@ -380,24 +439,47 @@ export function mergeRisk(localRisk: string, cross: CrossRepoImpact[]): string { return localRisk; } +function liftLocalRiskMeta( + local: unknown, + cross: CrossRepoImpact[], +): Pick { + const { riskSharedAxes, riskScale } = local as { + riskSharedAxes?: unknown; + riskScale?: GroupImpactResult['riskScale']; + }; + return { + ...(riskSharedAxes !== undefined + ? { riskSharedAxes: mergeRisk(asImpactRisk(riskSharedAxes), cross) } + : {}), + ...(riskScale !== undefined ? { riskScale } : {}), + }; +} + /** - * Build the truncation fields every `runGroupImpact` return path shares. + * Is this bridge's metadata unable to say where its contents came from? * - * `riskEpistemic` must follow `truncated` mechanically: it is the marker that - * tells a caller the `risk` value is a floor rather than a verdict, and - * `mergeRisk` can only under-report once a crossing is dropped. Attaching it at - * each return let two of the four paths set `truncated` without it, so a - * truncated result read as complete — deriving it in one place is what keeps - * the invariant from drifting again (#2787). + * The three reads are all about a `BridgeMeta` and stay OUT of + * `crossRepoCompleteness` on purpose (see its doc): they are how a caller that + * opened a bridge computes `provenanceUnknown`, not how every caller does. + * + * - `version === 0` — no readable meta.json at all (`readBridgeMeta` answers + * that for both "absent" and "unparseable"); + * - `repoListsUnreadable` — a meta.json that parsed but whose repo lists are + * not repo lists. A value we could not read is not a measurement of zero, + * so it may not be spent as one; + * - `pairedWithDatabase === false` — a meta.json that does not describe the + * database sitting beside it, which is what a sync interrupted between the + * swap and the metadata write leaves behind. Measured by + * `ensureBridgeReady` BEFORE the database is opened and carried on the + * meta; this only reads the answer (#3012). + * + * Treating any of them as complete is the fail-open the completeness channel + * exists to close. */ -function truncationFields( - truncated: boolean, - // Only read on the truncated branch, so the not-truncated call sites omit it - // rather than passing a reason that is thrown away. - reasonIfTruncated: GroupImpactTruncationReason = 'partial', -): Pick { - if (!truncated) return { truncated: false }; - return { truncated: true, truncationReason: reasonIfTruncated, riskEpistemic: 'lower-bound' }; +export function bridgeProvenanceUnknown(meta: BridgeMeta): boolean { + return ( + meta.version === 0 || meta.repoListsUnreadable === true || meta.pairedWithDatabase === false + ); } function addCrossImpact(cross: CrossRepoImpact[], candidate: CrossRepoImpact): void { @@ -418,7 +500,7 @@ function addCrossImpact(cross: CrossRepoImpact[], candidate: CrossRepoImpact): v export async function ensureBridgeReady( groupDir: string, -): Promise<{ handle: BridgeHandle } | { error: string }> { +): Promise<{ handle: BridgeHandle; meta: BridgeMeta } | { error: string }> { const meta = await readBridgeMeta(groupDir); if (meta.version > 0 && meta.version !== BRIDGE_SCHEMA_VERSION) { return { @@ -433,6 +515,13 @@ export async function ensureBridgeReady( error: `No bridge.lbug in this group directory. Run gitnexus group sync (schema ${BRIDGE_SCHEMA_VERSION}).`, }; } + // Pair the metadata to the database BEFORE opening it, and carry the answer. + // An unstamped pair is judged on the two files' write order, so any open that + // touched `bridge.lbug`'s mtime would silently convert "legacy but intact" + // into "provenance unknown" for every pre-stamp bridge on that platform. This + // ordering removes the question rather than betting on the answer. + meta.pairedWithDatabase = await bridgeMetaMatchesFile(groupDir, meta); + // Use the cached read-only handle if available — avoids reopening the same // bridge.lbug in a long-lived MCP server, which fails on Windows because // the OS handle isn't fully released before the next open races in. @@ -442,7 +531,7 @@ export async function ensureBridgeReady( error: `Could not open bridge.lbug read-only (schema ${BRIDGE_SCHEMA_VERSION}). Run gitnexus group sync.`, }; } - return { handle }; + return { handle, meta }; } function rowToNeighbor(r: Record): BridgeNeighborRow | null { @@ -516,6 +605,9 @@ export async function runGroupImpact( name, repoPath, target, + target_uid, + file_path, + kind, direction, maxDepth, crossDepth: _crossDepth, @@ -543,6 +635,14 @@ export async function runGroupImpact( const impactParams: Parameters[1] = { target, + // Selector params pass through to the member repo's impact (the port + // contract in service.ts documents them), so the single-repo tool's + // "re-call with target_uid to disambiguate" loop works unchanged in + // group mode. `undefined` keeps the call shape flat — same convention + // as the relationTypes line below. + target_uid, + file_path, + kind, direction, maxDepth, relationTypes: relationTypes && relationTypes.length > 0 ? relationTypes : undefined, @@ -576,6 +676,7 @@ export async function runGroupImpact( cross_repo_hits: 0, }, risk: 'UNKNOWN', + ...liftLocalRiskMeta(local, []), timeoutMs, crossDepthWarning, }; @@ -631,7 +732,8 @@ export async function runGroupImpact( modules_affected: s.modules_affected ?? 0, cross_repo_hits: 0, }, - risk: String((local as { risk?: string }).risk ?? 'LOW'), + risk: asImpactRisk((local as { risk?: unknown }).risk), + ...liftLocalRiskMeta(local, []), timeoutMs, crossDepthWarning, }; @@ -641,6 +743,25 @@ export async function runGroupImpact( if ('error' in bridgePrep) return { error: bridgePrep.error }; const handle = bridgePrep.handle; + // Repos the sync that built this bridge could not account for. Their + // contracts — and every cross-link touching them — are simply absent from + // bridge.lbug, and nothing else in this walk can notice that: the only + // incompleteness channel on the result is `truncationFields`, driven by + // fan-out state. Without folding these in, a query about a symbol whose one + // downstream consumer lives in an unreadable repo returns + // `{ cross: [], truncated: false }` — "complete: nothing depends on this" — + // which is a wrong answer, not an empty one, for a tool an agent uses to + // license a delete or a rename. + // + // The metadata read that answers it (`bridgeProvenanceUnknown`) happens + // INSIDE the `try` below, and the flag is initialized fail-closed here only + // because it outlives that block. The lease taken by `ensureBridgeReady` is + // released by the `finally` and nowhere else, so work done between the lease + // and the `try` is work whose every throw leaks a refcount the cached handle + // can never get back — which is how a malformed meta.json used to wedge the + // handle as well as crash the query. (The repo lists are folded in after the + // `finally`, where a throw can no longer strand the lease.) + let provenanceUnknown = true; const cross: CrossRepoImpact[] = []; const outOfScope: OutOfScopeLink[] = []; const truncatedRepos: string[] = []; @@ -650,6 +771,8 @@ export async function runGroupImpact( let fanoutTimedOut = false; try { + provenanceUnknown = bridgeProvenanceUnknown(bridgePrep.meta); + const neighbors = await resolveBridgeNeighbors(handle, { localRepo: repoPath, uids, @@ -780,9 +903,48 @@ export async function runGroupImpact( } const localSum = (local as { summary?: Record })?.summary || {}; - const localRisk = String((local as { risk?: string }).risk ?? 'LOW'); + const localRisk = asImpactRisk((local as { risk?: unknown }).risk); const localPartial = Boolean((local as { partial?: boolean }).partial); - const truncated = truncatedRepos.length > 0 || localPartial; + // The bridge's own incompleteness, in the shared vocabulary, read through + // what this query DECLARED. The fan-out above already drops every neighbour + // outside `subgroup`, so an incomplete repo the query excluded could not have + // contributed a crossing to this answer — marking the answer a floor because + // of it makes the marker fire on results it does not describe, which is how a + // caller learns to ignore it. An unscoped query passes `subgroup: undefined`, + // which `repoInSubgroup` answers true for, so the intersection is the whole + // set and that path is byte-for-byte the old behaviour. + // + // The declared scope is the subgroup PLUS the query's own repo (`exact` + // reuses the one membership helper for the equality, rather than growing a + // second notion of it): the walk starts from `repoPath`'s contracts in the + // bridge, so if THAT is the repo the sync could not read there are no + // crossings to find for any scope, and a subgroup excluding it must not turn + // that vacuum into a confident "complete". + // + // Declared scope, not traversed scope: an incomplete repo's contracts are + // absent from the bridge by definition, so it is never in the set the walk + // reached — filtering on what was traversed would empty the intersection on + // every query and silently restore the fail-open. + // + // Sound only while `MAX_SUPPORTED_CROSS_DEPTH` is 1. At depth 2+ an + // out-of-scope repo can sit BETWEEN two in-scope ones, so dropping it would + // convert a genuine lower bound into a confident complete answer; widen this + // predicate in the same change that raises the depth. + const bridge = crossRepoCompleteness({ + unreadableRepos: bridgePrep.meta.unreadableRepos, + missingRepos: bridgePrep.meta.missingRepos, + suppressedMatchStages: bridgePrep.meta.suppressedMatchStages, + provenanceUnknown, + inScope: (candidate) => + repoInSubgroup(candidate, subgroup) || repoInSubgroup(candidate, repoPath, true), + }); + // One predicate, read twice below. Written out at both sites, a third runtime + // cause added to the flag and forgotten at the reason would label a + // retry-able answer `incomplete-sync` — telling the operator to re-sync for + // something a retry fixes. That reason-vs-flag drift is what `truncationFields` + // exists to prevent. + const runtimeTruncated = truncatedRepos.length > 0 || localPartial; + const truncated = runtimeTruncated || bridge.truncated; const result: GroupImpactResult = { local, @@ -794,8 +956,25 @@ export async function runGroupImpact( // and under-reporting a blast radius is the unsafe direction (an agent told // LOW proceeds; told CRITICAL it stops). Marking the floor keeps the // warning intact while making the incompleteness legible. - ...truncationFields(truncated, fanoutTimedOut ? 'timeout' : 'partial'), - truncatedRepos: [...new Set(truncatedRepos)], + // Runtime limits first — they are what the caller can retry. Past those, the + // BRIDGE's own reason wins: it already distinguished an unreadable repo + // ('incomplete-sync', remedy: re-sync) from a stage the sync was asked to + // skip ('suppressed-stage', remedy: re-sync WITHOUT the flag). Hardcoding + // the fallback here overrode that and told every caller to repair a repo + // that read fine — and made the second value unreachable from this surface + // while the tool description promised it. `cross-trace.ts` re-spreads the + // bridge's fields for the same reason. + ...truncationFields( + truncated, + fanoutTimedOut + ? 'timeout' + : runtimeTruncated + ? 'partial' + : bridge.truncated + ? bridge.truncationReason + : 'incomplete-sync', + ), + truncatedRepos: [...new Set([...truncatedRepos, ...bridge.incompleteRepos])], summary: { direct: localSum.direct ?? 0, processes_affected: localSum.processes_affected ?? 0, @@ -803,6 +982,7 @@ export async function runGroupImpact( cross_repo_hits: cross.length, }, risk: mergeRisk(localRisk, cross), + ...liftLocalRiskMeta(local, cross), timeoutMs, crossDepthWarning, }; diff --git a/gitnexus/src/core/group/cross-trace.ts b/gitnexus/src/core/group/cross-trace.ts index 7c115de01..92c52e789 100644 --- a/gitnexus/src/core/group/cross-trace.ts +++ b/gitnexus/src/core/group/cross-trace.ts @@ -25,16 +25,29 @@ import { GroupNotFoundError, loadGroupConfig } from './config-parser.js'; import { getGroupDir } from './storage.js'; -import { ensureBridgeReady, MAX_SUPPORTED_CROSS_DEPTH } from './cross-impact.js'; +import { + bridgeProvenanceUnknown, + crossRepoCompleteness, + ensureBridgeReady, + MAX_SUPPORTED_CROSS_DEPTH, +} from './cross-impact.js'; +import type { CrossRepoCompleteness } from './completeness.js'; +import { truncationFields } from './completeness.js'; import { compareCodeUnits } from '../../lib/utils.js'; import { closeBridgeDb, queryBridge } from './bridge-db.js'; +import { repoInSubgroup } from './group-path-utils.js'; import type { GroupPdgFlowHop, GroupRepoHandle, GroupSymbolResolution, GroupToolPort, } from './service.js'; -import type { BridgeHandle, GroupConfig } from './types.js'; +import type { + BridgeHandle, + BridgeMeta, + GroupConfig, + GroupImpactTruncationReason, +} from './types.js'; // ── Result types (discriminated on `status`) ───────────────────────────── @@ -77,7 +90,29 @@ export interface GroupTraceEndpoint { repo: string; } -export interface GroupTraceOkResult { +/** + * The incompleteness vocabulary, verbatim from `GroupImpactResult` (KTD10). + * + * A cross-repo trace and a cross-repo impact can both be cut short by the same + * two kinds of cause — a runtime limit inside this walk, or a bridge that never + * held part of the group — and an agent must not have to learn a second + * vocabulary (or parse a `notes` string) to tell "no path exists" from "we + * could not have seen the path". Every field here means exactly what it means + * on `GroupImpactResult`; `notes` stays a human-readable ADDITION to them, + * never the machine-readable channel. + */ +export interface GroupTraceCompleteness { + /** True when this answer is a floor rather than a verdict. */ + truncated?: boolean; + /** Why, when `truncated` — runtime limit ('partial'/'timeout') before structure. */ + truncationReason?: GroupImpactTruncationReason; + /** Set with `truncated`: the answer under-reports, it never over-reports. */ + riskEpistemic?: 'lower-bound'; + /** In-scope repos absent from the bridge; omitted when none were measured. */ + truncatedRepos?: string[]; +} + +export interface GroupTraceOkResult extends GroupTraceCompleteness { status: 'ok'; group: string; from: GroupTraceEndpoint; @@ -89,7 +124,6 @@ export interface GroupTraceOkResult { edges: TraceEdge[]; /** Present only when PDG enrichment ran for at least one segment. */ dataFlow?: SegmentDataFlow[]; - truncated?: boolean; notes: string[]; } @@ -101,23 +135,23 @@ export interface GroupTraceCandidate { startLine: number; } -export interface GroupTraceNotFoundResult { +/** + * `truncated: true` here means the answer is NOT authoritative — either the + * crossing cap (`MAX_CROSSINGS_TO_TRY`) was hit so a connecting ContractLink + * ranked beyond it may have been skipped, or the bridge itself never held part + * of the group. Both read as "unknown", not as "no path exists"; + * `truncationReason` says which. + */ +export interface GroupTraceNotFoundResult extends GroupTraceCompleteness { status: 'not_found'; group: string; role?: 'from' | 'to'; query?: string; - /** - * True when the answer is NOT authoritative: the crossing cap - * (`MAX_CROSSINGS_TO_TRY`) was hit, so a connecting ContractLink ranked beyond - * the cap may have been skipped. A consumer should treat this as "unknown", - * not "no path exists". - */ - truncated?: boolean; notes: string[]; suggestion?: string; } -export interface GroupTraceAmbiguousResult { +export interface GroupTraceAmbiguousResult extends GroupTraceCompleteness { status: 'ambiguous'; group: string; role: 'from' | 'to'; @@ -187,6 +221,55 @@ export const TRACE_NOTES = { 'The candidates are listed; trace from the exact calling function or pass `to_uid`.', } as const; +/** + * Fold this bridge's completeness into the runtime-truncation flag a trace call + * site already computed, and answer in the shared vocabulary. + * + * Precedence mirrors `runGroupImpact`: a runtime limit wins the reason, because + * it is the cause the caller can act on (narrow the query, raise maxDepth), + * while `'incomplete-sync'` needs a different remedy — `gitnexus group sync` — + * and would otherwise mask it. + * + * Returns `{}` — not `{ truncated: false }` — when the answer is complete, so a + * clean trace result keeps the exact shape it has always had. + */ +function traceCompleteness( + bridge: CrossRepoCompleteness, + runtimeTruncated: boolean, +): GroupTraceCompleteness { + const repos = bridge.incompleteRepos.length > 0 ? { truncatedRepos: bridge.incompleteRepos } : {}; + // Through `truncationFields`, not hand-written: `riskEpistemic` must follow + // `truncated` mechanically, and a third writer of that pair is how the + // invariant drifts (#2787). The bridge branch re-spreads the helper's own + // output rather than naming its fields. + if (runtimeTruncated) return { ...truncationFields(true, 'partial'), ...repos }; + if (!bridge.truncated) return {}; + const { incompleteRepos: _incompleteRepos, ...fields } = bridge; + return { ...fields, ...repos }; +} + +/** + * The trace's declared scope for `crossRepoCompleteness`. + * + * A symbol-to-symbol trace asks about exactly two repos, so an unreadable third + * member cannot make its answer a floor. A DESTINATION trace declares no `to` + * at all — the call may land in any member — so every repo is in scope there, + * which is why the predicate is built per call site rather than derived from + * the endpoints inside the helper. + */ +function bridgeCompletenessFor( + meta: BridgeMeta, + inScope: (repoPath: string) => boolean, +): CrossRepoCompleteness { + return crossRepoCompleteness({ + unreadableRepos: meta.unreadableRepos, + missingRepos: meta.missingRepos, + suppressedMatchStages: meta.suppressedMatchStages, + provenanceUnknown: bridgeProvenanceUnknown(meta), + inScope, + }); +} + /** Repo-relative path equality, tolerant of a leading "./" / "/" or a repo prefix. */ function sameFile(a: string, b: string): boolean { if (!a || !b) return false; @@ -873,6 +956,23 @@ async function stitchCrossRepo( if (p.pdg) notes.push(TRACE_NOTES.pdgRequested); try { + // Inside the `try`, like `runGroupImpact`'s equivalent: the lease taken by + // `ensureBridgeReady` is released by this block's `finally` and nowhere + // else, so anything computed between the lease and the `try` is work whose + // every throw would strand a refcount the cached handle never gets back. + // + // Declared scope = the two endpoint repos. Whether either of them is a repo + // this bridge could not read decides whether "no ContractLink connects + // them" is a verdict or a floor. + const bridge = bridgeCompletenessFor( + bridgePrep.meta, + // `repoInSubgroup(..., exact)` rather than `===`: it normalizes separators + // and strips trailing slashes, which bare equality does not, so the same + // group.yaml spelling cannot be in scope for impact and out of scope here. + (repoPath) => + repoInSubgroup(repoPath, fromEp.member.repoPath, true) || + repoInSubgroup(repoPath, toEp.member.repoPath, true), + ); const { crossings, truncated: crossingsTruncated } = await listCrossingsBetween( handle, fromEp.member.repoPath, @@ -883,6 +983,10 @@ async function stitchCrossRepo( return { status: 'not_found', group: p.name, + // No crossings at all is exactly the answer a bridge that never held an + // endpoint's repo produces, so it is the one that most needs the floor + // marker. (Nothing was capped: there were zero rows to cap.) + ...traceCompleteness(bridge, false), notes, suggestion: 'The endpoints live in different repos with no ContractLink between them. ' + @@ -1016,6 +1120,13 @@ async function stitchCrossRepo( hopCount: edges.length, hops: [...hopsA, ...hopsB], edges, + // A found path is still an answer from this bridge: if its provenance is + // unknown, or an endpoint's repo never made it in, the path may be stale + // and it is certainly not the only one. An incompleteness channel that + // fires only on the empty answer teaches an agent that a non-empty one + // is always complete. The crossing cap is NOT folded in here — a path + // that connected is not a capped search — so this site passes `false`. + ...traceCompleteness(bridge, false), notes, ...(dataFlow.length > 0 ? { dataFlow } : {}), }; @@ -1028,7 +1139,7 @@ async function stitchCrossRepo( return { status: 'not_found', group: p.name, - ...(crossingsTruncated ? { truncated: true } : {}), + ...traceCompleteness(bridge, crossingsTruncated), notes, suggestion: crossingsTruncated ? `No connecting crossing among the ${MAX_CROSSINGS_TO_TRY} highest-confidence ` + @@ -1099,6 +1210,12 @@ async function stitchToDestination( if (p.crossDepthClamped) notes.push(TRACE_NOTES.crossDepthClamped); try { + // Inside the `try` for the lease reason above `stitchCrossRepo`'s copy. A + // destination trace declares NO `to`: the call may land in any member, so + // every repo is in the query's scope and no incomplete one can be filtered + // out. An unreadable provider repo is precisely how "no outgoing + // ContractLink leaves this repo" becomes a wrong answer, not an empty one. + const bridge = bridgeCompletenessFor(bridgePrep.meta, () => true); const { crossings, truncated } = await listCrossingsFrom(handle, fromEp.member.repoPath); if (crossings.length === 0) { notes.push(TRACE_NOTES.destinationNoLink); @@ -1107,6 +1224,8 @@ async function stitchToDestination( group: p.name, role: 'to', query: p.from_uid ?? p.from, + // Zero rows to cap, so only the bridge's own completeness can speak. + ...traceCompleteness(bridge, false), notes, suggestion: 'Pass a `to` symbol for a symbol-to-symbol trace, or run group_sync.', }; @@ -1224,7 +1343,9 @@ async function stitchToDestination( hopCount: edgesA.length + 1, hops: [...hopsA, providerHop], edges: [...edgesA, boundaryEdge], - ...(truncated ? { truncated: true } : {}), + // The cap already marked this result; the bridge's completeness folds + // into the same fields rather than beside them. + ...traceCompleteness(bridge, truncated), notes: resultNotes, }; }; @@ -1240,6 +1361,8 @@ async function stitchToDestination( group: p.name, role: 'to', candidates: candidatesFrom(precise), + // The candidate LIST is what an incomplete bridge shortens here. + ...traceCompleteness(bridge, truncated), notes: [...notes, TRACE_NOTES.destinationMultiple], }; } @@ -1255,6 +1378,7 @@ async function stitchToDestination( group: p.name, role: 'to', candidates: candidatesFrom(fileLevel), + ...traceCompleteness(bridge, truncated), notes: [...notes, TRACE_NOTES.destinationAmbiguousFile], }; } @@ -1265,7 +1389,7 @@ async function stitchToDestination( group: p.name, role: 'to', query: p.from_uid ?? p.from, - ...(truncated ? { truncated: true } : {}), + ...traceCompleteness(bridge, truncated), notes, suggestion: 'Trace from the function that issues the HTTP request, or pass a `to` symbol.', }; diff --git a/gitnexus/src/core/group/extractors/fs-utils.ts b/gitnexus/src/core/group/extractors/fs-utils.ts index 384f63203..7f02bbd1d 100644 --- a/gitnexus/src/core/group/extractors/fs-utils.ts +++ b/gitnexus/src/core/group/extractors/fs-utils.ts @@ -21,3 +21,94 @@ export function readSafe(repoPath: string, rel: string): string | null { return null; } } + +/** Read a regular in-repo file without buffering more than `maxBytes`. */ +export async function readSafeBounded( + repoPath: string, + rel: string, + maxBytes: number, +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) return null; + const abs = path.resolve(repoPath, rel); + const base = path.resolve(repoPath); + const relToBase = path.relative(base, abs); + if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; + + try { + const canonicalBase = await fs.promises.realpath(base); + const canonicalFile = await fs.promises.realpath(abs); + const canonicalRelative = path.relative(canonicalBase, canonicalFile); + if (canonicalRelative.startsWith('..') || path.isAbsolute(canonicalRelative)) return null; + const beforeOpen = await fs.promises.lstat(canonicalFile); + if (!beforeOpen.isFile() || beforeOpen.size > maxBytes) return null; + if (maxBytes === 0) return beforeOpen.size === 0 ? '' : null; + + return await new Promise((resolve) => { + const stream = fs.createReadStream(canonicalFile, { + flags: 'r', + start: 0, + end: maxBytes, + autoClose: true, + }); + const chunks: Buffer[] = []; + let totalBytes = 0; + let validated = false; + let settled = false; + + const finish = (value: string | null): void => { + if (settled) return; + settled = true; + resolve(value); + }; + + stream.pause(); + stream.once('open', (fd) => { + try { + const opened = fs.fstatSync(fd); + if (!opened.isFile() || opened.size > maxBytes) { + finish(null); + stream.destroy(); + return; + } + + const currentCanonical = fs.realpathSync(canonicalFile); + const currentRelative = path.relative(canonicalBase, currentCanonical); + const current = fs.statSync(currentCanonical); + if ( + currentRelative.startsWith('..') || + path.isAbsolute(currentRelative) || + opened.dev !== current.dev || + opened.ino !== current.ino + ) { + finish(null); + stream.destroy(); + return; + } + + validated = true; + stream.resume(); + } catch { + finish(null); + stream.destroy(); + } + }); + stream.on('data', (chunk: Buffer | string) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += bytes.length; + if (totalBytes > maxBytes) { + finish(null); + stream.destroy(); + return; + } + chunks.push(bytes); + }); + stream.once('end', () => { + finish(validated ? Buffer.concat(chunks, totalBytes).toString('utf8') : null); + }); + stream.once('error', () => finish(null)); + stream.once('close', () => finish(null)); + }); + } catch { + return null; + } +} diff --git a/gitnexus/src/core/group/extractors/graphql-extractor.ts b/gitnexus/src/core/group/extractors/graphql-extractor.ts new file mode 100644 index 000000000..efafe4e2e --- /dev/null +++ b/gitnexus/src/core/group/extractors/graphql-extractor.ts @@ -0,0 +1,707 @@ +import { glob } from 'glob'; +import { + Kind, + parse, + type DocumentNode, + type FragmentDefinitionNode, + type OperationDefinitionNode, + type SelectionSetNode, +} from 'graphql'; +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import { createIgnoreFilter } from '../../../config/ignore-service.js'; +import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { logger } from '../../logger.js'; +import { ParseTimeoutError, parseSourceSafe } from '../../tree-sitter/safe-parse.js'; +import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; +import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafeBounded } from './fs-utils.js'; + +const PROVIDER_GLOB = '**/*.{ts,tsx,mts,cts}'; +const DOCUMENT_GLOB = '**/*.{graphql,gql}'; +const NEST_GRAPHQL_PACKAGE = '@nestjs/graphql'; +const MAX_GRAPHQL_TOKENS = 100_000; +const MAX_GRAPHQL_DEFINITIONS = 5_000; +const MAX_GRAPHQL_OPERATIONS = 500; +const MAX_GRAPHQL_SELECTIONS = 10_000; +const MAX_GRAPHQL_TRAVERSAL_DEPTH = 64; +const MAX_PROVIDER_AST_NODES = 100_000; +const MAX_PROVIDER_AST_DEPTH = 256; +const GRAPHQL_NAME = /^[_A-Za-z][_0-9A-Za-z]*$/; + +type GraphqlOperationKind = 'query' | 'mutation' | 'subscription'; + +interface ResolvedSymbol { + uid: string; + name: string; + filePath: string; +} + +interface DecoratorBindings { + operations: Map; + resolvers: Set; +} + +type DecoratorFieldName = + | { kind: 'absent' } + | { kind: 'literal'; value: string } + | { kind: 'dynamic' }; + +type GeneratedSymbolIndex = Map; +type GeneratedIndexCache = Map>; + +export const RESOLVE_METHOD_QUERY = ` +MATCH (n) +WHERE labels(n) IN ['Method','Function','Property','CodeElement'] + AND n.name = $name AND n.filePath = $filePath AND n.startLine = $startLine AND n.id <> '' +RETURN n.id AS uid, n.name AS name, n.filePath AS filePath +ORDER BY n.id ASC +LIMIT 2`; + +// LadybugDB returns labels(n) as a scalar string, not Neo4j's string array. +// The real-db integration test executes this exact query and guards that dialect contract. +export const RESOLVE_GENERATED_SYMBOL_QUERY = ` +MATCH (n) +WHERE labels(n) IN ['Const','Variable','Function','Method','CodeElement'] + AND n.name = $name AND n.filePath <> '' AND n.id <> '' +RETURN n.id AS uid, n.name AS name, n.filePath AS filePath +ORDER BY n.id ASC +LIMIT 2`; + +function rowValue(row: Record, key: string, position: number): string { + return String(row[key] ?? row[position] ?? ''); +} + +function uniqueRealSymbol(rows: Record[]): ResolvedSymbol | null { + if (rows.length !== 1) return null; + const row = rows[0]; + const symbol = { + uid: rowValue(row, 'uid', 0), + name: rowValue(row, 'name', 1), + filePath: rowValue(row, 'filePath', 2).replace(/\\/g, '/'), + }; + return symbol.uid && symbol.name && symbol.filePath ? symbol : null; +} + +function unquote(text: string): string | null { + const trimmed = text.trim(); + if (trimmed.length < 2) return null; + const quote = trimmed[0]; + if ((quote !== "'" && quote !== '"' && quote !== '`') || trimmed.at(-1) !== quote) return null; + const value = trimmed.slice(1, -1); + return value.includes('${') ? null : value; +} + +function unwrapExpression(node: Parser.SyntaxNode): Parser.SyntaxNode { + let current = node; + while ( + ['as_expression', 'satisfies_expression', 'parenthesized_expression'].includes(current.type) && + current.namedChildren[0] + ) { + current = current.namedChildren[0]; + } + return current; +} + +function objectPairValue(node: Parser.SyntaxNode, key: string): Parser.SyntaxNode | null { + const object = unwrapExpression(node); + if (object.type !== 'object') return null; + for (const pair of object.namedChildren) { + if (pair.type !== 'pair') continue; + const keyNode = pair.childForFieldName('key'); + const pairKey = keyNode ? (unquote(keyNode.text) ?? keyNode.text) : null; + if (pairKey === key) return pair.childForFieldName('value'); + } + return null; +} + +function literalValue(node: Parser.SyntaxNode | null): string | null { + return node ? unquote(unwrapExpression(node).text) : null; +} + +function graphqlNameValue(node: Parser.SyntaxNode | null): string | null { + return node ? literalValue(objectPairValue(node, 'value')) : null; +} + +function withinGeneratedAstBudget(root: Parser.SyntaxNode): boolean { + const pending: Array<{ node: Parser.SyntaxNode; depth: number }> = [{ node: root, depth: 0 }]; + let visited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + visited++; + if (visited > MAX_PROVIDER_AST_NODES || current.depth > MAX_PROVIDER_AST_DEPTH) return false; + for (let index = current.node.namedChildren.length - 1; index >= 0; index--) { + const child = current.node.namedChildren[index]; + if (child) pending.push({ node: child, depth: current.depth + 1 }); + } + } + return true; +} + +function generatedRootFields( + selectionSet: Parser.SyntaxNode | null, + fragments: ReadonlyMap, +): Set | null { + const fields = new Set(); + if (!selectionSet) return null; + const seenFragments = new Set(); + const pending: Array<{ selectionSet: Parser.SyntaxNode; depth: number }> = [ + { selectionSet, depth: 0 }, + ]; + let selectionsVisited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + const selections = objectPairValue(current.selectionSet, 'selections'); + const array = selections ? unwrapExpression(selections) : null; + if (!array || array.type !== 'array') return null; + for (const item of array.namedChildren) { + selectionsVisited++; + if (selectionsVisited > MAX_GRAPHQL_SELECTIONS) return null; + const selection = unwrapExpression(item); + const kind = literalValue(objectPairValue(selection, 'kind')); + if (kind === 'Field') { + const field = graphqlNameValue(objectPairValue(selection, 'name')); + if (field) fields.add(field); + continue; + } + if (kind === 'InlineFragment') { + const nested = objectPairValue(selection, 'selectionSet'); + if (nested) pending.push({ selectionSet: nested, depth: current.depth + 1 }); + continue; + } + if (kind !== 'FragmentSpread') continue; + const name = graphqlNameValue(objectPairValue(selection, 'name')); + if (!name || seenFragments.has(name)) continue; + const fragment = fragments.get(name); + if (!fragment) continue; + const nested = objectPairValue(fragment, 'selectionSet'); + if (!nested) continue; + seenFragments.add(name); + pending.push({ selectionSet: nested, depth: current.depth + 1 }); + } + } + return fields; +} + +function parsedDocumentProof( + source: string, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], +): boolean { + let document: DocumentNode; + try { + document = parse(source, { noLocation: true, maxTokens: MAX_GRAPHQL_TOKENS }); + } catch { + return false; + } + if (document.definitions.length > MAX_GRAPHQL_DEFINITIONS) return false; + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind === Kind.FRAGMENT_DEFINITION) + fragments.set(definition.name.value, definition); + } + for (const definition of document.definitions) { + if (definition.kind !== Kind.OPERATION_DEFINITION) continue; + if (definition.operation !== operationKind || definition.name?.value !== operationName) + continue; + const fields = rootFields(definition.selectionSet, fragments); + return fields !== null && requiredFields.every((field) => fields.includes(field)); + } + return false; +} + +function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { + const value = unwrapExpression(initializer); + if (value.type === 'string') { + if (value.text.startsWith('"')) { + try { + return JSON.parse(value.text) as string; + } catch { + return null; + } + } + return unquote(value.text); + } + if (value.type === 'template_string') return unquote(value.text); + + if (value.type === 'call_expression') { + const template = value.namedChildren.find((child) => child.type === 'template_string'); + return template ? unquote(template.text) : null; + } + + if (value.type !== 'new_expression') return null; + const constructor = value.childForFieldName('constructor') ?? value.namedChildren[0]; + if (!constructor || !constructor.text.endsWith('TypedDocumentString')) return null; + const args = value.childForFieldName('arguments'); + const first = args?.namedChildren[0]; + return first ? staticGraphqlSource(first) : null; +} + +export function hasGeneratedDocumentProof( + initializer: Parser.SyntaxNode, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], +): boolean { + if (!withinGeneratedAstBudget(initializer)) return false; + const staticSource = staticGraphqlSource(initializer); + if (staticSource !== null) { + return parsedDocumentProof(staticSource, operationKind, operationName, requiredFields); + } + const document = unwrapExpression(initializer); + if (literalValue(objectPairValue(document, 'kind')) !== 'Document') return false; + const definitions = objectPairValue(document, 'definitions'); + const array = definitions ? unwrapExpression(definitions) : null; + if (!array || array.type !== 'array') return false; + + const fragments = new Map(); + for (const item of array.namedChildren) { + const definition = unwrapExpression(item); + if (literalValue(objectPairValue(definition, 'kind')) !== 'FragmentDefinition') continue; + const name = graphqlNameValue(objectPairValue(definition, 'name')); + if (name) fragments.set(name, definition); + } + + for (const item of array.namedChildren) { + const definition = unwrapExpression(item); + if (literalValue(objectPairValue(definition, 'kind')) !== 'OperationDefinition') continue; + if (literalValue(objectPairValue(definition, 'operation')) !== operationKind) continue; + if (graphqlNameValue(objectPairValue(definition, 'name')) !== operationName) continue; + const fields = generatedRootFields(objectPairValue(definition, 'selectionSet'), fragments); + if (fields && requiredFields.every((field) => fields.has(field))) return true; + } + return false; +} + +function importedDecoratorBindings(root: Parser.SyntaxNode): DecoratorBindings { + const operations = new Map(); + const resolvers = new Set(); + for (const child of root.namedChildren) { + if (child.type !== 'import_statement') continue; + const source = child.childForFieldName('source'); + if (!source || unquote(source.text) !== NEST_GRAPHQL_PACKAGE) continue; + + const namedImports = child.namedChildren + .find((node) => node.type === 'import_clause') + ?.namedChildren.find((node) => node.type === 'named_imports'); + if (!namedImports) continue; + + for (const specifier of namedImports.namedChildren) { + if (specifier.type !== 'import_specifier') continue; + const imported = specifier.childForFieldName('name')?.text; + const local = specifier.childForFieldName('alias')?.text ?? imported; + if (!imported || !local) continue; + const kind = imported.toLowerCase(); + if (kind === 'query' || kind === 'mutation' || kind === 'subscription') { + operations.set(local, kind); + } else if (imported === 'Resolver') { + resolvers.add(local); + } + } + } + return { operations, resolvers }; +} + +function decoratorKind( + decorator: Parser.SyntaxNode, + bindings: Map, +): { kind: GraphqlOperationKind; argumentsNode?: Parser.SyntaxNode } | null { + const expression = decorator.namedChildren[0]; + if (!expression) return null; + if (expression.type === 'identifier') { + const kind = bindings.get(expression.text); + return kind ? { kind } : null; + } + if (expression.type !== 'call_expression') return null; + const callee = expression.childForFieldName('function'); + if (!callee || callee.type !== 'identifier') return null; + const kind = bindings.get(callee.text); + if (!kind) return null; + return { kind, argumentsNode: expression.childForFieldName('arguments') ?? undefined }; +} + +function decoratorFieldName(argumentsNode: Parser.SyntaxNode | undefined): DecoratorFieldName { + if (!argumentsNode || argumentsNode.namedChildren.length === 0) return { kind: 'absent' }; + const args = argumentsNode.namedChildren; + if (args[0] && ['string', 'template_string'].includes(args[0].type)) { + const direct = unquote(args[0].text); + return direct === null ? { kind: 'dynamic' } : { kind: 'literal', value: direct }; + } + + let sawOptions = false; + + for (const arg of args) { + if (arg.type !== 'object') continue; + sawOptions = true; + for (const pair of arg.namedChildren) { + if (pair.type === 'spread_element' || pair.type.startsWith('shorthand_property_identifier')) { + return { kind: 'dynamic' }; + } + if (pair.type !== 'pair') continue; + const key = pair.childForFieldName('key')?.text.replace(/^['"]|['"]$/g, ''); + if (key !== 'name') continue; + const value = pair.childForFieldName('value'); + if (!value || !['string', 'template_string'].includes(value.type)) { + return { kind: 'dynamic' }; + } + const literal = unquote(value.text); + return literal === null ? { kind: 'dynamic' } : { kind: 'literal', value: literal }; + } + } + if (sawOptions || args.length === 1) return { kind: 'absent' }; + return { kind: 'dynamic' }; +} + +function topLevelResolverClassBodies( + root: Parser.SyntaxNode, + resolverBindings: ReadonlySet, +): Parser.SyntaxNode[] | null { + if (!withinGeneratedAstBudget(root)) return null; + const bodies: Parser.SyntaxNode[] = []; + for (const statement of root.namedChildren) { + const classNode = + statement.type === 'class_declaration' + ? statement + : statement.type === 'export_statement' + ? statement.namedChildren.find((child) => child.type === 'class_declaration') + : undefined; + if (!classNode) continue; + const decorators = [ + ...new Set([ + ...statement.namedChildren.filter((child) => child.type === 'decorator'), + ...classNode.namedChildren.filter((child) => child.type === 'decorator'), + ]), + ]; + const isResolver = decorators.some((decorator) => { + const expression = decorator.namedChildren[0]; + if (!expression) return false; + const callee = + expression.type === 'call_expression' + ? expression.childForFieldName('function') + : expression; + return callee?.type === 'identifier' && resolverBindings.has(callee.text); + }); + if (!isResolver) continue; + const body = classNode.childForFieldName('body'); + if (body) bodies.push(body); + } + return bodies; +} + +function rootFields( + selectionSet: SelectionSetNode, + fragments: ReadonlyMap, +): string[] | null { + const fields: string[] = []; + const seenFragments = new Set(); + const pending: Array<{ selectionSet: SelectionSetNode; depth: number }> = [ + { selectionSet, depth: 0 }, + ]; + let selectionsVisited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + if (current.depth > MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + for (const selection of current.selectionSet.selections) { + selectionsVisited++; + if (selectionsVisited > MAX_GRAPHQL_SELECTIONS) return null; + if (selection.kind === Kind.FIELD) { + fields.push(selection.name.value); + continue; + } + if (selection.kind === Kind.INLINE_FRAGMENT) { + pending.push({ selectionSet: selection.selectionSet, depth: current.depth + 1 }); + continue; + } + const name = selection.name.value; + if (seenFragments.has(name)) continue; + const fragment = fragments.get(name); + if (!fragment) continue; + seenFragments.add(name); + pending.push({ selectionSet: fragment.selectionSet, depth: current.depth + 1 }); + } + } + return fields; +} + +function generatedCandidates(operation: OperationDefinitionNode): string[] { + const name = operation.name?.value; + return name ? [`${name}Document`] : []; +} + +async function generatedDocumentMatches( + repoPath: string, + symbol: ResolvedSymbol, + operationKind: GraphqlOperationKind, + operationName: string, + requiredFields: readonly string[], + cache: GeneratedIndexCache, +): Promise { + const normalizedPath = symbol.filePath.replace(/\\/g, '/'); + let pendingIndex = cache.get(normalizedPath); + if (!pendingIndex) { + pendingIndex = buildGeneratedSymbolIndex(repoPath, normalizedPath); + cache.set(normalizedPath, pendingIndex); + } + const index = await pendingIndex; + const values = index?.get(symbol.name) ?? []; + return values.some((value) => + hasGeneratedDocumentProof(value, operationKind, operationName, requiredFields), + ); +} + +async function buildGeneratedSymbolIndex( + repoPath: string, + filePath: string, +): Promise { + const source = await readSafeBounded(repoPath, filePath, getMaxFileSizeBytes()); + if (source === null) return null; + const parser = new Parser(); + parser.setLanguage( + filePath.toLowerCase().endsWith('.tsx') ? TypeScript.tsx : TypeScript.typescript, + ); + let tree: Parser.Tree; + try { + tree = parseSourceSafe(parser, source, undefined, undefined, filePath); + } catch (error) { + if (error instanceof ParseTimeoutError) return null; + throw error; + } + + return indexGeneratedDeclarators(tree.rootNode); +} + +export function indexGeneratedDeclarators(root: Parser.SyntaxNode): GeneratedSymbolIndex { + const index: GeneratedSymbolIndex = new Map(); + const pending = [root]; + while (pending.length > 0) { + const node = pending.pop(); + if (!node) break; + if (node.type === 'variable_declarator') { + const name = node.childForFieldName('name')?.text; + const value = node.childForFieldName('value'); + if (name && value) { + const values = index.get(name) ?? []; + values.push(value); + index.set(name, values); + } + } + for (let child = node.namedChildren.length - 1; child >= 0; child--) { + pending.push(node.namedChildren[child]); + } + } + return index; +} + +function dedupe(contracts: ExtractedContract[]): ExtractedContract[] { + const seen = new Set(); + return contracts.filter((contract) => { + const key = `${contract.contractId}|${contract.role}|${contract.symbolUid}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +export class GraphqlExtractor implements ContractExtractor { + type = 'graphql' as const; + + async canExtract(_repo: RepoHandle): Promise { + return true; + } + + async extract( + dbExecutor: CypherExecutor | null, + repoPath: string, + _repo: RepoHandle, + ): Promise { + if (!dbExecutor) return []; + const ignore = await createIgnoreFilter(repoPath); + const [providerFiles, documentFiles] = await Promise.all([ + glob(PROVIDER_GLOB, { cwd: repoPath, ignore, nodir: true }), + glob(DOCUMENT_GLOB, { cwd: repoPath, ignore, nodir: true }), + ]); + const contracts = [ + ...(await this.extractProviders(dbExecutor, repoPath, providerFiles)), + ...(await this.extractConsumers(dbExecutor, repoPath, documentFiles)), + ]; + return dedupe(contracts); + } + + private async extractProviders( + dbExecutor: CypherExecutor, + repoPath: string, + files: string[], + ): Promise { + const parser = new Parser(); + const contracts: ExtractedContract[] = []; + const maxFileSizeBytes = getMaxFileSizeBytes(); + for (const rel of files) { + if (/\.(?:spec|test)\.[cm]?tsx?$/i.test(rel)) continue; + const source = await readSafeBounded(repoPath, rel, maxFileSizeBytes); + if (source === null || !source.includes(NEST_GRAPHQL_PACKAGE)) continue; + parser.setLanguage( + rel.toLowerCase().endsWith('.tsx') ? TypeScript.tsx : TypeScript.typescript, + ); + let tree: Parser.Tree; + try { + tree = parseSourceSafe(parser, source, undefined, undefined, rel); + } catch (error) { + if (error instanceof ParseTimeoutError) continue; + throw error; + } + const bindings = importedDecoratorBindings(tree.rootNode); + if (bindings.operations.size === 0 || bindings.resolvers.size === 0) continue; + const bodies = topLevelResolverClassBodies(tree.rootNode, bindings.resolvers); + if (bodies === null) continue; + for (const body of bodies) { + let decorators: Parser.SyntaxNode[] = []; + for (const member of body.namedChildren) { + if (member.type === 'comment') continue; + if (member.type === 'decorator') { + decorators.push(member); + continue; + } + if (member.type !== 'method_definition' && member.type !== 'public_field_definition') { + decorators = []; + continue; + } + const memberDecorators = [ + ...new Set([ + ...decorators, + ...member.namedChildren.filter((child) => child.type === 'decorator'), + ]), + ]; + const methodName = member.childForFieldName('name')?.text; + if (!methodName) { + decorators = []; + continue; + } + for (const decorator of memberDecorators) { + const operation = decoratorKind(decorator, bindings.operations); + if (!operation) continue; + const parsedField = decoratorFieldName(operation.argumentsNode); + if (parsedField.kind === 'dynamic') continue; + const field = parsedField.kind === 'literal' ? parsedField.value : methodName; + if (!GRAPHQL_NAME.test(field)) continue; + const filePath = rel.replace(/\\/g, '/'); + const symbol = uniqueRealSymbol( + await dbExecutor(RESOLVE_METHOD_QUERY, { + name: methodName, + filePath, + startLine: + member.type === 'public_field_definition' + ? (member.childForFieldName('value')?.startPosition.row ?? + member.startPosition.row) + 1 + : member.startPosition.row + 1, + }), + ); + if (!symbol) continue; + contracts.push({ + contractId: `graphql::${operation.kind}::${field}`, + type: 'graphql', + role: 'provider', + symbolUid: symbol.uid, + symbolRef: { filePath: symbol.filePath, name: symbol.name }, + symbolName: symbol.name, + confidence: 1, + meta: { + operationKind: operation.kind, + fieldName: field, + resolverPath: filePath, + extractionStrategy: 'nestjs_decorator', + }, + }); + } + decorators = []; + } + } + } + return contracts; + } + + private async extractConsumers( + dbExecutor: CypherExecutor, + repoPath: string, + files: string[], + ): Promise { + const contracts: ExtractedContract[] = []; + const generatedIndexCache: GeneratedIndexCache = new Map(); + const maxFileSizeBytes = getMaxFileSizeBytes(); + for (const rel of files) { + const source = await readSafeBounded(repoPath, rel, maxFileSizeBytes); + if (source === null) continue; + let document: DocumentNode; + try { + document = parse(source, { noLocation: true, maxTokens: MAX_GRAPHQL_TOKENS }); + } catch (error) { + logger.debug({ file: rel, error }, 'skipping invalid GraphQL document'); + continue; + } + if (document.definitions.length > MAX_GRAPHQL_DEFINITIONS) continue; + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind === Kind.FRAGMENT_DEFINITION) { + fragments.set(definition.name.value, definition); + } + } + const operations = document.definitions.filter( + (definition): definition is OperationDefinitionNode => + definition.kind === Kind.OPERATION_DEFINITION && definition.name !== undefined, + ); + if (operations.length > MAX_GRAPHQL_OPERATIONS) continue; + for (const definition of operations) { + const operationName = definition.name?.value; + if (!operationName) continue; + const documentPath = rel.replace(/\\/g, '/'); + const operationFields = rootFields(definition.selectionSet, fragments); + if (operationFields === null) continue; + const uniqueFields = [...new Set(operationFields)]; + let symbol: ResolvedSymbol | null = null; + for (const candidate of generatedCandidates(definition)) { + const resolved = uniqueRealSymbol( + await dbExecutor(RESOLVE_GENERATED_SYMBOL_QUERY, { name: candidate }), + ); + if ( + resolved && + (await generatedDocumentMatches( + repoPath, + resolved, + definition.operation, + operationName, + uniqueFields, + generatedIndexCache, + )) + ) { + symbol = resolved; + break; + } + } + if (!symbol) continue; + for (const field of uniqueFields) { + contracts.push({ + contractId: `graphql::${definition.operation}::${field}`, + type: 'graphql', + role: 'consumer', + symbolUid: symbol.uid, + symbolRef: { filePath: symbol.filePath, name: symbol.name }, + symbolName: symbol.name, + confidence: 1, + meta: { + operationKind: definition.operation, + operationName: definition.name.value, + fieldName: field, + documentPath, + extractionStrategy: 'graphql_ast', + }, + }); + } + } + } + return contracts; + } +} diff --git a/gitnexus/src/core/group/extractors/http-patterns/java.ts b/gitnexus/src/core/group/extractors/http-patterns/java.ts index 4eba0c1f1..8d216c50a 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/java.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/java.ts @@ -28,6 +28,16 @@ import { REQUEST_LINE_CONFIDENCE, EXCHANGE_CONFIDENCE, } from './spring-consumer-shared.js'; +import { + expandJavaWildcardStaticImports, + extractJavaModuleConstants, + foldJavaOperands, + isJavaConstantFile, + parseJavaConstOperands, + prepareJavaRouteConstants, + type JavaConstantIndex, + type RepoConstants, +} from '../../../ingestion/route-extractors/java-const-resolver.js'; import { extractStaticPathExpression, inferOkHttpMethod, @@ -165,6 +175,34 @@ const JAVA_ROUTE_ANNOTATION_PATTERNS = compilePatterns({ key: (identifier) @key value: [(string_literal) @value (element_value_array_initializer (string_literal) @value)])))) name: (identifier) @member) @node + (class_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list [(identifier) @value_expr (field_access) @value_expr (binary_expression) @value_expr])))) @node + (class_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key + value: [(identifier) @value_expr (field_access) @value_expr (binary_expression) @value_expr]))))) @node + (method_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list [(identifier) @value_expr (field_access) @value_expr (binary_expression) @value_expr]))) + name: (identifier) @member) @node + (method_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key + value: [(identifier) @value_expr (field_access) @value_expr (binary_expression) @value_expr])))) + name: (identifier) @member) @node ] `, }, @@ -469,6 +507,12 @@ interface MethodRouteAnnotation { rawPath: string; /** OpenFeign's single effective verb; null means its contract is invalid/ambiguous. */ feignHttpMethod?: string | null; + /** + * Non-literal path operands (constant ref or `+`-concat), captured when the + * annotation value is not a string literal. Resolved against the repo-wide + * Java constant map in scan(); a failed fold drops the route (skip floor). + */ + pathOperands?: readonly import('../../../ingestion/route-extractors/constant-resolver.js').Operand[]; } interface RequestLineAnnotation { @@ -484,6 +528,16 @@ interface RouteAnnotationScan { feignPrefixByInterfaceId: Map; /** Spring HTTP Interface `@HttpExchange(url|value)` type-level prefixes per class/interface node id. */ httpExchangePrefixByTypeId: Map; + /** + * Class node ids whose `@RequestMapping` prefix is a constant reference or + * concat rather than a literal. Folding a TYPE-level prefix would need the + * repo constant map inside `scanRouteAnnotations`, which has no access to it, + * so `scan()` suppresses every method route under such a class instead of + * emitting it with the prefix silently dropped (a wrong path, not a missing + * one). Ingestion's `extractSpringRoutes` applies the identical rule — R4 + * parity. + */ + typesWithUnfoldablePrefix: Set; /** Resolved Spring shortcut/`@RequestMapping` routes — paths × verbs yield one entry each. */ methodRoutes: MethodRouteAnnotation[]; /** One entry per OpenFeign `@RequestLine` whose value parses to a verb + path. */ @@ -511,6 +565,7 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { // feeds the OpenFeign *consumer* path in scan(). An interface carrying both // `@RequestMapping` and `@FeignClient(path)` lands a different value in each. const prefixByTypeId = new Map(); + const typesWithUnfoldablePrefix = new Set(); const feignPrefixByInterfaceId = new Map(); const httpExchangePrefixByTypeId = new Map(); const methodRoutes: MethodRouteAnnotation[] = []; @@ -527,7 +582,10 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { const annNode = captures.ann; const node = captures.node; const valueNode = captures.value; - if (!annNode || !node || !valueNode) continue; + // A non-literal annotation value (constant ref / `+`-concat) is captured + // as @value_expr instead of @value — one of the two must be present. + const valueExprNode = captures.value_expr; + if (!annNode || !node || (!valueNode && !valueExprNode)) continue; // Discrimination is on the trailing segment only (`simpleName`), so a // non-Spring annotation whose last segment collides with a route annotation // (e.g. `@com.evil.GetMapping("/x")`) is treated as a route. This is the @@ -550,7 +608,7 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { const feignHttpMethod = httpMethods.length === 1 ? (httpMethods[0] === '*' ? 'GET' : httpMethods[0]) : null; if (!isRouteMemberKey(keyNode)) continue; - const rawPath = unquoteLiteral(valueNode.text); + const rawPath = valueNode ? unquoteLiteral(valueNode.text) : null; if (rawPath !== null) { for (const httpMethod of httpMethods) { methodRoutes.push({ @@ -561,10 +619,33 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { feignHttpMethod, }); } + } else { + // Non-literal path (a constant reference or `+`-concatenation). + // Defer to scan(): the fold needs the repo-wide constant map built + // by prepareRepo. Capture the operand list now; resolution happens + // in scan() against JavaRepoContext, and an unresolvable operand + // list leaves the route skipped (KTD5 skip floor). + const operands = parseJavaConstOperands(valueExprNode); + if (operands !== null) { + for (const httpMethod of httpMethods) { + methodRoutes.push({ + methodNode: node, + methodName: captures.member?.text ?? null, + httpMethod, + rawPath: '', + feignHttpMethod, + pathOperands: operands, + }); + } + } } } else if (ann === 'RequestLine') { // Feign packs verb + path in one literal; its only named argument is `value`. if (keyNode && keyNode.text !== 'value') continue; + // A constant-valued `@RequestLine` arrives as @value_expr, not @value — + // `valueNode` is undefined in that shape. Skip rather than dereference + // (constant folding for Feign verb+path literals is out of scope here). + if (!valueNode) continue; const raw = unquoteLiteral(valueNode.text); const parsed = raw !== null ? parseRequestLine(raw) : null; if (parsed) { @@ -579,7 +660,7 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { // `url` or `value` attribute (or positionally); other attributes // (`accept`, `contentType`, …) are not routes. if (keyNode && keyNode.text !== 'url' && keyNode.text !== 'value') continue; - const rawPath = unquoteLiteral(valueNode.text); + const rawPath = valueNode ? unquoteLiteral(valueNode.text) : null; if (rawPath !== null) { exchangeRoutes.push({ methodNode: node, @@ -596,6 +677,11 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { // — on an interface — an OpenFeign `@FeignClient(path = "...")` prefix. if (ann === 'RequestMapping') { if (!isRouteMemberKey(keyNode)) continue; + if (!valueNode) { + // Constant-valued class prefix — see `typesWithUnfoldablePrefix`. + typesWithUnfoldablePrefix.add(node.id); + continue; + } const prefix = unquoteLiteral(valueNode.text); if (prefix !== null) { pushPrefix(prefixByTypeId, node.id, prefix); @@ -606,13 +692,13 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { } else if (ann === 'FeignClient' && node.type === 'interface_declaration') { // Feign's `name`/`value` identify a service, not a path — only `path` is a prefix. if (!keyNode || keyNode.text !== 'path') continue; - const prefix = unquoteLiteral(valueNode.text); + const prefix = valueNode ? unquoteLiteral(valueNode.text) : null; if (prefix !== null) pushPrefix(feignPrefixByInterfaceId, node.id, prefix); } else if (ann === 'HttpExchange') { // Spring HTTP Interface type-level prefix: the path lives in `url`/`value` // (or positionally). Applies to its `@(Get|...)Exchange` consumer methods. if (keyNode && keyNode.text !== 'url' && keyNode.text !== 'value') continue; - const prefix = unquoteLiteral(valueNode.text); + const prefix = valueNode ? unquoteLiteral(valueNode.text) : null; if (prefix !== null) pushPrefix(httpExchangePrefixByTypeId, node.id, prefix); } } @@ -662,6 +748,7 @@ function scanRouteAnnotations(tree: Parser.Tree): RouteAnnotationScan { return { prefixByTypeId, + typesWithUnfoldablePrefix, feignPrefixByInterfaceId, httpExchangePrefixByTypeId, methodRoutes: constrainedMethodRoutes, @@ -707,9 +794,20 @@ function collectImplementedInterfaces(typeNode: Parser.SyntaxNode): string[] { } function collectSpringTypes(filePath: string, tree: Parser.Tree): SharedSpringType[] { - const { prefixByTypeId, methodRoutes } = scanRouteAnnotations(tree); + const { prefixByTypeId, typesWithUnfoldablePrefix, methodRoutes } = scanRouteAnnotations(tree); const routesByMethodId = new Map>(); for (const route of methodRoutes) { + // Constant-valued class prefix: no single prefix string exists here, so the + // inheritance view would publish this route unprefixed. Skip — same rule as + // scan() and as ingestion (R4 parity). + const owner = findEnclosingClass(route.methodNode); + if (owner && typesWithUnfoldablePrefix.has(owner.id)) continue; + // A constant-referencing route still carries `rawPath: ''` here — folding + // happens in scan() against the repo constant map, which this + // inheritance-view collector has no access to. Emitting it as an empty + // path would publish `POST /`-shaped noise into the shared type view; + // skip instead (ingestion keeps the same skip floor — R4 parity). + if (route.pathOperands) continue; const routes = routesByMethodId.get(route.methodNode.id) ?? []; routes.push({ method: route.httpMethod, path: route.rawPath }); routesByMethodId.set(route.methodNode.id, routes); @@ -781,8 +879,76 @@ export const JAVA_HTTP_PLUGIN: HttpLanguagePlugin = { content, ); }, - scan(tree) { + prepareRepo(args) { + // Build the repo-wide Java string-constant map once per extract() run + // (mirrors the Python binding's cost-gated pre-pass). A cheap content + // gate keeps literal-only repos at zero parses: only files containing a + // `static final String` declaration are parsed for constants. + try { + // The orchestrator hands over a bare Parser (no language set yet); + // bind Java explicitly — Python's prepareRepo does the same — otherwise + // parseSourceSafe spins to its 15 s budget per file. + args.parser.setLanguage(Java); + } catch { + // fall through: a parser that rejects binding cannot produce a constant + // map; per-file try/catch below then skips everything harmlessly. + } + const constants = new Map< + string, + import('../../../ingestion/route-extractors/constant-resolver.js').ModuleConstants + >(); + for (const rel of args.files) { + if (!rel.endsWith('.java')) continue; + try { + const src = args.readFile(rel); + // Cheap content gate: only constant-DEFINITION candidates get parsed + // here (~hundreds of files). Import-only files (every controller) + // are deliberately NOT parsed in this pass — scan() lazily extracts + // the importing file's own import table from the tree it already + // holds when a constant-referencing route actually needs the fold. + // A gate that also matched `import ...;` would parse the entire + // repository here (tens of thousands of files) just to build import + // tables the fold can derive per-file on demand. + // + // The predicate is the SHARED one the ingestion provider uses, so the + // two subsystems agree on which files define constants. Its previous + // local spelling missed `final static String` and lowercase interface + // names, and admitted an interface that ingestion's gate rejected. + if (!src || !isJavaConstantFile(src)) { + continue; + } + const tree = args.parseSource(args.parser, src); + if (!tree) continue; + const mc = extractJavaModuleConstants(tree); + if ( + mc.literals.size > 0 || + mc.exprs.size > 0 || + mc.imports.size > 0 || + (mc.wildcardImports?.length ?? 0) > 0 + ) { + constants.set(rel, mc); + } + } catch { + // Per-file resilience: one unreadable/oversized/ill-formed file must + // not forfeit the whole repo's constant map (a missing constants + // class only degrades refs that pointed at it). + continue; + } + } + // On-demand static imports (`import static a.b.C.*`) were recorded as + // pending class FQNs during extraction; materialize their bare-name + // bindings now that the whole map exists. A wildcard's target is itself + // a constants file, so it is necessarily a map entry — anything else + // degrades to the fold's skip floor. In-place: each entry is owned by + // this map, and every file is expanded exactly once. + const constantIndex = prepareJavaRouteConstants(constants); + return { constants, constantIndex }; + }, + scan(tree, repoContext, fileRel) { const out: HttpDetection[] = []; + const javaCtx = repoContext as + | { constants: RepoConstants; constantIndex: JavaConstantIndex } + | undefined; // ─── Spring providers + OpenFeign consumers (one query pass) ──── // `scanRouteAnnotations` resolves every route-defining annotation — @@ -790,6 +956,7 @@ export const JAVA_HTTP_PLUGIN: HttpLanguagePlugin = { // `@RequestLine`s — from a single `matches()` pass over the tree. const { prefixByTypeId, + typesWithUnfoldablePrefix, feignPrefixByInterfaceId, httpExchangePrefixByTypeId, methodRoutes, @@ -802,7 +969,52 @@ export const JAVA_HTTP_PLUGIN: HttpLanguagePlugin = { // class is a Spring *provider*. A mapping on a non-Feign interface has no // enclosing class and is dropped here — interface→controller inheritance is // handled by `scanProject`. + // Lazy per-file constants view. prepareRepo only indexes constant- + // DEFINING files (cheap gate); an importing controller is absent from + // that map. When a route actually references a constant, extract THIS + // file's import table from the tree scan() already holds (zero extra + // parses) and overlay it for the fold. Files whose routes are all + // literal — the overwhelming majority — never pay this cost. + let foldConstants: RepoConstants | undefined; + const getFoldConstants = (): RepoConstants | undefined => { + if (foldConstants !== undefined) return foldConstants; + foldConstants = javaCtx?.constants; + if (!javaCtx?.constants || !fileRel) return foldConstants; + if (javaCtx.constants.has(fileRel)) return foldConstants; + try { + const mc = extractJavaModuleConstants(tree); + // A file carrying ONLY wildcard static imports has an empty import + // table pre-expansion — overlay it too, then materialize the promised + // bindings against the repo map before it becomes a fold target. + if (mc.imports.size > 0 || (mc.wildcardImports?.length ?? 0) > 0) { + const merged = new Map(javaCtx.constants); + expandJavaWildcardStaticImports(mc, fileRel, merged, javaCtx.constantIndex); + merged.set(fileRel, mc); + foldConstants = merged; + } + } catch { + // fold falls back to the repo-wide map (imports stay unresolved) + } + return foldConstants; + }; + for (const route of methodRoutes) { + // A constant-valued CLASS prefix cannot be folded here, so every method + // route under such a class is suppressed rather than emitted at a wrong + // (unprefixed) path — the same rule `classesWithArrayPrefix` already + // encodes for the array form, and the same rule ingestion applies. + const owner = findEnclosingClass(route.methodNode); + if (owner && typesWithUnfoldablePrefix.has(owner.id)) continue; + // Non-literal route path: fold the operand list against the repo-wide + // constant map. Skip (never a guessed path) when the fold fails or the + // repo context is absent (context-less fallback scanning). + if (route.pathOperands && javaCtx && fileRel) { + const resolved = foldJavaOperands(fileRel, route.pathOperands, getFoldConstants()!); + if (resolved === null) continue; + route.rawPath = resolved; + } else if (route.pathOperands) { + continue; + } const enclosingInterface = findEnclosingInterface(route.methodNode); if (enclosingInterface && hasAnnotation(enclosingInterface, 'FeignClient')) { if (!route.feignHttpMethod) continue; diff --git a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts index 10f195111..ec695a9b6 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts @@ -18,6 +18,19 @@ import { joinPath, type SharedSpringType, } from '../../../ingestion/route-extractors/spring-shared.js'; +import { + buildKotlinConstantIndex, + extractKotlinModuleConstants, + foldKotlinOperands, + isKotlinConstantFile, + overlayKotlinConstantIndex, + parseKotlinConstOperands, + unfoldableDeclarationsOf, + unquoteKotlinIdentifier, + type KotlinConstantIndex, + type ModuleConstants, + type RepoConstants, +} from '../../../ingestion/route-extractors/kotlin-const-resolver.js'; import { REST_TEMPLATE_TO_HTTP, WEB_CLIENT_SHORT_TO_HTTP, @@ -42,6 +55,24 @@ import { * named annotation arguments (`@GetMapping(value = "/x")` and * `@GetMapping(path = "/x")`) are supported. * + * A method path that is a CONSTANT rather than a literal — + * `@GetMapping(ApiPaths.ORDERS)`, `@PostMapping(value = ApiPaths.BASE + "/create")` — + * is folded against a repo-wide Kotlin constant map built once per `extract()` + * run by `prepareRepo`, mirroring what the Java plugin does for the same shape + * in `java.ts`. An unresolvable fold skips the route (never a guessed path), and + * a class prefix that resolves to NO literal at all suppresses every method + * route under that class — the rule `java.ts` applies too, because emitting + * those routes unprefixed would publish paths the application does not serve. + * A prefix that resolves only PARTLY (Kotlin's vararg spelling + * `@RequestMapping("/lit", ApiPaths.BASE)`) still publishes its resolvable arm: + * suppression exists to avoid wrong routes, not to discard right ones. An EMPTY + * path array (`@RequestMapping(arrayOf())`) is not a prefix at all and + * suppresses nothing — see `classifyPathArgument`. On a + * `@FeignClient` the same rule is applied to whichever prefix GOVERNS, in the + * "path wins" order the URL is assembled in — `@FeignClient(path)` first, then + * the interface's `@RequestMapping` — and to both consumer lanes, `@(Get|...)Mapping` + * and `@RequestLine`. + * * **Consumers** — four call-site patterns common in Kotlin * Spring projects: * @@ -131,6 +162,180 @@ const arrayOfArg = (cap: string): string => `(call_expression (simple_identifier) @arrayOf (#eq? @arrayOf "arrayOf") (call_suffix (value_arguments (value_argument (string_literal) ${cap}))))`; +/** + * Expression node types a METHOD route path can be FOLDED from. A + * `string_literal` is deliberately absent: literal paths are already captured by + * the dedicated literal patterns, so admitting one here would emit the same + * route twice. + * + * This is an allow-list on purpose, and only safe because it gates FOLDING: a + * shape missing from it yields no route, which is the skip floor. The + * unfoldable-CLASS-PREFIX analysis must not be written this way — there a shape + * missing from the list means "emit unprefixed", a wrong route — so it inverts + * the test instead (see `classifyPathArgument`). + */ +const FOLDABLE_PATH_EXPRESSIONS: ReadonlySet = new Set([ + 'simple_identifier', + 'navigation_expression', + 'additive_expression', +]); + +/** + * Repo-relative path in the POSIX form the Kotlin constant map is keyed by. + * + * The orchestrator's file list comes from glob v13, which has no `posix: true` + * option and joins with the platform separator, so on Windows `prepareRepo` + * receives `src\main\kotlin\com\example\ApiPaths.kt` and `scan` receives the + * same for `fileRel`. `resolveKotlinImport` turns an import specifier into + * `com/example/ApiPaths.kt` and asks whether a key ENDS WITH it — a test no + * backslashed key can pass. Left unnormalized, every cross-file constant fold + * returns null on Windows and on Windows only: the pre-pass still runs, the + * context is still built, and the feature is simply, silently absent. The unit + * fixtures build POSIX keys by hand, so CI cannot see it. + * + * Normalizing at this boundary — write side (the map keys below) and read side + * (`fileRel`) — is the same fix `node.ts` (`normalizeRel`) and `python.ts` + * (`fileShortKey` / `fileLongKey`) already apply for the same reason, and it is + * the only coherent place: the resolver returns the key it matched, so + * normalizing inside it would hand back a value that misses in a map nobody + * normalized. `readFile` still receives the ORIGINAL `rel`, since the filesystem + * wants the platform's own spelling. + */ +function normalizeRel(rel: string): string { + return rel.replace(/\\/g, '/').replace(/^\.\//, ''); +} + +/** + * The path expression carried by one route-annotation argument, or null when the + * argument does not designate a path. + * + * tree-sitter-kotlin gives positional and named arguments the same + * `value_argument` node, distinguished only by a leading `simple_identifier` and + * an `=` token — so the key must be read here rather than constrained in the + * query. Non-route keys (`produces`, `consumes`, `headers`, …) return null, + * matching the `#match? @key "^(path|value)$"` guard the literal patterns use. + */ +function kotlinRouteArgumentExpression(arg: Parser.SyntaxNode): Parser.SyntaxNode | null { + const first = arg.namedChild(0); + if (!first) return null; + if (!arg.children.some((c) => c.type === '=')) return first; // positional + if (first.type !== 'simple_identifier') return null; + if (first.text !== 'path' && first.text !== 'value') return null; + return arg.namedChild(1); +} + +/** + * The `path = …` expression of one `@FeignClient` argument, or null. + * + * Deliberately narrower than {@link kotlinRouteArgumentExpression}: on a Feign + * client the positional argument and `value =` name a SERVICE, not a path, so + * only the explicit `path` key contributes a URL prefix. This mirrors the + * `#eq? @key "path"` guard the literal `@FeignClient` patterns use, and the + * `keyNode.text !== 'path'` guard `java.ts` applies to the same annotation. + */ +function kotlinFeignPathArgumentExpression(arg: Parser.SyntaxNode): Parser.SyntaxNode | null { + const first = arg.namedChild(0); + if (!first || first.type !== 'simple_identifier') return null; + if (!arg.children.some((c) => c.type === '=')) return null; + if (first.text !== 'path') return null; + return arg.namedChild(1); +} + +/** + * Is `node` a string literal whose value is fully known at parse time — that is, + * a literal carrying no interpolation? + * + * tree-sitter-kotlin models `"$base/x"` and `"${base}/x"` as a `string_literal` + * whose named children INTERLEAVE `string_content` runs with interpolation nodes + * — `interpolation_identifier_start`/`interpolated_identifier` for the `$name` + * form, `interpolation_expression_start`/`interpolated_expression`/ + * `interpolation_expression_end` for `${…}` — so the test has to be `every`, not + * `some`: `"pre${A.B}post"` carries `string_content` too. The route layer + * unquotes the RAW TEXT, so treating one as a literal publishes the source + * spelling — `/${ApiPaths.BASE}/orders` — as though the application served it. + * Escape sequences are NOT separate nodes in this grammar (`"/a\nb"` is one + * `string_content`), so this accepts exactly what it accepted before; a future + * grammar that split them would floor to "unknown" rather than to a de-escaped + * guess. Same test the constant resolver's `stringLiteralValue` applies, so a + * path is either literal on both sides or folded on neither. + */ +function isPlainStringLiteral(node: Parser.SyntaxNode): boolean { + if (node.type !== 'string_literal') return false; + return node.namedChildren.every((child) => child.type === 'string_content'); +} + +/** + * Element expressions of a Kotlin `arrayOf(...)` call, or null when `node` is + * not one. The JS mirror of the {@link arrayOfArg} query fragment, so the + * unfoldable-prefix analysis inspects exactly the elements the literal prefix + * patterns harvest. + */ +function kotlinArrayOfElements(node: Parser.SyntaxNode): Parser.SyntaxNode[] | null { + if (node.type !== 'call_expression') return null; + const callee = node.namedChild(0); + if (callee?.type !== 'simple_identifier' || callee.text !== 'arrayOf') return null; + const suffix = node.namedChildren.find((c) => c.type === 'call_suffix'); + const args = suffix?.namedChildren.find((c) => c.type === 'value_arguments'); + if (!args) return null; + return args.namedChildren + .filter((c) => c.type === 'value_argument') + .map((c) => c.namedChild(0)) + .filter((c): c is Parser.SyntaxNode => c !== null); +} + +/** + * What a route-annotation path argument says about the prefix it designates. + * Only `'unresolvable'` may suppress a route: + * + * - `'literal'` — at least one element is a plain literal, already harvested by + * the literal prefix patterns, so there is nothing to suppress. + * - `'none'` — no prefix. Empty `arrayOf()` or `[]` is Spring's "map at the root". + * Kept distinct from `'unresolvable'` because conflating them suppressed even + * plain literal routes below such a class, which no constant fold was ever + * involved in. tree-sitter-kotlin (fwcd) represents empty `[]` with a + * zero-width recovery child; filtering it is required for route interfaces, + * which do parse as `class_declaration`. + * - `'unresolvable'` — a non-empty argument with no literal element + * (`ApiPaths.BASE`, `buildPath()`, a template). Served path is unknowable. + */ +type PathArgumentPrefix = 'literal' | 'none' | 'unresolvable'; + +function classifyPathArgument(expr: Parser.SyntaxNode): PathArgumentPrefix { + if (isPlainStringLiteral(expr)) return 'literal'; + if (expr.type === 'collection_literal') { + const elements = expr.namedChildren.filter((child) => child.text.length > 0); + if (elements.length === 0) return 'none'; + return elements.some(isPlainStringLiteral) ? 'literal' : 'unresolvable'; + } + const elements = kotlinArrayOfElements(expr); + if (elements) { + if (elements.length === 0) return 'none'; + return elements.some(isPlainStringLiteral) ? 'literal' : 'unresolvable'; + } + return 'unresolvable'; +} + +/** + * Type declarations enclosing `node`, innermost first, by qualified type path. + * + * The scope a bare constant in a route annotation is resolved against; passed to + * `foldKotlinOperands`, which applies it. Collects `class_declaration` (including + * interfaces) and `object_declaration`. A `companion_object` adds no link of + * its own — members are keyed under the enclosing class one hop up. For a node + * inside `Outer.Inner`, returns `['Outer.Inner', 'Outer']`, matching the keys + * produced by `extractKotlinModuleConstants`. Skips unnamed types rather than + * guessing. + */ +function kotlinEnclosingTypeNames(node: Parser.SyntaxNode): string[] { + const simpleNames: string[] = []; + for (let cur = node.parent; cur; cur = cur.parent) { + if (cur.type !== 'class_declaration' && cur.type !== 'object_declaration') continue; + const ident = cur.children.find((c) => c.type === 'type_identifier'); + if (ident) simpleNames.push(unquoteKotlinIdentifier(ident.text)); + } + return simpleNames.map((_, index) => simpleNames.slice(index).reverse().join('.')); +} + // ─── Kotlin OkHttp builder verb-walk (parity with java-static-path.ts) ── // Mirrors `inferOkHttpMethod`, adapted to the Kotlin grammar: a call `X.name(args)` // is a `call_expression` whose callee is a `navigation_expression` (receiver + @@ -399,6 +604,151 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { ], } satisfies LanguagePatterns>); + // ─── Provider: constant-valued @RequestMapping / @(Get|...)Mapping ──── + // The literal patterns above pin the path node itself (`(string_literal) @path`), + // which structurally cannot match `@GetMapping(ApiPaths.ORDERS)`. These two + // capture the whole `value_argument` instead and let + // `kotlinRouteArgumentExpression` sort out positional vs `path =`/`value =` + // in JS — a query-level split is not available here, because tree-sitter-kotlin + // uses one `value_argument` node for both forms and 0.21.x has no negation to + // test the `=` token with. + // + // These deliberately match LITERAL arguments too (any `value_argument` does). + // The method-route loop drops those via `FOLDABLE_PATH_EXPRESSIONS` so a + // literal route is emitted once, by the literal patterns; the class-prefix + // collector instead KEEPS them and tests them for literalness, which is how a + // prefix that no literal pattern could resolve gets noticed at all. + const SPRING_CONST_CLASS_PREFIX_PATTERNS = compilePatterns({ + name: 'kotlin-spring-const-class-prefix', + language, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + (modifiers + (annotation + (constructor_invocation + (user_type (type_identifier) @ann (#eq? @ann "RequestMapping")) + (value_arguments (value_argument) @arg)))) + (type_identifier) @cls) @class + `, + }, + ], + } satisfies LanguagePatterns>); + + const SPRING_CONST_METHOD_ROUTE_PATTERNS = compilePatterns({ + name: 'kotlin-spring-const-method-route', + language, + patterns: [ + { + meta: {}, + query: ` + (function_declaration + (modifiers + (annotation + (constructor_invocation + (user_type (type_identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")) + (value_arguments (value_argument) @arg)))) + (simple_identifier) @method_name) @method + `, + }, + ], + } satisfies LanguagePatterns>); + + const SPRING_CONST_FEIGN_PATH_PATTERNS = compilePatterns({ + name: 'kotlin-spring-const-feign-path', + language, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + (modifiers + (annotation + (constructor_invocation + (user_type (type_identifier) @ann (#eq? @ann "FeignClient")) + (value_arguments (value_argument) @arg))))) @class + `, + }, + ], + } satisfies LanguagePatterns>); + + /** + * Ids of classes whose `@RequestMapping` prefix cannot be resolved to any + * literal, so no route under them can be published at a path the application + * actually serves. + * + * The predicate is INVERTED rather than an allow-list of non-literal node + * types: a class is marked unless its `path`/`value` argument is provably + * literal (recursing into `[…]` and `arrayOf(…)` elements, and refusing an + * interpolated `string_literal`). An allow-list has to enumerate every + * non-literal spelling and silently passes the ones it forgot — + * `[ApiPaths.BASE]`, `arrayOf(ApiPaths.BASE)`, `buildPath()`, + * `if (…) "/a" else "/b"` — each of which then publishes its methods at their + * UNPREFIXED path, a route the application does not serve. `java.ts` gates on + * the ABSENCE of a literal (`if (!valueNode)`) for the same reason. + * + * `resolvedPrefixes` is the literal prefix map built by the pass ABOVE, and a + * class holding an entry there is deliberately NOT marked: Kotlin's vararg + * spelling `@RequestMapping("/lit", ApiPaths.BASE)` leaves a resolvable `/lit` + * behind, and suppressing it would drop a route that IS derivable — trading a + * wrong route for a missing one, which is not the bargain this suppression + * exists to make. The prefix set is then partial (the constant arm is absent) + * exactly as it was before constant folding existed. + * + * The prefix is never folded here: it also feeds the cross-file + * interface-inheritance pass, which has no repo context, so folding it in + * `scan` alone would make the two views disagree. Same rule `java.ts` applies + * (`typesWithUnfoldablePrefix`); folding class prefixes cross-file is a + * follow-up on both sides. Used by BOTH `scan` and the inheritance-view + * collector — with the prefix map each has already built — so the two cannot + * drift apart. + */ + const collectUnfoldablePrefixClassIds = ( + tree: Parser.Tree, + resolvedPrefixes: ReadonlyMap, + ): Set => { + const ids = new Set(); + for (const match of runCompiledPatterns(SPRING_CONST_CLASS_PREFIX_PATTERNS, tree)) { + const argNode = match.captures.arg; + const classNode = match.captures.class; + if (!argNode || !classNode) continue; + if ((resolvedPrefixes.get(classNode.id) ?? []).length > 0) continue; + const expr = kotlinRouteArgumentExpression(argNode); + if (!expr || classifyPathArgument(expr) !== 'unresolvable') continue; + ids.add(classNode.id); + } + return ids; + }; + + /** + * Ids of `@FeignClient` interfaces whose `path` argument is present but not + * resolvable to a literal. + * + * `collectUnfoldablePrefixClassIds` cannot see these: it matches + * `@RequestMapping` only, so `@FeignClient(path = ApiPaths.BASE)` fell through + * to the `['']` prefix fallback and published the consumer at its unprefixed + * path — a call the service never makes. Kept as its own set rather than + * merged into the `@RequestMapping` one because `path` OUTRANKS + * `@RequestMapping` on a Feign client: an unresolvable `path` is fatal + * whatever the `@RequestMapping` says, and a resolvable `path` rescues a route + * whose `@RequestMapping` is a constant. The consumer lanes therefore consult + * the two in that same "path wins" order. + */ + const collectFeignUnfoldablePathClassIds = (tree: Parser.Tree): Set => { + const ids = new Set(); + for (const match of runCompiledPatterns(SPRING_CONST_FEIGN_PATH_PATTERNS, tree)) { + const argNode = match.captures.arg; + const classNode = match.captures.class; + if (!argNode || !classNode) continue; + const expr = kotlinFeignPathArgumentExpression(argNode); + if (!expr || classifyPathArgument(expr) !== 'unresolvable') continue; + ids.add(classNode.id); + } + return ids; + }; + // ─── Consumer: Spring RestTemplate ──────────────────────────────────── // Kotlin call-site shape mirrors the Java plugin's // `REST_TEMPLATE_PATTERNS`, but goes through tree-sitter-kotlin's @@ -875,11 +1225,24 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { const prefixNode = match.captures.prefix; const classNode = match.captures.class; if (!prefixNode || !classNode) continue; + // An INTERPOLATED literal (`"${ApiPaths.BASE}"`) is not a path — unquoting + // its raw text would carry the source spelling into the shared type view + // as a served prefix. Refusing it here is also what lets the unfoldable + // analysis below mark such a class (it skips classes with a resolved + // prefix), so the two stay one decision rather than two. + if (!isPlainStringLiteral(prefixNode)) continue; const prefix = unquoteLiteral(prefixNode.text); if (prefix !== null) pushPrefix(prefixByClassId, classNode.id, prefix); } // Method @(Get|...)Mapping routes keyed by the function_declaration node id. + // + // Only LITERAL paths land here. A constant-valued path is folded in `scan` + // against the repo constant map, which this inheritance-view collector has + // no access to; publishing it as an empty path would put `POST /`-shaped + // noise into the shared type view, so it is left out — the same skip floor + // `java.ts`'s `collectSpringTypes` keeps. const routesByMethodId = new Map>(); + const unfoldablePrefixClassIds = collectUnfoldablePrefixClassIds(tree, prefixByClassId); for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const pathNode = match.captures.path; @@ -889,6 +1252,10 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { if (!httpMethod) continue; const rawPath = unquoteLiteral(pathNode.text); if (rawPath === null) continue; + // A constant class prefix leaves no single prefix string for the + // inheritance view to carry, so this route would be published unprefixed. + const owner = findEnclosingClass(methodNode); + if (owner && unfoldablePrefixClassIds.has(owner.id)) continue; const arr = routesByMethodId.get(methodNode.id) ?? []; arr.push({ method: httpMethod, path: rawPath }); routesByMethodId.set(methodNode.id, arr); @@ -931,8 +1298,92 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { return { name: 'kotlin-http', language, - scan(tree) { + prepareRepo(args) { + // Build the repo-wide Kotlin string-constant map and import index once per + // extract() run. The orchestrator hands over a bare Parser with no language + // bound; bind Kotlin explicitly or `parseSource` spins to its whole time + // budget on every file. + try { + args.parser.setLanguage(language); + } catch { + // A parser that rejects binding cannot produce a constant map; the + // per-file try/catch below then skips everything harmlessly. + } + const constants = new Map(); + for (const rel of args.files) { + if (!rel.endsWith('.kt') && !rel.endsWith('.kts')) continue; + try { + const src = args.readFile(rel); + // Cheap content gate: only constant-DEFINITION candidates are parsed + // here. Import-only files (every controller) are deliberately NOT + // parsed in this pass — `scan` extracts the importing file's own + // import table from the tree it already holds, on demand, for the + // rare file that actually references a constant. A gate that also + // matched `import …` would parse the entire repository here. + if (!src || !isKotlinConstantFile(src)) continue; + const tree = args.parseSource(args.parser, src); + if (!tree) continue; + const mc = extractKotlinModuleConstants(tree); + if ( + mc.literals.size > 0 || + mc.exprs.size > 0 || + mc.imports.size > 0 || + (mc.wildcardImports?.length ?? 0) > 0 || + unfoldableDeclarationsOf(mc).size > 0 + ) { + // POSIX key (see `normalizeRel`); `readFile` above got the raw `rel`. + constants.set(normalizeRel(rel), mc); + } + } catch { + // Per-file resilience: one unreadable/oversized/ill-formed file must + // not forfeit the whole repo's constant map. + continue; + } + } + return { constants, index: buildKotlinConstantIndex(constants) }; + }, + scan(tree, repoContext, fileRel) { const out: HttpDetection[] = []; + const kotlinCtx = repoContext as + | { constants: RepoConstants; index: KotlinConstantIndex } + | undefined; + + // Read side of the POSIX keying (see `normalizeRel`): the map `prepareRepo` + // built is keyed by normalized path, so every lookup and every fold entry + // point below uses `fileKey`, never the raw `fileRel`. + const fileKey = fileRel === undefined ? undefined : normalizeRel(fileRel); + + // Lazy per-file constants/index view. `prepareRepo` only indexes constant- + // DEFINING files, so an importing controller is absent from that map. When + // a route references a constant, extract THIS file's import table from the + // tree `scan` already holds and overlay it. Import-only overlays reuse the + // prepared package projections; files whose routes are all literal never + // pay this cost. + let foldIndex: KotlinConstantIndex | undefined; + const getFoldIndex = (): KotlinConstantIndex | undefined => { + if (foldIndex !== undefined) return foldIndex; + foldIndex = kotlinCtx?.index; + if (!kotlinCtx || !fileKey) return foldIndex; + if (kotlinCtx.constants.has(fileKey)) return foldIndex; + try { + const mc = extractKotlinModuleConstants(tree); + // Same admission test the pre-pass applies above. Keeping the complete + // test here also makes this overlay correct if a future gate safely + // excludes another declaration shape. + if ( + mc.literals.size > 0 || + mc.exprs.size > 0 || + mc.imports.size > 0 || + (mc.wildcardImports?.length ?? 0) > 0 || + unfoldableDeclarationsOf(mc).size > 0 + ) { + foldIndex = overlayKotlinConstantIndex(kotlinCtx.index, fileKey, mc); + } + } catch { + // fold falls back to the repo-wide map (imports stay unresolved) + } + return foldIndex; + }; // ─── Class prefixes ───────────────────────────────────────────── const prefixByClassId = new Map(); @@ -940,10 +1391,17 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { const prefixNode = match.captures.prefix; const classNode = match.captures.class; if (!prefixNode || !classNode) continue; + // An INTERPOLATED literal (`"${ApiPaths.BASE}"`) is not a path — see + // `isPlainStringLiteral`. Refusing it here also lets the unfoldable + // analysis below mark such a class, since that skips classes whose + // prefix already resolved. + if (!isPlainStringLiteral(prefixNode)) continue; const prefix = unquoteLiteral(prefixNode.text); if (prefix !== null) pushPrefix(prefixByClassId, classNode.id, prefix); } + const classesWithUnfoldablePrefix = collectUnfoldablePrefixClassIds(tree, prefixByClassId); + // ─── OpenFeign client interfaces + HTTP Interface type prefixes ── // In tree-sitter-kotlin an `interface` is a `class_declaration`, so a // `@FeignClient` interface's @(Get|...)Mapping methods would otherwise be @@ -956,11 +1414,12 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { if (!classNode) continue; feignClassIds.add(classNode.id); const prefixNode = match.captures.prefix; - if (prefixNode) { + if (prefixNode && isPlainStringLiteral(prefixNode)) { const prefix = unquoteLiteral(prefixNode.text); if (prefix !== null) pushPrefix(feignPrefixByClassId, classNode.id, prefix); } } + const feignClassesWithUnfoldablePath = collectFeignUnfoldablePathClassIds(tree); const httpExchangePrefixByClassId = new Map(); for (const match of runCompiledPatterns(SPRING_HTTP_EXCHANGE_CLASS_PATTERNS, tree)) { const classNode = match.captures.class; @@ -971,24 +1430,91 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { } // ─── Method routes (Spring providers) + OpenFeign consumers ───── + // Literal and constant-valued paths are normalized into one candidate list + // so both reach the same Feign/interface/prefix classification below. + const methodRoutes: Array<{ + httpMethod: string; + rawPath: string; + nameNode: Parser.SyntaxNode | undefined; + methodNode: Parser.SyntaxNode; + }> = []; for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { const annNode = match.captures.ann; const pathNode = match.captures.path; - const nameNode = match.captures.method_name; const methodNode = match.captures.method; if (!annNode || !pathNode || !methodNode) continue; const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; if (!httpMethod) continue; const rawPath = unquoteLiteral(pathNode.text); if (rawPath === null) continue; + methodRoutes.push({ + httpMethod, + rawPath, + nameNode: match.captures.method_name, + methodNode, + }); + } + for (const match of runCompiledPatterns(SPRING_CONST_METHOD_ROUTE_PATTERNS, tree)) { + const annNode = match.captures.ann; + const argNode = match.captures.arg; + const methodNode = match.captures.method; + if (!annNode || !argNode || !methodNode) continue; + const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; + if (!httpMethod) continue; + const expr = kotlinRouteArgumentExpression(argNode); + if (!expr || !FOLDABLE_PATH_EXPRESSIONS.has(expr.type)) continue; + // No repo context (context-less fallback scanning) means no constant map + // and therefore no honest answer — skip rather than guess a path. + if (!fileKey) continue; + const index = getFoldIndex(); + if (!index) continue; + const operands = parseKotlinConstOperands(expr); + if (operands === null) continue; + // A bare reference means whatever the ENCLOSING types bind it to before + // it means anything at file level — Kotlin's rule for a companion + // member, which is in scope unqualified only inside its own class body. + const rawPath = foldKotlinOperands( + fileKey, + operands, + index.repo, + kotlinEnclosingTypeNames(methodNode), + index, + ); + if (rawPath === null) continue; + methodRoutes.push({ + httpMethod, + rawPath, + nameNode: match.captures.method_name, + methodNode, + }); + } + + for (const { httpMethod, rawPath, nameNode, methodNode } of methodRoutes) { const enclosingClass = findEnclosingClass(methodNode); // A @(Get|...)Mapping inside a @FeignClient interface is an OpenFeign // consumer (a remote call), not a route this service serves. if (enclosingClass && feignClassIds.has(enclosingClass.id)) { + // Whichever prefix GOVERNS must be resolvable, or the remote URL is + // unknowable and an unprefixed consumer would be a call this service + // never makes. Checked in the same "path wins" order the fallback + // below resolves in, so an unresolvable `@RequestMapping` does not + // suppress a client whose literal `@FeignClient(path)` outranks it, + // and an unresolvable `path` is fatal even when `@RequestMapping` is + // a literal. + // + // This reaches a Feign INTERFACE at all because tree-sitter-kotlin + // models `interface` as a `class_declaration`, and it should: Spring + // Cloud prepends the governing prefix to every method of the client. + // Java diverges only by accident of its grammar — `findEnclosingClass` + // skips `interface_declaration`, so `java.ts` still emits such a + // consumer at its unprefixed path. Aligning Java is a change to Java's + // behavior and belongs in its own follow-up, not in the Kotlin binding. + if (feignClassesWithUnfoldablePath.has(enclosingClass.id)) continue; + const feignPrefixes = feignPrefixByClassId.get(enclosingClass.id); + if (!feignPrefixes && classesWithUnfoldablePrefix.has(enclosingClass.id)) continue; // @FeignClient(path) wins over @RequestMapping; a multi-element prefix // yields one consumer per (prefix × this route). - const prefixes = feignPrefixByClassId.get(enclosingClass.id) ?? - prefixByClassId.get(enclosingClass.id) ?? ['']; + const prefixes = feignPrefixes ?? prefixByClassId.get(enclosingClass.id) ?? ['']; for (const prefix of prefixes) { out.push({ role: 'consumer', @@ -1002,6 +1528,10 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { } continue; } + // An unresolvable class prefix leaves no path this service serves, so + // every route under such a class is dropped rather than emitted at a + // wrong (unprefixed) one — the rule `java.ts` applies for Java. + if (enclosingClass && classesWithUnfoldablePrefix.has(enclosingClass.id)) continue; // A @(Get|...)Mapping on a (non-Feign) interface declares a route // *contract*, not a route this service serves — the implementing // @RestController is the provider, emitted via scanProject's interface @@ -1171,13 +1701,21 @@ function buildKotlinPlugin(language: unknown): HttpLanguagePlugin { if (!parsed) continue; const enclosingClass = findEnclosingClass(methodNode); if (!enclosingClass || !isKotlinInterface(enclosingClass)) continue; + // The same governing-prefix resolvability guard the @(Get|...)Mapping-in-Feign + // lane applies, in the same "path wins" order — this loop resolves through + // the identical fallback chain, so an unresolvable governing prefix leaves + // the remote URL just as unknowable here. Without it a single interface + // could suppress its @(Get|...)Mapping routes and publish its @RequestLine + // routes under the very same unresolvable prefix. + if (feignClassesWithUnfoldablePath.has(enclosingClass.id)) continue; + const feignPrefixes = feignPrefixByClassId.get(enclosingClass.id); + if (!feignPrefixes && classesWithUnfoldablePrefix.has(enclosingClass.id)) continue; // Mirror java.ts (which pre-merges the @RequestMapping fallback into // feignPrefixByInterfaceId, "path wins"): @FeignClient(path) wins, else // the interface's class-level @RequestMapping prefix, else none. Without // the prefixByClassId fallback Kotlin dropped the class prefix that Java // applies — the same fallback chain the @GetMapping-in-Feign path uses above. - const prefixes = feignPrefixByClassId.get(enclosingClass.id) ?? - prefixByClassId.get(enclosingClass.id) ?? ['']; + const prefixes = feignPrefixes ?? prefixByClassId.get(enclosingClass.id) ?? ['']; for (const prefix of prefixes) { out.push({ role: 'consumer', diff --git a/gitnexus/src/core/group/extractors/http-patterns/node.ts b/gitnexus/src/core/group/extractors/http-patterns/node.ts index edd0921f1..7a198f595 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/node.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/node.ts @@ -9,15 +9,28 @@ import { type LanguagePatterns, type PatternSpec, } from '../tree-sitter-scanner.js'; -import type { HttpDetection, HttpLanguagePlugin } from './types.js'; +import type { HttpDetection, HttpLanguagePlugin, RepoContext } from './types.js'; +import { MAX_FOLD_LENGTH } from '../../../ingestion/route-extractors/constant-resolver.js'; import { DATA_ROUTE_TABLE_SOURCE, scanDataRouteTables, } from '../../../ingestion/route-extractors/data-route-table.js'; +import { extractNestRoutes } from '../../../ingestion/route-extractors/nest.js'; +import { normalizeExtractedRoutePath } from '../../../ingestion/route-extractors/route-path.js'; +import { + buildJsRepoFacts, + extractJsModuleFacts, + isAxiosNamespace, + isHttpClientRef, + resolveJsPathExpression, + type JsModuleFacts, + type JsRepoFacts, +} from '../../../ingestion/route-extractors/js-const-resolver.js'; /** * Node.js / TypeScript HTTP plugin family. Handles: - * - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods + * - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods, + * delegated wholesale to the indexer's `extractNestRoutes` * - Express `router.get(...)` / `app.post(...)` providers * - `fetch(url)` / `fetch(url, { method: 'POST' })` consumers * - `axios.get(url)` / `axios.delete(url)` consumers @@ -32,34 +45,8 @@ import { * same `scan` function but bind to different grammars. */ -// ─── Provider: NestJS — class-level @Controller('prefix') ──────────── -// In tree-sitter-typescript decorators are NOT children of -// class_declaration / method_definition — they're siblings in the -// surrounding class_body / program node. We therefore match the -// decorator standalone and walk to its related class/method in JS. -const NEST_CONTROLLER_SPEC: PatternSpec> = { - meta: {}, - query: ` - (decorator - (call_expression - function: (identifier) @dec (#eq? @dec "Controller") - arguments: (arguments . [(string) (template_string)] @prefix))) @ctrl_decorator - `, -}; - -// ─── Provider: NestJS — method-level @Get/@Post/... decorators ─────── -// Matches either `@Get('path')` or `@Get()`. The `@path` capture is -// optional — when the first argument isn't a string, the plugin falls -// back to '/' for the method-level path. -const NEST_METHOD_SPEC: PatternSpec> = { - meta: {}, - query: ` - (decorator - (call_expression - function: (identifier) @dec (#match? @dec "^(Get|Post|Put|Delete|Patch)$") - arguments: (arguments) @args)) @method_decorator - `, -}; +// NestJS providers are not queried here at all — see the `extractNestRoutes` +// call in `scanBundle`. // ─── Provider: Express — router.get/app.post/... ───────────────────── const EXPRESS_SPEC: PatternSpec> = { @@ -98,15 +85,28 @@ const FETCH_WITH_OPTIONS_SPEC: PatternSpec> = { `, }; -// ─── Consumer: axios.get/post/... ──────────────────────────────────── -const AXIOS_SPEC: PatternSpec> = { +// ─── Consumer: .get/post/... ───────────────────────────── +// Widened from a literal `axios` receiver with a literal path. Application +// code satisfies neither: it calls through a configured instance +// (`const api = axios.create({ baseURL })`, imported at the call site under +// whatever name the app chose) and passes the path by reference from a shared +// route table (`api.get(API_ROUTE_PATH.LINKS)`). The query therefore matches +// ANY identifier receiver with an HTTP-verb method and ANY first argument; +// `scanBundle` admits a match only after PROVING the receiver is an axios +// instance and resolving the argument to a path. +// +// The proof gate is load-bearing, not belt-and-braces: EXPRESS_SPEC above +// matches `router.get('/x', handler)` / `app.post(...)` as PROVIDERS. A +// receiver admitted on spelling alone would re-emit every Express route in the +// repo as a consumer of itself, on both sides of every cross-repo pair. +const HTTP_CLIENT_SPEC: PatternSpec> = { meta: {}, query: ` (call_expression function: (member_expression - object: (identifier) @obj (#eq? @obj "axios") + object: (identifier) @obj property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$")) - arguments: (arguments . [(string) (template_string)] @path)) + arguments: (arguments . (_) @path)) `, }; @@ -153,12 +153,10 @@ const AXIOS_OBJECT_SPEC: PatternSpec> = { }; interface NodePatternBundle { - controller: CompiledPatterns>; - methodDecorator: CompiledPatterns>; express: CompiledPatterns>; fetchNoOptions: CompiledPatterns>; fetchWithOptions: CompiledPatterns>; - axios: CompiledPatterns>; + httpClient: CompiledPatterns>; jqueryShorthand: CompiledPatterns>; jqueryAjax: CompiledPatterns>; axiosObject: CompiledPatterns>; @@ -172,12 +170,10 @@ function compileBundle(language: unknown, name: string): NodePatternBundle { patterns: [spec], } satisfies LanguagePatterns>); return { - controller: mk(NEST_CONTROLLER_SPEC, 'nest-controller'), - methodDecorator: mk(NEST_METHOD_SPEC, 'nest-method-decorator'), express: mk(EXPRESS_SPEC, 'express'), fetchNoOptions: mk(FETCH_NO_OPTIONS_SPEC, 'fetch-no-options'), fetchWithOptions: mk(FETCH_WITH_OPTIONS_SPEC, 'fetch-with-options'), - axios: mk(AXIOS_SPEC, 'axios'), + httpClient: mk(HTTP_CLIENT_SPEC, 'http-client'), jqueryShorthand: mk(JQUERY_SHORTHAND_SPEC, 'jquery-shorthand'), jqueryAjax: mk(JQUERY_AJAX_SPEC, 'jquery-ajax'), axiosObject: mk(AXIOS_OBJECT_SPEC, 'axios-object'), @@ -188,33 +184,6 @@ const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-http'); const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-http'); const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-http'); -const NEST_DECORATOR_TO_HTTP: Record = { - Get: 'GET', - Post: 'POST', - Put: 'PUT', - Delete: 'DELETE', - Patch: 'PATCH', -}; - -/** - * Find the nearest enclosing class_declaration for a node, or null. - */ -function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let cur: Parser.SyntaxNode | null = node.parent; - while (cur) { - if (cur.type === 'class_declaration') return cur; - cur = cur.parent; - } - return null; -} - -function joinPath(prefix: string, sub: string): string { - const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, ''); - const cleanSub = sub.replace(/^\/+/, ''); - if (!cleanPrefix) return `/${cleanSub}`; - return `/${cleanPrefix}/${cleanSub}`; -} - /** * Walk `pair` children of an `object` literal and return the unquoted * string/template_string value for the first pair whose key matches one @@ -237,68 +206,6 @@ function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string return null; } -/** - * For a standalone `decorator` node (child of class_body / program), - * find the related `class_declaration` node that it decorates. In - * tree-sitter-typescript the decorator is placed before the class - * declaration as a sibling (when decorating a class) or inside the - * class_body before a method_definition (when decorating a method); - * we walk the parent chain until we find the enclosing class. - */ -function findDecoratedClass(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null { - const parent = decoratorNode.parent; - if (!parent) return null; - // Case 1: decorator is a sibling of the class_declaration at program / - // export_statement level. Walk forward through siblings until we find - // the class_declaration this decorator belongs to. - for (let i = 0; i < parent.namedChildCount; i++) { - const child = parent.namedChild(i); - if (child && child.id === decoratorNode.id) { - for (let j = i + 1; j < parent.namedChildCount; j++) { - const next = parent.namedChild(j); - if (!next) continue; - if (next.type === 'decorator') continue; // adjacent decorators stack - if (next.type === 'class_declaration') return next; - if (next.type === 'export_statement') { - // `export class Foo { ... }` wraps the declaration. - for (let k = 0; k < next.namedChildCount; k++) { - const inner = next.namedChild(k); - if (inner?.type === 'class_declaration') return inner; - } - } - break; - } - break; - } - } - // Case 2: decorator is inside a class_body (decorating a method) — - // walk up to the enclosing class_declaration. - return findEnclosingClass(decoratorNode); -} - -/** - * For a method-level decorator node (child of class_body before a - * method_definition), find the method_definition it decorates. - */ -function findDecoratedMethod(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null { - const parent = decoratorNode.parent; - if (!parent || parent.type !== 'class_body') return null; - for (let i = 0; i < parent.namedChildCount; i++) { - const child = parent.namedChild(i); - if (child && child.id === decoratorNode.id) { - for (let j = i + 1; j < parent.namedChildCount; j++) { - const next = parent.namedChild(j); - if (!next) continue; - if (next.type === 'decorator') continue; - if (next.type === 'method_definition') return next; - return null; - } - return null; - } - } - return null; -} - /** * Map each named import's LOCAL binding to its DECLARED export name and source * module, by walking the file's `import { x as y } from 'm'` statements. Lets @@ -309,12 +216,22 @@ function findDecoratedMethod(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNod */ function buildImportMap(tree: Parser.Tree): Map { const map = new Map(); - const walk = (node: Parser.SyntaxNode): void => { + // Both walks are explicit-stack, not recursive. They visit EVERY node of the + // file, so their depth is the source's nesting depth — and `scan` may not + // throw: a `RangeError` here escapes to `sync.ts`, which records the repo as + // an unexplained "missing repo" and drops every contract of every kind for + // it, silently. A file nesting template substitutions ~4 000 deep (well + // inside what tree-sitter will parse) was enough. + const stack: Parser.SyntaxNode[] = [tree.rootNode]; + while (stack.length > 0) { + const node = stack.pop() as Parser.SyntaxNode; if (node.type === 'import_statement') { const sourceNode = node.childForFieldName('source'); const module = sourceNode ? unquoteLiteral(sourceNode.text) : null; if (module !== null) { - const collect = (n: Parser.SyntaxNode): void => { + const inner: Parser.SyntaxNode[] = [node]; + while (inner.length > 0) { + const n = inner.pop() as Parser.SyntaxNode; if (n.type === 'import_specifier') { const nameNode = n.childForFieldName('name'); const aliasNode = n.childForFieldName('alias'); @@ -323,81 +240,266 @@ function buildImportMap(tree: Parser.Tree): Map(); + +/** + * Skip ceiling for the pre-pass, mirroring the analyzer's default + * `--max-file-size`. A minified bundle is megabytes on one line and defines no + * route table a human wrote; parsing it costs far more than it can return. + */ +const MAX_PREPASS_FILE_BYTES = 512 * 1024; + +/** Repo-relative path in the same POSIX form the fact map is keyed by. */ +function normalizeRel(rel: string): string { + return rel.replace(/\\/g, '/').replace(/^\.\//, ''); +} + +/** The grammar a JS/TS-family file should be parsed with, or null if not one. */ +function grammarForFile(rel: string): unknown | null { + const lower = rel.toLowerCase(); + if (lower.endsWith('.tsx')) return TypeScript.tsx; + if (/\.[cm]?ts$/.test(lower)) return TypeScript.typescript; + if (/\.[cm]?jsx?$/.test(lower)) return JavaScript; + return null; +} + +function buildNodeRepoContext(args: { + files: string[]; + readFile: (rel: string) => string | null; + parseSource: (parser: Parser, src: string) => Parser.Tree | null; +}): NodeRepoContext { + const cached = REPO_CONTEXT_BY_FILE_LIST.get(args.files); + if (cached) return cached; + + const byFile = new Map(); + const parsers = new Map(); + const parserFor = (language: unknown): Parser => { + let parser = parsers.get(language); + if (!parser) { + parser = new Parser(); + parser.setLanguage(language as Parameters[0]); + parsers.set(language, parser); + } + return parser; + }; + + // Cost gate, in the spirit of the sibling `python.ts` pre-pass: every fact + // this map holds exists to prove a receiver is an axios instance or to fold a + // path for one. A repo where the string `axios` appears nowhere can prove no + // receiver, so every parse below is dead work — and parsing is the expensive + // half (measured 4.36 s / +258 MB RSS over 827 TypeScript files, on top of + // the parse `getScanInput` already does). + // Only the file's identity is carried between the passes, never its text: a + // large monorepo's whole source tree held in one array at once is the shape + // that produced the analyzer's scale problems, and the second read is cheap + // beside the parse it gates. + const eligible: Array<{ rel: string; language: unknown }> = []; + let sawAxios = false; + for (const rel of args.files) { + const language = grammarForFile(rel); + if (language === null) continue; + const content = args.readFile(rel); + // `MAX_PREPASS_FILE_BYTES` is a BYTE ceiling; `String.length` counts UTF-16 + // code units, which under-counts every multi-byte source. + if (content === null || Buffer.byteLength(content, 'utf8') > MAX_PREPASS_FILE_BYTES) continue; + if (!sawAxios && content.includes('axios')) sawAxios = true; + eligible.push({ rel, language }); + } + + if (sawAxios) { + for (const { rel, language } of eligible) { + try { + const content = args.readFile(rel); + if (content === null) continue; + // `parseSource` belongs INSIDE the guard: `safe-parse.ts` throws + // `ParseTimeoutError` and makes catching it a per-caller obligation, and + // `prepareRepo` is contractually non-throwing. One escape here left the + // fact map unwritten for the WHOLE repo — and, because the orchestrator + // caches per plugin NAME, made all three JS/TS plugins re-walk it and + // fail the same way before falling back to literal-only scanning. + const tree = args.parseSource(parserFor(language), content); + if (!tree) continue; + byFile.set(normalizeRel(rel), extractJsModuleFacts(tree)); + } catch { + // One malformed file must never abort the pre-pass — it simply stays + // unresolved, exactly as it is without this pass at all. + } + } + } + + const ctx: NodeRepoContext = { facts: buildJsRepoFacts(byFile) }; + REPO_CONTEXT_BY_FILE_LIST.set(args.files, ctx); + return ctx; +} + +/** The repo facts to resolve against, or null when there was no pre-pass. */ +function resolveFactsFor( + repoContext: RepoContext | undefined, + fileRel: string | undefined, +): JsRepoFacts | null { + const ctx = repoContext as NodeRepoContext | undefined; + if (!ctx || fileRel === undefined) return null; + return ctx.facts; +} + +/** + * Whether a folded first argument is plausibly a URL path. + * + * The query now captures ANY first argument, and "it folded to a string" is not + * "it is a path" — `normalizeConsumerPath` is a canonicalizer, not a validator, + * and it happily turns non-paths into contracts that exact-match real provider + * routes: + * + * api.get(CONFIG.TIMEOUT) // "5000" -> http::GET::/{param} + * api.post(MSG.ERROR) // "Could not reach the …" -> http::POST::/could not reach the server + * + * `/{param}` matches every one-segment provider route in the group, and + * `matching.exclude_links_param_only_paths` defaults to `false`. A path whose + * leading term is an unresolved placeholder is refused for the same reason — + * nothing pins where it starts. (`resolveJsPathExpression` already refuses those + * it folded itself; this also covers the literal fallback below.) + */ +function looksLikeHttpPath(path: string): boolean { + if (path === '') return false; + if (/^https?:\/\//i.test(path)) return true; + // A `${…}` term is a runtime value that `normalizeConsumerPath` rewrites to + // `{param}`; its SOURCE text can be any expression (`${draft ? 'a' : 'b'}`, + // `${id ?? ''}`), so the checks below have to run against the normalized + // shape. Testing the raw source dropped every partially folded path whose + // unresolved term happened to contain a space. + const shape = path.replace(/\$\{[^}]+\}/g, '{param}'); + if (/\s/.test(shape)) return false; + if (shape.startsWith('{param}')) return false; + // An all-digit string is a path only when it is written as one. A leading + // slash is that evidence: `client.get('/123')` is a route whose segment the + // consumer normalizer reads as `{param}`, while a bare `"5000"` folded out of + // `CONFIG.TIMEOUT` is a timeout that would match every one-segment provider. + if (!shape.startsWith('/')) return !/^\d+$/.test(shape); + return true; +} + +/** + * The path a consumer call's first argument denotes. + * + * Prefers full resolution against the repo facts; falls back to the raw + * literal for a string/template node so a repo with no pre-pass (or an + * unresolvable reference) behaves exactly as it did before. + * + * `fileKey` is already `normalizeRel`-ed by the caller — see `scanBundle`. + * + * `legacyShape` marks the exact combination this pattern matched BEFORE it was + * widened: the literal receiver `axios` with a string or template-string first + * argument. That combination keeps its old output verbatim, so this PR adds + * detections without removing any — `axios.get(`${API_BASE}/users`)` still + * yields `/{param}/users`. Everything the widened query NEWLY admits (any other + * receiver, or any non-literal argument) has to clear the gates. + */ +function resolveConsumerPath( + pathNode: Parser.SyntaxNode, + facts: JsRepoFacts | null, + fileKey: string | undefined, + legacyShape: boolean, +): string | null { + if (facts && fileKey !== undefined) { + const resolved = resolveJsPathExpression(fileKey, pathNode, facts); + if (resolved !== null && looksLikeHttpPath(resolved)) return resolved; + } + // The fallback is deliberately gated on node TYPE: `unquoteLiteral` returns + // unrecognized input unchanged, so handing it a `member_expression` would + // yield the literal text `API_ROUTE_PATH.LINKS` as if it were a URL path. + if (pathNode.type !== 'string' && pathNode.type !== 'template_string') return null; + const literal = unquoteLiteral(pathNode.text); + // The fold bails past `MAX_FOLD_LENGTH`; the raw source it falls back to has + // no such bound and lands in `contractId` and `meta.path` all the same. + if (literal === null || literal.length > MAX_FOLD_LENGTH) return null; + return legacyShape || looksLikeHttpPath(literal) ? literal : null; +} + +function scanBundle( + bundle: NodePatternBundle, + tree: Parser.Tree, + repoContext?: RepoContext, + fileRel?: string, +): HttpDetection[] { const out: HttpDetection[] = []; + // Repo-wide constant / HTTP-client facts, when the orchestrator ran the + // `prepareRepo` pre-pass. Absent for a bare `scan(tree)` call, in which case + // every cross-file resolution below floors to the literal-only behavior. + const facts = resolveFactsFor(repoContext, fileRel); + // The fact map is keyed by `normalizeRel(rel)`. Normalizing at ONE place and + // using that value for every read keeps the two sides in step: the receiver + // gate used to read the raw `fileRel`, and `isHttpClientRef` cannot tell a key + // miss from "not a client", so any non-POSIX path (glob v13 has no + // `posix: true` and its walker joins with the platform separator; graph rows + // are a second unnormalized source) silently returned zero consumers. + const fileKey = fileRel === undefined ? undefined : normalizeRel(fileRel); // Local-binding → { declared export name, module } for the file's named // imports, so an express handler that is an imported (possibly aliased) // symbol resolves to the real definition rather than its local alias text. const importMap = buildImportMap(tree); - // NestJS: collect `@Controller('prefix')` class decorators, keyed by - // the `class_declaration` they decorate. - const prefixByClassId = new Map(); - for (const match of runCompiledPatterns(bundle.controller, tree)) { - const prefixNode = match.captures.prefix; - const decoratorNode = match.captures.ctrl_decorator; - if (!prefixNode || !decoratorNode) continue; - const prefix = unquoteLiteral(prefixNode.text); - if (prefix === null) continue; - const classNode = findDecoratedClass(decoratorNode); - if (!classNode) continue; - prefixByClassId.set(classNode.id, prefix); - } - - // NestJS: method-level @Get/@Post/... decorators. The decorator's - // arguments list may be empty (`@Get()`), a string (`@Get('path')`), - // or something else (which we skip). - for (const match of runCompiledPatterns(bundle.methodDecorator, tree)) { - const decNode = match.captures.dec; - const argsNode = match.captures.args; - const decoratorNode = match.captures.method_decorator; - if (!decNode || !argsNode || !decoratorNode) continue; - const httpMethod = NEST_DECORATOR_TO_HTTP[decNode.text]; - if (!httpMethod) continue; - const methodNode = findDecoratedMethod(decoratorNode); - if (!methodNode) continue; - const enclosingClass = findEnclosingClass(methodNode); - // Only emit NestJS detections when the class actually has a - // @Controller decorator — without it, the match is almost certainly - // something else (e.g. an unrelated library using similar names). - if (!enclosingClass || !prefixByClassId.has(enclosingClass.id)) continue; - const prefix = prefixByClassId.get(enclosingClass.id) ?? ''; - - let rawPath = '/'; - const firstArg = argsNode.namedChild(0); - if (firstArg && (firstArg.type === 'string' || firstArg.type === 'template_string')) { - const unquoted = unquoteLiteral(firstArg.text); - if (unquoted !== null) rawPath = unquoted; - } - - // Get the method name from the decorated method_definition. - const methodNameNode = methodNode.childForFieldName('name'); - const name = methodNameNode?.text ?? null; - + // NestJS: delegated to the indexer's extractor rather than re-queried here. + // Two independent readings of the same decorators is how the layers drift: + // the local scan saw only `class_declaration` (never `abstract class`), only + // five of the nine verbs, only a positional string `@Controller('x')`, and + // — worst — INVENTED `/` for a method path it could not read, so + // `@Get(ROUTES.SEARCH)` became a `GET /venues` contract that the graph, which + // correctly drops it, has no Route node for. "A missing route is a coverage + // limit; an invented one is a lie" (ARCHITECTURE.md). Calling the extractor + // makes that divergence structurally impossible, exactly as the + // `scanDataRouteTables` call below already does for static route tables. + // + // `filePath` rides only on the returned struct and never reaches the + // `HttpDetection`, so a bare `scan(tree)` with no `fileRel` passes '' rather + // than losing the routes. `lineOffset` is 0: the group scanner parses whole + // files, so `lineNumber` is already the absolute 1-based line this + // `HttpDetection.line` wants. + for (const route of extractNestRoutes(tree, fileRel ?? '', 0)) { out.push({ role: 'provider', framework: 'nest', - method: httpMethod, - path: joinPath(prefix, rawPath), - name, - line: methodNode.startPosition.row + 1, + method: route.httpMethod, + // The prefix travels separately at the ingestion layer, so the join is + // ours to do — with ingestion's own joiner, so the two layers cannot + // disagree about the URL either. + path: normalizeExtractedRoutePath(route.routePath, route.prefix ?? null), + name: route.handlerName ?? null, + line: route.lineNumber, confidence: 0.8, }); } @@ -471,22 +573,57 @@ function scanBundle(bundle: NodePatternBundle, tree: Parser.Tree): HttpDetection }); } - // Consumer: axios.(url) - for (const match of runCompiledPatterns(bundle.axios, tree)) { + // Consumer: .(url) — `axios` itself, or any receiver the + // repo pre-pass proves is an axios instance. + for (const match of runCompiledPatterns(bundle.httpClient, tree)) { const methodNode = match.captures.http_method; const pathNode = match.captures.path; - if (!methodNode || !pathNode) continue; - const path = unquoteLiteral(pathNode.text); - if (path === null) continue; - out.push({ - role: 'consumer', - framework: 'axios', - method: methodNode.text.toUpperCase(), - path, - name: null, - line: pathNode.startPosition.row + 1, - confidence: 0.7, - }); + const objNode = match.captures.obj; + if (!methodNode || !pathNode || !objNode) continue; + + // Receiver gate. `axios.get(...)` needs no proof; anything else must be + // traced to an `axios.create(...)` binding, or it is not ours to claim. + const receiver = objNode.text; + + // Cross-file resolution is the only work in this file that walks a + // repo-wide graph, and `HttpLanguagePlugin.scan` may not throw: a single + // hostile call site must cost its own detection, not the repo's whole + // contract set (`sync.ts` catches a throw here as an unexplained "missing + // repo", silently, for every contract type). + try { + // The receiver is admitted when it IS the axios module — the bare + // spelling this pattern trusted before it was widened, or a declared + // import/require of 'axios' under any name — or when it traces to an + // `axios.create(...)` instance. Nothing else. + const isModule = + facts === null || fileKey === undefined + ? receiver === 'axios' + : isAxiosNamespace(fileKey, receiver, facts); + if (!isModule) { + if (!facts || fileKey === undefined) continue; + if (!isHttpClientRef(fileKey, receiver, facts)) continue; + } + + const path = resolveConsumerPath( + pathNode, + facts, + fileKey, + isModule && (pathNode.type === 'string' || pathNode.type === 'template_string'), + ); + if (path === null) continue; + + out.push({ + role: 'consumer', + framework: 'axios', + method: methodNode.text.toUpperCase(), + path, + name: null, + line: pathNode.startPosition.row + 1, + confidence: 0.7, + }); + } catch { + // Unresolvable is the same outcome as unresolved — skip this call site. + } } // Consumer: jQuery shorthand $.get(url) / $.post(url, ...) @@ -574,17 +711,20 @@ function scanBundle(bundle: NodePatternBundle, tree: Parser.Tree): HttpDetection export const JAVASCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'javascript-http', language: JavaScript, - scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree), + prepareRepo: buildNodeRepoContext, + scan: (tree, repoContext, fileRel) => scanBundle(JAVASCRIPT_BUNDLE, tree, repoContext, fileRel), }; export const TYPESCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'typescript-http', language: TypeScript.typescript, - scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree), + prepareRepo: buildNodeRepoContext, + scan: (tree, repoContext, fileRel) => scanBundle(TYPESCRIPT_BUNDLE, tree, repoContext, fileRel), }; export const TSX_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'tsx-http', language: TypeScript.tsx, - scan: (tree) => scanBundle(TSX_BUNDLE, tree), + prepareRepo: buildNodeRepoContext, + scan: (tree, repoContext, fileRel) => scanBundle(TSX_BUNDLE, tree, repoContext, fileRel), }; diff --git a/gitnexus/src/core/group/extractors/http-patterns/php.ts b/gitnexus/src/core/group/extractors/http-patterns/php.ts index bf2eb7aba..65a1896c9 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/php.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/php.ts @@ -15,20 +15,36 @@ import type { HttpDetection, HttpLanguagePlugin } from './types.js'; * Providers: * - Laravel `Route::get/post/...` * - * Consumers (string-literal URLs only): + * Consumers (string-literal URLs only, unless noted): * - Laravel HTTP client: `Http::get/post/put/delete/patch($url)` * - Guzzle / generic object method: `$client->get/post/...($url)` * - `file_get_contents($url)` + * - `new Request($method, $host . $resourcePath)` — the openapi-generator-php + * / swagger-codegen client shape. `$resourcePath` is resolved via a + * single-scope backward constant fold (see `resolveLocalStringLiteral`), + * not a string literal at the call site itself. * * The pipeline already uses `PHP.php_only` for ingesting plain `.php` * files (see `core/tree-sitter/parser-loader.ts`), and we do the same * here so Laravel route files are parsed with the right grammar dialect. * - * Scope notes: consumer patterns match string literals only. URLs built - * via binary concatenation (`$base . '/path'`), `sprintf`, or config - * lookup (`config('services.foo.base').'/path'`) are intentionally left - * for a follow-up — they require constant-folding the surrounding - * scope to be meaningful. + * Scope notes: consumer patterns match string literals only, with one + * narrow exception (above). URLs built via `sprintf`, config lookup + * (`config('services.foo.base').'/path'`), or a variable resolved from + * outside its own function/method body are intentionally left for a + * follow-up — they require constant-folding beyond one local scope to + * be meaningful. + * + * That narrow exception (`resolveLocalStringLiteral`) is a temporary, + * single-scope fallback, not this language's entry into the shared + * cross-file constant-fold used by the other languages in this plugin + * (`constant-resolver.ts`, wired in via `java-const-resolver.ts` / + * `python-const-resolver.ts` / `js-const-resolver.ts`). PHP has no such + * binding yet — adding one is a real, separate project (this repo's PHP + * import resolution for `use`-statements is its own multi-file subsystem + * under `ingestion/import-resolvers/php.ts`, built for symbol/scope + * resolution, not constant extraction) and is intentionally out of scope + * here. Tracked as a follow-up, not silently punted. */ const LARAVEL_ROUTE_SPEC: PatternSpec> = { @@ -71,11 +87,31 @@ const FILE_GET_CONTENTS_SPEC: PatternSpec> = { `, }; +/** + * `new Request($method, $host . $resourcePath)` — the shape swagger-codegen / + * openapi-generator-php emit for every operation of a generated API client + * (Guzzle's `\GuzzleHttp\Psr7\Request`, or a bare `Request` behind a `use` + * import). Matches both `(name)` and `(qualified_name)` class references; + * `scan()` below filters to the last path segment being exactly `Request` + * and resolves the concatenated path argument (see `resolveLocalStringLiteral`). + */ +const GUZZLE_REQUEST_CTOR_SPEC: PatternSpec> = { + meta: {}, + query: ` + (object_creation_expression + [(name) (qualified_name)] @class + (arguments + . (argument (_) @methodArg) + . (argument (_) @pathArg))) + `, +}; + interface PhpPatternBundle { laravelRoute: CompiledPatterns>; httpFacade: CompiledPatterns>; guzzleMember: CompiledPatterns>; fileGetContents: CompiledPatterns>; + guzzleRequestCtor: CompiledPatterns>; } const mk = (spec: PatternSpec>, suffix: string) => @@ -90,6 +126,7 @@ const PHP_PATTERNS: PhpPatternBundle = { httpFacade: mk(HTTP_FACADE_SPEC, 'http-facade'), guzzleMember: mk(GUZZLE_MEMBER_SPEC, 'guzzle-member'), fileGetContents: mk(FILE_GET_CONTENTS_SPEC, 'file-get-contents'), + guzzleRequestCtor: mk(GUZZLE_REQUEST_CTOR_SPEC, 'guzzle-request-ctor'), }; /** @@ -129,6 +166,183 @@ function isHttpUrlLiteral(path: string): boolean { return path.startsWith('http://') || path.startsWith('https://'); } +/** + * Last identifier segment of a class-name reference: `(name)` returns its + * own text, `(qualified_name)` returns the text of its last child (the + * unqualified class name — `\GuzzleHttp\Psr7\Request` → `Request`). + */ +function lastNameSegment(node: import('tree-sitter').SyntaxNode): string { + if (node.type === 'qualified_name') { + const last = node.child(node.childCount - 1); + return last ? last.text : node.text; + } + return node.text; +} + +/** + * Return the variable at the LAST position of a `.`-concatenation + * expression, if (and only if) that position is a plain variable — + * generated clients build ` . `, so the path segment + * is the one closest to the end. + * + * No fallback to an earlier operand: if the rightmost position is anything + * other than a variable, a parenthesized sub-expression, or a nested `.` + * concatenation (a literal, a function call, ...), that position is a real + * value we simply can't resolve — falling back to an EARLIER operand would + * silently substitute a different value (e.g. the host) for the one that's + * actually there. `null` here is a miss, not a signal to keep looking. + */ +function lastConcatVariable( + node: import('tree-sitter').SyntaxNode, +): import('tree-sitter').SyntaxNode | null { + if (node.type === 'variable_name') return node; + if (node.type === 'parenthesized_expression') { + const inner = node.namedChild(0); + return inner ? lastConcatVariable(inner) : null; + } + if (node.type === 'binary_expression') { + const operator = node.childForFieldName('operator'); + if (!operator || operator.text !== '.') return null; // not concatenation + const right = node.childForFieldName('right'); + return right ? lastConcatVariable(right) : null; + } + return null; +} + +/** + * True if `node`'s subtree assigns to `$target` ANYWHERE inside it, at any + * depth (including inside nested functions — deliberately over-broad: a + * false positive here only costs a miss in the caller, never a wrong + * answer, so there's no need to be precise about scoping inside the probe + * itself). + */ +function containsAssignmentTo(node: import('tree-sitter').SyntaxNode, target: string): boolean { + if (node.type === 'assignment_expression') { + const lhs = node.childForFieldName('left'); + if (lhs && lhs.type === 'variable_name' && lhs.text === target) return true; + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child && containsAssignmentTo(child, target)) return true; + } + return false; +} + +/** + * True if an `anonymous_function` node's `use (...)` clause lists + * `$target`. PHP closures capture NOTHING automatically — only variables + * named in `use (...)` are visible inside — unlike arrow functions + * (`fn() => ...`), which auto-capture everything by value and have no + * `compound_statement` body of their own, so they're never seen as a + * `scope` by the walk below in the first place. + */ +function anonymousFunctionCaptures( + anonFn: import('tree-sitter').SyntaxNode, + target: string, +): boolean { + for (let i = 0; i < anonFn.namedChildCount; i++) { + const child = anonFn.namedChild(i); + if (!child || child.type !== 'anonymous_function_use_clause') continue; + for (let j = 0; j < child.namedChildCount; j++) { + const v = child.namedChild(j); + if (v && v.type === 'variable_name' && v.text === target) return true; + } + return false; // has a use(...) clause, but $target isn't in it + } + return false; // no use(...) clause at all — nothing is captured +} + +/** + * Best-effort, single-scope constant fold: given a `variable_name` node + * referenced inside a `new Request(...)` argument, walk BACKWARD through + * the preceding statements of its immediately enclosing function/method + * body (or file scope, for top-level script code) looking for the nearest + * `$var = '';` assignment. + * + * "Enclosing body" is resolved level by level, not just the nearest + * `compound_statement` — a call site nested in `if`/`foreach`/`try` inside + * that function is still within the same function/method body, and a + * preceding assignment above that conditional must still be found. Each + * level searches only its own preceding siblings, then the search + * continues from the enclosing block itself one level up, UNLESS that + * block IS the body of a function/method/closure: + * - a regular `function_definition` or `method_declaration` boundary + * always stops the search — PHP gives a function or method no access + * to anything outside its own body (no automatic capture, no implicit + * global), so widening past one into the containing class or + * file-level scope would resolve a variable the call site could never + * actually see at runtime; + * - an `anonymous_function` boundary stops UNLESS `$target` is + * explicitly captured via `use (...)` — closures capture nothing + * automatically either. + * It stops at `program` regardless, for the case where the call site was + * at file/script scope all along. + * + * A preceding sibling that ISN'T a plain assignment but might reassign the + * target somewhere inside itself (an `if`/`foreach`/`try`/`switch`, ...) + * stops the search rather than being skipped over: whether that branch ran + * is unknown, so an older literal further back can't be trusted either. + * + * Deliberately conservative and bounded — no interprocedural resolution, + * no constant/property lookups. A miss just means the endpoint stays + * undetected, never a wrong one: this is exactly the class of case the + * module docblock flags as in-scope only for one local scope. + */ +function resolveLocalStringLiteral(varNode: import('tree-sitter').SyntaxNode): string | null { + const target = varNode.text; // includes the `$` sigil, e.g. "$resourcePath" + let cursor: import('tree-sitter').SyntaxNode = varNode; + + for (;;) { + let scope: import('tree-sitter').SyntaxNode | null = cursor.parent; + while (scope && scope.type !== 'compound_statement' && scope.type !== 'program') { + scope = scope.parent; + } + if (!scope) return null; + + let stmt: import('tree-sitter').SyntaxNode | null = cursor; + while (stmt && stmt.parent !== scope) stmt = stmt.parent; + if (!stmt) return null; + + let sibling = stmt.previousNamedSibling; + while (sibling) { + if (sibling.type === 'expression_statement') { + const inner = sibling.namedChild(0); + if (inner && inner.type === 'assignment_expression') { + const lhs = inner.childForFieldName('left'); + if (lhs && lhs.type === 'variable_name' && lhs.text === target) { + // The NEAREST assignment to this variable wins, full stop — an + // older literal further back is shadowed by this one even when + // this one isn't itself a resolvable string (`$v = f();`). + const rhs = inner.childForFieldName('right'); + return rhs && rhs.type === 'string' ? phpStringText(rhs) : null; + } + } + } else if (containsAssignmentTo(sibling, target)) { + return null; // reassigned somewhere inside a conditional/loop/try + } + sibling = sibling.previousNamedSibling; + } + + if (scope.type === 'program') return null; + const enclosing = scope.parent; + if (enclosing && enclosing.type === 'anonymous_function') { + // Closures capture nothing automatically — only what's use()'d. + if (!anonymousFunctionCaptures(enclosing, target)) return null; + } else if ( + enclosing && + (enclosing.type === 'function_definition' || enclosing.type === 'method_declaration') + ) { + // A regular function or method boundary — NOT a closure. PHP gives + // these no access to anything outside their own body (no automatic + // capture, no implicit global): widening past one into the + // containing class body or file-level scope would resolve a + // variable the call site could never actually see at runtime. + return null; + } + cursor = scope; // one block up: search resumes from this block's own position + } +} + export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'php-http', language: PHP.php_only, @@ -136,12 +350,16 @@ export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { // ingestion, so the graph is authoritative for PHP providers (#2138 Part 2). routeCoverage: 'complete', // Consumer signals scan() can detect: Laravel `Http::`, Guzzle client - // `->get/post/.../request(...)`, and `file_get_contents` of an HTTP URL. A - // provider-covered file with any of these must still be parsed (ingestion - // emits no FETCHES for PHP). Conservative — the `->verb(` shape over-matches - // ordinary method calls, which only costs a parse, never data. + // `->get/post/.../request(...)`, `file_get_contents` of an HTTP URL, and a + // generated-client `new ...Request(...)` constructor call. A provider-covered + // file with any of these must still be parsed (ingestion emits no FETCHES for + // PHP). Conservative — the `->verb(`/`new ...Request(` shapes over-match + // ordinary method calls and unrelated constructors, which only costs a + // parse, never data. hasConsumerSignals(content) { - return /Http::|file_get_contents|->\s*(get|post|put|delete|patch|request)\s*\(/i.test(content); + return /Http::|file_get_contents|->\s*(get|post|put|delete|patch|request)\s*\(|new\s+[\\\w]*Request\s*\(/i.test( + content, + ); }, scan(tree) { const out: HttpDetection[] = []; @@ -222,6 +440,62 @@ export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { }); } + for (const match of runCompiledPatterns(PHP_PATTERNS.guzzleRequestCtor, tree)) { + const classNode = match.captures.class; + const methodArg = match.captures.methodArg; + const pathArg = match.captures.pathArg; + if (!classNode || !methodArg || !pathArg) continue; + // PHP class names are case-insensitive at the language level, and + // `hasConsumerSignals` above matches case-insensitively (`/i`) for + // the same reason — this comparison must agree with it, or a valid + // `new request(...)` / `new \NS\REQUEST(...)` call would be waved + // through the parse-skip gate as a signal and then silently dropped + // here. + if (lastNameSegment(classNode).toLowerCase() !== 'request') continue; + + // Path: a direct string literal, or the last variable in a + // concatenation chain (see `lastConcatVariable`) resolved to a + // locally-assigned literal. + let path: string | null = null; + if (pathArg.type === 'string') { + path = phpStringText(pathArg); + } else { + const lastVar = lastConcatVariable(pathArg); + path = lastVar ? resolveLocalStringLiteral(lastVar) : null; + } + if (path === null || !isHttpClientPath(path)) continue; + + // The HTTP verb is a literal, a local variable resolved the same way + // as the path (see `resolveLocalStringLiteral` above), or — commonly + // in generated clients — a parameter of the enclosing builder method + // fixed by ITS caller, not by this call site. That last case needs + // the same interprocedural reach the module docblock rules out, so it + // falls through to a wildcard verb, matching this project's own + // convention for a contract whose verb isn't pinned (see manifest + // links, `http::*::`). + let method: string | null = null; + if (methodArg.type === 'string') { + method = phpStringText(methodArg); + } else if (methodArg.type === 'variable_name') { + method = resolveLocalStringLiteral(methodArg); + } + + out.push({ + role: 'consumer', + framework: 'guzzle-request-ctor', + method: method ? method.toUpperCase() : '*', + path, + name: null, + // Line of the path ARGUMENT, not the `new Request(` call — same + // choice the other three consumer patterns in this file make, but + // this is the one pattern where the two routinely differ (generated + // clients wrap the call across multiple lines). Line-span + // containment still resolves to the right symbol either way. + line: pathArg.startPosition.row + 1, + confidence: 0.6, + }); + } + return out; }, }; diff --git a/gitnexus/src/core/group/extractors/http-patterns/python.ts b/gitnexus/src/core/group/extractors/http-patterns/python.ts index 7d8bd99af..7a0fa2f7f 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/python.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/python.ts @@ -1137,13 +1137,17 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'python-http', language: Python, // routeCoverage intentionally LEFT at the default 'partial' (#2138 Part 2). - // It would be a no-op even if set to 'complete': FastAPI decorator routes set - // no handlerName (generic worker path) and Django sets methodName: null, so no - // Python file ever resolves a handlerSymbolId and none would be parse-skipped. - // Declaring 'complete' now is only a latent trap for the moment a follow-up - // gives FastAPI routes a handlerName. `hasConsumerSignals` is kept (and is a - // true superset of scan()'s consumer shapes) so the precondition already holds - // when Python is later flipped to 'complete'. + // 'complete' is now an active data-loss risk rather than a no-op: FastAPI and + // Flask decorator routes do carry a handlerName (Python's + // `decoratorRouteHandlerName` hook reads the `decorated_definition`), so their + // files can resolve every handlerSymbolId and become parse-skip candidates. + // The flag asserts more than that — it asserts ingestion emits a Route node + // for EVERY provider route this scan() finds, and it does not: Flask's + // imperative `add_url_rule('/p', view_func=handler)` registration below has no + // ingestion counterpart, so skipping a file that mixes it with resolved + // decorator routes would drop those providers. `hasConsumerSignals` is kept + // (and is a true superset of scan()'s consumer shapes) so the consumer half of + // the precondition already holds once provider parity is closed. // Consumer signals scan() can detect: `requests.`/`requests.request`, // `httpx` (sync/async client), the `uri=`/`url=` keyword/variable wrapper // calls, plus aiohttp/urllib. Conservative — over-matching only costs a parse. diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index 6aa4c8476..5def6a9a9 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -621,6 +621,7 @@ export class HttpRouteExtractor implements ContractExtractor { dbExecutor, getDetections, resolveDetectionSymbol, + loadFileSymbols, coveredFiles, ) : []; @@ -690,6 +691,7 @@ export class HttpRouteExtractor implements ContractExtractor { db: CypherExecutor, getDetections: (rel: string) => Promise, resolveSymbol: (filePath: string, d: HttpDetection) => Promise, + loadFileSymbols: (filePath: string) => Promise[]>, coveredFiles?: Set, ): Promise { const out: ExtractedContract[] = []; @@ -749,15 +751,11 @@ export class HttpRouteExtractor implements ContractExtractor { if (!method) method = 'GET'; symbolUid = handlerSymbolId; if (filePath) { - try { - const syms = await db(CONTAINING_QUERY, { filePath }); - const hit = syms.find((s) => String(s.uid ?? s[0]) === handlerSymbolId); - if (hit) { - symbolName = String(hit.name ?? hit[1]) || symbolName; - symPath = String(hit.filePath ?? hit[2]) || filePath; - } - } catch { - /* keep the authoritative uid + basename fallback */ + const syms = await loadFileSymbols(filePath); + const hit = syms.find((s) => String(s.uid ?? s[0]) === handlerSymbolId); + if (hit) { + symbolName = String(hit.name ?? hit[1]) || symbolName; + symPath = String(hit.filePath ?? hit[2]) || filePath; } } } else { diff --git a/gitnexus/src/core/group/extractors/java-workspace-extractor.ts b/gitnexus/src/core/group/extractors/java-workspace-extractor.ts index b6beed71c..fcbb06507 100644 --- a/gitnexus/src/core/group/extractors/java-workspace-extractor.ts +++ b/gitnexus/src/core/group/extractors/java-workspace-extractor.ts @@ -1,5 +1,6 @@ import fs from 'node:fs/promises'; import path from 'node:path'; +import { XMLParser } from 'fast-xml-parser'; import type { CypherExecutor } from '../contract-extractor.js'; import type { GroupManifestLink, ContractRole } from '../types.js'; import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js'; @@ -20,6 +21,21 @@ interface ImportedSymbol { filePath: string; } +type XmlNode = Record; + +// POMs are static metadata. Parse hierarchy with a real XML parser, but do not +// invoke Maven or resolve the effective model. Properties, profiles, and remote +// parent resolution remain outside this extractor's deterministic boundary. +const pomParser = new XMLParser({ + ignoreAttributes: true, + removeNSPrefix: true, + trimValues: true, + parseTagValue: false, + processEntities: false, + ignoreDeclaration: true, + ignorePiTags: true, +}); + async function parseJavaManifest( repoPath: string, ): Promise<{ groupId: string; artifactId: string; deps: string[] } | null> { @@ -28,14 +44,15 @@ async function parseJavaManifest( const content = await fs.readFile(pomPath, 'utf-8'); return parsePom(content); } catch { - // fall through to Gradle + // Missing pom.xml — fall through to Gradle. } + const gradleSidecars = await readGradleSidecars(repoPath); for (const name of ['build.gradle.kts', 'build.gradle']) { const gradlePath = path.join(repoPath, name); try { const content = await fs.readFile(gradlePath, 'utf-8'); - return parseGradle(content, repoPath); + return parseGradle(content, repoPath, gradleSidecars); } catch { continue; } @@ -44,59 +61,286 @@ async function parseJavaManifest( return null; } -function parsePom(content: string): { groupId: string; artifactId: string; deps: string[] } | null { - const projectGroupMatch = content.match(/]*>[\s\S]*?([^<]+)<\/groupId>/); - const projectArtifactMatch = content.match( - /]*>[\s\S]*?([^<]+)<\/artifactId>/, - ); - if (!projectGroupMatch || !projectArtifactMatch) return null; +interface GradleSidecars { + propertiesGroup?: string; + rootProjectName?: string; + catalogLibraries: Map; + catalogBundles: Map; +} - const groupId = projectGroupMatch[1].trim(); - const artifactId = projectArtifactMatch[1].trim(); +async function readIfPresent(filePath: string): Promise { + try { + return await fs.readFile(filePath, 'utf-8'); + } catch { + return undefined; + } +} - const deps: string[] = []; - const depBlocks = content.matchAll(/\s*([\s\S]*?)<\/dependency>/g); - for (const block of depBlocks) { - const gMatch = block[1].match(/([^<]+)<\/groupId>/); - const aMatch = block[1].match(/([^<]+)<\/artifactId>/); - if (gMatch && aMatch) { - deps.push(`${gMatch[1].trim()}:${aMatch[1].trim()}`); +async function readGradleSidecars(repoPath: string): Promise { + const [properties, settingsKts, settingsGroovy, catalog] = await Promise.all([ + readIfPresent(path.join(repoPath, 'gradle.properties')), + readIfPresent(path.join(repoPath, 'settings.gradle.kts')), + readIfPresent(path.join(repoPath, 'settings.gradle')), + readIfPresent(path.join(repoPath, 'gradle', 'libs.versions.toml')), + ]); + + const sidecars: GradleSidecars = { + catalogLibraries: new Map(), + catalogBundles: new Map(), + }; + + const groupMatch = properties?.match(/(?:^|\n)\s*group\s*=\s*([^\s#]+)/); + if (groupMatch) sidecars.propertiesGroup = groupMatch[1]; + + const settings = settingsKts ?? settingsGroovy; + const nameMatch = settings?.match(/rootProject\.name\s*=\s*['"]([^'"]+)['"]/); + if (nameMatch) sidecars.rootProjectName = nameMatch[1]; + + if (catalog) { + const parsed = parseGradleVersionCatalog(catalog); + sidecars.catalogLibraries = parsed.libraries; + sidecars.catalogBundles = parsed.bundles; + } + + return sidecars; +} + +function catalogAccessors(alias: string): string[] { + const dotted = alias.replace(/[-_]/g, '.'); + const camel = alias.replace(/[-_]+([A-Za-z0-9])/g, (_, char: string) => char.toUpperCase()); + return [...new Set([alias, dotted, camel])]; +} + +function projectAccessorToArtifactId(accessor: string): string { + const last = accessor.split('.').pop()!; + return last.replace(/[A-Z]/g, (char) => `-${char.toLowerCase()}`).replace(/^-/, ''); +} + +function moduleToGa(module: string): string | undefined { + const parts = module.split(':'); + return parts.length >= 2 ? `${parts[0]}:${parts[1]}` : undefined; +} + +function parseInlineTomlTable(rhs: string): Record { + const fields: Record = {}; + for (const match of rhs.matchAll(/([A-Za-z0-9_-]+)\s*=\s*['"]([^'"]+)['"]/g)) { + fields[match[1]] = match[2]; + } + return fields; +} + +/** Default Gradle catalog (`gradle/libs.versions.toml`) — aliases only, no version resolution. */ +function parseGradleVersionCatalog(toml: string): { + libraries: Map; + bundles: Map; +} { + const libraries = new Map(); + const bundles = new Map(); + let section: 'libraries' | 'bundles' | 'other' = 'other'; + + const addLibrary = (alias: string, ga: string) => { + for (const accessor of catalogAccessors(alias)) libraries.set(accessor, ga); + }; + + for (const raw of toml.split(/\r?\n/)) { + const line = raw.replace(/#.*$/, '').trim(); + if (!line) continue; + const header = line.match(/^\[([^\]]+)\]$/); + if (header) { + const name = header[1]; + section = + name === 'libraries' || name.endsWith('.libraries') + ? 'libraries' + : name === 'bundles' || name.endsWith('.bundles') + ? 'bundles' + : 'other'; + continue; + } + + if (section === 'libraries') { + const dottedModule = line.match(/^([A-Za-z0-9._-]+)\.module\s*=\s*['"]([^'"]+)['"]$/); + if (dottedModule) { + const ga = moduleToGa(dottedModule[2]); + if (ga) addLibrary(dottedModule[1], ga); + continue; + } + const assignment = line.match(/^([A-Za-z0-9._-]+)\s*=\s*(.+)$/); + if (!assignment) continue; + const alias = assignment[1]; + const rhs = assignment[2].trim(); + const quoted = rhs.match(/^['"]([^'"]+)['"]$/); + if (quoted) { + const ga = moduleToGa(quoted[1]); + if (ga) addLibrary(alias, ga); + continue; + } + const table = parseInlineTomlTable(rhs); + const ga = table.module + ? moduleToGa(table.module) + : table.group && table.name + ? `${table.group}:${table.name}` + : undefined; + if (ga) addLibrary(alias, ga); + continue; + } + + if (section === 'bundles') { + const assignment = line.match(/^([A-Za-z0-9._-]+)\s*=\s*\[([^\]]*)\]$/); + if (!assignment) continue; + const members = [...assignment[2].matchAll(/['"]([^'"]+)['"]/g)].map((match) => match[1]); + for (const accessor of catalogAccessors(assignment[1])) bundles.set(accessor, members); } } + return { libraries, bundles }; +} + +const GRADLE_GROUP_PATTERNS = [ + /(?:^|[\n{;])\s*(?:rootProject\.)?group\s*=\s*['"]([^'"]+)['"]/, + /(?:^|[\n{;])\s*group\s+['"]([^'"]+)['"]/, +]; + +const GRADLE_COORD_CONFIGS = + 'implementation|api|compileOnly|runtimeOnly|testImplementation|testApi|testCompileOnly|compile|kapt|ksp|commonMainImplementation|commonMainApi'; + +const CATALOG_ALIAS = '([A-Za-z0-9_]+(?:\\.[A-Za-z0-9_]+)*)(?:\\.get\\(\\)|\\.asProvider\\(\\))?'; + +function gradleDepRe(suffix: string): RegExp { + return new RegExp(`(?:${GRADLE_COORD_CONFIGS})\\s*${suffix}`, 'g'); +} + +function parseGradleGroup(content: string): string | undefined { + for (const pattern of GRADLE_GROUP_PATTERNS) { + const match = content.match(pattern); + if (match?.[1]) return match[1]; + } + return undefined; +} + +function asXmlNode(value: unknown): XmlNode | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as XmlNode) + : undefined; +} + +function xmlText(value: unknown): string | undefined { + if (typeof value === 'string' || typeof value === 'number') { + const text = String(value).trim(); + return text || undefined; + } + const nested = asXmlNode(value)?.['#text']; + if (nested === undefined) return undefined; + return xmlText(nested); +} + +function xmlChildText(node: XmlNode | undefined, name: string): string | undefined { + return node ? xmlText(node[name]) : undefined; +} + +function asList(value: unknown): unknown[] { + if (value === undefined || value === null) return []; + return Array.isArray(value) ? value : [value]; +} + +/** Direct project dependencies only — not BOM, profiles, or plugin classpath. */ +function collectProjectDependencies(project: XmlNode, deps: string[]): void { + const dependencies = asXmlNode(project.dependencies); + if (!dependencies) return; + for (const dep of asList(dependencies.dependency)) { + const depNode = asXmlNode(dep); + const groupId = xmlChildText(depNode, 'groupId'); + const artifactId = xmlChildText(depNode, 'artifactId'); + if (groupId && artifactId) deps.push(`${groupId}:${artifactId}`); + } +} + +function parsePom(content: string): { groupId: string; artifactId: string; deps: string[] } | null { + let parsed: unknown; + try { + // parseSourceSafe guards tree-sitter's Windows SIGSEGV by switching to a + // chunked input callback above 16 KB; XMLParser only accepts XML text, so + // routing POMs through it silently yields an empty document. + // eslint-disable-next-line gitnexus/require-safe-parse + parsed = pomParser.parse(content); + } catch { + return null; + } + + const project = asXmlNode(asXmlNode(parsed)?.project); + if (!project) return null; + + // Maven inherits groupId from , but artifactId is always the + // project's own direct child and must never fall back to parent.artifactId. + const groupId = + xmlChildText(project, 'groupId') ?? xmlChildText(asXmlNode(project.parent), 'groupId'); + const artifactId = xmlChildText(project, 'artifactId'); + if (!groupId || !artifactId) return null; + + const deps: string[] = []; + collectProjectDependencies(project, deps); return { groupId, artifactId, deps: [...new Set(deps)] }; } function parseGradle( content: string, repoPath: string, + sidecars: GradleSidecars = { catalogLibraries: new Map(), catalogBundles: new Map() }, ): { groupId: string; artifactId: string; deps: string[] } | null { - const groupMatch = content.match(/group\s*=\s*['"]([^'"]+)['"]/); - const dirName = path.basename(repoPath); - const groupId = groupMatch ? groupMatch[1] : ''; + // Static text + default catalog file. Do not execute Gradle. + const groupId = parseGradleGroup(content) ?? sidecars.propertiesGroup ?? ''; if (!groupId) return null; - const artifactId = dirName; + const artifactId = sidecars.rootProjectName ?? path.basename(repoPath); + const { catalogLibraries, catalogBundles } = sidecars; const deps: string[] = []; - // implementation("group:artifact:version") or api("group:artifact:version") - const depMatches = content.matchAll( - /(?:implementation|api|compileOnly|runtimeOnly)\s*\(\s*['"]([^'"]+)['"]\s*\)/g, + const pushCatalogAlias = (alias: string) => { + const ga = catalogLibraries.get(alias); + if (ga) deps.push(ga); + }; + + const namedPattern = gradleDepRe( + `(?:\\(\\s*)?(?:group\\s*=\\s*['"](?[^'"]+)['"]\\s*,\\s*name\\s*=\\s*['"](?[^'"]+)['"]|name\\s*=\\s*['"](?[^'"]+)['"]\\s*,\\s*group\\s*=\\s*['"](?[^'"]+)['"]|group:\\s*['"](?[^'"]+)['"]\\s*,\\s*name:\\s*['"](?[^'"]+)['"]|name:\\s*['"](?[^'"]+)['"]\\s*,\\s*group:\\s*['"](?[^'"]+)['"])`, ); - for (const m of depMatches) { - const parts = m[1].split(':'); - if (parts.length >= 2) { - deps.push(`${parts[0]}:${parts[1]}`); + for (const match of content.matchAll(namedPattern)) { + const group = + match.groups?.group1 ?? match.groups?.group2 ?? match.groups?.group3 ?? match.groups?.group4; + const name = + match.groups?.name1 ?? match.groups?.name2 ?? match.groups?.name3 ?? match.groups?.name4; + if (group && name) deps.push(`${group}:${name}`); + } + + for (const match of content.matchAll( + gradleDepRe(`(?:\\(\\s*)?libs(?:\\.libraries)?\\.(?!bundles\\.|plugins\\.)${CATALOG_ALIAS}`), + )) { + pushCatalogAlias(match[1]); + } + + for (const match of content.matchAll( + gradleDepRe(`(?:\\(\\s*)?libs\\.bundles\\.${CATALOG_ALIAS}`), + )) { + for (const member of catalogBundles.get(match[1]) ?? []) { + for (const accessor of catalogAccessors(member)) pushCatalogAlias(accessor); } } - // implementation(project(":subproject")) - const projDeps = content.matchAll( - /(?:implementation|api)\s*\(\s*project\s*\(\s*['"]([^'"]+)['"]\s*\)\s*\)/g, - ); - for (const m of projDeps) { - const subName = m[1].replace(/^:/, ''); - deps.push(`${groupId}:${subName}`); + for (const match of content.matchAll(gradleDepRe(`\\(\\s*projects\\.([A-Za-z][A-Za-z0-9.]*)`))) { + deps.push(`${groupId}:${projectAccessorToArtifactId(match[1])}`); + } + + for (const match of content.matchAll( + gradleDepRe(`(?:\\(\\s*['"]([^'"]+)['"]\\s*\\)|['"]([^'"]+)['"])`), + )) { + const coord = match[1] ?? match[2]; + if (!coord) continue; + const parts = coord.split(':'); + if (parts.length >= 2) deps.push(`${parts[0]}:${parts[1]}`); + } + + for (const match of content.matchAll( + gradleDepRe(`(?:\\(\\s*)?project\\s*\\(\\s*['"]([^'"]+)['"]\\s*\\)`), + )) { + deps.push(`${groupId}:${match[1].replace(/^:/, '')}`); } return { groupId, artifactId, deps: [...new Set(deps)] }; diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 272c27ce6..d4175442a 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -1,4 +1,9 @@ -import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js'; +import type { + CrossLink, + GroupManifestLink, + ManifestContractType, + StoredContract, +} from '../types.js'; import type { CypherExecutor } from '../contract-extractor.js'; import { logger } from '../../logger.js'; @@ -366,11 +371,11 @@ export class ManifestExtractor { * equality matching without requiring wildcard logic downstream. * * NOTE on exhaustiveness: the switch covers every current - * `ContractType` variant and falls through to a `never` assertion so + * manifest-declared contract type and falls through to a `never` assertion so * TypeScript fails the build if a new variant is added without a * corresponding case. */ - private buildContractId(type: ContractType, contract: string): string { + private buildContractId(type: ManifestContractType, contract: string): string { switch (type) { case 'http': { // Canonicalize method casing and path separators so logically diff --git a/gitnexus/src/core/group/extractors/python-workspace-extractor.ts b/gitnexus/src/core/group/extractors/python-workspace-extractor.ts index 4453852a6..c07e5d8cc 100644 --- a/gitnexus/src/core/group/extractors/python-workspace-extractor.ts +++ b/gitnexus/src/core/group/extractors/python-workspace-extractor.ts @@ -2,7 +2,11 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import type { CypherExecutor } from '../contract-extractor.js'; import type { GroupManifestLink, ContractRole } from '../types.js'; -import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js'; +import { + shouldIgnorePath, + loadIgnoreRules, + isHardcodedIgnoredDirectoryAtPath, +} from '../../../config/ignore-service.js'; import { logger } from '../../logger.js'; interface PythonPackageMeta { @@ -161,9 +165,11 @@ async function findPythonFiles(repoPath: string): Promise { for (const entry of entries) { const childRel = rel ? `${rel}/${entry.name}` : entry.name; if (entry.isDirectory()) { + const childPath = path.join(dir, entry.name); if (shouldIgnorePath(childRel)) continue; + if (isHardcodedIgnoredDirectoryAtPath(repoPath, childPath)) continue; if (ig && ig.ignores(childRel + '/')) continue; - await walk(path.join(dir, entry.name), childRel); + await walk(childPath, childRel); } else if (entry.name.endsWith('.py')) { if (shouldIgnorePath(childRel)) continue; if (ig && ig.ignores(childRel)) continue; diff --git a/gitnexus/src/core/group/group-lock.ts b/gitnexus/src/core/group/group-lock.ts new file mode 100644 index 000000000..cdaacb283 --- /dev/null +++ b/gitnexus/src/core/group/group-lock.ts @@ -0,0 +1,200 @@ +/** + * Cross-process single-writer lock for one group's persisted state (R9). + * + * A group sync ends by REPLACING `contracts.json` and rebuilding `bridge.lbug` + * from a snapshot it computed minutes earlier. Two syncs of the same group that + * overlap therefore do not merge — the second one's write simply overwrites the + * first one's, and whichever finishes last wins with a registry assembled from + * repo state the other run never saw. Nothing detects it afterwards: both runs + * report success, and the group's contracts silently describe a mixture that was + * never true at any instant. This module serializes that section so one sync at + * a time can be inside it. + * + * WHERE THE LOCK LIVES. On a dedicated `sync-lock` directory INSIDE the group + * directory — mirroring `withRegistryLock`, which locks a `registry-lock` + * directory beside the registry rather than the registry's own directory + * (repo-manager.ts). {@link acquireIndexLock} is NOT reentrant and its file + * backend writes `analyze.lock` into the directory it is handed, so pointing it + * at a directory that some other code path might also lock — or that already + * holds a per-repo index slot — reintroduces exactly the collision the registry + * lock's own comment warns about. `/sync-lock` is a namespace nothing + * else claims: group directories live under `~/.gitnexus/groups/` (or + * `$GITNEXUS_HOME`), never under a repo's `.gitnexus[/branches/]`. + * + * WHY IT FAILS CLOSED, like the registry lock. `withRegistryLock` also + * refuses to continue unlocked on timeout: a lost registry update can drop a + * concurrent registration. A group sync still fails closed for additional + * reasons — it is long, expensive, operator-initiated, and a lost update + * destroys contracts rather than a registry field. + * A sync that cannot be protected must not run at all, and there are three + * distinct ways it can fail to be protected; all three throw + * {@link GroupSyncLockError}: + * + * 1. TIMEOUT — the holder is still alive when the ceiling elapses. + * 2. LOCK-FREE DEGRADATION — `acquireIndexLock` answers a read-only or + * permission-denied filesystem with a no-op handle that is byte-identical + * to a real one at the API boundary. That is a deliberate tolerance for + * `analyze` (an unwritable index dir rejects every write anyway, so the + * lock is moot), but here it would hand back a handle that protects + * nothing while the sync went on to attempt its writes. The handle now + * carries {@link IndexLockHandle.lockFree}, so we can see it and refuse. + * 3. ANY OTHER ACQUIRE FAILURE — e.g. `sync-lock` cannot be created because a + * regular file already occupies the path. Silently proceeding on an error + * we did not anticipate is the same unprotected run under another name. + * + * WHY THE CEILING IS PASSED EXPLICITLY. The magnitude is not the point — 10 + * minutes deliberately matches `acquireIndexLock`'s own default, because a group + * sync is analyze-shaped and a legitimately queued second sync must be able to + * wait out a full first one (the registry lock's 5s is sized for a sub-second + * merge and is the wrong model here). The reason to pass it is + * `resolveTimeoutMs`: it prefers an explicit argument over + * `GITNEXUS_INDEX_LOCK_TIMEOUT_MS`, and that variable's `<= 0` case resolves to + * `Number.POSITIVE_INFINITY`. Inheriting it would let an environment turn this + * lock's fail-closed timeout into an unbounded hang. + * + * ACQUIRED EXACTLY ONCE, by `syncGroup`, around its whole persist section. + * Nothing it calls beneath that point — `writeContractRegistry`, + * `refreshPreservedBridgeMeta`, `writeBridgeUnlocked` — takes this lock; a + * second acquisition would deadlock a non-reentrant primitive on the HAPPY + * path, not on some edge case. `bridge-db.ts` exports the swap in both forms + * for exactly that reason: `writeBridgeUnlocked` for the held-lock caller + * (`syncGroup`), and the `writeBridge` wrapper, which acquires here, for direct + * callers that are outside the region. The same split `repo-manager.ts` uses + * for `registerRepoUnlocked` / `registerRepo`. + * + * SCOPE CAVEAT (recorded, not solved): the default socket backend uses Linux + * abstract sockets, which are network-namespace-scoped. Two containers that + * share a bind-mounted group directory but sit in separate netns will NOT + * contend, exactly as documented for the index lock itself; forcing + * `GITNEXUS_INDEX_LOCK_BACKEND=file` is what covers that deployment. + */ +import path from 'node:path'; +import { + acquireIndexLock, + IndexLockTimeoutError, + type IndexLockHandle, +} from '../../storage/index-lock.js'; +import { logger } from '../logger.js'; + +/** Lock-directory name inside the group directory. Never the group dir itself. */ +export const GROUP_SYNC_LOCK_DIRNAME = 'sync-lock'; + +/** The dedicated lock namespace for one group: `/sync-lock`. */ +export const getGroupSyncLockDir = (groupDir: string): string => + path.join(groupDir, GROUP_SYNC_LOCK_DIRNAME); + +/** + * Wait ceiling for the group sync lock (10 min). See the module header: the + * magnitude matches `acquireIndexLock`'s analyze-sized default on purpose; the + * reason it is passed EXPLICITLY is to keep `GITNEXUS_INDEX_LOCK_TIMEOUT_MS` + * (whose `<= 0` case means unbounded) from turning fail-closed into a hang. + */ +export const GROUP_SYNC_LOCK_TIMEOUT_MS = 600_000; + +/** Which of the three fail-closed exits produced a {@link GroupSyncLockError}. */ +export type GroupSyncLockFailure = 'timeout' | 'lock-free' | 'unavailable'; + +/** + * A group sync could not be protected, so it did not run. One class for all + * three exits so both callers — the CLI command and the MCP service — have a + * single thing to catch and report. + */ +export class GroupSyncLockError extends Error { + readonly reason: GroupSyncLockFailure; + readonly groupDir: string; + constructor(reason: GroupSyncLockFailure, groupDir: string, message: string, cause?: unknown) { + super(message, cause === undefined ? undefined : { cause }); + this.name = 'GroupSyncLockError'; + this.reason = reason; + this.groupDir = groupDir; + } +} + +/** + * Run `operation` as the only group sync touching `groupDir`, or throw + * {@link GroupSyncLockError} without running it at all. + * + * The lock is released in a `finally`, so it is dropped whether the operation + * succeeds or throws. + */ +export const withGroupSyncLock = async ( + groupDir: string, + operation: () => Promise, +): Promise => { + let handle: IndexLockHandle; + // The wrapper times the acquisition itself. `IndexLockTimeoutError` carries + // `holder` and `holderKnown` and nothing else — the elapsed wait exists only + // inside its inherited message string, so the figure has to be measured here + // to be reported without that message. `Date.now()` matches how the primitive + // measures its own wait. + const acquireStartedAt = Date.now(); + try { + handle = await acquireIndexLock(getGroupSyncLockDir(groupDir), { + timeoutMs: GROUP_SYNC_LOCK_TIMEOUT_MS, + // `acquireIndexLock`'s own `log` texts name an "analyze" holder, which + // misattributes a group-sync wait — the same reason `withRegistryLock` + // supplies its own line instead of passing `log` through. + onWaitStart: () => + logger.info( + { groupDir }, + 'Waiting for another GitNexus process to finish syncing this group…', + ), + }); + } catch (err) { + // The inherited message names "another gitnexus analyze" as the holder — + // a cause this detection path cannot establish. Nothing but a group sync + // ever locks `/sync-lock` (see the module header), and on the + // socket backend the holder is not identifiable at all. Re-word it around + // what IS known: which group, which operation, and how long we waited. + if (err instanceof IndexLockTimeoutError) { + throw new GroupSyncLockError( + 'timeout', + groupDir, + `Timed out after ${Date.now() - acquireStartedAt}ms waiting for the sync lock on ` + + `group "${path.basename(groupDir)}" (${getGroupSyncLockDir(groupDir)}). ` + + // `holderKnown` is false on the socket backend and on the file + // backend's malformed/vanished-lock timeouts, where `holder` is a + // placeholder (`pid -1`). Presenting that as a real owner would be the + // same unestablished claim in a new form. + (err.holderKnown + ? `Held by pid ${err.holder.pid} on ${err.holder.hostname} ` + + `(invocation ${err.holder.invocationId}). ` + : `The lock stayed held for the whole wait, but this lock backend ` + + `cannot identify the holder. `) + + `Nothing was written and this group was not synced. ` + + `Re-run once the other sync of this group has finished.`, + err, + ); + } + throw new GroupSyncLockError( + 'unavailable', + groupDir, + `Could not acquire the sync lock for this group (${getGroupSyncLockDir(groupDir)}): ` + + `${err instanceof Error ? err.message : String(err)}. Nothing was written.`, + err, + ); + } + + if (handle.lockFree) { + // A handle that owns nothing. Release it anyway (it is a no-op, but the + // contract is that every handle is released) and refuse to run: this sync + // would otherwise write `contracts.json` and `bridge.lbug` with no + // protection at all against a concurrent sync doing the same. + handle.release(); + throw new GroupSyncLockError( + 'lock-free', + groupDir, + `The sync lock for this group could not be created at ` + + `${getGroupSyncLockDir(groupDir)} (read-only or permission-denied filesystem), ` + + `so this sync cannot be protected against a concurrent one. Nothing was written. ` + + `Make the group directory writable and re-run.`, + undefined, + ); + } + + try { + return await operation(); + } finally { + handle.release(); + } +}; diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index eea6fc102..2647cc009 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -37,7 +37,13 @@ function buildNoisyContractFilter( : new Set(); const excludeParamOnly = matchingConfig?.exclude_links_param_only_paths === true; - return function isNoisyHttpContract(contractId: string): boolean { + return function isNoisyContract(contractId: string): boolean { + if (contractId.startsWith('graphql::')) { + const parts = contractId.split('::'); + if (parts.length < 3) return false; + const field = parts.slice(2).join('::'); + return excludePaths.has(field) || excludePaths.has(`/${field}`); + } if (!contractId.startsWith('http::')) return false; const parts = contractId.split('::'); if (parts.length < 3) return false; diff --git a/gitnexus/src/core/group/normalization.ts b/gitnexus/src/core/group/normalization.ts index c99d36850..50102415a 100644 --- a/gitnexus/src/core/group/normalization.ts +++ b/gitnexus/src/core/group/normalization.ts @@ -91,6 +91,61 @@ function crossLinkKey(link: CrossLink): string { ].join('\0'); } +/** + * True when a link endpoint carries no resolved graph symbol — empty + * `symbolUid` or a missing/empty `symbolRef`. + * + * Sync marks a cross-link `degraded: true` when this holds for the PROVIDER + * endpoint (`to`): the contract boundary is proven, but the empty uid can + * never match a Phase-1 impact symbol id, so cross-repo fan-out across the + * link silently yields nothing (the classic case is a provider whose handler + * failed to resolve, leaving `symbolName` degraded to the file name with one + * pseudo-symbol carrying every route in that file). Consumer-side (`from`) + * emptiness is deliberately NOT degraded — several extractors (topics, grpc) + * legitimately emit consumer contracts without a per-call symbol, and the + * anchor that matters for far-side fan-out is the provider's. + * + * Kept next to the endpoint merge logic because `dedupeCrossLinks` must + * re-derive the flag after a merge: `mergeEndpoints` backfills `symbolUid` + * from the losing twin, which can invalidate a flag carried in from the winner. + * + * NOT unresolved: a deterministic `manifest::::` synthetic + * uid (see `manifestSymbolUid`). Manifest endpoints fall back to it precisely + * when the graph has no symbol for them — its empty `symbolRef.filePath` would + * otherwise trip the check below — yet cross-impact anchors those links by + * design (#2722: the crossing is preserved with `fanout_status: + * 'not_attempted'` instead of silently yielding cross=0). The prefix is the + * canonical discriminator — real indexer uids never start with `manifest::` + * — and `cross-impact.ts` branches on the same test. Encoding the exemption + * HERE (not at the sync marking call site) keeps marking and the post-merge + * re-derivation from drifting apart, and keeps the flag's meaning exactly what + * `types.ts` documents: "distinct from manifest::… synthetic UIDs". + */ +export function isUnresolvedEndpoint(endpoint: CrossLinkEndpoint): boolean { + if (endpoint.symbolUid.startsWith('manifest::')) return false; + return ( + !endpoint.symbolUid || + !endpoint.symbolRef || + !endpoint.symbolRef.filePath || + !endpoint.symbolRef.name + ); +} + +/** + * Derive `degraded` from the provider endpoint. Present (`true`) only when + * unresolved; deleted otherwise so contracts.json stays "carried only when + * meaningful" (`'degraded' in link === false` for anchored links). + */ +export function applyDegradedFlag(link: CrossLink): CrossLink { + const next: CrossLink = { ...link }; + if (isUnresolvedEndpoint(next.to)) { + next.degraded = true; + } else { + delete next.degraded; + } + return next; +} + export function dedupeContracts(items: StoredContract[]): StoredContract[] { const deduped = new Map(); for (const contract of items) { @@ -113,12 +168,15 @@ export function dedupeCrossLinks(items: CrossLink[]): CrossLink[] { const keepIncoming = link.confidence > existing.confidence; const primary = keepIncoming ? link : existing; const secondary = keepIncoming ? existing : link; - deduped.set(key, { + const merged: CrossLink = { ...primary, confidence: Math.max(existing.confidence, link.confidence), from: mergeEndpoints(primary.from, secondary.from), to: mergeEndpoints(primary.to, secondary.to), - }); + }; + // Re-derive after mergeEndpoints: a richer twin can backfill `to.symbolUid` + // and must not leave a stale `degraded` flag on an now-anchored link. + deduped.set(key, applyDegradedFlag(merged)); } - return [...deduped.values()]; + return [...deduped.values()].map(applyDegradedFlag); } diff --git a/gitnexus/src/core/group/service.ts b/gitnexus/src/core/group/service.ts index f1356dcec..6f41d6586 100644 --- a/gitnexus/src/core/group/service.ts +++ b/gitnexus/src/core/group/service.ts @@ -6,7 +6,16 @@ import fsp from 'node:fs/promises'; import path from 'node:path'; import { checkStaleness } from '../git-staleness.js'; -import { loadMeta, type RepoMeta } from '../../storage/repo-manager.js'; +import { + canonicalizePath, + loadMeta, + readRegistryStrict, + registryPathEquals, + type RegistryEntry, + type RepoMeta, +} from '../../storage/repo-manager.js'; +import { crossRepoCompleteness } from './completeness.js'; +import { recordedMatchStages, recordedRepoList } from './completeness.js'; import { GroupNotFoundError, loadGroupConfig } from './config-parser.js'; import { fileMatchesServicePrefix, @@ -42,6 +51,17 @@ export interface GroupToolPort { repo: GroupRepoHandle, params: { target: string; + /** + * Target-selector params, same semantics as the single-repo `impact` + * tool: `target_uid` is the zero-ambiguity lookup (it wins over the + * name), `file_path`/`kind` narrow a name shared by several symbols + * (e.g. same-named Api/Impl/Controller layers). The port implementation + * consumes them directly; the Phase-1 caller in cross-impact.ts is + * responsible for threading them from the MCP `impact` args. + */ + target_uid?: string; + file_path?: string; + kind?: string; direction: 'upstream' | 'downstream'; maxDepth?: number; relationTypes?: string[]; @@ -222,10 +242,39 @@ function isCrossLink(raw: unknown): raw is CrossLink { return typeof o.contractId === 'string' && typeof o.type === 'string'; } +/** + * Does the global registry hold a row for this configured group member? + * + * Consulted only once resolution has ALREADY failed, to choose which of the + * two failures `group status` reports. It mirrors the two tiers + * `LocalBackend.resolveRepo` matches a bare group-config value on — the + * registry `name`, case-insensitively, and the repo `path` — and deliberately + * stops short of its hashed-id and partial-name tiers: those exist to be + * generous about what an operator typed, while this predicate only decides + * between two labels, and a looser match here would relabel a genuine registry + * miss as an unresolvable row. That is the same conflation this reporting + * exists to remove, pointed the other way. + */ +function registryIdentifies(entries: RegistryEntry[], registryName: string): boolean { + const wantedName = registryName.toLowerCase(); + // Path equality goes through the registry's own rule rather than a local + // `resolve` + platform-case compare. `canonicalizePath` also follows symlinks, + // so a row registered through one and looked up through the other still + // matches — and there is one definition of registry path identity instead of + // a third, weaker copy of it living in a group module nobody would grep. + const wantedPath = canonicalizePath(registryName); + return entries.some((entry) => { + if (typeof entry.name === 'string' && entry.name.toLowerCase() === wantedName) return true; + if (typeof entry.path !== 'string') return false; + return registryPathEquals(canonicalizePath(entry.path), wantedPath); + }); +} + async function loadContractRegistryResilient( groupDir: string, ): Promise< - { ok: true; registry: ContractRegistry; skippedCorrupt: number } | { ok: false; error: string } + | { ok: true; registry: ContractRegistry; skippedCorrupt: number; suppressionUnreadable: boolean } + | { ok: false; error: string } > { const filePath = path.join(groupDir, 'contracts.json'); let raw: string; @@ -288,6 +337,17 @@ async function loadContractRegistryResilient( } } + // Bound once: the gate is a full array scan and the ternary below used it twice. + const recordedUnreadable = recordedRepoList(base.unreadableRepos); + const recordedSuppressed = recordedMatchStages(base.suppressedMatchStages); + // Present-but-unreadable is NOT the same as absent. `recordedMatchStages` is + // all-or-nothing, so garbage collapses to `undefined` — and a consumer that + // reads `undefined` as "nothing was suppressed" would throw that safety away + // and report a registry it could not parse as complete. Absent stays + // legitimate (a registry predating the field); only a value that was there + // and unreadable forces the answer to a floor. + const suppressionUnreadable = + base.suppressedMatchStages !== undefined && recordedSuppressed === undefined; const registry: ContractRegistry = { version: typeof base.version === 'number' ? base.version : 0, generatedAt: typeof base.generatedAt === 'string' ? base.generatedAt : '', @@ -295,12 +355,89 @@ async function loadContractRegistryResilient( base.repoSnapshots && typeof base.repoSnapshots === 'object' && base.repoSnapshots !== null ? (base.repoSnapshots as Record) : {}, - missingRepos: Array.isArray(base.missingRepos) ? (base.missingRepos as string[]) : [], + // Same gate as `groupStatus` uses on the same field, for the same reason: + // `Array.isArray` alone waves through `[{repo:'x'}]`, and `groupContracts` + // now returns this list AND folds it into its completeness answer, so a + // value we could not read would be reported as a repo name. `missingRepos` + // has always been required, so — unlike `unreadableRepos` below — there is + // no "not recorded" state to preserve: an unreadable value degrades to empty. + missingRepos: recordedRepoList(base.missingRepos) ?? [], + // Spread, not `?? []`. `ContractRegistry.unreadableRepos` documents absence + // as "not recorded", and a registry written before the field existed has no + // opinion about which indexes were readable. Normalizing that to `[]` hands + // the caller "the last sync found none unreadable" — an unmeasured state + // rendered as a clean result, which is the same conflation this whole + // change removes. + ...(recordedUnreadable ? { unreadableRepos: recordedUnreadable } : {}), + // Same omit-when-unrecorded rule. This reader rebuilds the envelope field + // by field with no spread of `base`, so a new on-disk field is dropped + // unless it is named here. + ...(recordedSuppressed ? { suppressedMatchStages: recordedSuppressed } : {}), contracts, crossLinks, }; - return { ok: true, registry, skippedCorrupt }; + return { ok: true, registry, skippedCorrupt, suppressionUnreadable }; +} + +/** + * Validate a boolean MCP parameter — reject, never coerce. + * + * `Boolean(params.x)` is the trap this exists to close: the string `"false"` + * is truthy, and an LLM caller emitting JSON produces that shape routinely. + * While `exactOnly` was inert the coercion was harmless; now that it gates a + * matching stage, a coerced `"false"` suppresses that stage and persists a + * registry with fewer cross-links than the caller asked for. + * + * Absent stays absent-as-false (the unchanged default). Anything that is not + * a real boolean returns a structured `{ error }`, mirroring + * `validateImpactMode` — the established shape for this boundary, and the one + * `groupSync`'s other guards already use. + */ +function validateBooleanParam(name: string, raw: unknown): { value: boolean } | { error: string } { + if (raw === undefined) return { value: false }; + if (typeof raw === 'boolean') return { value: raw }; + return { error: `Invalid "${name}": expected true or false, got ${describeValue(raw)}.` }; +} + +/** + * Render an untrusted value for an error message, without throwing. + * + * `JSON.stringify` is the right shape here — it distinguishes the string + * `"false"` from the boolean, which is the whole point of the message — but it + * throws on a BigInt and on a cyclic object. A validator whose ERROR path can + * throw does not return the structured `{ error }` it promises: the caller gets + * a rejected promise instead of feedback it can act on, and `callTool` is + * reachable directly, so neither input is hypothetical. + */ +function describeValue(raw: unknown): string { + try { + const rendered = JSON.stringify(raw); + // `undefined`, a function, or a symbol serialize to `undefined`. + return rendered ?? String(raw); + } catch { + return typeof raw === 'bigint' ? `${raw}n` : Object.prototype.toString.call(raw); + } +} + +/** + * Refuse parameters this tool used to accept and no longer does. + * + * The CLI rejects a removed flag outright because commander errors on an + * unknown option. The MCP path had no equivalent, so an agent working from a + * cached tool schema kept sending a retired key and was told nothing — the + * removal took away discoverability, not acceptance. Naming the parameter is + * what lets the caller correct itself on the next call. + */ +function rejectRetiredSyncParams(params: Record): { error: string } | null { + for (const retired of ['skipEmbeddings', 'allowStale']) { + if (params[retired] !== undefined) { + return { + error: `"${retired}" was removed and is no longer accepted. Drop it from the call.`, + }; + } + } + return null; } export class GroupService { @@ -332,6 +469,13 @@ export class GroupService { async groupSync(params: Record): Promise { const name = String(params.name ?? '').trim(); if (!name) return { error: 'name is required' }; + // Before anything reads the group off disk: the MCP SDK does not enforce a + // tool's advertised `inputSchema` and `callTool` is reachable directly, so + // this method is the real validation boundary. + const exactOnly = validateBooleanParam('exactOnly', params.exactOnly); + if ('error' in exactOnly) return exactOnly; + const retired = rejectRetiredSyncParams(params); + if (retired) return retired; const groupDir = getGroupDir(getDefaultGitnexusDir(), name); let config: GroupConfig; try { @@ -346,19 +490,52 @@ export class GroupService { // group tools never need it — so deferring it here keeps that closure off // MCP server startup entirely and off every non-sync group call. The CLI // already does exactly this at `cli/group.ts`'s sync command. - const { syncGroup } = await import('./sync.js'); - const result = await syncGroup(config, { - groupDir, - exactOnly: Boolean(params.exactOnly), - skipEmbeddings: Boolean(params.skipEmbeddings), - allowStale: Boolean(params.allowStale), - verbose: Boolean(params.verbose), - }); + const { syncGroup, formatGroupSyncAmbiguousError } = await import('./sync.js'); + const { GroupSyncLockError } = await import('./group-lock.js'); + const { RegistryAmbiguousTargetError } = await import('../../storage/repo-manager.js'); + let result: Awaited>; + try { + result = await syncGroup(config, { + groupDir, + exactOnly: exactOnly.value, + // `verbose` is deliberately NOT accepted here. It gates diagnostics on + // the server's logger, which an MCP caller cannot observe — advertising + // it would be exactly the kind of knob that does not do what the caller + // expects. `SyncOptions.verbose` stays for the CLI, which can see them. + }); + } catch (err) { + if (err instanceof RegistryAmbiguousTargetError) { + return { error: formatGroupSyncAmbiguousError(err) }; + } + // Fails closed (R9): this sync could not be protected against a concurrent + // one, so it did not run and wrote nothing. Return it through the same + // error channel a missing group uses — NEVER as a success payload of zeroes, + // which an agent would read as "the group genuinely has no contracts". + if (!(err instanceof GroupSyncLockError)) throw err; + return { error: err.message }; + } return { contracts: result.contracts.length, crossLinks: result.crossLinks.length, unmatched: result.unmatched.length, missingRepos: result.missingRepos, + unreadableRepos: result.unreadableRepos, + // The agent-facing half of the skipped-stage signal. A human sees it in + // the CLI summary; without this an agent would have to issue a second + // `group_contracts` call to discover its own sync was narrowed. + suppressedMatchStages: result.suppressedMatchStages, + // An agent that calls group_sync and then group_contracts a moment later + // can otherwise see contract counts that disagree with this payload, with + // nothing here explaining why the write was skipped. + registryOutcome: result.registryOutcome, + // Data-quality signals surfaced from the sync run: links whose provider + // endpoint never resolved to a graph symbol, per-repo extraction + // failures with reasons, and operator warnings (e.g. bridge.lbug write + // failed after contracts.json was written). Always present so MCP + // consumers can branch on them without existence checks. + degradedLinks: result.degradedLinks, + failedRepos: result.failedRepos, + warnings: result.warnings, }; } @@ -386,7 +563,50 @@ export class GroupService { ); contracts = contracts.filter((c) => !matchedIds.has(`${c.repo}::${c.contractId}`)); } - const out: Record = { contracts, crossLinks: registry.crossLinks }; + // `loadContractRegistryResilient` already applied `recordedRepoList` to + // both: `undefined` here is "the last sync recorded no opinion" (a registry + // written before the field existed, or a value we could not read), which is + // NOT the same answer as the measured empty list. + const { unreadableRepos, missingRepos } = registry; + // `incompleteRepos` is dropped on this surface only because the two lists it + // is derived from are returned verbatim right below; the truncation triple is + // the part that has no other channel here. + const { incompleteRepos: _incompleteRepos, ...truncation } = crossRepoCompleteness({ + unreadableRepos, + missingRepos, + suppressedMatchStages: registry.suppressedMatchStages, + // An unrecorded `unreadableRepos` means this listing cannot say which + // repos the sync failed to read — so it cannot claim to be complete. + // Either kind of unreadable provenance forces the floor: a sync that + // could not say which repos it read, or a suppression record that was + // present and could not be parsed. Reading the second as "nothing was + // suppressed" would report an unparseable registry as complete. + provenanceUnknown: unreadableRepos === undefined || loaded.suppressionUnreadable, + // A contract LISTING declares no scope to intersect with: it is the whole + // registry, so every configured repo is in scope by construction. The + // `type`/`repo`/`unmatchedOnly` filters above narrow which rows are shown, + // not which repos the sync had to read to produce them. + inScope: () => true, + }); + const out: Record = { + contracts, + crossLinks: registry.crossLinks, + missingRepos, + // Omitted rather than `[]` when the registry never recorded it — the same + // convention `skippedCorrupt` follows below, and the difference between + // "the sync measured zero unreadable repos" and "the sync never said". + ...(unreadableRepos ? { unreadableRepos } : {}), + // Same omit-when-unrecorded rule, and deliberately NOT folded into the + // truncation triple below: that triple reports limits a run hit by + // accident, whose remedy is to fix the repo. A suppressed stage was + // asked for, and its remedy is to re-sync without that flag. + ...(registry.suppressedMatchStages + ? { suppressedMatchStages: registry.suppressedMatchStages } + : {}), + // The structured triple, verbatim from the impact surface (KTD10): + // `truncated` always, `truncationReason` + `riskEpistemic` with it. + ...truncation, + }; if (skippedCorrupt > 0) out.skippedCorrupt = skippedCorrupt; return out; } @@ -573,17 +793,80 @@ export class GroupService { } const registry = await readContractRegistry(groupDir); + /** + * The STRICT global-registry read, deliberately — this is the one caller + * that has to tell "the registry says nothing about this repo" apart from + * "the registry could not be read at all", and only the strict mode can. + * `readRegistry`'s `catch { return [] }` collapses a malformed registry + * into an empty one, which is indistinguishable from a genuine absence and + * would report every configured repo as having no entry — the exact + * conflation the two labels below exist to remove. + * + * The consequence is accepted knowingly: the strict read rejects the WHOLE + * registry when any single row fails to identify a repo, so one malformed + * row renders every member of the group unresolvable, including members + * whose own rows are fine. That is the honest verdict — a registry the + * resolver cannot trust row-wise cannot be trusted about any row — and it + * is reported as an unresolved state, never as a clean one. + * + * ENOENT is not a failure in either mode: no registry file genuinely means + * nothing has been registered yet, so every repo is legitimately missing. + */ + let registryEntries: RegistryEntry[] | null = null; + let registryReadError: string | null = null; + try { + registryEntries = await readRegistryStrict(); + } catch (err) { + registryReadError = err instanceof Error ? err.message : String(err); + } + const repoStatuses: Record< string, { indexStale: boolean; contractsStale: boolean; + /** + * Unchanged meaning: this repo has no usable status. It stays `true` + * for BOTH failures below, so a consumer written before the split + * still sees every unusable repo flagged. Reporting an unresolvable + * repo as `missing: false` would hand that consumer `indexStale: + * false` for a repo nothing was ever read from — a false all-clear. + */ missing: boolean; + /** + * Which failure `missing` means: `false` is a genuine registry miss, + * `true` is an entry the resolver could not turn into a repo. Additive + * — always present on every row, so an agent can branch on it without + * having to treat an absent key as either answer. + */ + unresolvable: boolean; + /** Set only when `unresolvable`; says what could not be resolved. */ + unresolvableReason?: string; commitsBehind?: number; } > = {}; for (const [repoPath, registryName] of Object.entries(config.repos)) { + if (registryEntries === null) { + repoStatuses[repoPath] = { + indexStale: false, + contractsStale: false, + missing: true, + unresolvable: true, + unresolvableReason: `the global registry could not be read: ${registryReadError}`, + }; + continue; + } + // Only `resolveRepo` is inside the try that produces the + // "did not resolve" label, so the label is earned rather than assumed. + // `loadMeta` and `checkStaleness` cannot throw — the first returns null on + // every error, the second catches everything — but the reading below them + // can, and did: `registry.repoSnapshots` is read off a bare + // `JSON.parse(...) as ContractRegistry` with no shape check, so a + // contracts.json missing that field threw a TypeError into this catch and + // reported every repo as an unresolvable GLOBAL-registry entry. That sent + // the operator to repair the wrong file. The optional chain below closes + // the crash; this split stops the next one being mislabelled the same way. try { const repoObj = await this.port.resolveRepo(registryName); const meta: Partial> = @@ -593,7 +876,7 @@ export class GroupService { ? checkStaleness(repoObj.repoPath, meta.lastCommit) : { isStale: true, commitsBehind: -1 }; - const snapshot = registry?.repoSnapshots[repoPath]; + const snapshot = registry?.repoSnapshots?.[repoPath]; const contractsStale = snapshot && meta.indexedAt ? snapshot.indexedAt !== meta.indexedAt : !snapshot; @@ -601,17 +884,49 @@ export class GroupService { indexStale: staleness.isStale, contractsStale: Boolean(contractsStale), missing: false, + unresolvable: false, commitsBehind: staleness.commitsBehind, }; - } catch { - repoStatuses[repoPath] = { indexStale: false, contractsStale: false, missing: true }; + } catch (err) { + // The registry read succeeded, so its answer about this row is + // trustworthy: a row that is there and still would not resolve is a + // different fact from a row that was never there, and the operator's + // next move differs (repair the entry vs. index the repo). + const known = registryIdentifies(registryEntries, registryName); + const reason = err instanceof Error ? err.message : String(err); + repoStatuses[repoPath] = { + indexStale: false, + contractsStale: false, + missing: true, + unresolvable: known, + ...(known + ? { unresolvableReason: `registry entry "${registryName}" did not resolve: ${reason}` } + : {}), + }; } } return { group: name, lastSync: registry?.generatedAt || null, - missingRepos: registry?.missingRepos || [], + // `readContractRegistry` is a bare `JSON.parse(...) as ContractRegistry`, + // so both of these are whatever the file happened to hold — the + // validation in `loadContractRegistryResilient` never runs on this path. + // A `contracts.json` carrying a string here reached `cli/group.ts` and + // died in `.join(', ')`, i.e. an unreadable registry crashing the command + // whose job is to explain unreadable things. + // + // `missingRepos` has always been required, so there is no "not recorded" + // state to preserve for it — an unreadable value degrades to empty. + missingRepos: recordedRepoList(registry?.missingRepos) ?? [], + // `unreadableRepos` does have one: absent means "not recorded", not + // "none" (see ContractRegistry), and a value we could not read is equally + // unrecorded. Reporting either as an empty list is the same conflation. + unreadableRepos: recordedRepoList(registry?.unreadableRepos), + // Same tri-state, same reason: `group status` is where an operator goes + // to ask "is this group's answer trustworthy right now", and a registry + // narrowed on purpose is a different answer from a complete one. + suppressedMatchStages: recordedMatchStages(registry?.suppressedMatchStages), repos: repoStatuses, }; } diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index 6e568cd6e..e196095ef 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -5,7 +5,7 @@ import * as os from 'node:os'; import type { ContractRegistry } from './types.js'; import { writeFileAtomic } from '../../storage/fs-atomic.js'; -const CONTRACTS_FILE = 'contracts.json'; +export const CONTRACTS_FILE = 'contracts.json'; export function getDefaultGitnexusDir(): string { return process.env.GITNEXUS_HOME || path.join(os.homedir(), '.gitnexus'); @@ -30,6 +30,11 @@ export function getGroupDir(gitnexusDir: string, groupName: string): string { return path.join(gitnexusDir, 'groups', groupName); } +/** The registry path, so callers that stat or watch the file do not respell its name. */ +export function getContractRegistryPath(groupDir: string): string { + return path.join(groupDir, CONTRACTS_FILE); +} + export async function writeContractRegistry( groupDir: string, registry: ContractRegistry, @@ -91,15 +96,11 @@ packages: {} detect: http: true + graphql: false grpc: true topics: true - shared_libs: true - embedding_fallback: true matching: - bm25_threshold: 0.7 - embedding_threshold: 0.65 - max_candidates_per_step: 3 # exclude_links_paths: [/ping, /health, /healthcheck] # exclude_links_param_only_paths: false `; diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index a329500be..de7ce5c9e 100644 Binary files a/gitnexus/src/core/group/sync.ts and b/gitnexus/src/core/group/sync.ts differ diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index db9d1989f..b0ba1a1dc 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,5 +1,16 @@ -export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include'; -export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding'; +import type { ImpactRisk, ImpactRiskResult } from 'gitnexus-shared'; + +export type ContractType = + | 'http' + | 'graphql' + | 'grpc' + | 'thrift' + | 'topic' + | 'lib' + | 'custom' + | 'include'; +export type ManifestContractType = Exclude; +export type MatchType = 'exact' | 'manifest' | 'wildcard'; export type ContractRole = 'provider' | 'consumer'; export interface GroupConfig { @@ -16,32 +27,29 @@ export interface GroupConfig { export interface GroupManifestLink { from: string; to: string; - type: ContractType; + type: ManifestContractType; contract: string; role: ContractRole; } export interface DetectConfig { http: boolean; + graphql?: boolean; grpc: boolean; thrift: boolean; topics: boolean; - shared_libs: boolean; - embedding_fallback: boolean; includes: boolean; workspace_deps: boolean; } export interface MatchingConfig { - bm25_threshold: number; - embedding_threshold: number; - max_candidates_per_step: number; /** - * HTTP paths to exclude from cross-link matching. Contracts at these paths + * HTTP paths or GraphQL root fields to exclude from cross-link matching. Contracts at these paths * are still extracted and visible in the registry, but they don't produce * cross-repo links. Useful for health-check endpoints (`/ping`, `/health`) * that every service exposes and would otherwise create N×M false links. - * Trailing slashes are normalized before comparison. + * Trailing slashes are normalized before comparison. GraphQL fields may be + * written as `health` or `/health`. * @default [] */ exclude_links_paths?: string[]; @@ -89,6 +97,19 @@ export interface CrossLink { contractId: string; matchType: MatchType; confidence: number; + /** + * `true` when the PROVIDER endpoint (`to`) has no resolved graph symbol — + * empty `symbolUid` / `symbolRef` at sync time (e.g. the handler failed to + * resolve and `symbolName` degraded to the file name). The contract boundary + * is still proven, but the link cannot anchor a cross-impact fan-out: an + * empty provider uid never matches a Phase-1 symbol id, and a downstream + * fan-out into it has no neighbor symbol to resolve. Derived once at the + * sync persistence boundary (`isUnresolvedEndpoint` in normalization.ts) and + * re-derived by `dedupeCrossLinks` when a merge backfills the uid. Absent on + * fully-anchored links. Distinct from manifest `manifest::…` synthetic UIDs, + * which have their own `fanout_status: 'not_attempted'` channel downstream. + */ + degraded?: boolean; } export interface RepoSnapshot { @@ -100,7 +121,34 @@ export interface ContractRegistry { version: number; generatedAt: string; repoSnapshots: Record; + /** Configured repos with no entry in the registry. */ missingRepos: string[]; + /** + * Configured repos that ARE registered but that this sync could not extract + * from — the index would not open (version skew, lock, corruption), or an + * extractor threw partway through. The two are one bucket because the + * consequence is one thing: NONE of that repo's contracts are in this + * registry. Distinct from `missingRepos`, which is "no entry in the + * registry at all" and needs a different answer from the operator. + * + * Optional so a registry written before this field existed still parses — + * absent means "not recorded", not "none". + */ + unreadableRepos?: string[]; + /** + * Matching stages this sync was ASKED to skip, so a later reader can tell a + * short cross-link list from a complete one. `--exact-only` / `exactOnly` + * suppresses the wildcard stage, and the registry it writes is otherwise + * indistinguishable from one where that stage ran and matched nothing. + * + * Same tri-state as `unreadableRepos` and for the same reason: absent means + * "not recorded" (written before this field existed), `[]` means "measured, + * nothing was suppressed", and a populated list names the stages. Distinct + * from `truncated` / `truncationReason`, which report limits this run hit by + * accident — a suppressed stage is a deliberate request, and its remedy is + * "re-sync without exactOnly", not "fix the unreadable repo". + */ + suppressedMatchStages?: MatchType[]; contracts: StoredContract[]; crossLinks: CrossLink[]; } @@ -117,8 +165,36 @@ export interface RepoHandle { storagePath: string; } -/** Why local impact or fan-out stopped early (e.g. wall-clock budget exhausted). */ -export type GroupImpactTruncationReason = 'timeout' | 'partial'; +/** + * Why local impact or fan-out stopped early (e.g. wall-clock budget exhausted). + * + * `'timeout'` and `'partial'` are runtime limits — the same query can succeed on + * a retry. `'incomplete-sync'` is structural: the bridge itself was built from a + * sync that could not read every configured repo, so those repos' contracts are + * absent from every query against it until `gitnexus group sync` succeeds. + * `'suppressed-stage'` is structural too but has its own remedy: the sync was + * ASKED to skip a matching stage (`--exact-only`), so cross-links that stage + * would have found are absent by request. Retrying returns the same floor, and + * so does re-running the sync — the fix is to re-run it WITHOUT the flag. Kept a + * separate member rather than folded into `'incomplete-sync'` precisely because + * that remedy differs; telling an agent to repair a repo it read fine is the + * failure this distinction exists to prevent. + * + * A runtime array rather than a bare type union: every value here has to be + * explained on the agent-facing surface that returns it, and only an enumerable + * list lets a guard test assert that. A test that hand-lists the members passes + * forever once a fourth is added — which is the exact drift the guard exists to + * catch, so the list an agent is promised and the list the code can emit have + * to come from the same place. + */ +export const GROUP_IMPACT_TRUNCATION_REASONS = [ + 'timeout', + 'partial', + 'incomplete-sync', + 'suppressed-stage', +] as const; + +export type GroupImpactTruncationReason = (typeof GROUP_IMPACT_TRUNCATION_REASONS)[number]; export interface GroupImpactResult { local: unknown; @@ -133,7 +209,17 @@ export interface GroupImpactResult { modules_affected: number; cross_repo_hits: number; }; - risk: string; + risk: ImpactRisk; + /** + * Two-axis (direct + total) risk from the local leg, then `mergeRisk` with + * crossings — compare File vs symbol here, not via top-level `risk`. + */ + riskSharedAxes?: ImpactRisk; + /** + * Local-leg scale metadata (File / skipped enrichment). Crossings do not + * invent process/module membership for File nodes. + */ + riskScale?: ImpactRiskResult['riskScale']; /** * `'lower-bound'` when the fan-out was cut short, so `risk` is a FLOOR, not a * verdict. Same vocabulary as single-repo `impact`'s `epistemic` field. @@ -222,5 +308,118 @@ export interface BridgeHandle { export interface BridgeMeta { version: number; generatedAt: string; + /** + * Size and mtime of the `bridge.lbug` this metadata was written for, so a + * reader can tell whether the two still belong together. + * + * `writeBridge` replaces the database and writes this file as two operations; + * a sync that stops between them leaves the PREVIOUS sync's metadata beside a + * new database, and `runGroupImpact` reads completeness from that metadata. + * Stamping the pair is what lets `bridgeMetaMatchesFile` reject the mismatch + * without anything having to be deleted — deleting the old metadata up front + * would lose it permanently on a swap that fails with the old database still + * in place, which is a normal Windows outcome when a read-only handle is held. + * + * Optional: metadata written before this existed carries no stamp. Such a + * file is not waved through — `bridgeMetaMatchesFile` falls back to comparing + * the two files' modification times, since a successful write orders the + * database rename before the metadata write and a database NEWER than the + * metadata beside it therefore cannot be the one it describes. + * + * That fallback proves WRITE ORDER, not provenance, and is wrong in both + * directions — a non-monotonic clock can make a mis-paired set read as + * ordered, and any copy or restore that rewrites the database's times after + * the metadata's demotes an intact legacy pair to a lower bound until the + * next sync re-stamps it. A stamped pair never reaches that fallback, which + * is the reason to prefer stamping over widening the heuristic. Both + * directions are spelled out at `bridgeMetaMatchesFile`. + */ + bridgeSize?: number; + bridgeMtimeMs?: number; + /** + * Reader-side only: true when `meta.json` parsed but one of its repo lists + * held a value that was not a list of repo paths. + * + * NEVER PERSISTED. `readBridgeMeta` sets it to describe what it found in the + * file; `writeBridgeMeta`'s only caller builds a fresh literal, so it cannot + * round-trip back to disk. It lives on this interface rather than on a + * reader-only subtype so that `readBridgeMeta` keeps the exact signature + * every caller already compiles against. + * + * The unusable value is dropped rather than normalized, so `missingRepos: []` + * on such a result is inert filler — this flag, not the empty list, is what + * says the bridge's provenance is unknown. + */ + repoListsUnreadable?: boolean; + /** + * Reader-side only: did this metadata pair with the `bridge.lbug` beside it, + * measured BEFORE anything opened that database? + * + * NEVER PERSISTED, for the same reason as `repoListsUnreadable`. + * + * The measurement has to happen before the open, and the answer has to be + * carried rather than recomputed. `runGroupImpact` and `runGroupTrace` open + * the bridge and only then ask about provenance, so a platform where a + * read-only open advances the database's mtime would fail every unstamped + * pair the moment it was read — turning back-compat for pre-stamp bridges + * into a repo-wide "everything is a lower bound". Whether any given + * LadybugDB build and OS does that is not something a reader should have to + * know, and it cannot be observed on Windows, where the in-process + * write→read reopen this would need is a documented limitation. Ordering the + * check ahead of the open makes the question moot on every platform instead + * of true on the ones that happen to be testable. + */ + pairedWithDatabase?: boolean; + /** + * PERSISTED, unlike the two fields above: the writer of this metadata could + * not establish that it describes the `bridge.lbug` beside it, and no reader + * may conclude otherwise from the files alone. + * + * Written by `refreshPreservedBridgeMeta` — the preserve path in `syncGroup`, + * which refreshes the diagnostic lists of a bridge it deliberately does NOT + * rebuild. That refresh rewrites `meta.json` ATOMICALLY, so this file's mtime + * becomes now while the database's stays old; and "metadata newer than the + * database beside it" is exactly the write order that + * `unstampedMetaPairsByWriteOrder` accepts. A refresh that simply carried the + * old fields forward would therefore convert a pair that check had been + * REJECTING into one it waves through — laundering unknown provenance into + * verified provenance, which is the fail-open this whole channel exists to + * close. + * + * "Just don't write a stamp" is not a substitute, and is worse: an unstamped + * metadata file is judged on the two file times, and the refresh has already + * moved them into the accepting order. The verdict has to be recorded IN the + * file, because the write that records it is itself what destroys the + * evidence a reader would otherwise use. + * + * `bridgeMetaMatchesFile` rejects on this ahead of both the stamp and the + * write-order heuristic, so `ensureBridgeReady` answers + * `pairedWithDatabase: false` and `bridgeProvenanceUnknown` reports the + * cross-repo answer as a lower bound. That is the ONE enforcement point; do + * not add a second reader for this field. + * + * Self-clearing: a successful `writeBridge` builds fresh metadata from a + * literal and never sets it, so the next good sync retires the marker without + * anything having to delete it. + */ + provenanceUnknown?: boolean; missingRepos: string[]; + /** + * Configured repos the sync that produced this bridge could not extract from + * (see `ContractRegistry.unreadableRepos`). Their contracts and every + * cross-link touching them are absent from `bridge.lbug`, so a cross-repo + * impact query against this bridge is a lower bound, not a verdict — + * `runGroupImpact` folds a non-empty value into its truncation fields for + * exactly that reason. + * Optional: a bridge written before this field existed does not record it. + */ + unreadableRepos?: string[]; + /** + * Matching stages the sync that built this bridge was asked to skip. + * PERSISTED, like `unreadableRepos` and unlike `repoListsUnreadable` — a + * later `group_impact` or `trace` reads this bridge with no access to the run + * that produced it, and a narrowed graph is otherwise indistinguishable from + * a complete one. Same tri-state: absent is "not recorded". + */ + suppressedMatchStages?: MatchType[]; } diff --git a/gitnexus/src/core/incremental/derived-writeback.ts b/gitnexus/src/core/incremental/derived-writeback.ts new file mode 100644 index 000000000..9cec191eb --- /dev/null +++ b/gitnexus/src/core/incremental/derived-writeback.ts @@ -0,0 +1,83 @@ +/** + * Incremental derived-layer writeback helpers (#3016). + * + * The derived layers — Leiden communities, execution flows, and the FTS + * indexes — are graph-wide, so every analyze run rebuilt all three in full no + * matter how small the diff. A surgical incremental write can instead: + * - drop and rebuild only the FTS indexes whose tables hold rows in the + * write set (LadybugDB still cannot DML a table with a live FTS index — + * #2589 — so a table being written must still lose its index first); + * - leave the untouched tables' rows alone, so their indexes stay live; + * - reuse persisted Community/Process rows only when the file-hash diff is + * empty (no added, changed, or deleted files). Any content change can + * add, rename, or retarget symbols that Leiden and flow extraction + * consume — a no-deletion edit is not a validity proof. + */ +import { FTS_INDEXES } from '../search/fts-schema.js'; +import type { KnowledgeGraph } from '../graph/types.js'; +import type { FileHashDiff } from '../../storage/file-hash.js'; + +const FTS_TABLE_NAMES: ReadonlySet = new Set(FTS_INDEXES.map((i) => i.table)); + +/** The FTS-backed members of `tables`. */ +export const ftsTablesAmong = (tables: Iterable): Set => { + const out = new Set(); + for (const table of tables) { + if (FTS_TABLE_NAMES.has(table)) out.add(table); + } + return out; +}; + +/** + * Whether a surgical incremental write may reuse the persisted derived layer. + * + * Deletions disqualify it: the persisted Community/Process rows and their + * MEMBER_OF / STEP_IN_PROCESS edges can reference nodes that no longer exist + * after this run, and nothing short of re-deriving can tell which. + * + * Added or content-changed files also disqualify it: they can introduce, + * rename, or retarget symbols and CALLS edges that Leiden and flow extraction + * consume. File-deletion-only was too weak a proof that the derived graph is + * still valid. + */ +export const shouldPreservePersistedDerivedGraph = ( + diff: Pick, +): boolean => diff.deleted.length === 0 && diff.added.length === 0 && diff.changed.length === 0; + +/** + * FTS-backed node tables that the fresh graph will WRITE rows into for + * `fileSet` — the inserting half of the DML. + * + * Callers must union this with a DB probe for the deleting half + * (`nodeTablesWithRowsForFiles`): a table whose last row in these files was + * just removed by the edit has nothing here, but still holds a stale row that + * the writeback must delete, and deleting it means taking its index down too. + */ +export const incrementalFtsTablesFromGraph = ( + graph: KnowledgeGraph, + fileSet: ReadonlySet, +): Set => { + const touched = new Set(); + graph.forEachNode((n) => { + const filePath = n.properties?.filePath as string | undefined; + if (!filePath || !fileSet.has(filePath)) return; + if (FTS_TABLE_NAMES.has(n.label)) touched.add(n.label); + }); + return touched; +}; + +/** + * The node tables an incremental DETACH DELETE should target, given the FTS + * tables this run is rebuilding. + * + * Every non-FTS table (Folder, CodeElement, …) deletes as before. An FTS-backed + * table only deletes when its index is being rebuilt anyway, because deleting + * from it otherwise would mean DML against a live FTS index (#2589). + */ +export const nodeTablesForIncrementalDelete = ( + allNodeTables: readonly string[], + rebuildingFtsTables: ReadonlySet, +): string[] => + allNodeTables.filter( + (tableName) => !FTS_TABLE_NAMES.has(tableName) || rebuildingFtsTables.has(tableName), + ); diff --git a/gitnexus/src/core/incremental/spring-config-drift.ts b/gitnexus/src/core/incremental/spring-config-drift.ts new file mode 100644 index 000000000..958f33924 --- /dev/null +++ b/gitnexus/src/core/incremental/spring-config-drift.ts @@ -0,0 +1,57 @@ +import type { KnowledgeGraph } from '../graph/types.js'; +import { SPRING_CONFIG_UNRESOLVED_PREFIX } from '../ingestion/frameworks/spring/config-bindings.js'; + +export interface PersistedSpringConfigConsumerRow { + readonly id?: unknown; + readonly description?: unknown; +} + +const CONSUMER_LABELS = new Set(['Property', 'Class', 'Record']); + +function unresolvedKeys(description: unknown): readonly string[] { + if (typeof description !== 'string') return []; + return description + .split(';') + .map((part) => part.trim()) + .filter((part) => part.startsWith(SPRING_CONFIG_UNRESOLVED_PREFIX)) + .map((part) => part.slice(SPRING_CONFIG_UNRESOLVED_PREFIX.length)) + .sort(); +} + +/** + * Find unchanged Spring consumer files whose unresolved markers changed. + * + * A removed config key also removes the old USES edge from the fresh graph, so + * ordinary new-graph boundary expansion cannot discover the consumer file. + */ +export function collectSpringConfigConsumerDriftFiles( + graph: KnowledgeGraph, + persistedRows: readonly PersistedSpringConfigConsumerRow[], +): Set { + const persistedById = new Map(); + for (const row of persistedRows) { + if (typeof row.id !== 'string') continue; + persistedById.set(row.id, unresolvedKeys(row.description)); + } + + const driftFiles = new Set(); + graph.forEachNode((node) => { + if (!CONSUMER_LABELS.has(node.label)) return; + const filePath = node.properties.filePath; + if (typeof filePath !== 'string') return; + const description = node.properties.description; + const persisted = persistedById.get(node.id); + if ( + persisted === undefined && + (typeof description !== 'string' || !description.includes(SPRING_CONFIG_UNRESOLVED_PREFIX)) + ) { + return; + } + const current = unresolvedKeys(description); + const prior = persisted ?? []; + if (current.length !== prior.length || current.some((key, index) => key !== prior[index])) { + driftFiles.add(filePath); + } + }); + return driftFiles; +} diff --git a/gitnexus/src/core/incremental/subgraph-extract.ts b/gitnexus/src/core/incremental/subgraph-extract.ts index e0f0e41eb..64751d99e 100644 --- a/gitnexus/src/core/incremental/subgraph-extract.ts +++ b/gitnexus/src/core/incremental/subgraph-extract.ts @@ -6,9 +6,10 @@ * replaced, produce a smaller KnowledgeGraph that contains: * * - Every node whose `properties.filePath` is in `toWriteSet`. - * - Every graph-wide node (Community, Process, and Spring metadata - * placeholders) — these are regenerated each run and must be fully - * rewritten. + * - Graph-wide Community/Process nodes unless `includeDerivedGraphWide` + * is false (#3016 incremental preserve). Spring metadata placeholders + * and `Destination` nodes are always included — their owning phase + * delete-alls them unconditionally before the writeback. * - Every relationship where AT LEAST ONE endpoint is in the writable * set above. Relationships entirely between unchanged-file nodes * are skipped — their rows are still in the DB and re-inserting @@ -57,9 +58,31 @@ import { } from '../ingestion/frameworks/spring/auto-configuration.js'; import { isSpringAopEvidenceNode } from '../ingestion/frameworks/spring/aop.js'; +/** + * `Destination` is graph-wide for the same reason as the Spring AOP evidence + * nodes: the layer is recomputed in full on every run and deleted in full + * before the writeback (`deleteAllDestinations`), so it must be re-included in + * full or it is simply lost. + * + * The endpoint-writability rule cannot carry it. A RESOLVED destination stores + * no `filePath` at all — deliberately, so an incremental delete keyed on + * `filePath IN [...]` cannot cut a node shared across files — and the include + * test below starts from exactly that property. The result was a defect in both + * directions: a newly added file publishing to a new topic reported + * `added=1, exit 0` and silently put neither the destination nor the + * publisher's edge into the graph, so after the first index every new topic was + * invisible until a full rebuild; and a destination whose last referrer stopped + * referring to it survived forever as an edgeless orphan still carrying + * `address`, the cross-repository join key. + * + * Unresolved destinations DO carry a file path and would ride the ordinary + * rule, but they are included here too: the delete-all removes them as well, so + * anything not re-included would be dropped rather than merely stale. + */ const isGraphWideNode = (node: GraphNode): boolean => node.label === 'Community' || node.label === 'Process' || + node.label === 'Destination' || isSpringAopEvidenceNode(node) || isSpringAutoConfigurationSyntheticClass(node); @@ -122,13 +145,18 @@ const indexNodeFilePaths = (fullGraph: KnowledgeGraph): Map => { export const extractChangedSubgraph = ( fullGraph: KnowledgeGraph, toWriteSet: ReadonlySet, + options?: { includeDerivedGraphWide?: boolean }, ): KnowledgeGraph => { const sub = createKnowledgeGraph(); const writableNodeIds = new Set(); + const includeDerivedGraphWide = options?.includeDerivedGraphWide !== false; + fullGraph.forEachNode((n: GraphNode) => { const filePath = n.properties?.filePath as string | undefined; - const include = (filePath && toWriteSet.has(filePath)) || isGraphWideNode(n); + const derivedWide = + includeDerivedGraphWide || (n.label !== 'Community' && n.label !== 'Process'); + const include = (filePath && toWriteSet.has(filePath)) || (isGraphWideNode(n) && derivedWide); if (include) { sub.addNode(n); writableNodeIds.add(n.id); diff --git a/gitnexus/src/core/index-content-drift.ts b/gitnexus/src/core/index-content-drift.ts new file mode 100644 index 000000000..3599db7b1 --- /dev/null +++ b/gitnexus/src/core/index-content-drift.ts @@ -0,0 +1,170 @@ +/** + * Does the index still reflect the files it actually covers? + * + * `status` used to answer this with a repo-wide `git status --porcelain` + * boolean, which says something different: whether the working tree differs + * from HEAD. Those two questions diverge in both directions. A scratch file, + * a build artifact, or a tracked file under a tool directory the indexer + * never reads makes the tree dirty while every indexed file is byte-current — + * and because `analyze` cannot commit or delete that file, the resulting + * "stale (re-run gitnexus analyze)" verdict was unclearable (#3077). It also + * misses the reverse case: reverting a file that was indexed while dirty + * leaves a clean tree over an index holding the pre-revert content. + * + * `meta.fileHashes` already records the exact set of files the last run + * covered, so the question can be answered directly. This module recomputes + * the coverage set with the same `walkRepositoryPaths` scan (ignore rules and + * dotfile handling stay shared) and the large-file cap recorded in + * `meta.indexCoverage`, hashes only the paths that can actually have changed + * since that run, and diffs against what was recorded. + */ + +import { constants as fsConstants } from 'node:fs'; +import { access } from 'node:fs/promises'; +import path from 'node:path'; +import { walkRepositoryPaths } from './ingestion/filesystem-walker.js'; +import { computeFileHashesDetailed } from '../storage/file-hash.js'; +import { listWorkingTreeDirtyPaths } from '../storage/git.js'; +import { isGitNexusManagedPath } from '../storage/gitnexus-managed-paths.js'; +import { chunk } from '../lib/utils.js'; +import { logger } from './logger.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; + +/** Why the recorded coverage set could not be compared against disk at all. */ +export type IndexContentUnmeasurableReason = + /** Metadata predates per-file hashes, or the run recorded none (non-git). */ + | 'no-file-hashes' + /** The repository scan or hashing pass threw. */ + | 'scan-failed'; + +/** + * A three-way verdict. `'unmeasurable'` is kept apart from `'current'` on + * purpose: it means the comparison never ran, which is not evidence the index + * is fresh. Legacy metadata without hashes still falls back to the working-tree + * check; a failed scan must not. + */ +export type IndexContentDrift = + | { kind: 'current'; coveredFileCount: number } + | { kind: 'drifted'; changed: string[]; added: string[]; deleted: string[] } + | { kind: 'unmeasurable'; reason: IndexContentUnmeasurableReason }; + +export type IndexCoveragePolicy = NonNullable; + +const HASH_BATCH = 100; + +const collectUnreadablePaths = async ( + repoPath: string, + relPaths: readonly string[], +): Promise => { + const unreadable: string[] = []; + for (const batch of chunk(relPaths, HASH_BATCH)) { + await Promise.all( + batch.map(async (rel) => { + try { + await access(path.join(repoPath, rel), fsConstants.R_OK); + } catch { + unreadable.push(rel); + } + }), + ); + } + unreadable.sort(); + return unreadable; +}; + +/** + * Compare the files recorded in `fileHashes` against the current working tree. + * + * `added` covers files the index would pick up but has never seen, so a new + * source file still reports stale — the index is genuinely incomplete then, + * and comparing only the recorded entries would wave that through. + */ +export const detectIndexContentDrift = async ( + repoPath: string, + fileHashes: Readonly> | undefined, + coverage?: IndexCoveragePolicy, +): Promise => { + if (!fileHashes || Object.keys(fileHashes).length === 0) { + return { kind: 'unmeasurable', reason: 'no-file-hashes' }; + } + + // Excluded from BOTH sides, or GitNexus's own output guarantees a mismatch: + // analyze rewrites AGENTS.md/CLAUDE.md after recording hashes, so they read + // as `added` on a first run and `changed` on every run after that — a fresh + // index would report itself stale forever. + const recorded = Object.fromEntries( + Object.entries(fileHashes).filter(([rel]) => !isGitNexusManagedPath(rel)), + ); + if (Object.keys(recorded).length === 0) { + return { kind: 'unmeasurable', reason: 'no-file-hashes' }; + } + + try { + const scanned = await walkRepositoryPaths(repoPath, undefined, { + quiet: true, + maxFileSizeBytes: coverage?.maxFileSizeBytes, + }); + const scannedPaths = scanned.map((file) => file.path).filter((p) => !isGitNexusManagedPath(p)); + const scannedSet = new Set(scannedPaths); + const recordedSet = new Set(Object.keys(recorded)); + + // Legacy indexes have `fileHashes` but no `indexCoverage`. A later default + // cap would omit a still-present hashed file and call it deleted. Recorded + // paths that still exist stay in the coverage set even if this walk skipped + // them for size. + const recovered = new Set(); + for (const rel of recordedSet) { + if (scannedSet.has(rel)) continue; + try { + await access(path.join(repoPath, rel), fsConstants.R_OK); + recovered.add(rel); + scannedSet.add(rel); + } catch { + // Missing or unreadable: stays deleted / changed below. + } + } + + const added = scannedPaths.filter((p) => !recordedSet.has(p)).sort(); + const deleted = [...recordedSet].filter((p) => !scannedSet.has(p)).sort(); + const intersection = [...recordedSet].filter((p) => scannedSet.has(p)); + + const dirtyNow = listWorkingTreeDirtyPaths(repoPath); + const dirtyAtIndex = coverage?.dirtyPaths; + const dirtyNowSet = dirtyNow === null ? null : new Set(dirtyNow); + const dirtyAtIndexSet = dirtyAtIndex === undefined ? undefined : new Set(dirtyAtIndex); + const hashCandidates = + dirtyNowSet === null || dirtyAtIndexSet === undefined + ? intersection + : intersection.filter( + (p) => dirtyAtIndexSet.has(p) || dirtyNowSet.has(p) || recovered.has(p), + ); + + const hashCandidateSet = new Set(hashCandidates); + const skipHash = intersection.filter((p) => !hashCandidateSet.has(p)); + const unreadableFromAccess = await collectUnreadablePaths(repoPath, skipHash); + const unreadableSet = new Set(unreadableFromAccess); + const { hashes: hashed, unreadable: unreadableFromHash } = await computeFileHashesDetailed( + repoPath, + hashCandidates, + ); + for (const p of unreadableFromHash) unreadableSet.add(p); + const changed: string[] = []; + for (const p of intersection) { + if (unreadableSet.has(p)) { + changed.push(p); + continue; + } + const currentHash = hashed.get(p) ?? recorded[p]; + if (currentHash !== recorded[p]) changed.push(p); + } + changed.sort(); + + if (changed.length === 0 && added.length === 0 && deleted.length === 0) { + return { kind: 'current', coveredFileCount: scannedSet.size }; + } + return { kind: 'drifted', changed, added, deleted }; + } catch (err) { + logger.warn({ err, repoPath }, 'index content drift scan failed'); + return { kind: 'unmeasurable', reason: 'scan-failed' }; + } +}; diff --git a/gitnexus/src/core/index-freshness.ts b/gitnexus/src/core/index-freshness.ts index ea577f834..52e62e8d8 100644 --- a/gitnexus/src/core/index-freshness.ts +++ b/gitnexus/src/core/index-freshness.ts @@ -1,11 +1,14 @@ import { checkpointKind } from './embedding-checkpoint.js'; import type { RepoMeta } from '../storage/repo-manager.js'; +import { scopeExtractionFailureTotal } from './ingestion/scope-resolution/scope-extraction-failures.js'; export const INDEX_INCOMPLETE_REASONS = [ 'incremental-in-progress', 'embedding-checkpoint-pending', 'embedding-count-unverified', 'graph-write-collapsed', + 'scope-extraction-unverified', + 'scope-extraction-failed', ] as const; export type IndexIncompleteReason = (typeof INDEX_INCOMPLETE_REASONS)[number]; @@ -130,7 +133,14 @@ export function detectGraphWriteCollapse( /** Stable machine-readable reasons an index cannot be certified complete. */ export function getIndexIncompleteReasons( meta: - | Pick + | Pick< + RepoMeta, + | 'incrementalInProgress' + | 'embeddingCheckpoint' + | 'graphWriteCollapsed' + | 'scopeExtractionFailures' + | 'scopeExtractionReceipt' + > | null | undefined, ): IndexIncompleteReason[] { @@ -142,6 +152,13 @@ export function getIndexIncompleteReasons( // answers from a graph missing most of its edges, which is indistinguishable // from a codebase that genuinely has no such relationships. if (meta?.graphWriteCollapsed) reasons.push('graph-write-collapsed'); + if (meta?.scopeExtractionReceipt !== 1) { + reasons.push('scope-extraction-unverified'); + } else { + const total = scopeExtractionFailureTotal(meta.scopeExtractionFailures); + if (total === undefined) reasons.push('scope-extraction-unverified'); + else if (total > 0) reasons.push('scope-extraction-failed'); + } if (meta?.embeddingCheckpoint) { // The three checkpoint kinds are not one operator-facing state. GUARDRAILS // and the runbook document `embedding-checkpoint-pending` as "N node(s) diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 2a7f450d7..610280c4f 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -406,7 +406,9 @@ export function resolveRouteHandlerSymbols( httpMethod: string | null | undefined, symbolId: string | undefined, ) => { - if (!routePath) return; + // An empty path is a valid, pathless mapping and normalizes to either `/` + // or its class/router prefix. Only null means the extractor had no route. + if (routePath === null) return; const url = normalizeExtractedRoutePath(routePath, prefix); const key = routeNodeKey(normalizeRouteMethod(httpMethod), url); if (claimed.has(key)) return; // first-writer-wins: later same-key routes can't override diff --git a/gitnexus/src/core/ingestion/destination-key.ts b/gitnexus/src/core/ingestion/destination-key.ts new file mode 100644 index 000000000..31a76c307 --- /dev/null +++ b/gitnexus/src/core/ingestion/destination-key.ts @@ -0,0 +1,62 @@ +/** + * Shared destination-identity keying — the async counterpart of `routeNodeKey` + * in `route-extractors/route-path.ts`. + * + * Deliberately OUTSIDE `frameworks/spring/`, and for the same reason + * `routeNodeKey` sits outside the routes phase: the identity has to be mintable + * by anything that names a broker address, so a Node Kafka client or a Celery + * task queue can land on the very node a Spring publisher minted. A key that + * lived in the Spring module would force every other producer to import Spring, + * or — worse — let each one invent its own spelling, and two spellings of one + * address is precisely the missed connection this overlay exists to make. + * + * `broker` is a plain `string`, NOT the Spring `SpringDestinationBroker` union. + * Importing that union here is the dependency this module exists to avoid, and + * widening it costs nothing that matters: the union is a subtype of `string`, + * so a Spring caller passes its own values unchanged, while a future + * non-Spring caller stays free to attest to a broker Spring has no name for. + * The trade is real but small — this signature cannot reject a misspelled + * broker — and it is the same trade `routeNodeKey` makes by taking `method` as + * a `string` rather than an HTTP-verb union. Pure string logic, no + * dependencies. + */ + +/** + * The `Destination` node identity: `(broker, address)` when the broker is + * known, falling back to the address alone when it is not. + * + * The broker belongs IN the key, exactly as the HTTP verb belongs in + * `routeNodeKey`. `GET /x` and `POST /x` are two nodes, both fully joinable, + * and neither is punished for the other's existence; `kafka orders` and + * `rabbit orders` are two nodes on the same terms. A Kafka topic and a Rabbit + * queue that happen to share a name are two places, and one node for both would + * report a publisher and a subscriber as connected when nothing connects them. + * + * The known objection is that the broker is INFERRED — from a receiver's name, + * from an annotation table — so a wrong guess splits a pair that is really one. + * That is true and it is the cost. It is worth paying because the alternative + * tried first was worse: withdrawing the address from every site that named it + * split the pair even when the guess was RIGHT, since one unrelated third party + * writing the same word anywhere in the repository was enough to disconnect + * everybody on that spelling. Putting the broker in the key bounds the damage + * of a wrong guess to the one pair it was wrong about, instead of spreading it + * to every pair that shares an address with a stranger. + * + * ── THE ADDRESS-ONLY FALLBACK IS UNREACHABLE TODAY ────────────────────── + * + * `SpringDestinationCandidate.broker` is REQUIRED, and every annotation rule + * and every producer template supplies one, so no Spring caller can reach the + * `undefined` branch. It is written anyway, and on purpose: the parameter shape + * is the contract this module offers the next language, and the next language + * may well capture an address without being able to attest to a broker (a bare + * `queue.publish(name)` in a dynamic language, a binding that names only a + * channel). Degrading to address-only is the right answer there — silence about + * the broker is not a claim about it, and refusing to key such a site at all + * would lose a real destination over a value nobody disagreed about. + * + * Because the branch is dead, it is covered by testing THIS function directly + * rather than by a pipeline test staged to look as though a phase reached it. + */ +export function destinationNodeKey(broker: string | undefined, address: string): string { + return broker ? `${broker} ${address}` : address; +} diff --git a/gitnexus/src/core/ingestion/entry-point-scoring.ts b/gitnexus/src/core/ingestion/entry-point-scoring.ts index 58cf9389c..30a9c78c1 100644 --- a/gitnexus/src/core/ingestion/entry-point-scoring.ts +++ b/gitnexus/src/core/ingestion/entry-point-scoring.ts @@ -13,6 +13,7 @@ import { detectFrameworkFromPath } from './framework-detection.js'; import { SupportedLanguages } from 'gitnexus-shared'; import { providers } from './languages/index.js'; +import { isTestFilePath } from './utils/test-file-path.js'; // ============================================================================ // NAME PATTERNS @@ -164,54 +165,15 @@ export function calculateEntryPointScore( // ============================================================================ /** - * Check if a file path is a test file (should be excluded from entry points) - * Covers common test file patterns across all supported languages + * Check if a file path is a test file (should be excluded from entry points). + * + * Delegates to the shared predicate in `utils/test-file-path.ts`. This used to be + * a second, hand-maintained copy that had drifted from the one backing the MCP + * `includeTests` flag — see that module's header. Re-exported under this name so + * existing importers are unaffected. */ export function isTestFile(filePath: string): boolean { - const p = filePath.toLowerCase().replace(/\\/g, '/'); - - return ( - // JavaScript/TypeScript test patterns - p.includes('.test.') || - p.includes('.spec.') || - p.includes('__tests__/') || - p.includes('__mocks__/') || - // Generic test folders - p.includes('/test/') || - p.includes('/tests/') || - p.includes('/testing/') || - // Python test patterns - p.endsWith('_test.py') || - p.includes('/test_') || - // Go test patterns - p.endsWith('_test.go') || - // Java test patterns - p.includes('/src/test/') || - // Rust test patterns (inline tests are different, but test files) - p.includes('/tests/') || - // Swift/iOS test patterns - p.endsWith('tests.swift') || - p.endsWith('test.swift') || - p.includes('uitests/') || - // C# test patterns - p.endsWith('tests.cs') || - p.endsWith('test.cs') || - p.includes('.tests/') || - p.includes('.test/') || - p.includes('.integrationtests/') || - p.includes('.unittests/') || - p.includes('/testproject/') || - // PHP/Laravel test patterns - p.endsWith('test.php') || - p.endsWith('spec.php') || - p.includes('/tests/feature/') || - p.includes('/tests/unit/') || - // Ruby test patterns - p.endsWith('_spec.rb') || - p.endsWith('_test.rb') || - p.includes('/spec/') || - p.includes('/test/fixtures/') - ); + return isTestFilePath(filePath); } /** diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index fc65cda09..7e41c07c2 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -22,6 +22,27 @@ export interface FilePath { const READ_CONCURRENCY = 32; const ANALYZE_PROGRESS_ACTIVE_ENV = 'GITNEXUS_ANALYZE_PROGRESS_ACTIVE'; +const DECLARATION_COMPANION_SUFFIXES = [ + { declaration: '.d.ts', implementations: ['.ts', '.tsx'] }, + { declaration: '.d.mts', implementations: ['.mts'] }, + { declaration: '.d.cts', implementations: ['.cts'] }, +] as const; + +const hasImplementationSibling = ( + declarationPath: string, + scannedPaths: ReadonlySet, +): boolean => { + const companion = DECLARATION_COMPANION_SUFFIXES.find(({ declaration }) => + declarationPath.endsWith(declaration), + ); + if (!companion) return false; + + // Keep standalone declarations. Only suppress declaration output that sits + // beside an implementation with the corresponding module suffix. + const stem = declarationPath.slice(0, -companion.declaration.length); + return companion.implementations.some((suffix) => scannedPaths.has(`${stem}${suffix}`)); +}; + const warnLargeFileSkip = (message: string): void => { if (process.env[ANALYZE_PROGRESS_ACTIVE_ENV] === '1') { // analyze.ts routes console.warn through the progress bar logger while @@ -36,16 +57,49 @@ const warnLargeFileSkip = (message: string): void => { logger.warn(message); }; +export interface WalkRepositoryOptions { + /** + * Suppress the operator-facing large-file notice. Set by read-only callers + * such as `status`, which reuse this scan purely to learn which files the + * index covers and must not emit analyze's progress commentary. + */ + quiet?: boolean; + /** + * Override the large-file cap. `status` replays the bytes recorded at + * analyze time so `--max-file-size` / `GITNEXUS_MAX_FILE_SIZE` cannot + * silently drop a file that the index actually covers. + */ + maxFileSizeBytes?: number; +} + /** * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded. * Memory: ~10MB for 100K files vs ~1GB+ with content. */ +const assertWalkRootIsDirectory = async (repoPath: string): Promise => { + let st; + try { + st = await fs.stat(repoPath); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT' || code === 'ENOTDIR') { + throw new Error(`walkRepositoryPaths: path does not exist: ${repoPath}`); + } + throw err; + } + if (!st.isDirectory()) { + throw new Error(`walkRepositoryPaths: not a directory: ${repoPath}`); + } +}; + export const walkRepositoryPaths = async ( repoPath: string, onProgress?: (current: number, total: number, filePath: string) => void, + options: WalkRepositoryOptions = {}, ): Promise => { + await assertWalkRootIsDirectory(repoPath); const ignoreFilter = await createIgnoreFilter(repoPath); - const maxFileSizeBytes = getMaxFileSizeBytes(); + const maxFileSizeBytes = options.maxFileSizeBytes ?? getMaxFileSizeBytes(); const filtered = await glob('**/*', { cwd: repoPath, @@ -84,12 +138,19 @@ export const walkRepositoryPaths = async ( } } + const scannedPaths = new Set(entries.map((entry) => entry.path)); + const deduplicatedEntries = entries.filter( + (entry) => !hasImplementationSibling(entry.path, scannedPaths), + ); + // Filesystem/glob traversal order is not stable across filesystems or repeated // scans. Canonicalize once at the scan boundary so every downstream phase sees // the same repository order. - entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0)); + deduplicatedEntries.sort((left, right) => + left.path < right.path ? -1 : left.path > right.path ? 1 : 0, + ); - if (skippedLarge > 0) { + if (skippedLarge > 0 && !options.quiet) { const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; const isOverrideUnset = !process.env.GITNEXUS_MAX_FILE_SIZE; const suffix = isDefault ? ', likely generated/vendored' : ''; @@ -123,7 +184,7 @@ export const walkRepositoryPaths = async ( } } - return entries; + return deduplicatedEntries; }; /** diff --git a/gitnexus/src/core/ingestion/frameworks/spring/actuator-runtime.ts b/gitnexus/src/core/ingestion/frameworks/spring/actuator-runtime.ts new file mode 100644 index 000000000..4af7073cb --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/actuator-runtime.ts @@ -0,0 +1,968 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { GraphNode } from 'gitnexus-shared'; +import { generateId } from '../../../../lib/utils.js'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import { SPRING_DI_PROVIDER_PROPERTY } from '../../di-extractors/spring.js'; +import { + normalizeExtractedRoutePath, + normalizeRouteMethod, + routeNodeKey, +} from '../../route-extractors/route-path.js'; +import { stripBidiAndZeroWidth } from '../../utils/ast-helpers.js'; +import { SPRING_CONFIG_DESCRIPTION } from './config-bindings.js'; +import { getProviderForFile } from '../../languages/index.js'; +import type { RuntimeCallableIdentity } from '../../language-provider.js'; + +export const ACTUATOR_ENDPOINTS = [ + 'mappings', + 'beans', + 'conditions', + 'configprops', + 'env', +] as const; +type ActuatorEndpoint = (typeof ACTUATOR_ENDPOINTS)[number]; + +const MAX_ACTUATOR_PAYLOAD_BYTES = 16 * 1024 * 1024; +export const MAX_RUNTIME_RECORDS = 50_000; +const MAX_RUNTIME_DEPTH = 64; +const RUNTIME_FILE_PREFIX = 'spring-actuator:'; + +type JsonObject = Record; + +export interface SpringActuatorImportStats { + readonly payloads: number; + readonly mappings: number; + readonly beans: number; + readonly conditions: number; + readonly configProperties: number; + readonly environmentProperties: number; + /** Endpoint categories that exceeded the bounded import size. */ + readonly truncatedEndpoints: readonly ActuatorEndpoint[]; +} + +interface MutableImportStats { + payloads: number; + mappings: number; + beans: number; + conditions: number; + configProperties: number; + environmentProperties: number; + truncatedEndpoints: ActuatorEndpoint[]; +} + +interface ImportResult { + readonly count: number; + readonly truncated: boolean; +} + +export class SpringActuatorImportError extends Error { + constructor(message: string) { + super(message); + this.name = 'SpringActuatorImportError'; + } +} + +function objectValue(value: unknown): JsonObject | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as JsonObject) + : undefined; +} + +function safeText(value: unknown, maxLength = 1024): string | undefined { + if (typeof value !== 'string') return undefined; + const sanitized = stripBidiAndZeroWidth(value) + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + return sanitized.length === 0 ? undefined : sanitized.slice(0, maxLength); +} + +function safeStrings(value: unknown, limit = 100): string[] { + if (!Array.isArray(value)) return []; + const strings: string[] = []; + for (const item of value.slice(0, limit)) { + const text = safeText(item); + if (text !== undefined) strings.push(text); + } + return strings; +} + +async function readPayloadFile(filePath: string, label: string): Promise { + // Size gate and read share one handle so both observe the same inode. + // Re-resolving the path for the read would let a swapped file bypass the + // payload cap (CodeQL js/file-system-race). + let handle: Awaited> | undefined; + let raw: string; + try { + handle = await fs.open(filePath, 'r'); + const stat = await handle.stat(); + if (!stat.isFile()) { + throw new SpringActuatorImportError(`Spring Actuator ${label} input must be a JSON file.`); + } + if (stat.size > MAX_ACTUATOR_PAYLOAD_BYTES) { + throw new SpringActuatorImportError( + `Spring Actuator ${label} payload exceeds the ${MAX_ACTUATOR_PAYLOAD_BYTES / 1024 / 1024} MiB limit.`, + ); + } + const buffer = Buffer.alloc(MAX_ACTUATOR_PAYLOAD_BYTES + 1); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const result = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + if (bytesRead > MAX_ACTUATOR_PAYLOAD_BYTES) { + throw new SpringActuatorImportError( + `Spring Actuator ${label} payload exceeds the ${MAX_ACTUATOR_PAYLOAD_BYTES / 1024 / 1024} MiB limit.`, + ); + } + raw = buffer.subarray(0, bytesRead).toString('utf8'); + } catch (err) { + if (err instanceof SpringActuatorImportError) throw err; + throw new SpringActuatorImportError(`Spring Actuator ${label} input could not be read.`); + } finally { + await handle?.close().catch(() => {}); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + // Do not include JSON.parse's message: newer runtimes may quote source text, + // which could disclose an env/configprops value in CLI output. + throw new SpringActuatorImportError(`Spring Actuator ${label} payload is not valid JSON.`); + } + const object = objectValue(parsed); + if (object === undefined) { + throw new SpringActuatorImportError(`Spring Actuator ${label} payload must be a JSON object.`); + } + return object; +} + +async function loadPayloads( + repoPath: string, + configuredPath: string, +): Promise> { + const inputPath = path.resolve(repoPath, configuredPath); + let stat; + try { + stat = await fs.stat(inputPath); + } catch { + throw new SpringActuatorImportError( + 'Spring Actuator input path does not exist or is unreadable.', + ); + } + + const payloads = new Map(); + if (stat.isDirectory()) { + for (const endpoint of ACTUATOR_ENDPOINTS) { + const filePath = path.join(inputPath, `${endpoint}.json`); + try { + const endpointStat = await fs.stat(filePath); + if (!endpointStat.isFile()) continue; + } catch { + continue; + } + payloads.set(endpoint, await readPayloadFile(filePath, endpoint)); + } + } else if (stat.isFile()) { + const parsed = await readPayloadFile(inputPath, 'bundle'); + const endpointFromName = ACTUATOR_ENDPOINTS.find( + (endpoint) => path.basename(inputPath).toLowerCase() === `${endpoint}.json`, + ); + if (endpointFromName !== undefined) { + payloads.set(endpointFromName, parsed); + } else { + for (const endpoint of ACTUATOR_ENDPOINTS) { + const payload = objectValue(parsed[endpoint]); + if (payload !== undefined) payloads.set(endpoint, payload); + } + } + } else { + throw new SpringActuatorImportError( + 'Spring Actuator input must be a JSON bundle or a directory of endpoint JSON files.', + ); + } + + if (payloads.size === 0) { + throw new SpringActuatorImportError( + 'Spring Actuator input contains none of mappings, beans, conditions, configprops, or env.', + ); + } + return payloads; +} + +function evidenceFile(graph: KnowledgeGraph, endpoint: ActuatorEndpoint): GraphNode { + const filePath = `${RUNTIME_FILE_PREFIX}${endpoint}`; + const id = generateId('File', filePath); + const existing = graph.getNode(id); + if (existing !== undefined) return existing; + const node: GraphNode = { + id, + label: 'File', + properties: { name: `${endpoint}.json`, filePath }, + }; + graph.addNode(node); + return node; +} + +function appendRuntimeMarker(node: GraphNode, marker: string): void { + const current = + typeof node.properties.description === 'string' ? node.properties.description : ''; + if (current.includes(marker)) return; + node.properties.description = current.length === 0 ? marker : `${current}; ${marker}`; +} + +function markRuntimeEvidence( + graph: KnowledgeGraph, + endpoint: ActuatorEndpoint, + target: GraphNode, + status: string = 'runtime-confirmed', + confirmed: boolean = true, +): void { + // Only Route declares structured runtime columns in the persisted schema. + // Other labels retain the same evidence durably through their description + // plus the DECLARES edge below; setting undeclared properties would make the + // in-memory graph promise data that CSV/LadybugDB silently drops. + if (target.label === 'Route') { + // Confirmation is conflict-dominant. Once any runtime observation + // disagrees with static or runtime ownership, a later duplicate must not + // restore authoritative status. + target.properties.runtimeConfirmed = + target.properties.runtimeConfirmed === false ? false : confirmed; + // Source records provenance, not authority. Consumers MUST use + // runtimeConfirmed === true before treating runtime evidence as confirmed. + target.properties.runtimeSource = 'spring-actuator'; + const previousStatus = safeText(target.properties.runtimeStatus); + target.properties.runtimeStatus = [...new Set([...(previousStatus?.split(',') ?? []), status])] + .sort() + .join(','); + } + const marker = `Spring Actuator ${endpoint} ${status}`; + appendRuntimeMarker(target, marker); + + const evidence = evidenceFile(graph, endpoint); + graph.addRelationship({ + id: generateId('DECLARES', `${evidence.id}->${target.id}:${status}`), + sourceId: evidence.id, + targetId: target.id, + type: 'DECLARES', + confidence: 1, + reason: `spring-actuator:${endpoint}:${status}`, + }); +} + +function normalizedQualifiedName(value: string): string { + return value + .replace(/\$\$(?:SpringCGLIB|EnhancerBySpringCGLIB|FastClassBySpringCGLIB).*$/, '') + .replaceAll('$', '.'); +} + +function uniqueIndexAdd(index: Map, key: string, node: GraphNode): void { + const existing = index.get(key); + if (existing === undefined) index.set(key, node); + else if (existing !== null && existing.id !== node.id) index.set(key, null); +} + +interface RuntimeNodeIndexes { + readonly classesByQualifiedName: Map; + readonly classesByRuntimeAlias: Map; + readonly classesBySimpleName: Map; + readonly beanProvidersByName: Map; + readonly methodsByOwnerId: Map; + readonly callablesByRuntimeOwner: Map; + readonly routeOwnerFileIdsByRouteId: Map>; +} + +function addRuntimeCallable( + index: Map, + ownerName: string, + node: GraphNode, +): void { + const normalizedOwner = normalizedQualifiedName(ownerName); + const nodes = index.get(normalizedOwner) ?? []; + if (!nodes.some((candidate) => candidate.id === node.id)) nodes.push(node); + index.set(normalizedOwner, nodes); +} + +function buildRuntimeNodeIndexes(graph: KnowledgeGraph): RuntimeNodeIndexes { + const allNodes = [...graph.iterNodes()]; + const classesByQualifiedName = new Map(); + const classesByRuntimeAlias = new Map(); + const classesBySimpleName = new Map(); + const beanProvidersByName = new Map(); + const nodesById = new Map(allNodes.map((node) => [node.id, node])); + const methodsByOwnerId = new Map(); + const callablesByRuntimeOwner = new Map(); + const routeOwnerFileIdsByRouteId = new Map>(); + for (const node of allNodes) { + if (node.label === 'Class' || node.label === 'Record') { + const qualified = safeText(node.properties.qualifiedName); + if (qualified !== undefined) { + uniqueIndexAdd(classesByQualifiedName, normalizedQualifiedName(qualified), node); + } + uniqueIndexAdd(classesBySimpleName, String(node.properties.name), node); + } + const provider = objectValue(node.properties[SPRING_DI_PROVIDER_PROPERTY]); + for (const name of safeStrings(provider?.names)) + uniqueIndexAdd(beanProvidersByName, name, node); + } + const ownedNodeIds = new Set(); + for (const relationshipType of ['HAS_METHOD', 'HAS_PROPERTY'] as const) { + for (const relationship of graph.iterRelationshipsByType(relationshipType)) { + const member = nodesById.get(relationship.targetId); + const owner = nodesById.get(relationship.sourceId); + if ( + member === undefined || + !['Method', 'Function', 'Property'].includes(member.label) || + owner === undefined + ) { + continue; + } + ownedNodeIds.add(member.id); + if (member.label === 'Method' || member.label === 'Function') { + const methods = methodsByOwnerId.get(relationship.sourceId) ?? []; + methods.push(member); + methodsByOwnerId.set(relationship.sourceId, methods); + } + const ownerQualifiedName = safeText(owner.properties.qualifiedName); + if (ownerQualifiedName !== undefined) { + addRuntimeCallable(callablesByRuntimeOwner, ownerQualifiedName, member); + } + const strategy = getProviderForFile( + String(member.properties.filePath), + )?.runtimeSymbolStrategy; + for (const alias of strategy?.callableOwnerAliases?.(member, owner) ?? []) { + addRuntimeCallable(callablesByRuntimeOwner, alias, member); + if ( + (owner.label === 'Class' || owner.label === 'Record') && + ownerQualifiedName !== undefined && + normalizedQualifiedName(alias) !== normalizedQualifiedName(ownerQualifiedName) + ) { + uniqueIndexAdd(classesByRuntimeAlias, normalizedQualifiedName(alias), owner); + } + } + } + } + for (const node of allNodes) { + if ( + ownedNodeIds.has(node.id) || + (node.label !== 'Function' && node.label !== 'Method' && node.label !== 'Property') + ) { + continue; + } + const strategy = getProviderForFile(String(node.properties.filePath))?.runtimeSymbolStrategy; + for (const alias of strategy?.callableOwnerAliases?.(node, undefined) ?? []) { + addRuntimeCallable(callablesByRuntimeOwner, alias, node); + } + } + for (const relationship of graph.iterRelationshipsByType('HANDLES_ROUTE')) { + const owners = routeOwnerFileIdsByRouteId.get(relationship.targetId) ?? new Set(); + owners.add(relationship.sourceId); + routeOwnerFileIdsByRouteId.set(relationship.targetId, owners); + } + return { + classesByQualifiedName, + classesByRuntimeAlias, + classesBySimpleName, + beanProvidersByName, + methodsByOwnerId, + callablesByRuntimeOwner, + routeOwnerFileIdsByRouteId, + }; +} + +function resolveClass( + indexes: RuntimeNodeIndexes, + rawType: string | undefined, +): GraphNode | undefined { + if (rawType === undefined) return undefined; + const type = normalizedQualifiedName(rawType.replace(/\[\]$/, '')); + const exact = indexes.classesByQualifiedName.get(type); + if (exact !== null && exact !== undefined) return exact; + const alias = indexes.classesByRuntimeAlias.get(type); + if (alias !== null && alias !== undefined) return alias; + // A qualified runtime name is authoritative. Falling back to a unique class + // with the same simple name can bind a stale snapshot to a different package + // and then mint confidence-1 handler evidence for the wrong source. + if (type.includes('.')) return undefined; + const simple = type.slice(type.lastIndexOf('.') + 1); + const fallback = indexes.classesBySimpleName.get(simple); + return fallback === null ? undefined : fallback; +} + +function providerMatchesRuntimeType( + indexes: RuntimeNodeIndexes, + providerNode: GraphNode, + runtimeType: string | undefined, +): boolean { + if (runtimeType === undefined) return true; + const provider = objectValue(providerNode.properties[SPRING_DI_PROVIDER_PROPERTY]); + const providerType = + safeText(provider?.providedTypeName) ?? + (providerNode.label === 'Class' || providerNode.label === 'Record' + ? safeText(providerNode.properties.qualifiedName) + : undefined); + if (providerType === undefined) return true; + + const providerClass = resolveClass(indexes, providerType); + const runtimeClass = resolveClass(indexes, runtimeType); + if (providerClass !== undefined && runtimeClass !== undefined) { + return providerClass.id === runtimeClass.id; + } + + const normalizedProvider = normalizedQualifiedName(providerType); + const normalizedRuntime = normalizedQualifiedName(runtimeType); + if (normalizedProvider.includes('.')) return normalizedProvider === normalizedRuntime; + return normalizedProvider === normalizedRuntime.slice(normalizedRuntime.lastIndexOf('.') + 1); +} + +function descriptorParameterTypes(descriptor: string | undefined): string[] | undefined { + if (descriptor === undefined || descriptor.charAt(0) !== '(') return undefined; + const types: string[] = []; + for (let index = 1; index < descriptor.length && descriptor.charAt(index) !== ')'; ) { + let arrayDimensions = 0; + while (descriptor.charAt(index) === '[') { + arrayDimensions++; + index++; + } + const arraySuffix = '[]'.repeat(arrayDimensions); + if (descriptor.charAt(index) === 'L') { + const end = descriptor.indexOf(';', index); + if (end === -1) return undefined; + types.push(`${descriptor.slice(index + 1, end)}${arraySuffix}`); + index = end + 1; + } else { + const primitive = descriptor.charAt(index); + if (!'BCDFIJSZ'.includes(primitive)) return undefined; + types.push(`${primitive}${arraySuffix}`); + index++; + } + } + return descriptor.includes(')') ? types : undefined; +} + +function matchesRuntimeCallable(node: GraphNode, runtime: RuntimeCallableIdentity): boolean { + const strategy = getProviderForFile(String(node.properties.filePath))?.runtimeSymbolStrategy; + if (strategy !== undefined) return strategy.matchesCallable(node, runtime); + return ( + (node.label === 'Method' || node.label === 'Function') && + node.properties.name === runtime.name && + (runtime.descriptorParameterTypes === undefined || + node.properties.parameterCount === runtime.descriptorParameterTypes.length) + ); +} + +function resolveHandlerNode( + indexes: RuntimeNodeIndexes, + handlerMethod: JsonObject | undefined, +): GraphNode | undefined { + const className = safeText(handlerMethod?.className); + const methodName = safeText(handlerMethod?.name); + if (methodName === undefined) return resolveClass(indexes, className); + if (className === undefined) return undefined; + const owner = resolveClass(indexes, className); + const runtime: RuntimeCallableIdentity = { + name: methodName, + descriptorParameterTypes: descriptorParameterTypes(safeText(handlerMethod?.descriptor)), + }; + const ownerCandidates = owner === undefined ? [] : (indexes.methodsByOwnerId.get(owner.id) ?? []); + const aliasCandidates = + indexes.callablesByRuntimeOwner.get(normalizedQualifiedName(className)) ?? []; + const candidates = [...ownerCandidates, ...aliasCandidates] + .filter((node, index, all) => all.findIndex((candidate) => candidate.id === node.id) === index) + .filter((node) => matchesRuntimeCallable(node, runtime)); + return candidates.length === 1 ? candidates[0] : undefined; +} + +function predicateParts(predicate: string | undefined): { + readonly methods: string[]; + readonly patterns: string[]; +} { + if (predicate === undefined) return { methods: [], patterns: [] }; + const methodListEnd = predicate.indexOf('['); + const methodRegion = methodListEnd === -1 ? predicate : predicate.slice(0, methodListEnd); + const methods = [ + ...methodRegion.matchAll(/\b(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS|TRACE|CONNECT)\b/g), + ] + .map((match) => match[1]) + .filter((method): method is string => method !== undefined); + const patterns = [...predicate.matchAll(/(?:^|[\s[(])((?:\/)[^\s\]),}]+)/g)] + .map((match) => safeText(match[1])) + .filter((pattern): pattern is string => pattern !== undefined); + return { methods: [...new Set(methods)], patterns: [...new Set(patterns)] }; +} + +function mappingEntries(payload: JsonObject): { + entries: JsonObject[]; + truncated: boolean; +} { + const entries: JsonObject[] = []; + const contexts = objectValue(payload.contexts); + if (contexts === undefined) return { entries, truncated: false }; + for (const context of Object.values(contexts)) { + const mappings = objectValue(objectValue(context)?.mappings); + if (mappings === undefined) continue; + for (const groupName of ['dispatcherServlets', 'dispatcherHandlers']) { + const groups = objectValue(mappings[groupName]); + if (groups === undefined) continue; + for (const group of Object.values(groups)) { + if (!Array.isArray(group)) continue; + for (const entry of group) { + const object = objectValue(entry); + if (object !== undefined) entries.push(object); + if (entries.length > MAX_RUNTIME_RECORDS) { + entries.pop(); + return { entries, truncated: true }; + } + } + } + } + } + return { entries, truncated: false }; +} + +interface RuntimeMappingCandidate { + readonly key: string; + readonly method: string | undefined; + readonly url: string; + readonly handler: GraphNode | undefined; +} + +function importMappings( + graph: KnowledgeGraph, + payload: JsonObject, + indexes: RuntimeNodeIndexes, +): ImportResult { + let imported = 0; + const payloadEntries = mappingEntries(payload); + let truncated = payloadEntries.truncated; + const candidatesByKey = new Map(); + for (const entry of payloadEntries.entries) { + const details = objectValue(entry.details); + const conditions = objectValue(details?.requestMappingConditions); + const predicate = predicateParts(safeText(entry.predicate)); + const patterns = safeStrings(conditions?.patterns); + const methods = safeStrings(conditions?.methods) + .map(normalizeRouteMethod) + .filter((method): method is string => method !== undefined); + const effectivePatterns = patterns.length > 0 ? patterns : predicate.patterns; + const effectiveMethods = methods.length > 0 ? methods : predicate.methods; + if (effectivePatterns.length === 0) continue; + + const handler = resolveHandlerNode(indexes, objectValue(details?.handlerMethod)); + for (const rawPattern of effectivePatterns) { + const url = normalizeExtractedRoutePath(rawPattern, null); + for (const method of effectiveMethods.length > 0 ? effectiveMethods : [undefined]) { + const normalizedMethod = normalizeRouteMethod(method); + const key = routeNodeKey(normalizedMethod, url); + const candidate = { key, method: normalizedMethod, url, handler }; + const existing = candidatesByKey.get(key); + if (existing === undefined) { + if (candidatesByKey.size >= MAX_RUNTIME_RECORDS) { + truncated = true; + continue; + } + candidatesByKey.set(key, [candidate]); + } else { + existing.push(candidate); + } + } + } + } + + for (const candidates of candidatesByKey.values()) { + const first = candidates[0]; + if (first === undefined) continue; + const { key, method: normalizedMethod, url } = first; + const resolvedHandlers = new Map( + candidates + .map((candidate) => candidate.handler) + .filter((handler): handler is GraphNode => handler !== undefined) + .map((handler) => [handler.id, handler]), + ); + const runtimeHandlerConflict = resolvedHandlers.size > 1; + const handler = runtimeHandlerConflict ? undefined : resolvedHandlers.values().next().value; + const exactId = generateId('Route', key); + const fallbackId = generateId('Route', url); + let route = graph.getNode(exactId) ?? graph.getNode(fallbackId); + if (route?.label !== 'Route') route = undefined; + const routeWasPresent = route !== undefined; + if (route === undefined) { + route = { + id: exactId, + label: 'Route', + properties: { + name: url, + filePath: handler?.properties.filePath ?? `${RUNTIME_FILE_PREFIX}mappings`, + ...(normalizedMethod === undefined ? {} : { method: normalizedMethod }), + ...(handler === undefined ? {} : { handlerSymbolId: handler.id }), + }, + }; + graph.addNode(route); + } + const existingHandlerId = safeText(route.properties.handlerSymbolId); + const handlerFilePath = + handler !== undefined && typeof handler.properties.filePath === 'string' + ? handler.properties.filePath + : undefined; + const handlerFileId = + handlerFilePath === undefined ? undefined : generateId('File', handlerFilePath); + const staticOwnerFileIds = indexes.routeOwnerFileIdsByRouteId.get(route.id); + const conflictsWithStaticOwner = + routeWasPresent && + handlerFileId !== undefined && + staticOwnerFileIds !== undefined && + [...staticOwnerFileIds].some((ownerFileId) => ownerFileId !== handlerFileId); + if ( + runtimeHandlerConflict || + (handler !== undefined && + ((existingHandlerId !== undefined && existingHandlerId !== handler.id) || + conflictsWithStaticOwner)) + ) { + // Static ownership and runtime ownership disagree. Preserve the + // static handler, persist an explicit conflict, and do not mint an + // authoritative HANDLES_ROUTE edge from the runtime candidate. + markRuntimeEvidence(graph, 'mappings', route, 'handler-conflict', false); + imported++; + continue; + } + if (handler !== undefined && existingHandlerId === undefined) { + route.properties.handlerSymbolId = handler.id; + } + markRuntimeEvidence(graph, 'mappings', route); + if (handler !== undefined && handlerFileId !== undefined) { + if (graph.getNode(handlerFileId) !== undefined) { + graph.addRelationship({ + id: generateId('HANDLES_ROUTE', `${handlerFileId}->${route.id}`), + sourceId: handlerFileId, + targetId: route.id, + type: 'HANDLES_ROUTE', + confidence: 1, + reason: 'spring-actuator:runtime-confirmed', + }); + } + } + imported++; + } + return { count: imported, truncated }; +} + +function contextObjects(payload: JsonObject): JsonObject[] { + const contexts = objectValue(payload.contexts); + if (contexts === undefined) return []; + return Object.values(contexts) + .map(objectValue) + .filter((context): context is JsonObject => context !== undefined); +} + +function importBeans( + graph: KnowledgeGraph, + payload: JsonObject, + indexes: RuntimeNodeIndexes, +): ImportResult { + let imported = 0; + const seen = new Set(); + for (const [contextIndex, context] of contextObjects(payload).entries()) { + const beans = objectValue(context.beans); + if (beans === undefined) continue; + for (const [rawBeanName, rawBean] of Object.entries(beans)) { + if (imported >= MAX_RUNTIME_RECORDS) return { count: imported, truncated: true }; + const beanName = safeText(rawBeanName, 512); + const bean = objectValue(rawBean); + if (beanName === undefined || bean === undefined) continue; + const identity = `${contextIndex}:${beanName}`; + if (seen.has(identity)) continue; + seen.add(identity); + const type = safeText(bean.type, 1024); + const named = indexes.beanProvidersByName.get(beanName); + let target = named === null ? undefined : named; + if (target !== undefined && !providerMatchesRuntimeType(indexes, target, type)) { + target = undefined; + } + target ??= resolveClass(indexes, type); + if (target === undefined) { + const id = generateId('CodeElement', `spring-runtime-bean:${identity}`); + target = graph.getNode(id); + if (target === undefined) { + const scope = safeText(bean.scope, 128); + target = { + id, + label: 'CodeElement', + properties: { + name: beanName, + filePath: `${RUNTIME_FILE_PREFIX}beans`, + description: + `Spring runtime Bean ${beanName}` + + (type === undefined ? '' : ` of type ${type}`) + + (scope === undefined ? '' : ` (${scope})`), + ...(type === undefined ? {} : { qualifiedName: normalizedQualifiedName(type) }), + }, + }; + graph.addNode(target); + } + } + markRuntimeEvidence(graph, 'beans', target); + imported++; + } + } + return { count: imported, truncated: false }; +} + +function resolveConditionOwner( + indexes: RuntimeNodeIndexes, + rawName: string, +): GraphNode | undefined { + const separator = rawName.lastIndexOf('#'); + return resolveHandlerNode(indexes, { + className: separator === -1 ? rawName : rawName.slice(0, separator), + ...(separator === -1 ? {} : { name: rawName.slice(separator + 1) }), + }); +} + +function importConditions( + graph: KnowledgeGraph, + payload: JsonObject, + indexes: RuntimeNodeIndexes, +): ImportResult { + let imported = 0; + const seen = new Set(); + for (const context of contextObjects(payload)) { + for (const [field, status] of [ + ['positiveMatches', 'matched'], + ['negativeMatches', 'not-matched'], + ] as const) { + const matches = objectValue(context[field]); + if (matches === undefined) continue; + for (const rawName of Object.keys(matches)) { + if (imported >= MAX_RUNTIME_RECORDS) return { count: imported, truncated: true }; + const name = safeText(rawName); + if (name === undefined || seen.has(`${status}:${name}`)) continue; + seen.add(`${status}:${name}`); + const owner = resolveConditionOwner(indexes, name); + if (owner === undefined) continue; + // Actuator reports this status for the aggregate owner entry. Its child + // details may contain a mix of matched and not-matched conditions, but + // do not carry a stable identifier that maps to our CONDITIONAL_ON + // targets. Keep the aggregate on the owner instead of guessing. + markRuntimeEvidence(graph, 'conditions', owner, status); + imported++; + } + } + } + return { count: imported, truncated: false }; +} + +function relaxedPropertyName(value: string): string { + return value.toLowerCase().replace(/[-_.\[\]]/g, ''); +} + +interface RuntimePropertyIndex { + readonly exact: Map; + readonly relaxed: Map; +} + +function buildRuntimePropertyIndex(graph: KnowledgeGraph): RuntimePropertyIndex { + const exact = new Map(); + const relaxed = new Map(); + for (const node of graph.iterNodes()) { + if (node.label !== 'Property') continue; + const description = safeText(node.properties.description) ?? ''; + if (!description.startsWith(SPRING_CONFIG_DESCRIPTION) && !node.id.includes('spring-runtime')) { + continue; + } + const name = String(node.properties.name); + uniqueIndexAdd(exact, name, node); + uniqueIndexAdd(relaxed, relaxedPropertyName(name), node); + } + return { exact, relaxed }; +} + +function ensureRuntimeProperty( + graph: KnowledgeGraph, + index: RuntimePropertyIndex, + endpoint: 'configprops' | 'env', + rawName: string, +): GraphNode | undefined { + const name = safeText(rawName, 1024); + if (name === undefined) return undefined; + const exact = index.exact.get(name); + let node = exact === null ? undefined : exact; + if (node === undefined) { + const relaxed = index.relaxed.get(relaxedPropertyName(name)); + node = relaxed === null ? undefined : relaxed; + } + if (node === undefined) { + const id = generateId('Property', `spring-runtime-config:${name}`); + node = graph.getNode(id); + if (node === undefined) { + node = { + id, + label: 'Property', + properties: { + name, + filePath: `${RUNTIME_FILE_PREFIX}${endpoint}`, + description: `${SPRING_CONFIG_DESCRIPTION}; imported from Spring Actuator ${endpoint}`, + }, + }; + graph.addNode(node); + } + uniqueIndexAdd(index.exact, name, node); + uniqueIndexAdd(index.relaxed, relaxedPropertyName(name), node); + } + markRuntimeEvidence(graph, endpoint, node); + return node; +} + +function configInputPaths(inputs: unknown): { + paths: string[]; + truncated: boolean; +} { + const out: string[] = []; + const stack: Array<{ value: unknown; prefix: string; depth: number }> = [ + { value: inputs, prefix: '', depth: 0 }, + ]; + while (stack.length > 0 && out.length < MAX_RUNTIME_RECORDS) { + const current = stack.pop(); + if (current === undefined || current.depth > MAX_RUNTIME_DEPTH) continue; + const object = objectValue(current.value); + if (object === undefined) { + if (current.prefix.length > 0) out.push(current.prefix); + continue; + } + const keys = Object.keys(object); + const metadataLeaf = + keys.length === 0 || keys.every((key) => key === 'value' || key === 'origin'); + if (metadataLeaf) { + if (current.prefix.length > 0) out.push(current.prefix); + continue; + } + for (let index = keys.length - 1; index >= 0; index--) { + const rawKey = keys[index]; + if (rawKey === undefined) continue; + const key = safeText(rawKey, 256); + if (key === undefined) continue; + stack.push({ + value: object[rawKey], + prefix: current.prefix.length === 0 ? key : `${current.prefix}.${key}`, + depth: current.depth + 1, + }); + } + } + return { paths: out, truncated: stack.length > 0 }; +} + +function importConfigProperties( + graph: KnowledgeGraph, + payload: JsonObject, + propertyIndex: RuntimePropertyIndex, +): ImportResult { + let imported = 0; + let truncated = false; + const seen = new Set(); + for (const context of contextObjects(payload)) { + const beans = objectValue(context.beans); + if (beans === undefined) continue; + for (const rawBean of Object.values(beans)) { + const bean = objectValue(rawBean); + const prefix = safeText(bean?.prefix, 512)?.replace(/\.+$/, ''); + if (bean === undefined || prefix === undefined) continue; + const inputPaths = configInputPaths(bean.inputs); + truncated ||= inputPaths.truncated; + const names = + inputPaths.paths.length === 0 + ? [prefix] + : inputPaths.paths.map((entry) => `${prefix}.${entry}`); + for (const name of names) { + if (imported >= MAX_RUNTIME_RECORDS) return { count: imported, truncated: true }; + if (seen.has(name)) continue; + seen.add(name); + if (ensureRuntimeProperty(graph, propertyIndex, 'configprops', name) !== undefined) + imported++; + } + } + } + return { count: imported, truncated }; +} + +function importEnvironmentProperties( + graph: KnowledgeGraph, + payload: JsonObject, + propertyIndex: RuntimePropertyIndex, +): ImportResult { + let imported = 0; + const seen = new Set(); + if (!Array.isArray(payload.propertySources)) return { count: imported, truncated: false }; + for (const rawSource of payload.propertySources) { + const properties = objectValue(objectValue(rawSource)?.properties); + if (properties === undefined) continue; + // Deliberately enumerate keys only. Never read, retain, interpolate, or log + // the corresponding {value, origin} objects. + for (const rawName of Object.keys(properties)) { + if (imported >= MAX_RUNTIME_RECORDS) return { count: imported, truncated: true }; + const name = safeText(rawName, 1024); + if (name === undefined || seen.has(name)) continue; + seen.add(name); + if (ensureRuntimeProperty(graph, propertyIndex, 'env', name) !== undefined) imported++; + } + } + return { count: imported, truncated: false }; +} + +/** + * Import explicitly supplied Spring Boot Actuator snapshots. Runtime evidence + * is additive: it confirms existing static nodes where possible and creates + * conservative synthetic Route/Bean/Property nodes otherwise. Raw payloads, + * condition messages, config values, env values, origins, and source names are + * never copied into graph properties or logs. + */ +export async function importSpringActuatorRuntime( + graph: KnowledgeGraph, + repoPath: string, + configuredPath: string, +): Promise { + const payloads = await loadPayloads(repoPath, configuredPath); + const stats: MutableImportStats = { + payloads: payloads.size, + mappings: 0, + beans: 0, + conditions: 0, + configProperties: 0, + environmentProperties: 0, + truncatedEndpoints: [], + }; + const indexes = buildRuntimeNodeIndexes(graph); + const propertyIndex = buildRuntimePropertyIndex(graph); + + const mappings = payloads.get('mappings'); + if (mappings !== undefined) { + const result = importMappings(graph, mappings, indexes); + stats.mappings = result.count; + if (result.truncated) stats.truncatedEndpoints.push('mappings'); + } + const beans = payloads.get('beans'); + if (beans !== undefined) { + const result = importBeans(graph, beans, indexes); + stats.beans = result.count; + if (result.truncated) stats.truncatedEndpoints.push('beans'); + } + const conditions = payloads.get('conditions'); + if (conditions !== undefined) { + const result = importConditions(graph, conditions, indexes); + stats.conditions = result.count; + if (result.truncated) stats.truncatedEndpoints.push('conditions'); + } + const configprops = payloads.get('configprops'); + if (configprops !== undefined) { + const result = importConfigProperties(graph, configprops, propertyIndex); + stats.configProperties = result.count; + if (result.truncated) stats.truncatedEndpoints.push('configprops'); + } + const env = payloads.get('env'); + if (env !== undefined) { + const result = importEnvironmentProperties(graph, env, propertyIndex); + stats.environmentProperties = result.count; + if (result.truncated) stats.truncatedEndpoints.push('env'); + } + return stats; +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts b/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts index 928f3e805..9754d40c8 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/annotation-arguments.ts @@ -124,7 +124,13 @@ export function parseSpringAnnotationArguments( const body = annotationText.slice(open + 1, close).trim(); if (body.length === 0) return []; const rawArguments = splitTopLevel(body, ','); - if (rawArguments === null || rawArguments.some((argument) => argument.length === 0)) return null; + if (rawArguments === null) return null; + // Kotlin (and some formatters) allow a trailing comma. An empty *middle* + // argument is still invalid and fail-closed. + while (rawArguments.at(-1)?.length === 0) { + rawArguments.pop(); + } + if (rawArguments.some((argument) => argument.length === 0)) return null; const parsed: SpringAnnotationArgument[] = []; for (const raw of rawArguments) { diff --git a/gitnexus/src/core/ingestion/frameworks/spring/argument-facts.ts b/gitnexus/src/core/ingestion/frameworks/spring/argument-facts.ts new file mode 100644 index 000000000..cb7c9cf9d --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/argument-facts.ts @@ -0,0 +1,140 @@ +/** + * One argument of a Spring annotation or of a messaging-template call, captured + * exactly as it is written in source. + * + * Capture-time facts are deliberately UNRESOLVED. When these facts are produced + * the file's imports are not finalized, constants declared in sibling files do + * not exist yet, and no configuration source has been read — so a captured + * `text` may be a string literal, a constant reference (`Destinations.ORDERS`), + * a property placeholder (`"${app.orders.topic}"`), or an arbitrary expression. + * Turning any of those into an address is a separate, later phase; nothing here + * may call a resolver. + * + * NOT the same thing as `SpringAnnotationArgument` in `annotation-arguments.ts`, + * and the two are deliberately not merged: + * + * - Source. This fact is built from AST nodes while the tree is in hand; + * `parseSpringAnnotationArguments` re-parses an annotation's `text` much + * later, from a string, with a hand-written delimiter scanner. + * - Failure. The text parser returns `null` when its scanner cannot balance + * the input, and a caller must decide what that means. There is no such + * state here: the grammar has already decided where each argument begins + * and ends. + * - Absence. The text parser answers `[]` both for `@Scheduled` and for + * `@Scheduled()`, because a string cannot tell "no list" from "empty list" + * without re-deriving it. Capture keeps the two apart — absent versus `[]` — + * so downstream code can rely on the distinction wherever arguments were + * read at all. A capture that reads them for only some of its facts says so + * on its own `args` field. + * - Scope. This fact also describes CALL arguments (`template.send(topic, p)`), + * which the annotation parser has no notion of. + * + * Collapsing them would mean giving the text parser a failure mode it cannot + * produce, or taking the three-state distinction away from capture. + */ +export interface SpringArgumentFact { + /** + * Argument name for a named argument, absent for a positional one. + * + * Both forms occur, and where the destination sits differs by construct. An + * annotation names it (`@KafkaListener(topics = ...)` versus + * `@RabbitListener(queues = ...)`). A call normally gives it by position + * (`kafkaTemplate.send(topic, payload)`) — always so in Java, which has no + * named arguments — but a Kotlin call may name its arguments whenever the + * callee is itself declared in Kotlin, and then the key is captured too. + */ + readonly name?: string; + /** + * Argument value in its source spelling — quotes, braces and casts intact, + * nothing resolved — after `normalizeSpringFactText`. That pass trims the + * text and collapses whitespace around the dots of a multi-line expression, + * so one destination written two ways yields one fact. It is the only + * rewrite; see the function for why formatting must not reach the data. + */ + readonly text: string; +} + +/** + * Join an expression that the source wrapped across lines, so that one + * expression has one spelling no matter where it was written. + * + * A receiver chain written as `outer\n .inner\n .kafkaTemplate`, and an + * argument written as `Destinations\n .ORDERS`, are the same expressions as + * their single-line spellings. Raw node text would carry the newline and the + * ENCLOSING BLOCK's indentation across the worker boundary, so the same + * expression at two nesting depths — or in a CRLF checkout — would not compare + * equal downstream. Receivers and arguments get the identical treatment on + * purpose: an inconsistent rule inside one fact is a trap for the phase that + * has to match a publish against a subscription. + * + * Only a run of whitespace that CONTAINS A NEWLINE and sits next to a dot is + * removed, and only OUTSIDE a string literal. Single-line spacing is left + * alone, so `registry.get("a . b").template` keeps its argument exactly as + * written; literal-awareness extends that to Java text blocks and Kotlin raw + * strings, whose embedded newlines are part of the value and must survive + * (`"""line-a\n.line-b"""` is not the same string as `"""line-a.line-b"""`). + * + * Wraps that are not adjacent to a dot (`"a" +\n "b"`) are left as written: + * normalizing them would have to reason about operators, and the same + * conservatism already applies to receivers. + */ +export function normalizeSpringFactText(text: string): string { + const trimmed = text.trim(); + // Fast path: the overwhelming majority of captured text is single-line. + if (!trimmed.includes('\n') && !trimmed.includes('\r')) return trimmed; + + let out = ''; + let index = 0; + let quote: '"""' | '"' | "'" | null = null; + while (index < trimmed.length) { + const char = trimmed[index] as string; + if (quote === '"""') { + if (trimmed.startsWith('"""', index)) { + out += '"""'; + index += 3; + quote = null; + continue; + } + out += char; + index += 1; + continue; + } + if (quote !== null) { + // A backslash escape is copied whole so that `"\\"` ends the literal and + // `"\""` does not. + if (char === '\\' && index + 1 < trimmed.length) { + out += trimmed.slice(index, index + 2); + index += 2; + continue; + } + if (char === quote) quote = null; + out += char; + index += 1; + continue; + } + if (trimmed.startsWith('"""', index)) { + quote = '"""'; + out += '"""'; + index += 3; + continue; + } + if (char === '"' || char === "'") { + quote = char; + out += char; + index += 1; + continue; + } + if (char === '.' || /\s/.test(char)) { + const separator = /^\s*\.\s*/.exec(trimmed.slice(index)); + if (separator !== null) { + const matched = separator[0]; + out += matched.includes('\n') ? '.' : matched; + index += matched.length; + continue; + } + } + out += char; + index += 1; + } + return out; +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/config-bindings.ts b/gitnexus/src/core/ingestion/frameworks/spring/config-bindings.ts index 0baba4e88..e08273167 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/config-bindings.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/config-bindings.ts @@ -3,6 +3,7 @@ import type { KnowledgeGraph } from '../../../graph/types.js'; import { generateId } from '../../../../lib/utils.js'; export const SPRING_CONFIG_DESCRIPTION = 'Spring configuration property'; +export const SPRING_CONFIG_UNRESOLVED_PREFIX = 'Spring config unresolved: '; export interface SpringValueConsumer { readonly kind: 'value'; @@ -41,7 +42,7 @@ function closestNode( } function markUnresolved(node: GraphNode, key: string): void { - const marker = `Spring config unresolved: ${key}`; + const marker = `${SPRING_CONFIG_UNRESOLVED_PREFIX}${key}`; const existing = typeof node.properties.description === 'string' ? node.properties.description : ''; if (existing.includes(marker)) return; diff --git a/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts b/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts new file mode 100644 index 000000000..f02386192 --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/destinations.ts @@ -0,0 +1,1216 @@ +import type { SpringArgumentFact } from './argument-facts.js'; +import type { SpringMessageProducerTemplate } from './message-producers.js'; + +/** + * Resolution of Spring async messaging DESTINATIONS — the broker address a + * `@KafkaListener` reads from or a `kafkaTemplate.send(...)` writes to. + * + * The capture layer records the destination argument exactly as written and + * resolves nothing (see `argument-facts.ts`). This module is the other half: + * it decides WHICH argument names the destination, then walks a four-step + * cascade to turn that argument's source text into an address. It is pure — + * no graph, no filesystem, no parser — so every rule below is unit-testable + * against a string, and `pipeline-phases/spring-destinations.ts` is left with + * only node and edge emission. + * + * ── THE INVARIANT THIS MODULE EXISTS TO PROTECT ────────────────────────── + * + * An address that could NOT be resolved must never become a shared identity. + * Two unrelated services that each merely write + * + * @KafkaListener(topics = "${app.topic}") + * + * have said nothing about each other. If the graph keyed a destination node on + * that placeholder text, they would land on one node and READ AS CONNECTED — + * and a false edge is worse than a missing one, because a missing edge is + * visible as a gap while a false one enters reports as a fact. + * + * So this module never returns a placeholder, a constant name, or any other + * unresolved spelling as an `address`. An unresolved candidate comes back as + * `{ kind: 'unresolved', reason }` with no address at all, and the phase keys + * such a node by its SOURCE LOCATION. A status flag would not have been + * enough: the two services would still share whatever key the node was minted + * from. Only withholding the key prevents the join. + * + * ── REFUSAL IS DATA ────────────────────────────────────────────────────── + * + * Every path that declines to produce an address records WHY, from a closed + * set ({@link SpringDestinationRefusal}). The measure of this feature is the + * unresolved fraction, so a silent `continue` would hide precisely the number + * that says whether it works. + */ + +/** + * Broker family behind a destination, as far as the syntax can attest. + * + * Part of the `Destination` node IDENTITY, not merely a label on it: the phase + * keys a resolved node by `(broker, address)` via the framework-neutral + * `ingestion/destination-key.ts`, so two brokers claiming one address are two + * ordinary nodes. Adding or renaming a member here therefore re-keys every node + * it applies to, which a full re-index absorbs and an incremental one does not + * — the destination layer is delete-alled and rebuilt graph-wide on every + * incremental writeback for exactly this class of reason. + * + * A member is only added when the SYNTAX attests to it. A guess here becomes a + * guess in the identity, and the cost of a wrong one is a real pair split in + * two (see `destinationNodeKey` for why that cost is nonetheless the cheaper + * of the two failures available). + */ +export type SpringDestinationBroker = + | 'kafka' + | 'rabbit' + | 'jms' + | 'pulsar' + | 'sqs' + | 'stream' + | 'integration'; + +/** + * Why a candidate produced no address. Closed set: each member is a distinct, + * countable diagnosis, and no path may decline without naming one. + * + * Members are split rather than merged wherever the two causes are different + * FACTS about the repository. The unresolved fraction is only useful if its + * breakdown says what to go and fix, and a bucket that means "either the + * capture could not read this or the source really did write it that way" + * answers neither question. + */ +export type SpringDestinationRefusal = + /** The annotation is a recognized listener but its arguments were never read + * — a CAPTURE limitation, not a statement about the source. */ + | 'annotation-arguments-unavailable' + /** The annotation's argument list was read and it was EMPTY: `@KafkaListener` + * with no elements at all. A real source-level gap, and deliberately not the + * same bucket as `annotation-arguments-unavailable` — see + * `SpringNonHttpHandlerAnnotationFact.args`, which keeps absent and `[]` + * apart precisely so a consumer of the fact does not have to guess. */ + | 'annotation-arguments-empty' + /** Recognized listener, argument list present, no element names a destination. */ + | 'no-destination-argument' + /** `@KafkaListeners({@KafkaListener(...), ...})` and its siblings. The + * container's single argument is a list of NESTED annotations, and capture + * does not descend into them, so their destinations are unreadable here. + * Recorded rather than skipped: a repository using repeated-listener + * containers loses real destinations, and that has to show up in the count + * instead of looking like a repository with no listeners. */ + | 'repeated-listener-container' + /** `@KafkaListener(topicPattern = ...)` — a regex over topics, not an address. */ + | 'topic-pattern' + /** A destination form this module deliberately does not read, e.g. + * `@RabbitListener(bindings = @QueueBinding(...))` or `topicPartitions`. */ + | 'unsupported-annotation-argument' + /** A Kotlin trailing-lambda call: the publish has no argument list at all. */ + | 'producer-arguments-unavailable' + /** The call's arity matches none of the overloads that carry a destination. */ + | 'producer-arity-unrecognized' + /** The call used NAMED arguments and none of them names a destination + * parameter this module knows. Selecting by position instead would read + * whatever the author happened to write first — see + * {@link selectProducerDestinationArguments}. */ + | 'producer-named-argument-unrecognized' + /** `rabbitTemplate.convertAndSend(message)` — default exchange, empty routing + * key. There is no address in the source to record. */ + | 'rabbit-default-exchange' + /** The argument in the destination position is not shaped like an address + * (not a string literal, not a constant reference) — most often because the + * overload actually taken has the payload there. */ + | 'producer-argument-not-address-shaped' + /** Two overloads fit the call, they disagree about which slot is the address, + * and the argument is spelled the same way under both readings. The + * archetype is `convertAndSend("orders.rk", "body", correlationData)`: it is + * `(exchange, routingKey, message)` with the address `"body"`, or + * `(routingKey, message, correlationData)` with the address `"orders.rk"` + * and `"body"` as a String PAYLOAD. Both are real overloads spelled + * (String, String, ref). + * + * Distinct from `producer-argument-not-address-shaped`, which says the slot + * cannot hold an address at all. This one says it can, twice, and the module + * will not pick — a payload published as an address joins a consumer of a + * queue that happens to be named after the payload's text. */ + | 'ambiguous-producer-overload' + /** `topics = {}` / `topics = []` / `arrayOf()`. */ + | 'empty-destination-list' + /** The element is an expression this module will not evaluate — a + * concatenation, a call, a ternary. */ + | 'not-a-literal-or-constant' + /** A constant reference no constant resolver could fold to a string. */ + | 'unresolved-constant' + /** `#{...}` — a SpEL expression, evaluated by the container against beans and + * the environment at RUNTIME. `#{@kafkaProps.ordersTopic}` is the archetypal + * unresolvable address: nothing in the source says what it evaluates to, and + * two services that merely wrote the same expression have said nothing about + * each other. */ + | 'spel-expression' + /** An unescaped `$` interpolation in a language whose string literals + * interpolate. In Kotlin `"orders-$env"` and `"orders-${env}"` are STRING + * TEMPLATES evaluated at runtime, not addresses and not Spring placeholders + * — the escaped `"\${app.topic}"` is how a Spring placeholder has to be + * written there. Java does not interpolate, so `$` is an ordinary character + * and this never fires for it. */ + | 'unescaped-interpolation' + /** `${key}` with no default. The KEY is recorded; the VALUE is deliberately + * absent from the graph (config values may hold credentials — see the header + * of `pipeline-phases/spring-config.ts`), so this can never resolve here. */ + | 'unresolved-config-key' + /** `${key:default}`. The default IS written in the source, and it is kept on + * the node — but it is not an IDENTITY. It holds only while the key is not + * overridden in configuration, and configuration VALUES are deliberately + * absent from this graph, so the code cannot know whether it holds. Keying + * on it merges every service that copy-pasted the same fallback: `${a:events}` + * and `${b:events}` are two different addresses that happen to share a + * default. Both the key and the default text survive as properties, so the + * case stays countable and distinguishable from a bare `${key}`. */ + | 'overridable-config-default' + /** `${}` — a placeholder that names no key. There is nothing to record and + * nothing to look up; kept separate so an empty key never reaches the + * `Property` lookup as if it were a real one. */ + | 'empty-config-key' + /** A string literal that is empty or nothing but whitespace. An empty address + * addresses nothing, and letting it through would give every such site one + * shared `''` identity — the same false join the placeholder rule prevents. */ + | 'empty-literal-address' + /** A constant reference that folded to an empty or whitespace-only string. + * Same outcome as `empty-literal-address`, different repository fact: there + * the source wrote `""`, here a constant declaration did. */ + | 'empty-constant-address'; + +/** + * How an address was arrived at, kept on the node for provenance. + * + * There is deliberately no `config-default` member. A `${key:default}` does not + * resolve — see `overridable-config-default` — so no address can be reached + * that way. + */ +export type SpringDestinationVia = 'literal' | 'constant' | 'specification'; + +export type SpringDestinationRole = 'consumer' | 'producer'; + +/** + * One argument element that has been ACCEPTED as naming a destination, before + * any attempt to resolve it. An array-valued argument yields one candidate per + * element: `topics = ["a", "b"]` really is two destinations, and each gets its + * own node and its own edge (see the phase for why no group node is minted). + */ +export interface SpringDestinationCandidate { + readonly role: SpringDestinationRole; + /** Annotation simple name (`KafkaListener`) or producer template (`kafka`). */ + readonly source: string; + readonly broker: SpringDestinationBroker; + /** Index of the argument this element came from, in source order. */ + readonly argIndex: number; + /** Argument name when the call/annotation named it (`topics`, `queues`). */ + readonly argName?: string; + /** Index within an array-valued argument; `0` for a scalar. */ + readonly elementIndex: number; + /** The element's source text, exactly as captured. */ + readonly rawText: string; + /** Companion provenance that is not itself an address — currently only the + * Rabbit exchange that accompanies a routing key. */ + readonly exchange?: string; +} + +/** A candidate that was declined before resolution was even attempted. */ +export interface SpringDestinationRefusalRecord { + readonly role: SpringDestinationRole; + readonly source: string; + readonly broker: SpringDestinationBroker; + readonly reason: SpringDestinationRefusal; + /** Source text that provoked the refusal, when there was one. */ + readonly rawText?: string; + readonly argIndex?: number; + readonly argName?: string; +} + +export interface SpringDestinationSelection { + readonly candidates: readonly SpringDestinationCandidate[]; + readonly refusals: readonly SpringDestinationRefusalRecord[]; +} + +export type SpringDestinationResolution = + | { readonly kind: 'resolved'; readonly address: string; readonly via: SpringDestinationVia } + | { + readonly kind: 'unresolved'; + readonly reason: SpringDestinationRefusal; + /** Configuration key named by an unresolvable `${...}` placeholder. Lets + * the phase link the node to the `Property` nodes for that key without + * ever learning the key's value. */ + readonly configKey?: string; + /** Default text of a `${key:default}`, exactly as the source wrote it. + * Kept as PROVENANCE only — it is never an address and never a key, for + * the reason `overridable-config-default` gives. */ + readonly configDefault?: string; + }; + +/** + * The cascade's pluggable steps plus the one language capability it needs. + * + * The steps are supplied by the phase, which owns the language-specific + * machinery; keeping them as callbacks is what lets this module stay + * language-neutral and testable with a plain map. + */ +export interface SpringDestinationResolvers { + /** + * Whether the owning language INTERPOLATES string literals — Kotlin does, + * Java does not. A capability, deliberately not a language name: shared + * ingestion code may not branch on a language (see AGENTS.md), and the + * capability is also the thing that actually matters. Supplied alongside + * `getSpringMessagingFacts` by the provider and threaded in by the phase. + * + * When true, an unescaped `$` inside a literal is a runtime template and the + * candidate is refused. When false (the default) `$` is an ordinary + * character and `"${app.topic}"` is a Spring placeholder. + */ + readonly interpolatesStringLiterals?: boolean; + /** + * Step 2 — fold a constant reference (`Topics.ORDERS`, `ORDERS`) to its + * string value, or `null` when it cannot be folded. Backed by + * `resolveJavaConstant` / `resolveKotlinConstant`. + */ + readonly constant?: (name: string) => string | null; + /** + * Step 4 — SEAM, DELIBERATELY NOT IMPLEMENTED. + * + * Some destinations are named nowhere in the source: the address lives in a + * published API specification (AsyncAPI / springwolf) that the service + * generates, and the code only names a binding. Resolving those means reading + * an artifact that is not a source file, deciding which specification belongs + * to which module, and trusting a generated document — a different problem + * from the three syntactic steps above, with a different failure mode. + * + * The hook exists so that work has a defined place to land and so the cascade + * order is fixed now rather than renegotiated later. Nothing supplies it + * today, so step 4 is a no-op and such destinations stay unresolved with the + * reason the earlier step recorded. + */ + readonly specification?: (candidate: SpringDestinationCandidate) => string | null; +} + +// ── Consumer side: which annotation argument names the destination ───────── + +interface ConsumerAnnotationRule { + readonly broker: SpringDestinationBroker; + /** Argument names that carry an address, in preference order. */ + readonly addressArgs: readonly string[]; + /** + * A bare positional argument is the annotation's `value` element. Accepted + * only where `value` really is the destination: `@SqsListener("q")` and + * `@StreamListener("ch")`. `@KafkaListener`, `@RabbitListener`, `@JmsListener` + * and `@ServiceActivator` declare no `value` alias for their destination, so + * a positional argument on one of those is something else entirely and is + * refused rather than guessed at. + */ + readonly positionalIsAddress: boolean; + /** Arguments that are patterns over addresses, not addresses. */ + readonly patternArgs?: readonly string[]; + /** Arguments that name a destination in a shape this module will not read. */ + readonly unsupportedArgs?: readonly string[]; +} + +/** + * Recognized listener annotations, keyed by SIMPLE name. + * + * Simple names, not fully-qualified ones, because a pipeline phase runs after + * scope resolution has finished and no longer has the import tables that + * `createSpringAnnotationNameResolver` needs. The capture layer already gates + * on simple names for the same reason (`CAPTURE_RELEVANT_SIMPLE_NAMES` in + * `non-http-handlers.ts`), so nothing reaches this map that was not already + * admitted on that basis; matching on the FQN here would only reject facts the + * capture had already accepted, never admit more. + * + * DELIBERATELY ABSENT: `@MessageMapping` and `@SubscribeMapping`. Both are + * recognized by `non-http-handlers.ts` as message handlers, and both are + * WebSocket/STOMP routes — an application-level destination inside a + * server-managed session, not an address on a broker. Modelling `/topic/prices` + * as a `Destination` would put a STOMP path in the same namespace as a Kafka + * topic and let the cross-service joiner match them. + */ +const CONSUMER_ANNOTATIONS: ReadonlyMap = new Map([ + [ + 'KafkaListener', + { + broker: 'kafka' as const, + addressArgs: ['topics'], + positionalIsAddress: false, + patternArgs: ['topicPattern'], + unsupportedArgs: ['topicPartitions'], + }, + ], + [ + 'PulsarListener', + { + broker: 'pulsar' as const, + addressArgs: ['topics'], + positionalIsAddress: false, + patternArgs: ['topicPattern'], + }, + ], + [ + 'RabbitListener', + { + broker: 'rabbit' as const, + addressArgs: ['queues'], + positionalIsAddress: false, + unsupportedArgs: ['bindings', 'queuesToDeclare'], + }, + ], + [ + 'JmsListener', + { broker: 'jms' as const, addressArgs: ['destination'], positionalIsAddress: false }, + ], + [ + 'ServiceActivator', + { broker: 'integration' as const, addressArgs: ['inputChannel'], positionalIsAddress: false }, + ], + ['SqsListener', { broker: 'sqs' as const, addressArgs: ['value'], positionalIsAddress: true }], + [ + 'StreamListener', + { broker: 'stream' as const, addressArgs: ['value'], positionalIsAddress: true }, + ], +]); + +/** + * Plural container annotations (`@KafkaListeners`, `@RabbitListeners`, …) wrap + * repeated listeners. Their single argument is a list of nested annotations, + * whose own arguments the capture does not descend into, so there is nothing + * here to read. + * + * They are recognized rather than ignored so the loss is COUNTED. A repository + * that declares its listeners this way really does lose those destinations, and + * returning an empty selection would make it indistinguishable from a + * repository with no listeners at all — the module header promises that every + * path which declines to produce an address records why, and an empty + * `refusals` array records nothing. The broker comes from the container's own + * name, which is the one thing the annotation does state. + */ +const CONSUMER_CONTAINER_ANNOTATIONS: ReadonlyMap = new Map([ + ['KafkaListeners', 'kafka' as const], + ['RabbitListeners', 'rabbit' as const], + ['JmsListeners', 'jms' as const], + ['PulsarListeners', 'pulsar' as const], +]); + +function simpleName(name: string): string { + const separator = name.lastIndexOf('.'); + return separator === -1 ? name : name.slice(separator + 1); +} + +/** + * Choose the destination-bearing arguments of one listener annotation. + * + * Returns `null` when the annotation is not a broker listener at all — that is + * not a refusal, there was nothing to refuse. A recognized annotation always + * returns a selection, even when every path in it declined, so the caller can + * count what was seen against what resolved. + */ +export function selectConsumerDestinationArguments( + annotationName: string, + args: readonly SpringArgumentFact[] | undefined, +): SpringDestinationSelection | null { + const name = simpleName(annotationName); + const containerBroker = CONSUMER_CONTAINER_ANNOTATIONS.get(name); + if (containerBroker !== undefined) { + return { + candidates: [], + refusals: [ + { + role: 'consumer', + source: name, + broker: containerBroker, + reason: 'repeated-listener-container', + ...(args === undefined || args[0] === undefined ? {} : { rawText: args[0].text }), + }, + ], + }; + } + const rule = CONSUMER_ANNOTATIONS.get(name); + if (rule === undefined) return null; + + const refusals: SpringDestinationRefusalRecord[] = []; + const refuse = ( + reason: SpringDestinationRefusal, + extra: Omit = {}, + ): void => { + refusals.push({ role: 'consumer', source: name, broker: rule.broker, reason, ...extra }); + }; + + // ABSENT arguments are a capture limitation: the annotation was recognized + // but its argument list was never read (see + // `SpringNonHttpHandlerAnnotationFact.args`). An empty ARRAY is a different + // fact entirely — an argument list WAS read and it was empty, so the source + // really does declare a listener that names no destination. Capture keeps the + // two apart on purpose, the producer side of this module already does, and + // merging them here would file a source-level gap under a tooling gap and + // corrupt the one breakdown this feature is measured on. + if (args === undefined) { + refuse('annotation-arguments-unavailable'); + return { candidates: [], refusals }; + } + if (args.length === 0) { + refuse('annotation-arguments-empty'); + return { candidates: [], refusals }; + } + + const candidates: SpringDestinationCandidate[] = []; + let sawDestinationArgument = false; + for (const [argIndex, arg] of args.entries()) { + const argName = arg.name; + if (argName === undefined) { + // Positional. Only the annotations whose `value` element IS the + // destination accept it; on the others a positional argument is a + // different element entirely and gets no guess. + if (!rule.positionalIsAddress) continue; + sawDestinationArgument = true; + pushElements(candidates, refusals, { + role: 'consumer', + source: name, + broker: rule.broker, + argIndex, + rawText: arg.text, + }); + continue; + } + if (rule.patternArgs?.includes(argName)) { + sawDestinationArgument = true; + refuse('topic-pattern', { rawText: arg.text, argIndex, argName }); + continue; + } + if (rule.unsupportedArgs?.includes(argName)) { + sawDestinationArgument = true; + refuse('unsupported-annotation-argument', { rawText: arg.text, argIndex, argName }); + continue; + } + if (!rule.addressArgs.includes(argName)) continue; + sawDestinationArgument = true; + pushElements(candidates, refusals, { + role: 'consumer', + source: name, + broker: rule.broker, + argIndex, + argName, + rawText: arg.text, + }); + } + + // A listener whose arguments were read and named `groupId` and + // `containerFactory` but no destination is a real, countable gap — most often + // a form this module has not learned. It must not be silent. + if (!sawDestinationArgument) refuse('no-destination-argument'); + return { candidates, refusals }; +} + +// ── Producer side: which call argument names the destination ─────────────── + +/** + * Parameter names that carry a destination, per template, for calls that pass + * their arguments BY NAME. + * + * Kotlin call sites may name arguments, and a named argument list is in source + * order, not parameter order — `send(data = payload, topic = "orders")` is + * legal and puts the payload in slot 0. Reading slot 0 there publishes the + * PAYLOAD as an address. The name is captured + * ({@link SpringArgumentFact.name}), so the honest rule is to use it: select by + * name when there is one, and refuse when the names present say nothing this + * module recognizes. Selecting by position while ignoring a name that + * contradicts it is the one option that is never defensible. + * + * `exchange` is listed for rabbit but is NOT an address — it is the companion + * provenance the routing key carries (see the arity notes below). + */ +const PRODUCER_DESTINATION_PARAMETERS: Readonly< + Record +> = { + kafka: ['topic'], + // `RabbitTemplate.convertAndSend(String exchange, String routingKey, Object message, …)`. + rabbit: ['routingKey'], + // `JmsTemplate.convertAndSend(Destination destination, …)` and the + // `String destinationName` overloads. + jms: ['destination', 'destinationName'], + // `StreamBridge.send(String bindingName, Object data, …)`. + 'stream-bridge': ['bindingName'], +}; + +/** Rabbit's exchange parameter, carried as provenance rather than as an address. */ +const RABBIT_EXCHANGE_PARAMETER = 'exchange'; + +/** + * Choose the destination-bearing arguments of one messaging-template publish. + * + * A NAME beats a position, arity decides where it can decide, and shape decides + * where it cannot. + * + * When any argument is passed by name, {@link PRODUCER_DESTINATION_PARAMETERS} + * decides — position is not consulted at all, because a named argument list + * need not be in parameter order. When the slot this module would have read + * positionally is itself named with something it does not recognize, that is a + * contradiction and the publish is refused rather than read. + * + * `KafkaTemplate.send` and `StreamBridge.send` put the destination first in + * every multi-argument positional overload they have, so once such a call has + * two or more arguments its slot 0 is the destination and nothing further needs + * deciding. Those slots use the PERMISSIVE gate ({@link isAddressShaped}): a + * bare identifier is let through to the cascade, which refuses it by name if no + * constant folds. That keeps `unresolved-constant` — a thing we tried to + * resolve — distinct from `producer-argument-not-address-shaped`, a thing we + * declined to read at all. + * + * The `convertAndSend` families are different. Both admit trailing + * `MessagePostProcessor` and `CorrelationData` parameters, and arity does not + * separate the overloads in EITHER direction: + * + * jms (destination, message) 2 vs (message, postProcessor) 2 + * rabbit (routingKey, message) 2 vs (message, postProcessor) 2 + * rabbit (exchange, routingKey, message) 3 vs (routingKey, message, pp) 3 + * vs (routingKey, message, correlation) 3 + * rabbit (exchange, routingKey, message, pp) 4 vs (routingKey, message, pp, corr) 4 + * + * So the tie is broken by the STRICT gate ({@link isConfidentAddressShape}) — a + * string literal, a qualified reference, or a screaming-snake constant, all of + * which a payload variable is not. A lowercase bare identifier is NOT confident + * evidence, so `convertAndSend(topic, payload)` is refused rather than read: + * the same spelling is how a payload variable looks, and nothing in the syntax + * separates them. That refusal is the deliberate cost. A refusal is counted and + * recoverable; a wrong address enters reports as a fact. + * + * There is NO positional fallback at rabbit arity 3+. An earlier revision fell + * back to accepting slot 0 when slot 1 was not confident, which turned the + * ordinary `convertAndSend(EXCHANGE, routingKey, event)` — routing key in a + * variable — into a destination whose address was the EXCHANGE NAME. That is + * the worst possible outcome: an address that looks entirely plausible and can + * join a `@RabbitListener(queues = "orders")` that has nothing to do with it. + * + * ── THE ONE AMBIGUITY, REFUSED RATHER THAN GUESSED ─────────────────────── + * + * `convertAndSend("orders.rk", "body", correlationData)` fits two overloads at + * once and they disagree about which slot is the address: + * + * (exchange, routingKey, message) → the address is `"body"` + * (routingKey, message, correlationData) → the address is `"orders.rk"` + * + * Both are real, both are spelled (String, String, ref), and no rule over the + * syntax separates them. Picking either one publishes the OTHER reading's + * payload as an address, where a consumer of a queue named after that text + * joins a publisher that never wrote to it. So neither is picked: the call is + * refused as `ambiguous-producer-overload` and yields no candidate and no edge. + * + * The refusal is narrow on purpose, because over-refusing here costs the + * ordinary case, and a suppression that eats correct results is the more + * expensive mistake. It fires ONLY on a STRING LITERAL in slot 1, at the + * arities where a competing overload exists: + * + * - A literal is no evidence at all. An address and a payload are BOTH + * ordinarily written as literals, so the spelling distinguishes nothing. + * - A CONSTANT or QUALIFIED reference is evidence, which is the whole premise + * of {@link isConfidentAddressShape}: `ORDERS_ROUTING_KEY` and + * `Topics.ORDERS_KEY` are how a configured NAME is written, not how a + * payload computed at the call site is. Those keep resolving. + * - Arity 5 has no competing overload at all — + * `(exchange, routingKey, message, pp, correlationData)` is the only + * five-argument form — so slot 1 there is the routing key whatever it is + * spelled like, and the refusal must not reach it. + * - A NAMED argument settles the reading outright, and the name-beats-position + * pre-pass above has already returned by then. + */ +export function selectProducerDestinationArguments(fact: { + readonly template: SpringMessageProducerTemplate; + readonly methodName: string; + readonly args?: readonly SpringArgumentFact[]; +}): SpringDestinationSelection { + const broker: SpringDestinationBroker = + fact.template === 'stream-bridge' ? 'stream' : fact.template; + const source = fact.template; + const refusals: SpringDestinationRefusalRecord[] = []; + const refuse = ( + reason: SpringDestinationRefusal, + extra: Omit = {}, + ): void => { + refusals.push({ role: 'producer', source, broker, reason, ...extra }); + }; + + const args = fact.args; + if (args === undefined) { + refuse('producer-arguments-unavailable'); + return { candidates: [], refusals }; + } + if (args.length === 0) { + refuse('producer-arity-unrecognized'); + return { candidates: [], refusals }; + } + + const candidates: SpringDestinationCandidate[] = []; + const accept = (argIndex: number, exchange?: string): void => { + const arg = args[argIndex] as SpringArgumentFact; + pushElements(candidates, refusals, { + role: 'producer', + source, + broker, + argIndex, + ...(arg.name === undefined ? {} : { argName: arg.name }), + rawText: arg.text, + ...(exchange === undefined ? {} : { exchange }), + }); + }; + /** + * Accept a slot chosen by POSITION. + * + * Refuses when the argument in that slot carries a name — a named list need + * not be in parameter order, so a name in the destination slot that is not a + * destination parameter contradicts the position, and reading the position + * anyway is how `send(data = "payload", topic = "orders")` published the + * payload. The name-matching pre-pass above has already had its chance. + */ + const acceptPositional = (argIndex: number, exchange?: string): void => { + const arg = args[argIndex] as SpringArgumentFact; + if (arg.name !== undefined) { + refuse('producer-named-argument-unrecognized', { + rawText: arg.text, + argIndex, + argName: arg.name, + }); + return; + } + accept(argIndex, exchange); + }; + const textAt = (index: number): string => (args[index] as SpringArgumentFact).text; + const confident = (index: number): boolean => + index < args.length && isConfidentAddressShape(textAt(index)); + const refuseShape = (index: number): void => { + refuse('producer-argument-not-address-shaped', { rawText: textAt(index), argIndex: index }); + }; + + // ── A name beats a position ───────────────────────────────────────────── + // When an argument names a destination parameter, that argument IS the + // destination wherever it sits in the list. Only when no name matches does + // the positional reasoning below run, and `acceptPositional` then refuses if + // the slot it lands on turns out to be named after something else. + const destinationNames = PRODUCER_DESTINATION_PARAMETERS[fact.template]; + const namedIndex = args.findIndex( + (arg) => arg.name !== undefined && destinationNames.includes(arg.name), + ); + if (namedIndex !== -1) { + const exchangeIndex = + fact.template === 'rabbit' + ? args.findIndex((arg) => arg.name === RABBIT_EXCHANGE_PARAMETER) + : -1; + accept( + namedIndex, + exchangeIndex === -1 ? undefined : unquoteForProvenance(textAt(exchangeIndex)), + ); + return { candidates, refusals }; + } + + if (fact.template === 'rabbit') { + // `convertAndSend` overloads, by what occupies the leading slots: + // (message) → default exchange, no address + // (routingKey, message) → arg0 is the routing key + // (message, postProcessor) → NO address, same arity + // (exchange, routingKey, message) → arg0 + arg1 + // (routingKey, message, postProcessor) → arg0 only, same arity + // (routingKey, message, correlationData) → arg0 only, same arity + // (exchange, routingKey, message, pp) → arg0 + arg1 + // (routingKey, message, pp, correlationData) → arg0 only, same arity + // (exchange, routingKey, message, pp, corr) → arg0 + arg1 + if (args.length === 1) { + refuse('rabbit-default-exchange', { rawText: textAt(0), argIndex: 0 }); + return { candidates, refusals }; + } + if (args.length === 2) { + // (routingKey, message) versus (message, postProcessor). + if (confident(0)) { + acceptPositional(0); + return { candidates, refusals }; + } + refuseShape(0); + return { candidates, refusals }; + } + // Three arguments and up. Arity separates almost nothing here — three and + // four both admit an exchange form and a routing-key form — so the ONLY + // acceptance is confident evidence in slot 1, and there is no positional + // fallback. `convertAndSend(EXCHANGE, routingKey, event)` fails that test + // and is refused; the discarded fallback published `EXCHANGE` as the + // address, which is a wrong answer wearing the costume of a right one. + // + // And confident evidence in slot 1 is not enough when the evidence is a + // STRING LITERAL: under the competing overload that same literal is the + // String PAYLOAD, and the two readings are spelled identically. See the + // ambiguity section in this function's doc comment for why this is a + // refusal rather than a choice, and for each of the three cases it must not + // touch — a constant in slot 1 (spelling that IS evidence), arity 5 (no + // competing overload exists), and a named argument (already returned + // above, and its name settles the reading). + const competingOverload = args.length === 3 || args.length === 4; + if ( + competingOverload && + args[1]?.name === undefined && + parseSpringStringLiteral(textAt(1)) !== null + ) { + refuse('ambiguous-producer-overload', { rawText: textAt(1), argIndex: 1 }); + return { candidates, refusals }; + } + if (confident(1)) { + // The ADDRESS is the routing key. The exchange rides along as provenance + // on the edge rather than becoming part of the address: composing + // `exchange/routingKey` would invent a spelling no consumer ever writes, + // and a `@RabbitListener` names a QUEUE, so the two sides do not join on + // the exchange anyway. Which queue an exchange/key pair reaches is decided + // by bindings this index does not read. + acceptPositional(1, unquoteForProvenance(textAt(0))); + return { candidates, refusals }; + } + refuseShape(1); + return { candidates, refusals }; + } + + // kafka `send(topic, …)`, jms `convertAndSend(destination, message, …)` and + // stream-bridge `send(binding, …)` all put the destination first and all + // require at least one further argument for the payload. A single-argument + // call is therefore one of the payload-only overloads — + // `send(ProducerRecord)`, `send(Message)`, `convertAndSend(Object)` — which + // carries its destination inside an object this module does not open. + if (args.length < 2) { + refuse('producer-arity-unrecognized', { rawText: textAt(0), argIndex: 0 }); + return { candidates, refusals }; + } + // Two-argument `convertAndSend` is the one JMS arity that collides with the + // post-processor overload, so only there does slot 0 need confident evidence. + const strict = fact.template === 'jms' && args.length === 2; + if (strict ? !confident(0) : !isAddressShaped(textAt(0))) { + refuseShape(0); + return { candidates, refusals }; + } + acceptPositional(0); + return { candidates, refusals }; +} + +// ── Array / literal / placeholder text handling ──────────────────────────── + +/** + * Split an array-valued destination argument into its elements. + * + * Both languages hand this module ONE unsplit string per argument: capture + * records an argument's source text, and `topics = {"a", "b"}` is a single + * argument whose text happens to be a list. So the list is parsed here, in the + * three spellings the two languages use — Java `{…}`, Kotlin `[…]`, and Kotlin + * `arrayOf(…)`. + * + * Anything else comes back as a single element, unchanged: a scalar argument, + * and equally an expression that merely starts with a brace. The split tracks + * nesting and string literals, so a comma inside a literal or inside a nested + * call does not split the list. + * + * Returns `[]` for an empty list, which the caller must distinguish from a + * one-element list — `topics = {}` names no destination at all. + */ +export function splitSpringDestinationList(text: string): readonly string[] { + const trimmed = text.trim(); + let inner: string | null = null; + if (trimmed.startsWith('{') && trimmed.endsWith('}')) inner = trimmed.slice(1, -1); + else if (trimmed.startsWith('[') && trimmed.endsWith(']')) inner = trimmed.slice(1, -1); + else if (/^arrayOf\s*\(/.test(trimmed) && trimmed.endsWith(')')) { + inner = trimmed.slice(trimmed.indexOf('(') + 1, -1); + } + if (inner === null) return [trimmed]; + if (inner.trim() === '') return []; + + const elements: string[] = []; + let current = ''; + let depth = 0; + let quote: '"""' | '"' | "'" | null = null; + for (let index = 0; index < inner.length; index += 1) { + const char = inner[index] as string; + if (quote === '"""') { + current += char; + if (inner.startsWith('"""', index)) { + current += '""'; + index += 2; + quote = null; + } + continue; + } + if (quote !== null) { + if (char === '\\' && index + 1 < inner.length) { + current += inner.slice(index, index + 2); + index += 1; + continue; + } + current += char; + if (char === quote) quote = null; + continue; + } + if (inner.startsWith('"""', index)) { + current += '"""'; + index += 2; + quote = '"""'; + continue; + } + if (char === '"' || char === "'") { + quote = char; + current += char; + continue; + } + if (char === '(' || char === '[' || char === '{') depth += 1; + else if (char === ')' || char === ']' || char === '}') depth -= 1; + if (char === ',' && depth === 0) { + elements.push(current.trim()); + current = ''; + continue; + } + current += char; + } + elements.push(current.trim()); + return elements.filter((element) => element !== ''); +} + +/** + * ONE string literal, whole. + * + * The triple-quoted alternative excludes `"""` from its body rather than + * matching greedily: `"""a""" + """b"""` is a concatenation, not a literal, and + * a greedy body swallowed the operator and folded it to the single address + * `a""" + """b`. Excluding the terminator makes the whole-string anchor fail + * there, so the text falls through to the constant test and is refused as + * `not-a-literal-or-constant`, which is what it is. + */ +const STRING_LITERAL = + /^(?:"""((?:(?!""")[\s\S])*)"""|"((?:[^"\\]|\\[\s\S])*)"|'((?:[^'\\]|\\[\s\S])*)')$/; + +/** + * Unquote a string literal to its value, or `null` when the text is not a + * single literal. + * + * Escapes are undone only for the sequences that can appear inside a + * destination: `\"`, `\\`, and Kotlin's `\$`. That last one matters more than it + * looks — a Spring placeholder written in Kotlin MUST escape the dollar + * (`"\${app.topic}"`) or the compiler reads it as a string template, so without + * undoing it every Kotlin placeholder would fail the `${` test below and be + * misfiled as a plain literal address named `\${app.topic}`. + * + * The unescaping is also why {@link hasUnescapedStringInterpolation} has to run + * against the RAW text: once `\$` has become `$`, the escaped placeholder and + * the runtime template are the same string. + */ +export function parseSpringStringLiteral(text: string): string | null { + const match = STRING_LITERAL.exec(text.trim()); + if (match === null) return null; + const raw = match[1] ?? match[2] ?? match[3] ?? ''; + return raw.replace(/\\(["'\\$nrt])/g, (_all, escaped: string) => { + if (escaped === 'n') return '\n'; + if (escaped === 'r') return '\r'; + if (escaped === 't') return '\t'; + return escaped; + }); +} + +/** `$` followed by a brace or an identifier start — the two template forms. */ +const INTERPOLATION_START = /^[{A-Za-z_]/; + +/** + * Whether a string literal contains an UNESCAPED interpolation, for a language + * whose literals interpolate. + * + * Only meaningful for such a language; Java never calls it. In Kotlin: + * + * "orders-$env" template — the value is decided at runtime + * "orders-${env}" template — NOT a Spring placeholder + * "\${app.topic}" escaped — this is how a Spring placeholder is written + * """orders-$env""" template — raw strings interpolate and cannot escape + * + * Reads the RAW literal text on purpose: {@link parseSpringStringLiteral} + * resolves `\$` to `$`, after which the second and third rows above are the + * same string and the distinction is gone. A raw (`"""`) literal has no + * backslash escapes at all — `${'$'}` is the only way to write a dollar there — + * so every `$` in one is an interpolation. + */ +export function hasUnescapedStringInterpolation(text: string): boolean { + const trimmed = text.trim(); + const match = STRING_LITERAL.exec(trimmed); + if (match === null) return false; + const raw = match[1] ?? match[2] ?? match[3] ?? ''; + const escapable = match[1] === undefined; + for (let index = 0; index < raw.length; index += 1) { + const char = raw[index] as string; + if (escapable && char === '\\') { + index += 1; + continue; + } + if (char === '$' && INTERPOLATION_START.test(raw.slice(index + 1))) return true; + } + return false; +} + +/** `#{...}` — a SpEL expression the container evaluates at runtime. */ +function containsSpelExpression(value: string): boolean { + return value.includes('#{'); +} + +/** A dotted or bare identifier — the only non-literal shape read as a constant. */ +const CONSTANT_REFERENCE = /^[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\.\s*[A-Za-z_$][A-Za-z0-9_$]*)*$/; + +/** + * PERMISSIVE gate — true when the text could name an address: a string literal, + * or any reference a constant resolver could plausibly fold. Says nothing about + * whether that reference actually resolves; the cascade decides that and + * records `unresolved-constant` when it does not. + * + * Used where the overload set already fixes which slot holds the destination. + */ +export function isAddressShaped(text: string): boolean { + const trimmed = text.trim(); + if (parseSpringStringLiteral(trimmed) !== null) return true; + return CONSTANT_REFERENCE.test(trimmed); +} + +/** A reference whose spelling is evidence in itself: qualified (`Topics.ORDERS`) + * or a screaming-snake constant (`ORDERS_TOPIC`). `this.x` is excluded — the + * qualifier says nothing about the member. */ +const CONFIDENT_REFERENCE = /^(?!this\s*\.)[A-Za-z_$][A-Za-z0-9_$]*\s*\.\s*[A-Za-z0-9_$.\s]+$/; +const SCREAMING_SNAKE = /^[A-Z][A-Z0-9_$]*$/; + +/** + * STRICT gate — true only when the spelling is confident evidence of an + * address, not merely compatible with one. + * + * The difference from {@link isAddressShaped} is the lowercase bare identifier. + * `convertAndSend(topic, payload)` and `convertAndSend(message, processor)` are + * the same syntax; only a human reading the names can tell which slot is the + * destination, and a name is not something this module is willing to rank as + * evidence. So a bare `topic` fails here and the publish is refused, while + * `"orders"`, `Topics.ORDERS` and `ORDERS_TOPIC` pass. + * + * Used ONLY at the arities where a trailing `MessagePostProcessor` overload + * collides with the destination-carrying one. Everywhere else the permissive + * gate applies, so this stricter rule costs nothing outside the ambiguity. + */ +export function isConfidentAddressShape(text: string): boolean { + const trimmed = text.trim(); + if (parseSpringStringLiteral(trimmed) !== null) return true; + if (!CONSTANT_REFERENCE.test(trimmed)) return false; + return CONFIDENT_REFERENCE.test(trimmed) || SCREAMING_SNAKE.test(trimmed); +} + +/** Best-effort display form for provenance text; never used as an identity. */ +function unquoteForProvenance(text: string): string { + return parseSpringStringLiteral(text) ?? text.trim(); +} + +export interface SpringPlaceholderResult { + /** True when the text contained no `${…}` at all. */ + readonly plain: boolean; + /** Key of the FIRST placeholder, in source order. Present whenever `plain` + * is false; the empty string when the placeholder named no key (`${}`). */ + readonly key?: string; + /** Default text of that placeholder, exactly as written, when it had one. + * Absent for a bare `${key}`. The empty string for `${key:}`, which is a + * default that was written and is empty. */ + readonly defaultValue?: string; +} + +/** + * Read the FIRST Spring property placeholder out of an already-unquoted value. + * + * NOTHING IS SUBSTITUTED, and that is the rule, not an omission. + * + * `${key}` cannot resolve: the value lives in a configuration file this index + * deliberately does not read into the graph (values may hold credentials — see + * `pipeline-phases/spring-config.ts`). The KEY comes back instead, so the + * caller can link the node to the `Property` nodes for that key without ever + * learning its value. + * + * `${key:default}` cannot resolve EITHER, which is a correction to this + * module's original rule. The default is written in the source, so reading it + * is legitimate and it is returned — but it is provenance, never an identity. + * A default holds only while the key is not overridden, and whether it is + * overridden is a fact about configuration VALUES, which are absent from this + * graph by design. Substituting it made `${a.topic:events}` and + * `${b.topic:events}` one node and reported a producer/consumer pair between + * two services that shared nothing but a copy-pasted fallback. The same + * reasoning applies to any placeholder-derived value: a value the configuration + * can override is not an identity. + * + * Only the first placeholder is read because there is nothing to do with the + * rest — the value is already unresolvable, and the first key is the one a + * reader would look up. A nested default (`${a:${b}}`) needs no special case + * under this rule: `a` is the key and `${b}` is the default text, both reported + * as written. + */ +export function resolveSpringPlaceholders(value: string): SpringPlaceholderResult { + const start = value.indexOf('${'); + if (start === -1) return { plain: true }; + let depth = 1; + let cursor = start + 2; + while (cursor < value.length && depth > 0) { + if (value.startsWith('${', cursor)) { + depth += 1; + cursor += 2; + continue; + } + if (value[cursor] === '}') depth -= 1; + cursor += 1; + } + // An unterminated `${` is not a placeholder this module can read. Treating + // the tail as a literal would mint an address containing `${`; treating it as + // a key at least names the thing the author was reaching for. + if (depth > 0) return { plain: false, key: value.slice(start + 2).trim() }; + const body = value.slice(start + 2, cursor - 1); + const separator = body.indexOf(':'); + // Spring splits on the FIRST colon, so `${a:b:c}` defaults to `b:c`. + if (separator === -1) return { plain: false, key: body.trim() }; + return { + plain: false, + key: body.slice(0, separator).trim(), + defaultValue: body.slice(separator + 1), + }; +} + +// ── The cascade ──────────────────────────────────────────────────────────── + +/** + * Resolve one candidate to an address, or to a named refusal. + * + * Four steps, in this order, each of which may decline: + * + * 1. literal — `"orders.v1"`, including one element of an array form. + * 2. constant — `Topics.ORDERS`, through the supplied constant resolver. + * 3. configuration — neither `${app.topic}` nor `${app.topic:orders}` + * resolves; the key, and the default text when there is + * one, are reported instead. + * 4. specification — the deferred seam; see {@link SpringDestinationResolvers}. + * + * Steps 1 and 2 both feed step 3: a literal may be a placeholder, and so may + * the value a constant folds to (`static final String TOPIC = "${app.topic}"` + * is an ordinary way to write one). Skipping step 3 after step 2 would file + * that constant's placeholder text as a resolved address — the exact false + * identity this module exists to prevent, arrived at one step later. + * + * Two classes of text are rejected BEFORE step 3, because they are not + * addresses in any configuration: a SpEL expression, which the container + * evaluates against live beans, and an unescaped string-template interpolation + * in a language that interpolates. Order matters between them and the + * placeholder rule — `"#{'${app.topics}'.split(',')}"` contains a `${` and + * would otherwise be filed under a configuration key that is not really what + * it is. + * + * WHITESPACE. An address is kept exactly as the source wrote it, `" orders "` + * included, so `" orders "` is its own node and does not join `"orders"`. That + * is a missing connection rather than a false one, which is the trade this + * module makes everywhere. The emptiness test below trims, because a + * whitespace-only address addresses nothing — the two rules disagree on + * purpose, and this is the statement of it. + */ +export function resolveSpringDestination( + candidate: SpringDestinationCandidate, + resolvers: SpringDestinationResolvers = {}, +): SpringDestinationResolution { + const specification = (): SpringDestinationResolution | null => { + const resolved = resolvers.specification?.(candidate); + if (resolved === undefined || resolved === null || resolved === '') return null; + return { kind: 'resolved', address: resolved, via: 'specification' }; + }; + + const literal = parseSpringStringLiteral(candidate.rawText); + if (literal !== null) { + // The raw spelling, not the unquoted value: unquoting has already turned + // `\$` into `$` and the escaped placeholder into the runtime template. + if ( + resolvers.interpolatesStringLiterals === true && + hasUnescapedStringInterpolation(candidate.rawText) + ) { + return specification() ?? { kind: 'unresolved', reason: 'unescaped-interpolation' }; + } + return finish(literal, 'literal', specification); + } + + const trimmed = candidate.rawText.trim(); + if (CONSTANT_REFERENCE.test(trimmed)) { + const folded = resolvers.constant?.(trimmed.replace(/\s*\.\s*/g, '.')) ?? null; + if (folded === null) + return specification() ?? { kind: 'unresolved', reason: 'unresolved-constant' }; + // A folded value has already lost its escapes, so an interpolating language + // cannot tell `"\${app.topic}"` from `"${app.topic}"` here the way the + // literal branch can. Both are unresolved either way, so the cost is a + // reason filed under `unescaped-interpolation` that might have belonged + // under `unresolved-config-key` — never a false address. + // + // A LIVE PATH, not a guard for the future. Kotlin both interpolates and + // supplies a constant fold — `languages/kotlin.ts` declares + // `extractModuleConstants` and `foldRoutePathOperands`, and + // `spring-destinations.ts` hands the fold to this cascade — so a Kotlin + // constant reaching this branch is an ordinary occurrence and the misfiled + // reason above is a cost actually paid. Fixing it means teaching the fold + // to report whether the value it returned was escaped at its declaration, + // which the shared `ModuleConstants` shape does not carry. + if (resolvers.interpolatesStringLiterals === true && /\$[{A-Za-z_]/.test(folded)) { + return specification() ?? { kind: 'unresolved', reason: 'unescaped-interpolation' }; + } + return finish(folded, 'constant', specification); + } + + return specification() ?? { kind: 'unresolved', reason: 'not-a-literal-or-constant' }; +} + +function finish( + value: string, + via: 'literal' | 'constant', + specification: () => SpringDestinationResolution | null, +): SpringDestinationResolution { + const decline = ( + reason: SpringDestinationRefusal, + extra: { configKey?: string; configDefault?: string } = {}, + ): SpringDestinationResolution => + specification() ?? { + kind: 'unresolved', + reason, + ...(extra.configKey === undefined ? {} : { configKey: extra.configKey }), + ...(extra.configDefault === undefined ? {} : { configDefault: extra.configDefault }), + }; + + // Before the placeholder rule: a SpEL expression may CONTAIN a `${…}`, and + // calling that a configuration key would name the wrong diagnosis. + if (containsSpelExpression(value)) return decline('spel-expression'); + + const placeholders = resolveSpringPlaceholders(value); + if (!placeholders.plain) { + const key = placeholders.key ?? ''; + if (key === '') return decline('empty-config-key'); + if (placeholders.defaultValue !== undefined) { + return decline('overridable-config-default', { + configKey: key, + configDefault: placeholders.defaultValue, + }); + } + return decline('unresolved-config-key', { configKey: key }); + } + + if (value.trim() === '') { + return decline(via === 'literal' ? 'empty-literal-address' : 'empty-constant-address'); + } + return { kind: 'resolved', address: value, via }; +} + +/** + * Expand one accepted argument into per-element candidates. + * + * An empty list is a refusal rather than zero silent candidates: `topics = {}` + * is a listener that names nothing, which is a finding, not an absence. + */ +function pushElements( + candidates: SpringDestinationCandidate[], + refusals: SpringDestinationRefusalRecord[], + base: Omit, +): void { + const elements = splitSpringDestinationList(base.rawText); + if (elements.length === 0) { + refusals.push({ + role: base.role, + source: base.source, + broker: base.broker, + reason: 'empty-destination-list', + rawText: base.rawText, + argIndex: base.argIndex, + ...(base.argName === undefined ? {} : { argName: base.argName }), + }); + return; + } + for (const [elementIndex, element] of elements.entries()) { + candidates.push({ ...base, rawText: element, elementIndex }); + } +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/dynamic-lookups.ts b/gitnexus/src/core/ingestion/frameworks/spring/dynamic-lookups.ts new file mode 100644 index 000000000..dfcfbb7c0 --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/dynamic-lookups.ts @@ -0,0 +1,177 @@ +import type { ParsedFile, Range, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { DiInjectionMatch } from '../../di-extractors/index.js'; +import { SPRING_DI_INJECTION_SITES_PROPERTY } from '../../di-extractors/spring.js'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { + resolveCallerGraphId, + resolveDefGraphId, +} from '../../scope-resolution/graph-bridge/ids.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import { isClassLike, lookupBindingsAt } from '../../scope-resolution/scope/walkers.js'; + +const COLLECTION_LOOKUP_METHODS = new Set(['getBeans', 'getBeansOfType']); +const SINGLE_LOOKUP_METHODS = new Set(['getBean']); + +/** + * Distinctive utility names plus conventional Spring context variable names. + * Generic locals remain recall-oriented because repositories often omit the + * third-party context type from the index; AST call/class-literal gates and + * import-aware target resolution prevent the raw-text false-positive class. + */ +const KNOWN_RECEIVERS = new Set([ + 'SpringContextUtil', + 'SpringContextHolder', + 'SpringBeanUtil', + 'ApplicationContextProvider', + 'BeanFactoryProvider', + 'ApplicationContext', + 'BeanFactory', + 'ListableBeanFactory', + 'applicationContext', + 'context', + 'ctx', + 'appContext', + 'beanFactory', +]); + +export interface SpringDynamicLookupFact { + readonly ownerScopeId: ScopeId; + readonly ownerRange: Range; + readonly receiverName: string; + readonly methodName: string; + readonly targetTypeName: string; +} + +export function springDynamicLookupCardinality( + receiverName: string, + methodName: string, +): DiInjectionMatch['cardinality'] | null { + const receiverSimpleName = receiverName.slice(receiverName.lastIndexOf('.') + 1); + if (!KNOWN_RECEIVERS.has(receiverSimpleName)) return null; + if (COLLECTION_LOOKUP_METHODS.has(methodName)) return 'collection'; + if (SINGLE_LOOKUP_METHODS.has(methodName)) return 'single'; + return null; +} + +function visibleTypeDefinitions( + fact: SpringDynamicLookupFact, + indexes: ScopeResolutionIndexes, +): readonly SymbolDefinition[] { + const simpleName = fact.targetTypeName.slice(fact.targetTypeName.lastIndexOf('.') + 1); + let scopeId: ScopeId | null = fact.ownerScopeId; + + while (scopeId !== null) { + const visible = lookupBindingsAt(scopeId, simpleName, indexes) + .map(({ def }) => def) + .filter((def) => isClassLike(def.type)) + .filter( + (def) => !fact.targetTypeName.includes('.') || def.qualifiedName === fact.targetTypeName, + ); + if (visible.length > 0) { + const unique = new Map(visible.map((def) => [def.nodeId, def])); + return [...unique.values()]; + } + scopeId = indexes.scopeTree.getScope(scopeId)?.parent ?? null; + } + + return []; +} + +function resolveTargetTypeName( + graph: KnowledgeGraph, + fact: SpringDynamicLookupFact, + callerLanguage: string | undefined, + nodeLookup: GraphNodeLookup, + indexes: ScopeResolutionIndexes, +): string | undefined { + const graphIds = new Set(); + for (const definition of visibleTypeDefinitions(fact, indexes)) { + const graphId = resolveDefGraphId(definition.filePath, definition, nodeLookup); + if (graphId === undefined) continue; + const node = graph.getNode(graphId); + if ( + (node?.label === 'Class' || + node?.label === 'Interface' || + node?.label === 'Record' || + node?.label === 'Enum') && + node.properties.language === callerLanguage + ) { + graphIds.add(graphId); + } + } + if (graphIds.size !== 1) return undefined; + + const targetId = graphIds.values().next().value; + if (targetId === undefined) return undefined; + const target = graph.getNode(targetId); + if (target === undefined) return undefined; + const qualifiedName = target.properties.qualifiedName; + return typeof qualifiedName === 'string' ? qualifiedName : target.properties.name; +} + +export interface SpringDynamicLookupMetadataAdapter { + getFacts(filePath: string): readonly SpringDynamicLookupFact[]; +} + +/** + * Attach AST-captured programmatic Spring lookups to the framework-neutral DI + * resolver. Java/Kotlin own syntax capture; this shared JVM/Spring seam owns + * import-aware type binding and metadata attachment. + */ +export function createSpringDynamicLookupMetadataAttacher( + adapter: SpringDynamicLookupMetadataAdapter, +) { + return ( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, + indexes: ScopeResolutionIndexes, + ): void => { + for (const parsed of parsedFiles) { + for (const fact of adapter.getFacts(parsed.filePath)) { + const cardinality = springDynamicLookupCardinality(fact.receiverName, fact.methodName); + if (cardinality === null) continue; + + const callerId = resolveCallerGraphId(fact.ownerScopeId, indexes, nodeLookup, { + startLine: fact.ownerRange.startLine, + startCol: fact.ownerRange.startCol, + }); + if (callerId === undefined) continue; + const caller = graph.getNode(callerId); + if ( + caller === undefined || + (caller.label !== 'Function' && + caller.label !== 'Method' && + caller.label !== 'Constructor') + ) { + continue; + } + + const targetTypeName = resolveTargetTypeName( + graph, + fact, + caller.properties.language, + nodeLookup, + indexes, + ); + if (targetTypeName === undefined) continue; + + const match: DiInjectionMatch = { + targetTypeName, + cardinality, + edgeSource: 'site', + reason: `Spring dynamic lookup: ${fact.receiverName}.${fact.methodName}(${fact.targetTypeName})`, + }; + // Singular lookups intentionally use the shared DI selection policy: + // a unique/@Primary candidate wins; unresolved multiplicity is an + // explicit 0.5-confidence fan-out rather than a guessed runtime winner. + const existing = caller.properties[SPRING_DI_INJECTION_SITES_PROPERTY]; + caller.properties[SPRING_DI_INJECTION_SITES_PROPERTY] = [ + ...(Array.isArray(existing) ? existing : []), + match, + ]; + } + } + }; +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/message-producers.ts b/gitnexus/src/core/ingestion/frameworks/spring/message-producers.ts new file mode 100644 index 000000000..4a4a731dc --- /dev/null +++ b/gitnexus/src/core/ingestion/frameworks/spring/message-producers.ts @@ -0,0 +1,140 @@ +import type { Range, ScopeId } from 'gitnexus-shared'; +import type { SpringArgumentFact } from './argument-facts.js'; + +/** + * Outbound side of Spring messaging: the template calls that publish to a + * broker destination, mirroring the inbound `@KafkaListener` / `@RabbitListener` + * family already recognized in `non-http-handlers.ts`. + * + * Recognition is purely syntactic and happens while the language's own scope + * query already has the call node in hand. The receiver's declared type is NOT + * consulted: at capture time the field may be inherited, injected from another + * file, or typed through an import that is not finalized yet. Matching on the + * receiver's simple name instead keeps the capture cheap and resolver-free; a + * later phase that owns type information can refine or discard a fact. + */ +export type SpringMessageProducerTemplate = 'kafka' | 'rabbit' | 'jms' | 'stream-bridge'; + +interface ProducerSignature { + readonly template: SpringMessageProducerTemplate; + /** + * Simple type name of the template bean, matched case-insensitively as a + * SUBSTRING of the receiver's folded simple name. The classifier below states + * which decorations that accepts, and what it does when one receiver name + * contains the type names of two different templates. + */ + readonly typeName: string; + readonly methodName: string; +} + +const PRODUCER_SIGNATURES: readonly ProducerSignature[] = [ + { template: 'kafka', typeName: 'KafkaTemplate', methodName: 'send' }, + { template: 'rabbit', typeName: 'RabbitTemplate', methodName: 'convertAndSend' }, + { template: 'jms', typeName: 'JmsTemplate', methodName: 'convertAndSend' }, + { template: 'stream-bridge', typeName: 'StreamBridge', methodName: 'send' }, +]; + +const PRODUCER_METHOD_NAMES: ReadonlySet = new Set( + PRODUCER_SIGNATURES.map((signature) => signature.methodName), +); + +/** A receiver we can attribute; `templates["k"]` or `getTemplate()` cannot be. */ +const PLAIN_IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/; + +/** + * Fold a receiver's simple name to the form the type-name match runs against. + * + * `_` and `$` are word separators in the spellings this has to accept, not part + * of the words: `KAFKA_TEMPLATE` and `kafka_template` are the same bean name as + * `kafkaTemplate`, written to the constant and snake conventions. Digits stay, + * because they are part of a name (`kafkaTemplate2`), never a separator. + */ +function foldReceiverName(receiverSimpleName: string): string { + return receiverSimpleName.replace(/[_$]/g, '').toLowerCase(); +} + +/** + * Cheap pre-filter usable before any receiver text is materialized. Both + * languages visit every member call, so the common case must cost one set + * lookup on the method name. + */ +export function isSpringMessageProducerMethod(methodName: string): boolean { + return PRODUCER_METHOD_NAMES.has(methodName); +} + +/** + * Classify a `receiver.method(...)` call as a messaging producer, or `null`. + * + * `receiverName` is the receiver expression as written; only its last + * dot-separated segment participates, so `this.kafkaTemplate` and + * `outer.inner.kafkaTemplate` match while `templates.get("k")` does not. + * + * The PLAIN_IDENTIFIER gate runs BEFORE the fold and is load-bearing, because + * the last-dot split is textual: in `config.get("a.kafkaTemplate")` it yields + * `kafkaTemplate")`, which folds to something a name match would accept. Only + * an identifier survives the gate, which is also what rejects `templates["k"]`, + * `getTemplate()`, and a receiver whose dot is separated by a comment. + * + * The folded segment then matches case-insensitively when it CONTAINS the + * template type name, so every convention a template bean is really declared + * with is recognized — decorated by prefix (`orderKafkaTemplate`), by suffix + * (`kafkaTemplateDlq`, `kafkaTemplateV2`, `rabbitTemplate1`), or written as a + * constant (`KAFKA_TEMPLATE`, `STREAM_BRIDGE`). A suffix-only rule accepted + * one of those and silently dropped the rest, which are exactly the publishes + * this capture exists to find. A receiver named only `template` still does not + * match: without type information that would attribute any `send` in the + * repository to Kafka. + * + * The bare type name (`KafkaTemplate.send(...)`) contains itself and so is + * accepted. That is left as it is: the match is by NAME, a name equal to the + * type is the strongest evidence the rule has, and a later phase that owns type + * information can discard a static-looking receiver. + * + * A substring rule also lets ONE receiver satisfy TWO signatures, which a + * suffix rule could not: `KafkaTemplate` and `StreamBridge` both publish + * through `send`, and `RabbitTemplate` and `JmsTemplate` both through + * `convertAndSend`, so `streamBridgeKafkaTemplate.send(...)` matches two + * templates at once. Such a receiver yields NO fact. Nothing here can break the + * tie honestly: the receiver's TYPE is deliberately not resolved, and the name + * is not ranked evidence — neither the longest match, nor the last one, nor the + * order of this list says whether that bean is a KafkaTemplate fronted by a + * stream binding or a StreamBridge named after the broker behind it. Returning + * the first match published an arbitrary choice as a definite broker + * attribution, the one outcome a consumer cannot tell from a fact. Silence + * costs a rare publish and stays recoverable by a phase that owns types. + */ +export function springMessageProducerTemplateOf( + receiverName: string, + methodName: string, +): SpringMessageProducerTemplate | null { + if (!isSpringMessageProducerMethod(methodName)) return null; + const receiverSimpleName = receiverName.slice(receiverName.lastIndexOf('.') + 1).trim(); + if (!PLAIN_IDENTIFIER.test(receiverSimpleName)) return null; + const folded = foldReceiverName(receiverSimpleName); + let matched: SpringMessageProducerTemplate | null = null; + for (const signature of PRODUCER_SIGNATURES) { + if (signature.methodName !== methodName) continue; + if (!folded.includes(signature.typeName.toLowerCase())) continue; + // A second match makes the receiver ambiguous; see above for why it is not + // resolved by preferring one of them. + if (matched !== null) return null; + matched = signature.template; + } + return matched; +} + +export interface SpringMessageProducerFact { + /** Callable that performs the publish; the enclosing method or function. */ + readonly ownerScopeId: ScopeId; + readonly ownerRange: Range; + readonly template: SpringMessageProducerTemplate; + /** Receiver expression as written, for example `this.orderKafkaTemplate`. */ + readonly receiverName: string; + readonly methodName: string; + /** + * Call arguments in source order, or absent when the call site has no + * argument list at all (a Kotlin trailing-lambda call). An empty array means + * an empty argument list was written — a different fact from no list. + */ + readonly args?: readonly SpringArgumentFact[]; +} diff --git a/gitnexus/src/core/ingestion/frameworks/spring/non-http-handlers.ts b/gitnexus/src/core/ingestion/frameworks/spring/non-http-handlers.ts index 8f3144d01..ae43f3eac 100644 --- a/gitnexus/src/core/ingestion/frameworks/spring/non-http-handlers.ts +++ b/gitnexus/src/core/ingestion/frameworks/spring/non-http-handlers.ts @@ -3,6 +3,7 @@ import type { KnowledgeGraph } from '../../../graph/types.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; import { resolveCallerGraphId } from '../../scope-resolution/graph-bridge/ids.js'; import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import type { SpringArgumentFact } from './argument-facts.js'; import { createSpringAnnotationNameResolver } from './bean-candidates.js'; import { SPRING_BEAN_ANNOTATION } from './bean-factories.js'; @@ -14,6 +15,34 @@ export interface SpringNonHttpHandlerAnnotationFact { readonly name: string; /** Kotlin use-site targets describe generated/property elements, not the callable. */ readonly useSiteTarget?: string; + /** + * Annotation arguments in source order. An empty array always means an empty + * list was written (`@Scheduled()`), which is a different fact from absence — + * but absence has TWO causes, and only one of them is a statement about the + * source. Either the annotation was written without an argument list + * (`@Scheduled`), or arguments were never read for this callable. + * + * They are read only for a callable that carries a handler annotation. Java + * produces facts for no other callable, so there absence does mean "no list + * was written". Kotlin also produces a fact for a merely annotated function — + * it captures those without a name prefilter so an import alias cannot hide a + * handler — and on those facts arguments are absent however the annotation + * was written. + * + * The values keep their source spelling, with one deliberate exception: + * `normalizeSpringFactText` trims them and collapses whitespace around the + * dots of a multi-line expression, so `Destinations.ORDERS` and the same + * reference wrapped across lines produce equal facts. Without that, source + * formatting — including the enclosing block's indentation, which is not a + * property of the expression at all — would leak into the data and make two + * spellings of one destination compare unequal downstream. + * + * Nothing else is touched. `@KafkaListener(topics = ...)` and + * `@RabbitListener(queues = ...)` name the destination differently, and a + * destination may be a literal, a constant reference, or a `${...}` + * placeholder; resolving any of those belongs to a later phase. + */ + readonly args?: readonly SpringArgumentFact[]; } export interface SpringNonHttpHandlerFact< diff --git a/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts b/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts index 712359a41..5750da917 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts @@ -19,7 +19,7 @@ import fs from 'fs/promises'; import path from 'path'; import { createRequire } from 'node:module'; -import { isHardcodedIgnoredDirectory } from '../../../config/ignore-service.js'; +import { isHardcodedIgnoredDirectoryAtPath } from '../../../config/ignore-service.js'; import { logger } from '../../logger.js'; import { resolveFile } from '../languages/typescript/file-candidates.js'; @@ -361,9 +361,10 @@ export async function loadNodeWorkspacePackages( for (const entry of entries) { if (entry.isDirectory()) { - if (isHardcodedIgnoredDirectory(entry.name)) continue; + const childDir = path.join(dir, entry.name); + if (isHardcodedIgnoredDirectoryAtPath(repoRoot, childDir)) continue; if (depth < SCAN_MAX_DEPTH) { - queue.push({ dir: path.join(dir, entry.name), depth: depth + 1 }); + queue.push({ dir: childDir, depth: depth + 1 }); } continue; } diff --git a/gitnexus/src/core/ingestion/import-resolvers/php.ts b/gitnexus/src/core/ingestion/import-resolvers/php.ts index 6652ecbfa..72acba2f0 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/php.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/php.ts @@ -49,13 +49,29 @@ export function resolvePhpImportInternal( if (composerConfig) { const sorted = getSortedPsr4(composerConfig); + const authoritativePsr4 = + composerConfig.authoritativePsr4 ?? new Set(sorted.map(([namespace]) => namespace)); + let matchedAuthoritativeNamespace = false; + let hasAuthoritativeCatchAllNamespace = false; + const ownershipPath = normalized.replace(/^\/+/, ''); + for (const [nsPrefix, dirPrefix] of sorted) { - const nsPrefixSlash = nsPrefix.replace(/\\/g, '/'); - if (normalized.startsWith(nsPrefixSlash + '/') || normalized === nsPrefixSlash) { - const remainder = normalized.slice(nsPrefixSlash.length).replace(/^\//, ''); + const nsPrefixSlash = nsPrefix.replace(/\\/g, '/').replace(/\/+$/, ''); + const isCatchAll = nsPrefixSlash === ''; + if ( + isCatchAll || + ownershipPath.startsWith(nsPrefixSlash + '/') || + ownershipPath === nsPrefixSlash + ) { + const isAuthoritative = authoritativePsr4.has(nsPrefix); + matchedAuthoritativeNamespace ||= isAuthoritative; + hasAuthoritativeCatchAllNamespace ||= isAuthoritative && isCatchAll; + const remainder = ownershipPath.slice(nsPrefixSlash.length).replace(/^\//, ''); // 1. Try class-style PSR-4: full path → file (e.g. App\Models\User → app/Models/User.php) - const filePath = dirPrefix + (remainder ? '/' + remainder : '') + '.php'; + const mappedPath = + dirPrefix === '' ? remainder : dirPrefix + (remainder ? '/' + remainder : ''); + const filePath = mappedPath + '.php'; if (allFiles.has(filePath)) return filePath; if (index) { const result = index.getInsensitive(filePath); @@ -64,45 +80,64 @@ export function resolvePhpImportInternal( // 2. Function/constant fallback: strip last segment (symbol name), scan namespace directory. // e.g. App\Models\getUser → directory app/Models/, find first .php file in that dir. - const lastSlash = remainder.lastIndexOf('/'); - const nsDir = lastSlash >= 0 ? dirPrefix + '/' + remainder.slice(0, lastSlash) : dirPrefix; + // A root/catch-all mapping cannot safely infer a symbol's declaring + // file from an arbitrary sibling. The higher-level PHP resolver has + // parsed symbol-kind and declaration evidence for function/const + // imports; class imports must not inherit this directory heuristic. + if (!isCatchAll && dirPrefix !== '') { + const lastSlash = remainder.lastIndexOf('/'); + const relativeNamespace = lastSlash >= 0 ? remainder.slice(0, lastSlash) : ''; + const nsDir = relativeNamespace === '' ? dirPrefix : `${dirPrefix}/${relativeNamespace}`; - // Prefer SuffixIndex directory lookup (O(log n + matches)) over linear scan. - // - // An EMPTY bucket is a final answer, not a miss to retry with the scan - // below — which is what the `else` restores, and what this comment - // always claimed. Re-scanning on empty was the last per-import - // workspace traversal left in PHP resolution after #2901: any `use` - // matching a PSR-4 prefix whose directory holds no direct `.php` child - // (`App\Legacy\Ghost`) paid a full pass, measured at 201 traversals for - // 200 imports. - // - // The bucket is a superset of what the scan can find, for BOTH index - // shapes that reach here. A root-anchored direct child `nsDir/.php` - // has its directory exactly equal to `nsDir`, and `nsDir` is always one - // of that directory's own suffixes — so the shared `dirMap` (keyed on - // every directory suffix) necessarily contains it, as does the - // root-anchored parity index `languages/php/import-target.ts` builds. - // Empty superset therefore implies empty scan, and control falls - // through to the next PSR-4 prefix exactly as before. - if (index) { - const candidates = index.getFilesInDir(nsDir, '.php'); - if (candidates.length > 0) return candidates[0]; - } else { - // Linear scan, only when a SuffixIndex is genuinely unavailable. - const nsDirPrefix = nsDir.endsWith('/') ? nsDir : nsDir + '/'; - for (const f of allFiles) { - if ( - f.startsWith(nsDirPrefix) && - f.endsWith('.php') && - !f.slice(nsDirPrefix.length).includes('/') - ) { - return f; + // Prefer SuffixIndex directory lookup (O(log n + matches)) over linear scan. + // + // An EMPTY bucket is a final answer, not a miss to retry with the scan + // below — which is what the `else` restores, and what this comment + // always claimed. Re-scanning on empty was the last per-import + // workspace traversal left in PHP resolution after #2901: any `use` + // matching a PSR-4 prefix whose directory holds no direct `.php` child + // (`App\Legacy\Ghost`) paid a full pass, measured at 201 traversals for + // 200 imports. + // + // The bucket is a superset of what the scan can find, for BOTH index + // shapes that reach here. A root-anchored direct child `nsDir/.php` + // has its directory exactly equal to `nsDir`, and `nsDir` is always one + // of that directory's own suffixes — so the shared `dirMap` (keyed on + // every directory suffix) necessarily contains it, as does the + // root-anchored parity index `languages/php/import-target.ts` builds. + // Empty superset therefore implies empty scan, and control falls + // through to the next PSR-4 prefix exactly as before. + if (index) { + const candidates = index.getFilesInDir(nsDir, '.php'); + if (candidates.length > 0) return candidates[0]; + } else { + // Linear scan, only when a SuffixIndex is genuinely unavailable. + const nsDirPrefix = nsDir.endsWith('/') ? nsDir : nsDir + '/'; + for (const f of allFiles) { + if ( + f.startsWith(nsDirPrefix) && + f.endsWith('.php') && + !f.slice(nsDirPrefix.length).includes('/') + ) { + return f; + } } } } } } + + // A non-empty PSR-4 map is authoritative for namespaces it does not own. + // Preserve the existing mapped-namespace fallback behavior; #2962 is the + // conservative external-namespace gate, not a rewrite of mapped lookup. + // A catch-all owns every namespace, so its misses remain authoritative. + if ( + authoritativePsr4.size > 0 && + !composerConfig.hasUnmodeledAutoload && + (!matchedAuthoritativeNamespace || hasAuthoritativeCatchAllNamespace) + ) { + return null; + } } // Fallback: suffix matching (works without composer.json) diff --git a/gitnexus/src/core/ingestion/language-config.ts b/gitnexus/src/core/ingestion/language-config.ts index a50ce16e6..b1f53c1a9 100644 --- a/gitnexus/src/core/ingestion/language-config.ts +++ b/gitnexus/src/core/ingestion/language-config.ts @@ -30,11 +30,103 @@ export interface GoModuleConfig { export interface ComposerConfig { /** Map of namespace prefix -> directory (e.g., "App\\" -> "app/") */ psr4: Map; + /** Production `autoload.psr-4` prefixes that may gate external namespaces. + * Absent on legacy/manual configs, where every mapping remains authoritative. */ + authoritativePsr4?: ReadonlySet; + /** True when Composer also declares an autoload mechanism this resolver does not model. */ + hasUnmodeledAutoload?: boolean; /** PSR-4 entries sorted by namespace length descending (longest match wins). * Cached once at config load time to avoid re-sorting on every import. */ psr4Sorted?: readonly [string, string][]; } +function normalizeComposerDirectory(baseDir: string, directory: string): string { + const normalizedBase = baseDir.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+$/, ''); + const normalizedDirectory = directory + .replace(/\\/g, '/') + .replace(/^(?:\.\/)+/, '') + .replace(/\/+$/, ''); + if (normalizedBase === '') return normalizedDirectory; + if (normalizedDirectory === '') return normalizedBase; + return path.posix.normalize(`${normalizedBase}/${normalizedDirectory}`); +} + +/** Parse one Composer manifest without performing I/O. */ +export function parseComposerConfig(value: unknown, baseDir = ''): ComposerConfig | null { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return null; + + const composer = value as Record; + const autoload = composer.autoload; + const autoloadDev = composer['autoload-dev']; + if (autoload === undefined && autoloadDev === undefined) return null; + + const psr4 = new Map(); + const authoritativePsr4 = new Set(); + let hasUnmodeledAutoload = false; + + const addSection = (sectionValue: unknown, authoritative: boolean): void => { + if (typeof sectionValue !== 'object' || sectionValue === null || Array.isArray(sectionValue)) { + return; + } + const section = sectionValue as Record; + if ('psr-0' in section || 'classmap' in section) hasUnmodeledAutoload = true; + + const rawPsr4 = section['psr-4']; + if (typeof rawPsr4 !== 'object' || rawPsr4 === null || Array.isArray(rawPsr4)) return; + + for (const [namespace, directories] of Object.entries(rawPsr4)) { + const stringDirectories = Array.isArray(directories) + ? directories.filter((entry): entry is string => typeof entry === 'string') + : typeof directories === 'string' + ? [directories] + : []; + if (stringDirectories.length === 0) continue; + if (stringDirectories.length > 1) hasUnmodeledAutoload = true; + + const normalizedNamespace = namespace.replace(/\\+$/, ''); + const normalizedDirectory = normalizeComposerDirectory(baseDir, stringDirectories[0]); + const existing = psr4.get(normalizedNamespace); + if (existing !== undefined && existing !== normalizedDirectory) { + hasUnmodeledAutoload = true; + continue; + } + if (existing === undefined) psr4.set(normalizedNamespace, normalizedDirectory); + if (authoritative) authoritativePsr4.add(normalizedNamespace); + } + }; + + // Production mappings win duplicate prefixes. Development mappings remain + // usable for test code but do not establish authority for the external gate. + addSection(autoload, true); + addSection(autoloadDev, false); + + return { psr4, authoritativePsr4, hasUnmodeledAutoload }; +} + +/** Merge package-local Composer manifests into one repository-relative config. */ +export function mergeComposerConfigs(configs: readonly ComposerConfig[]): ComposerConfig | null { + if (configs.length === 0) return null; + + const psr4 = new Map(); + const authoritativePsr4 = new Set(); + let hasUnmodeledAutoload = false; + for (const config of configs) { + hasUnmodeledAutoload ||= config.hasUnmodeledAutoload === true; + for (const [namespace, directory] of config.psr4) { + const existing = psr4.get(namespace); + if (existing !== undefined && existing !== directory) { + hasUnmodeledAutoload = true; + continue; + } + if (existing === undefined) psr4.set(namespace, directory); + } + for (const namespace of config.authoritativePsr4 ?? config.psr4.keys()) { + authoritativePsr4.add(namespace); + } + } + return { psr4, authoritativePsr4, hasUnmodeledAutoload }; +} + /** C# project config parsed from .csproj files */ export interface CSharpProjectConfig { /** Root namespace from or assembly name (default: project directory name) */ @@ -196,22 +288,13 @@ export async function loadComposerConfig(repoRoot: string): Promise(); - for (const [ns, dir] of Object.entries(merged)) { - const nsNorm = (ns as string).replace(/\\+$/, ''); - const dirNorm = (dir as string).replace(/\\/g, '/').replace(/\/+$/, ''); - psr4.set(nsNorm, dirNorm); - } + const config = parseComposerConfig(JSON.parse(raw)); + if (config === null) return null; if (isDev) { - logger.info(`📦 Loaded ${psr4.size} PSR-4 mappings from composer.json`); + logger.info(`📦 Loaded ${config.psr4.size} PSR-4 mappings from composer.json`); } - return { psr4 }; + return config; } catch { return null; } diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index 90e42f9e8..c8ddd4bd2 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -36,16 +36,90 @@ import type { VariableExtractor } from './variable-types.js'; import type { ImportResolverFn } from './import-resolvers/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; import type { CfgVisitor } from './cfg/types.js'; -import type { NodeLabel } from 'gitnexus-shared'; +import type { GraphNode, NodeLabel } from 'gitnexus-shared'; import type { ExtractedRoute } from './route-extractors/laravel.js'; import type { SharedSpringType } from './route-extractors/spring-shared.js'; +import type { + ModuleConstants, + Operand, + RepoConstants, +} from './route-extractors/constant-resolver.js'; import type Parser from 'tree-sitter'; import type { ExtractedDecoratorRoute } from './workers/parse-worker.js'; +import type { SpringNonHttpHandlerFact } from './frameworks/spring/non-http-handlers.js'; +import type { SpringMessageProducerFact } from './frameworks/spring/message-producers.js'; + +/** One file's captured Spring async messaging facts, in both directions. */ +export interface SpringMessagingFacts { + /** Callables carrying a listener annotation — the inbound side. */ + readonly handlers: readonly SpringNonHttpHandlerFact[]; + /** Messaging-template publishes — the outbound side. */ + readonly producers: readonly SpringMessageProducerFact[]; +} // ── Shared type aliases ──────────────────────────────────────────────────── /** Tree-sitter query captures: capture name → AST node (or undefined if not captured). */ export type CaptureMap = Record; +export interface DefinitionPropertiesContext { + readonly nodeLabel: NodeLabel; + readonly nodeName: string; + readonly filePath: string; + readonly definitionNode: SyntaxNode; + readonly parsedImports: readonly ParsedImport[]; + readonly isExported: boolean; +} + +export type DefinitionPropertiesExtractor = ( + context: DefinitionPropertiesContext, +) => Readonly> | undefined; + +export interface RuntimeCallableIdentity { + readonly name: string; + readonly descriptorParameterTypes: readonly string[] | undefined; +} + +/** + * Optional language-owned bridge from runtime/compiler symbol identities to + * source graph symbols. Framework importers use this instead of naming + * languages or reproducing compiler conventions in shared ingestion code. + */ +export interface RuntimeSymbolStrategy { + /** Runtime owner names that may contain this callable/property. */ + readonly callableOwnerAliases?: ( + node: GraphNode, + owner: GraphNode | undefined, + ) => readonly string[]; + /** Whether a runtime callable identity can conservatively identify a node. */ + readonly matchesCallable: (node: GraphNode, runtime: RuntimeCallableIdentity) => boolean; +} + +/** Run optional provider enrichment without allowing one hook failure to drop + * the rest of the worker's language batch. */ +export function runDefinitionPropertiesExtractor( + extractor: DefinitionPropertiesExtractor, + context: DefinitionPropertiesContext, + onError: (error: unknown) => void, +): Readonly> | undefined { + try { + return extractor(context); + } catch (error) { + onError(error); + return undefined; + } +} + +/** Provider metadata is additive; graph identity and source-location fields + * supplied by the worker remain authoritative. */ +export function mergeCanonicalDefinitionProperties< + TCanonical extends Readonly>, +>( + providerProperties: Readonly>, + canonicalProperties: TCanonical, +): Record & TCanonical { + return { ...providerProperties, ...canonicalProperties } as Record & TCanonical; +} + // ── Strategy tag types ───────────────────────────────────────────────────── // NOTE: `MroStrategy` is defined in `gitnexus-shared` and re-exported above // so `core/ingestion/model/resolve.ts` can consume it without importing from @@ -64,6 +138,25 @@ export interface AstFrameworkPatternConfig { * Required fields must be explicitly set; optional fields have defaults * applied by defineLanguage(). */ +/** + * Should the parse worker run {@link LanguageProviderConfig.extractModuleConstants} + * on this file? + * + * Exported so the DECISION is testable without booting a worker. It encodes the + * one rule that is easy to get backwards: a provider that declares no + * `moduleConstantHeuristic` harvests unconditionally. Writing the gate as + * `provider.moduleConstantHeuristic?.(content)` reads `undefined` as "skip" and + * silently disables the hook for every provider without a heuristic — which is + * exactly how Python's already-shipped harvest was turned off (#2391/#2980). + */ +export function shouldHarvestModuleConstants( + provider: Pick, + content: string, +): boolean { + if (!provider.extractModuleConstants) return false; + return !provider.moduleConstantHeuristic || provider.moduleConstantHeuristic(content); +} + interface LanguageProviderConfig { // ── Identity ────────────────────────────────────────────────────── readonly id: SupportedLanguages; @@ -130,6 +223,13 @@ interface LanguageProviderConfig { */ readonly preprocessSource?: (sourceText: string, filePath: string) => string; + /** + * Runtime/compiler identity reconciliation for framework metadata. The + * central importer owns ambiguity handling; providers only supply aliases + * and language-specific callable compatibility. + */ + readonly runtimeSymbolStrategy?: RuntimeSymbolStrategy; + // ── Core (required) ─────────────────────────────────────────────── /** Type extraction: declarations, initializers, for-loop bindings */ readonly typeConfig: LanguageTypeConfig; @@ -289,6 +389,10 @@ interface LanguageProviderConfig { * constant, and static declarations. Produces VariableInfo with type, visibility, * isConst, isStatic, isMutable metadata. Default: undefined (no variable extraction). */ readonly variableExtractor?: VariableExtractor; + /** Add language-owned, structured properties to a definition node. Values + * cross the worker boundary and must therefore be structured-clone-safe. + * Shared ingestion code treats these properties as opaque. */ + readonly definitionPropertiesExtractor?: DefinitionPropertiesExtractor; /** Class/type extractor for deriving canonical qualified names for class-like symbols. * Uses the same provider-driven strategy pattern as method/field extraction so * namespace/package/module rules stay language-specific. */ @@ -356,6 +460,28 @@ interface LanguageProviderConfig { lineOffset: number, ) => ExtractedDecoratorRoute[]; + /** + * Name of the function a route decorator captured by the worker's generic + * `@decorator` query applies to, given the decorator's own AST node. + * + * The worker knows a decorator is a route decorator but not how this + * language's grammar attaches it to a definition, so it hands the node over + * unchanged and takes whatever the language returns. Only languages that + * declare route handlers through the generic decorator captures need this; + * languages with a dedicated {@link extractDecoratorRoutes} extractor + * (JS/TS via `nest.ts`, Java via `spring.ts`) already set + * `ExtractedDecoratorRoute.handlerName` there and should leave this undefined. + * + * Implementations must read their own decorated-definition shape directly and + * return undefined for anything else — never climb ancestors to find a name, + * since a decorator that is not attached to a function has no handler and a + * borrowed enclosing name resolves `handlerSymbolId` to the wrong symbol. The + * routes phase treats undefined as "fall back to the file-level edge". + * + * Default: undefined (no handler name from generic decorator captures). + */ + readonly decoratorRouteHandlerName?: (decoratorNode: SyntaxNode) => string | undefined; + /** * Collect a project-wide, language-agnostic view of route-defining * class/interface declarations (`SharedSpringType`) from a parsed file. @@ -373,6 +499,150 @@ interface LanguageProviderConfig { filePath: string, ) => SharedSpringType[]; + /** + * Optional post-capture emission of synthetic structure members (nodes, + * symbols, ownership edges) that have no AST method node — e.g. Lombok + * accessors. Called once per file after the capture loop, at the same + * post-capture site as {@link extractDecoratorRoutes}. + * + * `classOwnersByNodeId` maps in-memory tree-sitter node ids of type + * declarations materialized in THIS file's capture loop to their graph + * node ids. Keys are never persisted; they exist only for the duration + * of the worker pass. + * + * Default: undefined (no synthetic structure members). + */ + readonly synthesizeStructureMembers?: ( + tree: Parser.Tree, + filePath: string, + classOwnersByNodeId: ReadonlyMap, + ) => { + nodes: ReadonlyArray<{ + id: string; + label: string; + properties: Record; + }>; + symbols: ReadonlyArray<{ + filePath: string; + name: string; + nodeId: string; + type: string; + ownerId?: string; + parameterCount?: number; + requiredParameterCount?: number; + parameterTypes?: string[]; + returnType?: string; + visibility?: string; + isStatic?: boolean; + isAbstract?: boolean; + isFinal?: boolean; + }>; + relationships: ReadonlyArray<{ + id: string; + sourceId: string; + targetId: string; + type: string; + confidence: number; + reason: string; + }>; + }; + + /** + * Harvest this file's module-level string constants (#2391 core, #2980 Java + * parity) into the language-agnostic {@link ModuleConstants} shape, so the + * parse phase can resolve non-literal decorator route paths cross-file. + * + * The worker calls this when BOTH hold: + * - the provider declares no `moduleConstantHeuristic`, or the one it + * declares matched — syntax-driven, e.g. a `static final String` field or + * a constants-bearing import; NEVER a class-name pattern like + * `*Constants`, which silently drops route constants living in classes + * named e.g. `ApiPaths`/`Routes`, and + * - the extraction yields something resolvable (a literal, an expression, or + * an import binding), keeping the aggregate bounded on large repos. + * + * Default: undefined (no constant harvest; non-literal route paths of this + * language floor to skip). + */ + readonly extractModuleConstants?: (tree: Parser.Tree) => ModuleConstants; + + /** + * Cheap content heuristic deciding whether the worker should run + * {@link extractModuleConstants} on a file. Guards the harvest cost on huge + * repos: files that cannot contribute (no constant-bearing syntax) are not + * walked. Must be syntax-driven (field/import shape), not identifier + * pattern-matching on class names. + * + * Default: undefined — harvest EVERY file of this language. A gate is opt-in + * because getting it wrong silently drops routes that already resolve, and a + * missed gate only costs time. Declare one only where the cost bites (Java's + * Maven monorepos) and only after checking it against every shape + * {@link extractModuleConstants} accepts. + */ + readonly moduleConstantHeuristic?: (content: string) => boolean; + + /** + * Prepare this language's harvested constants once the complete repo map is + * available and before route operands are folded. The parse phase passes only + * entries owned by this provider, so implementations can build one reusable + * language-specific index and may materialize deferred bindings in place. + * + * Default: undefined (the harvested constants are already fold-ready). + */ + readonly prepareRouteConstants?: (repo: RepoConstants) => void; + + /** + * Spring async messaging facts captured for one file — the listener + * annotations that subscribe to a broker destination and the template calls + * that publish to one. + * + * Both families are collected during capture and restored on the main thread + * by {@link LanguageProviderConfig.applyCaptureSideChannel}, so they are only + * readable AFTER scope resolution has run. The `springDestinations` phase is + * the caller; routing through a provider hook is what keeps that phase from + * naming a language to reach a per-language fact store. + * + * Default: undefined — this language captures no Spring messaging facts, and + * the phase contributes nothing for its files. + */ + readonly getSpringMessagingFacts?: (filePath: string) => SpringMessagingFacts; + + /** + * Whether this language INTERPOLATES its string literals — Kotlin's + * `"orders-$env"` and `"orders-${env}"` are string templates evaluated at + * runtime, while Java's are ordinary characters. + * + * A capability rather than a language name, because shared ingestion code may + * not branch on a language (see AGENTS.md) and because the capability is what + * the consumer actually needs. Spring destination resolution is the caller: + * in an interpolating language an unescaped `$` in a destination literal is a + * runtime value and must be refused, and `"${app.topic}"` is a TEMPLATE, not + * a Spring property placeholder — the placeholder has to be written + * `"\${app.topic}"` there. Reading either as an address gives two unrelated + * services one shared destination node. + * + * Default: false — literals are literal, `$` is a character. + */ + readonly interpolatesStringLiterals?: boolean; + + /** + * Fold one file's non-literal route-path operand list + * (`routePathExpr`/`routePathOperands` of an `ExtractedDecoratorRoute`) + * against the repo-wide, file-path-keyed constant map, or null when it cannot + * be fully folded (skip floor — never a phantom path). Languages whose + * qualified refs resolve through class imports (`Outer.CONST`, + * `com.example.ApiPaths.USERS`) need this hook because the shared fold has no + * notion of qualified names; Python's bare-name refs use the shared default. + * + * Default: undefined (the parse phase falls back to the shared + * language-agnostic operand fold). + */ + readonly foldRoutePathOperands?: ( + filePath: string, + operands: readonly Operand[], + repo: RepoConstants, + ) => string | null; + // ── Noise filtering ──────────────────────────────────────────────── /** Built-in/stdlib names that should be filtered from the call graph for this language. * Default: undefined (no language-specific filtering). */ @@ -773,6 +1043,34 @@ export interface LanguageProvider extends Omit boolean; } +/** + * Run each provider's repo-constant preparation hook once over only the files + * that provider owns. Values are shared with `repo`, so in-place preparation + * is visible to the subsequent fold without copying the complete map. + */ +export function prepareRouteConstantsByProvider( + repo: RepoConstants, + providerForFile: (filePath: string) => Pick | null, +): void { + const slices = new Map< + Pick, + Map + >(); + for (const [filePath, constants] of repo) { + const provider = providerForFile(filePath); + if (!provider?.prepareRouteConstants) continue; + let slice = slices.get(provider); + if (!slice) { + slice = new Map(); + slices.set(provider, slice); + } + slice.set(filePath, constants); + } + for (const [provider, slice] of slices) { + provider.prepareRouteConstants?.(slice); + } +} + const DEFAULTS: Pick = { mroStrategy: 'first-wins', }; diff --git a/gitnexus/src/core/ingestion/languages/csharp/razor-view-components.ts b/gitnexus/src/core/ingestion/languages/csharp/razor-view-components.ts new file mode 100644 index 000000000..0e7ea55a4 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/csharp/razor-view-components.ts @@ -0,0 +1,955 @@ +/** + * ASP.NET Core ViewComponent convention support. + * + * Same bound as Spring Boot DI in Java/Kotlin: do not resolve into the SDK + * (`Microsoft.AspNetCore.Mvc.ViewComponent`, `IViewComponentHelper`, + * `Component.InvokeAsync` itself). Those types live outside the workspace. + * The only hop worth taking is the framework convention that lands on an + * **in-repo** class — `InvokeAsync("Foo")` → workspace `FooViewComponent`, + * just as a Spring `@Autowired IFoo` fans out to an in-repo `@Service`, + * not to `ApplicationContext`. + * + * Razor templates are not parsed as C# (markup + code would poison + * tree-sitter-c-sharp). A small Razor state machine extracts C# islands and + * markup tag helpers; C# files use a string/comment-aware lexer so attributes + * and literals are not mistaken for helper calls. Literal names are enough + * because the target catalog is already built from parsed `.cs` classes. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { glob } from 'glob'; +import type { ParsedFile } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import { createIgnoreFilter } from '../../../../config/ignore-service.js'; +import { generateId } from '../../../../lib/utils.js'; +import { getMaxFileSizeBytes } from '../../utils/max-file-size.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import { resolveDefGraphId } from '../../scope-resolution/graph-bridge/ids.js'; +import { definitionIdPosition } from '../../scope-resolution/utils/definition-id.js'; + +const VIEW_COMPONENT_SUFFIX = 'ViewComponent'; +const VIEW_COMPONENT_TAG_RE = /<\s*vc:([a-z][a-z0-9-]*)\b/gi; +const COMPONENT_NAME_RE = /^[A-Za-z_][A-Za-z0-9_.-]*$/; +const TYPE_MODIFIERS = new Set([ + 'public', + 'internal', + 'protected', + 'private', + 'abstract', + 'sealed', + 'partial', + 'static', + 'new', + 'file', + 'required', + 'unsafe', + 'readonly', +]); +const RAZOR_BLOCK_KEYWORDS = new Set([ + 'if', + 'for', + 'foreach', + 'while', + 'using', + 'switch', + 'try', + 'lock', + 'functions', + 'helper', + 'code', + 'section', + 'do', +]); + +export interface RazorViewComponentConfig { + /** Repo-relative `.cshtml` path → extracted invocation names. */ + readonly views: ReadonlyMap; +} + +export interface ViewComponentAliasBind { + readonly className: string; + /** 1-based line of the type declaration (including leading attributes). */ + readonly startLine: number; + /** 0-based column of the type declaration (including leading attributes). */ + readonly startCol: number; + readonly aliases: readonly string[]; +} + +class SourceCursor { + i = 0; + line = 1; + col = 0; + + constructor(readonly source: string) {} + + get length(): number { + return this.source.length; + } + + get done(): boolean { + return this.i >= this.source.length; + } + + peek(n = 0): string { + return this.source[this.i + n] ?? ''; + } + + startsWith(value: string): boolean { + return this.source.startsWith(value, this.i); + } + + snapshot(): { i: number; line: number; col: number } { + return { i: this.i, line: this.line, col: this.col }; + } + + restore(pos: { i: number; line: number; col: number }): void { + this.i = pos.i; + this.line = pos.line; + this.col = pos.col; + } + + advance(count = 1): void { + const end = Math.min(this.i + count, this.source.length); + while (this.i < end) { + const ch = this.source[this.i]!; + this.i += 1; + if (ch === '\n') { + this.line += 1; + this.col = 0; + } else { + this.col += 1; + } + } + } +} + +function isIdentStart(ch: string): boolean { + return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch === '_' || ch === '@'; +} + +function isIdentPart(ch: string): boolean { + return isIdentStart(ch) || (ch >= '0' && ch <= '9'); +} + +function skipWhitespace(cur: SourceCursor): void { + while (!cur.done) { + const ch = cur.peek(); + if (ch !== ' ' && ch !== '\t' && ch !== '\n' && ch !== '\r' && ch !== '\f' && ch !== '\v') + break; + cur.advance(); + } +} + +/** Skip line comments and block comments. Returns true if a comment was consumed. */ +function skipCsharpComment(cur: SourceCursor): boolean { + if (cur.startsWith('//')) { + while (!cur.done && cur.peek() !== '\n') cur.advance(); + return true; + } + if (cur.startsWith('/*')) { + cur.advance(2); + while (!cur.done && !cur.startsWith('*/')) cur.advance(); + if (cur.startsWith('*/')) cur.advance(2); + return true; + } + return false; +} + +function skipCsharpTrivia(cur: SourceCursor): void { + for (;;) { + skipWhitespace(cur); + if (!skipCsharpComment(cur)) return; + } +} + +function skipRegularString(cur: SourceCursor, interpolated: boolean): void { + cur.advance(); // opening " + while (!cur.done) { + const ch = cur.peek(); + if (ch === '\\') { + cur.advance(2); + continue; + } + if (interpolated && ch === '{') { + if (cur.peek(1) === '{') { + cur.advance(2); + continue; + } + skipInterpolation(cur); + continue; + } + cur.advance(); + if (ch === '"') return; + } +} + +function skipVerbatimString(cur: SourceCursor, interpolated: boolean): void { + cur.advance(2); // @" + while (!cur.done) { + const ch = cur.peek(); + if (ch === '"') { + if (cur.peek(1) === '"') { + cur.advance(2); + continue; + } + cur.advance(); + return; + } + if (interpolated && ch === '{') { + if (cur.peek(1) === '{') { + cur.advance(2); + continue; + } + skipInterpolation(cur); + continue; + } + cur.advance(); + } +} + +function skipRawString(cur: SourceCursor): void { + let quoteCount = 0; + while (cur.peek() === '"') { + quoteCount += 1; + cur.advance(); + } + while (!cur.done) { + if (cur.peek() !== '"') { + cur.advance(); + continue; + } + let seen = 0; + while (cur.peek() === '"') { + seen += 1; + cur.advance(); + } + if (seen >= quoteCount) return; + } +} + +function skipInterpolation(cur: SourceCursor): void { + cur.advance(); // { + let depth = 1; + while (!cur.done && depth > 0) { + skipCsharpTrivia(cur); + if (cur.done) return; + if (skipCsharpString(cur)) continue; + const ch = cur.peek(); + if (ch === '{') depth += 1; + else if (ch === '}') depth -= 1; + cur.advance(); + } +} + +function skipCsharpString(cur: SourceCursor): boolean { + const ch = cur.peek(); + if (ch === "'") { + cur.advance(); + if (cur.peek() === '\\') cur.advance(2); + else cur.advance(); + if (cur.peek() === "'") cur.advance(); + return true; + } + if (ch === '"') { + if (cur.peek(1) === '"' && cur.peek(2) === '"') skipRawString(cur); + else skipRegularString(cur, false); + return true; + } + if (ch === '$' && cur.peek(1) === '@' && cur.peek(2) === '"') { + cur.advance(); + skipVerbatimString(cur, true); + return true; + } + if (ch === '@' && cur.peek(1) === '$' && cur.peek(2) === '"') { + cur.advance(2); + skipVerbatimString(cur, true); + return true; + } + if (ch === '@' && cur.peek(1) === '"') { + skipVerbatimString(cur, false); + return true; + } + if (ch === '$' && cur.peek(1) === '"') { + if (cur.peek(2) === '"' && cur.peek(3) === '"') { + cur.advance(); + skipRawString(cur); + } else { + cur.advance(); + skipRegularString(cur, true); + } + return true; + } + return false; +} + +function readIdent(cur: SourceCursor): string | undefined { + if (!isIdentStart(cur.peek())) return undefined; + const start = cur.i; + if (cur.peek() === '@') cur.advance(); + if (!isIdentStart(cur.peek()) && !(cur.peek() >= 'A' && cur.peek() <= 'z')) { + cur.i = start; + return undefined; + } + while (isIdentPart(cur.peek()) && cur.peek() !== '@') cur.advance(); + const raw = cur.source.slice(start, cur.i); + return raw.startsWith('@') ? raw.slice(1) : raw; +} + +function tryReadIdent(cur: SourceCursor): string | undefined { + skipCsharpTrivia(cur); + return readIdent(cur); +} + +function decodeCsharpString(cur: SourceCursor): string | undefined { + skipCsharpTrivia(cur); + const start = cur.snapshot(); + const ch = cur.peek(); + if (ch === '$') return undefined; + if (ch === '@' && cur.peek(1) === '"') { + cur.advance(2); + let value = ''; + while (!cur.done) { + if (cur.peek() === '"') { + if (cur.peek(1) === '"') { + value += '"'; + cur.advance(2); + continue; + } + cur.advance(); + return value; + } + value += cur.peek(); + cur.advance(); + } + cur.restore(start); + return undefined; + } + if (ch === '"' && cur.peek(1) === '"' && cur.peek(2) === '"') { + let quoteCount = 0; + while (cur.peek() === '"') { + quoteCount += 1; + cur.advance(); + } + const bodyStart = cur.i; + while (!cur.done) { + if (cur.peek() !== '"') { + cur.advance(); + continue; + } + const closeStart = cur.i; + let seen = 0; + while (cur.peek() === '"') { + seen += 1; + cur.advance(); + } + if (seen >= quoteCount) { + return cur.source.slice(bodyStart, closeStart); + } + } + cur.restore(start); + return undefined; + } + if (ch === '"') { + cur.advance(); + let value = ''; + while (!cur.done) { + const next = cur.peek(); + if (next === '\\') { + cur.advance(); + const esc = cur.peek(); + cur.advance(); + const map: Record = { + n: '\n', + r: '\r', + t: '\t', + '"': '"', + '\\': '\\', + '0': '\0', + }; + value += map[esc] ?? esc; + continue; + } + if (next === '"') { + cur.advance(); + return value; + } + value += next; + cur.advance(); + } + cur.restore(start); + return undefined; + } + return undefined; +} + +function skipBalanced(cur: SourceCursor, open: string, close: string): boolean { + skipCsharpTrivia(cur); + if (cur.peek() !== open) return false; + let depth = 0; + while (!cur.done) { + skipCsharpTrivia(cur); + if (cur.done) return false; + if (skipCsharpString(cur)) continue; + const ch = cur.peek(); + if (ch === open) depth += 1; + else if (ch === close) { + depth -= 1; + cur.advance(); + if (depth === 0) return true; + continue; + } + cur.advance(); + } + return false; +} + +function componentNameFromLiteral(value: string | undefined): string | undefined { + if (value === undefined || !COMPONENT_NAME_RE.test(value)) return undefined; + return value; +} + +function isViewComponentAttributeName(name: string): boolean { + return name === 'ViewComponent' || name === 'ViewComponentAttribute'; +} + +function readQualifiedTail(cur: SourceCursor): string | undefined { + skipCsharpTrivia(cur); + let name = readIdent(cur); + if (name === undefined) return undefined; + for (;;) { + skipCsharpTrivia(cur); + if (cur.peek() === '.' || (cur.peek() === ':' && cur.peek(1) === ':')) { + cur.advance(cur.peek() === ':' ? 2 : 1); + skipCsharpTrivia(cur); + const next = readIdent(cur); + if (next === undefined) return name; + name = next; + continue; + } + return name; + } +} + +function readViewComponentNameArgument(cur: SourceCursor): string | undefined { + skipCsharpTrivia(cur); + if (cur.peek() !== '(') return undefined; + cur.advance(); + let alias: string | undefined; + while (!cur.done && cur.peek() !== ')') { + skipCsharpTrivia(cur); + if (cur.peek() === ')') break; + const beforeArg = cur.snapshot(); + const ident = readIdent(cur); + skipCsharpTrivia(cur); + if (ident === 'Name' && cur.peek() === '=') { + cur.advance(); + alias = componentNameFromLiteral(decodeCsharpString(cur)); + } else { + cur.restore(beforeArg); + skipCsharpTrivia(cur); + if (cur.peek() === '"' || cur.peek() === '@') { + // Positional string arguments are not ViewComponentAttribute.Name. + skipCsharpString(cur); + } else if (cur.peek() === '(' || cur.peek() === '[' || cur.peek() === '{') { + const open = cur.peek(); + const close = open === '(' ? ')' : open === '[' ? ']' : '}'; + skipBalanced(cur, open, close); + } else { + while (!cur.done && cur.peek() !== ',' && cur.peek() !== ')') { + if (skipCsharpString(cur)) continue; + if (skipCsharpComment(cur)) continue; + cur.advance(); + } + } + } + skipCsharpTrivia(cur); + if (cur.peek() === ',') cur.advance(); + } + if (cur.peek() === ')') cur.advance(); + return alias; +} + +function collectInvokeAfterIdent( + ident: string, + cur: SourceCursor, + previous: string | undefined, + memberReceiver: string | undefined, + names: Set, +): void { + skipCsharpTrivia(cur); + const hasMvcReceiver = previous !== '.' || memberReceiver === 'this' || memberReceiver === 'base'; + if (ident === 'ViewComponent' && cur.peek() === '(') { + if (previous === '[' || previous === ',' || !hasMvcReceiver) return; + cur.advance(); + const name = componentNameFromLiteral(decodeCsharpString(cur)); + if (name !== undefined) names.add(name); + return; + } + if (ident !== 'Component' || cur.peek() !== '.' || !hasMvcReceiver) return; + const afterDot = cur.snapshot(); + cur.advance(); + skipCsharpTrivia(cur); + if (readIdent(cur) !== 'InvokeAsync') { + cur.restore(afterDot); + return; + } + skipCsharpTrivia(cur); + if (cur.peek() !== '(') return; + cur.advance(); + const name = componentNameFromLiteral(decodeCsharpString(cur)); + if (name !== undefined) names.add(name); +} + +/** In-repo C# `Component.InvokeAsync("X")` / `ViewComponent("X")` literals. */ +export function extractCsharpViewComponentInvocations(source: string): string[] { + if (!source.includes('ViewComponent') && !source.includes('InvokeAsync')) return []; + const names = new Set(); + const cur = new SourceCursor(source); + let previous: string | undefined; + let memberReceiver: string | undefined; + let squareDepth = 0; + while (!cur.done) { + skipCsharpTrivia(cur); + if (cur.done) break; + if (skipCsharpString(cur)) { + previous = 'string'; + continue; + } + const ident = readIdent(cur); + if (ident !== undefined) { + const inAttribute = squareDepth > 0; + collectInvokeAfterIdent(ident, cur, inAttribute ? '[' : previous, memberReceiver, names); + previous = ident; + memberReceiver = undefined; + continue; + } + const ch = cur.peek(); + if (ch === '[') squareDepth += 1; + else if (ch === ']' && squareDepth > 0) squareDepth -= 1; + memberReceiver = ch === '.' ? previous : undefined; + previous = ch; + cur.advance(); + } + return [...names]; +} + +function parseAttributeListBody(cur: SourceCursor): string[] { + const aliases: string[] = []; + skipCsharpTrivia(cur); + const specifier = cur.snapshot(); + const specifierName = readIdent(cur); + skipCsharpTrivia(cur); + if (specifierName !== undefined && cur.peek() === ':' && cur.peek(1) !== ':') { + cur.advance(); + } else { + cur.restore(specifier); + } + while (!cur.done && cur.peek() !== ']') { + skipCsharpTrivia(cur); + if (cur.peek() === ']') break; + const tail = readQualifiedTail(cur); + skipCsharpTrivia(cur); + if (tail !== undefined && isViewComponentAttributeName(tail) && cur.peek() === '(') { + const alias = readViewComponentNameArgument(cur); + if (alias !== undefined) aliases.push(alias); + } else if (cur.peek() === '(') { + skipBalanced(cur, '(', ')'); + } + skipCsharpTrivia(cur); + if (cur.peek() === ',') cur.advance(); + else break; + } + if (cur.peek() === ']') cur.advance(); + return aliases; +} + +/** + * Explicit `[ViewComponent(Name = "...")]` aliases keyed to the following + * class declaration. Positional constructor arguments are ignored: the MVC + * attribute only exposes `Name` as a property. + */ +export function extractViewComponentAliasBinds(source: string): ViewComponentAliasBind[] { + if (!source.includes('ViewComponent')) return []; + const binds: ViewComponentAliasBind[] = []; + const cur = new SourceCursor(source); + const pending: { startLine: number; startCol: number; aliases: string[] }[] = []; + + const flushPending = (className: string, startLine: number, startCol: number): void => { + const aliases = pending.flatMap((entry) => entry.aliases); + const start = pending[0]; + binds.push({ + className, + startLine: start?.startLine ?? startLine, + startCol: start?.startCol ?? startCol, + aliases: [...new Set(aliases)], + }); + pending.length = 0; + }; + + while (!cur.done) { + skipCsharpTrivia(cur); + if (cur.done) break; + if (skipCsharpString(cur)) continue; + const startLine = cur.line; + const startCol = cur.col; + if (cur.peek() === '[') { + cur.advance(); + const aliases = parseAttributeListBody(cur); + pending.push({ startLine, startCol, aliases }); + continue; + } + const ident = readIdent(cur); + if (ident === undefined) { + pending.length = 0; + cur.advance(); + continue; + } + if (TYPE_MODIFIERS.has(ident)) continue; + if (ident === 'class' || ident === 'record') { + let className = tryReadIdent(cur); + if (ident === 'record' && (className === 'class' || className === 'struct')) { + className = tryReadIdent(cur); + } + if (className !== undefined && pending.some((entry) => entry.aliases.length > 0)) { + flushPending(className, startLine, startCol); + } else { + pending.length = 0; + } + continue; + } + pending.length = 0; + } + return binds; +} + +/** Extract explicit `[ViewComponent(Name = "...")]` aliases by class name. */ +export function extractViewComponentAliases( + source: string, +): ReadonlyMap { + const aliases = new Map(); + for (const bind of extractViewComponentAliasBinds(source)) { + if (bind.aliases.length === 0) continue; + const existing = aliases.get(bind.className); + if (existing) { + for (const alias of bind.aliases) { + if (!existing.includes(alias)) existing.push(alias); + } + } else { + aliases.set(bind.className, [...bind.aliases]); + } + } + return aliases; +} + +function tagNameToComponentName(tagName: string): string { + return tagName + .split('-') + .filter(Boolean) + .map((part) => part[0]!.toUpperCase() + part.slice(1)) + .join(''); +} + +function collectVcTags(span: string, names: Set): void { + VIEW_COMPONENT_TAG_RE.lastIndex = 0; + for (const match of span.matchAll(VIEW_COMPONENT_TAG_RE)) { + names.add(tagNameToComponentName(match[1]!)); + } +} + +function skipRazorComment(cur: SourceCursor): boolean { + if (!cur.startsWith('@*')) return false; + cur.advance(2); + while (!cur.done && !cur.startsWith('*@')) cur.advance(); + if (cur.startsWith('*@')) cur.advance(2); + return true; +} + +function countAtRun(cur: SourceCursor): number { + let count = 0; + while (cur.peek() === '@') { + count += 1; + cur.advance(); + } + return count; +} + +function scanCsharpSpan(span: string, names: Set): void { + for (const name of extractCsharpViewComponentInvocations(span)) names.add(name); +} + +function skipOptionalParens(cur: SourceCursor): void { + skipWhitespace(cur); + if (cur.peek() === '(') skipBalanced(cur, '(', ')'); +} + +function consumeRazorCodeBlock(cur: SourceCursor, names: Set): void { + skipCsharpTrivia(cur); + skipOptionalParens(cur); + skipCsharpTrivia(cur); + if (cur.peek() !== '{') { + const start = cur.i; + while (!cur.done && cur.peek() !== '\n' && cur.peek() !== '{') { + if (skipCsharpString(cur) || skipCsharpComment(cur)) continue; + cur.advance(); + } + scanCsharpSpan(cur.source.slice(start, cur.i), names); + if (cur.peek() === '{') consumeRazorCodeBlock(cur, names); + return; + } + const bodyStart = cur.i + 1; + if (!skipBalanced(cur, '{', '}')) return; + scanCsharpSpan(cur.source.slice(bodyStart, cur.i - 1), names); +} + +function consumeImplicitExpression(cur: SourceCursor, names: Set): void { + const start = cur.i; + skipCsharpTrivia(cur); + if (cur.peek() === '(') { + const innerStart = cur.i + 1; + if (skipBalanced(cur, '(', ')')) { + scanCsharpSpan(cur.source.slice(innerStart, cur.i - 1), names); + } + return; + } + // Implicit expressions: `@await Component.InvokeAsync("X")` / `@Component.InvokeAsync(...)`. + while (!cur.done) { + skipCsharpTrivia(cur); + if (cur.done) break; + if (skipCsharpString(cur)) continue; + if (cur.peek() === '(') { + skipBalanced(cur, '(', ')'); + continue; + } + if (cur.peek() === '{') { + skipBalanced(cur, '{', '}'); + continue; + } + const ch = cur.peek(); + if (ch === '<' || ch === '\n') break; + if (ch === '@') break; + if (!isIdentPart(ch) && ch !== '.' && ch !== '?') { + if (ch === ';') cur.advance(); + break; + } + cur.advance(); + } + scanCsharpSpan(cur.source.slice(start, cur.i), names); +} + +function consumeRazorTransition(cur: SourceCursor, names: Set): void { + skipWhitespace(cur); + if (cur.peek() === '{') { + consumeRazorCodeBlock(cur, names); + return; + } + if (cur.peek() === '(') { + consumeImplicitExpression(cur, names); + return; + } + const identStart = cur.snapshot(); + const ident = readIdent(cur); + if (ident === undefined) { + consumeImplicitExpression(cur, names); + return; + } + if (ident === 'await' || ident === 'Component') { + cur.restore(identStart); + consumeImplicitExpression(cur, names); + return; + } + if (RAZOR_BLOCK_KEYWORDS.has(ident)) { + if (ident === 'section' || ident === 'helper') tryReadIdent(cur); + consumeRazorCodeBlock(cur, names); + return; + } + cur.restore(identStart); + consumeImplicitExpression(cur, names); +} + +/** Extract statically resolvable ViewComponent names from one Razor template. */ +export function extractRazorViewComponentInvocations(source: string): string[] { + // Most views do not invoke a ViewComponent. Avoid the character-by-character + // Razor scan unless one of the two supported invocation spellings is present. + // This is only a coarse gate; the state machine below still decides whether a + // token is executable markup/C# or a comment/string/escaped transition. + if (!source.includes('InvokeAsync') && !/<\s*vc:/i.test(source)) return []; + + const names = new Set(); + const cur = new SourceCursor(source); + let markupStart = 0; + const flushMarkup = (): void => { + if (cur.i > markupStart) collectVcTags(source.slice(markupStart, cur.i), names); + }; + + while (!cur.done) { + if (cur.peek() !== '@') { + cur.advance(); + continue; + } + flushMarkup(); + if (skipRazorComment(cur)) { + markupStart = cur.i; + continue; + } + const atCount = countAtRun(cur); + const leftover = atCount % 2; + if (leftover === 0) { + markupStart = cur.i; + continue; + } + consumeRazorTransition(cur, names); + markupStart = cur.i; + } + flushMarkup(); + return [...names]; +} + +/** + * Read Razor views once per C# resolution pass. The same ignore rules and file + * size ceiling as repository scanning are applied, and edge emission later + * additionally requires a live File node. This prevents ignored, oversized, + * or concurrently removed templates from entering the graph. + */ +export async function loadRazorViewComponentConfig( + repoRoot: string, +): Promise { + const ignore = await createIgnoreFilter(repoRoot); + const paths = await glob('**/*.cshtml', { + cwd: repoRoot, + nodir: true, + dot: false, + ignore, + }); + paths.sort(); + + const maxBytes = getMaxFileSizeBytes(); + const views = new Map(); + for (const rawPath of paths) { + const filePath = rawPath.replace(/\\/g, '/'); + // The size gate and the read go through one handle so both observe the same + // inode. Re-resolving the path for the read would let a template swapped in + // between them be read unchecked (CodeQL js/file-system-race). + let handle: fs.FileHandle | undefined; + try { + handle = await fs.open(path.join(repoRoot, filePath), 'r'); + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > maxBytes) continue; + const source = await handle.readFile('utf8'); + views.set(filePath, extractRazorViewComponentInvocations(source)); + } catch { + // A view may disappear between glob/open/read during watch mode. + } finally { + await handle?.close().catch(() => {}); + } + } + return { views }; +} + +function addCandidate( + candidates: Map>, + invocationName: string, + targetId: string, +): void { + const key = invocationName.toLocaleLowerCase('en-US'); + const existing = candidates.get(key); + if (existing) { + existing.add(targetId); + } else { + candidates.set(key, new Set([targetId])); + } +} + +function bindAliasesForClass( + binds: readonly ViewComponentAliasBind[], + className: string, + nodeId: string, + filePath: string, +): readonly string[] | undefined { + const matches = binds.filter((bind) => bind.className === className); + if (matches.length === 0) return undefined; + if (matches.length === 1) return matches[0]!.aliases; + const pos = definitionIdPosition(nodeId, filePath); + if (pos === undefined) return undefined; + const atPosition = matches.filter( + (bind) => bind.startLine === pos.line && bind.startCol === pos.column, + ); + if (atPosition.length === 1) return atPosition[0]!.aliases; + return undefined; +} + +/** + * Emit workspace File → in-repo ViewComponent Class CALLS edges. + * + * Targets are only Class nodes produced from this repo's `.cs` files. There is + * no lookup of ASP.NET SDK types; `: ViewComponent` in source is a naming + * hint, not a resolved EXTENDS edge to `Microsoft.AspNetCore.Mvc.ViewComponent`. + * + * Ambiguous component names fail closed: two in-repo classes claiming the + * same name is not evidence for picking either one. + */ +export function emitRazorViewComponentEdges( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, + config: RazorViewComponentConfig | undefined, + csharpSources: ReadonlyMap, +): void { + if (!config) return; + + const candidates = new Map>(); + for (const parsed of parsedFiles) { + if (!parsed.filePath.endsWith('.cs')) continue; + const source = csharpSources.get(parsed.filePath) ?? ''; + const binds = source.includes('ViewComponent') ? extractViewComponentAliasBinds(source) : []; + for (const def of parsed.localDefs) { + if (def.type !== 'Class') continue; + const className = def.qualifiedName?.split('.').pop() ?? def.nodeId.split(':').pop() ?? ''; + const conventionalName = className.endsWith(VIEW_COMPONENT_SUFFIX) + ? className.slice(0, -VIEW_COMPONENT_SUFFIX.length) + : undefined; + const explicitAliases = bindAliasesForClass(binds, className, def.nodeId, parsed.filePath); + if (!conventionalName && (explicitAliases === undefined || explicitAliases.length === 0)) { + continue; + } + + const targetId = resolveDefGraphId(parsed.filePath, def, nodeLookup); + if (!targetId || !graph.getNode(targetId)) continue; + // An explicit [ViewComponent(Name = "...")] replaces the suffix name, + // matching ASP.NET. Never register the SDK base type as a candidate. + if (explicitAliases !== undefined && explicitAliases.length > 0) { + for (const alias of explicitAliases) addCandidate(candidates, alias, targetId); + } else if (conventionalName) { + addCandidate(candidates, conventionalName, targetId); + } + } + } + + const emitFromFile = (filePath: string, invocationNames: readonly string[]): void => { + const sourceId = generateId('File', filePath); + if (!graph.getNode(sourceId)) return; + for (const invocationName of invocationNames) { + const matches = candidates.get(invocationName.toLocaleLowerCase('en-US')); + if (!matches || matches.size !== 1) continue; + const targetId = matches.values().next().value; + if (typeof targetId !== 'string' || !graph.getNode(targetId)) continue; + graph.addRelationship({ + id: generateId('CALLS', `${sourceId}:razor-view-component:${targetId}`), + sourceId, + targetId, + type: 'CALLS', + confidence: 0.9, + reason: 'aspnet-razor-view-component', + }); + } + }; + + for (const [viewPath, invocationNames] of config.views) { + emitFromFile(viewPath, invocationNames); + } + for (const [filePath, source] of csharpSources) { + if (!filePath.endsWith('.cs')) continue; + if (!source.includes('ViewComponent') && !source.includes('InvokeAsync')) continue; + emitFromFile(filePath, extractCsharpViewComponentInvocations(source)); + } +} diff --git a/gitnexus/src/core/ingestion/languages/csharp/resolution-config.ts b/gitnexus/src/core/ingestion/languages/csharp/resolution-config.ts index 9ea232c05..714be8c1f 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/resolution-config.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/resolution-config.ts @@ -12,19 +12,29 @@ import { type CSharpProjectConfig, type CSharpNamespaceEvidence, } from '../../language-config.js'; +import { + loadRazorViewComponentConfig, + type RazorViewComponentConfig, +} from './razor-view-components.js'; export interface CsharpResolutionConfig { readonly csharpConfigs: readonly CSharpProjectConfig[]; /** In-repo declared-namespace evidence gating suffix-fallback resolution (#1881). */ readonly namespaces?: CSharpNamespaceEvidence; + /** Razor views scanned for ASP.NET ViewComponent invocation conventions. */ + readonly razorViewComponents?: RazorViewComponentConfig; } export async function loadCsharpResolutionConfig( repoRoot: string, ): Promise { - const scan = await scanCSharpProject(repoRoot); + const [scan, razorViewComponents] = await Promise.all([ + scanCSharpProject(repoRoot), + loadRazorViewComponentConfig(repoRoot), + ]); return { csharpConfigs: scan.configs, namespaces: csharpScanToEvidence(scan), + razorViewComponents, }; } diff --git a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts index 4b50efc67..6d200de9b 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts @@ -22,6 +22,7 @@ import { import { populateCsharpNamespaceSiblings } from './namespace-siblings.js'; import { loadCsharpResolutionConfig, type CsharpResolutionConfig } from './resolution-config.js'; import { unwrapCsharpElementType } from './accessor-unwrap.js'; +import { emitRazorViewComponentEdges } from './razor-view-components.js'; const csharpScopeResolver: ScopeResolver = { // Construction is keyword-prefixed: `new Service(db).doWork()` (#2708). @@ -106,6 +107,20 @@ const csharpScopeResolver: ScopeResolver = { // `IValidator` and `IValidator` are one instantiation, so the // dispatch fan-out must not read them as two (#2912). See the alias table. normalizeTypeArgument: normalizeCsharpTypeArgument, + + // Razor views stay out of the C# parser. Bind literal ViewComponent names + // only onto in-repo classes (Spring-style: skip the SDK type, hop to the + // workspace implementor). + emitPostResolutionEdges: (graph, parsedFiles, nodeLookup, _indexes, ctx) => { + const config = ctx.resolutionConfig as CsharpResolutionConfig | undefined; + emitRazorViewComponentEdges( + graph, + parsedFiles, + nodeLookup, + config?.razorViewComponents, + ctx.fileContents, + ); + }, }; /** diff --git a/gitnexus/src/core/ingestion/languages/java.ts b/gitnexus/src/core/ingestion/languages/java.ts index 317a853ac..45386dd7b 100644 --- a/gitnexus/src/core/ingestion/languages/java.ts +++ b/gitnexus/src/core/ingestion/languages/java.ts @@ -15,6 +15,12 @@ import type { AstFrameworkPatternConfig } from '../language-provider.js'; import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import { javaTypeConfig } from '../type-extractors/jvm.js'; import { extractSpringRoutes, extractSpringTypes } from '../route-extractors/spring.js'; +import { + extractJavaModuleConstants, + foldJavaOperands, + isJavaConstantFile, + prepareJavaRouteConstants, +} from '../route-extractors/java-const-resolver.js'; import { javaExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; import { javaImportConfig } from '../import-resolvers/configs/jvm.js'; @@ -27,12 +33,17 @@ import { createVariableExtractor } from '../variable-extractors/generic.js'; import { javaVariableConfig } from '../variable-extractors/configs/jvm.js'; import { createJavaCfgVisitor } from '../cfg/visitors/java.js'; import { assertCloneable } from '../workers/clone-safety.js'; -import { collectJavaCaptureSideChannel } from './java/capture-side-channel.js'; +import { + collectJavaCaptureSideChannel, + getJavaSpringMessageProducerFacts, + getJavaSpringNonHttpHandlerFacts, +} from './java/capture-side-channel.js'; import type { SymbolDefinition } from 'gitnexus-shared'; import { javaRecordMethodExtractor, shouldSkipJavaRecordComponentDefinition, } from './java/record-components.js'; +import { synthesizeLombokAccessors } from './java/lombok-synthesizer.js'; import { emitJavaScopeCaptures, interpretJavaImport, @@ -44,6 +55,7 @@ import { javaArityCompatibility, resolveJavaImportTarget, } from './java/index.js'; +import { javaRuntimeSymbolStrategy } from './java/spring-actuator.js'; /** * Java names the platform owns, matched against a BARE IDENTIFIER — a dropped @@ -192,6 +204,7 @@ export const javaProvider = defineLanguage({ shouldSkipDefinitionCapture: shouldSkipJavaRecordComponentDefinition, variableExtractor: createVariableExtractor(javaVariableConfig), classExtractor: createClassExtractor(javaClassConfig), + runtimeSymbolStrategy: javaRuntimeSymbolStrategy, // ── Javadoc → description (issue #2270) ── descriptionExtractor: createLeadingDocDescriptionExtractor(), @@ -216,4 +229,34 @@ export const javaProvider = defineLanguage({ // ── Route extraction ── extractDecoratorRoutes: extractSpringRoutes, extractRouteInheritanceTypes: extractSpringTypes, + + synthesizeStructureMembers: synthesizeLombokAccessors, + + // ── #2980: constant harvest + qualified-ref fold for non-literal mapping + // paths (`@PostMapping(ApiPaths.SAVE_V1)`) — kept behind provider hooks so + // the shared ingestion layers stay language-agnostic. The heuristic is + // SYNTAX-driven (field/import shape), never a class-name pattern: constant + // classes are routinely named `ApiPaths`/`Routes`/`Paths`, which a + // `*Constants`-style gate would silently drop (review round-2 High finding). + extractModuleConstants: extractJavaModuleConstants, + // One gate, shared with the group side's `prepareRepo` pre-pass so the two + // subsystems cannot disagree about which files define constants (see + // JAVA_CONSTANT_FILE_RE — the previous divergence dropped constant + // INTERFACES on this side only, which cost the graph its Route nodes while + // the group still published the contract). + moduleConstantHeuristic: (content) => + isJavaConstantFile(content) || + // Class imports and static (including on-demand) imports can bind a + // constant ref. Ordinary `import a.b.*;` is not a Java type import and is + // not expanded by extractJavaModuleConstants, so it must not harvest. + /\bimport\s+(?:static\s+[\w.]+(?:\.\*)?|[\w.]+)\s*;/.test(content), + prepareRouteConstants: prepareJavaRouteConstants, + foldRoutePathOperands: foldJavaOperands, + // Async messaging facts for the `springDestinations` phase. Both stores are + // repopulated on the main thread by `applyJavaCaptureSideChannel`, so this + // answers for cache hits and misses alike. + getSpringMessagingFacts: (filePath) => ({ + handlers: getJavaSpringNonHttpHandlerFacts(filePath), + producers: getJavaSpringMessageProducerFacts(filePath), + }), }); diff --git a/gitnexus/src/core/ingestion/languages/java/analysis-features.ts b/gitnexus/src/core/ingestion/languages/java/analysis-features.ts index 73969670d..64ff16294 100644 --- a/gitnexus/src/core/ingestion/languages/java/analysis-features.ts +++ b/gitnexus/src/core/ingestion/languages/java/analysis-features.ts @@ -5,17 +5,24 @@ function isSpringApplicationConfig(filePath: string): boolean { return /^application(?:-[^.]+)?\.(?:properties|ya?ml)$/i.test(base); } -/** Durable completeness contract for Java Spring configuration bindings. */ +/** Durable completeness contract for Java and Kotlin Spring configuration bindings. */ export const SPRING_CONFIG_BINDINGS_FEATURE: AnalysisFeatureDescriptor = { id: 'spring.config-bindings', - version: 1, - // Java sources need consumer extraction even without config files (missing - // placeholders still get unresolved markers). Config-only repositories also - // need a one-time rebuild to backfill language-agnostic Property nodes. + version: 2, + // Java and Kotlin sources need consumer extraction even without config files + // (missing placeholders still get unresolved markers). Config-only + // repositories also need a one-time rebuild to backfill language-agnostic + // Property nodes. Gradle Kotlin DSL is not a consumer source. appliesTo: (filePaths) => - filePaths.some( - (filePath) => filePath.toLowerCase().endsWith('.java') || isSpringApplicationConfig(filePath), - ), + filePaths.some((filePath) => { + const normalized = filePath.replaceAll('\\', '/').toLowerCase(); + if (normalized.endsWith('.gradle.kts')) return false; + return ( + normalized.endsWith('.java') || + normalized.endsWith('.kt') || + isSpringApplicationConfig(filePath) + ); + }), }; /** Durable completeness contract for implicit Java record-component accessors. */ diff --git a/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts index 91a910fa5..a8c5e268e 100644 --- a/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts +++ b/gitnexus/src/core/ingestion/languages/java/capture-side-channel.ts @@ -13,6 +13,8 @@ import type { JavaSpringConfigConsumerFact } from './spring-config-bindings.js'; import type { JavaSpringAopFact } from './spring-aop.js'; import type { JavaSpringConditionalFact } from './spring-conditionals.js'; import type { JavaSpringDiClassFact } from './spring-di.js'; +import type { SpringDynamicLookupFact } from '../../frameworks/spring/dynamic-lookups.js'; +import type { SpringMessageProducerFact } from '../../frameworks/spring/message-producers.js'; import type { JavaSpringNonHttpHandlerFact } from './spring-non-http-handlers.js'; export type JavaClassAnnotationFact = ClassAnnotationFact; @@ -25,7 +27,9 @@ export interface JavaCaptureSideChannel { readonly springConfigConsumers?: readonly JavaSpringConfigConsumerFact[]; readonly springConditionalFacts?: readonly JavaSpringConditionalFact[]; readonly springDiFacts?: readonly JavaSpringDiClassFact[]; + readonly springDynamicLookupFacts?: readonly SpringDynamicLookupFact[]; readonly springNonHttpHandlerFacts?: readonly JavaSpringNonHttpHandlerFact[]; + readonly springMessageProducerFacts?: readonly SpringMessageProducerFact[]; } const classAnnotations = createClassAnnotationFactStore(); @@ -33,7 +37,9 @@ const springAopFacts = new Map(); const springConfigConsumers = new Map(); const springConditionalFacts = new Map(); const springDiFacts = new Map(); +const springDynamicLookupFacts = new Map(); const springNonHttpHandlerFacts = new Map(); +const springMessageProducerFacts = new Map(); /** Clear facts retained by a prior workspace pass in a long-lived process. */ export function clearJavaClassAnnotationFacts(): void { @@ -42,7 +48,9 @@ export function clearJavaClassAnnotationFacts(): void { springConfigConsumers.clear(); springConditionalFacts.clear(); springDiFacts.clear(); + springDynamicLookupFacts.clear(); springNonHttpHandlerFacts.clear(); + springMessageProducerFacts.clear(); } export function setJavaSpringAopFacts(filePath: string, facts: readonly JavaSpringAopFact[]): void { @@ -102,6 +110,20 @@ export function getJavaSpringDiFacts(filePath: string): readonly JavaSpringDiCla return springDiFacts.get(filePath) ?? []; } +export function setJavaSpringDynamicLookupFacts( + filePath: string, + facts: readonly SpringDynamicLookupFact[], +): void { + if (facts.length === 0) springDynamicLookupFacts.delete(filePath); + else springDynamicLookupFacts.set(filePath, facts); +} + +export function getJavaSpringDynamicLookupFacts( + filePath: string, +): readonly SpringDynamicLookupFact[] { + return springDynamicLookupFacts.get(filePath) ?? []; +} + export function setJavaSpringNonHttpHandlerFacts( filePath: string, facts: readonly JavaSpringNonHttpHandlerFact[], @@ -116,6 +138,20 @@ export function getJavaSpringNonHttpHandlerFacts( return springNonHttpHandlerFacts.get(filePath) ?? []; } +export function setJavaSpringMessageProducerFacts( + filePath: string, + facts: readonly SpringMessageProducerFact[], +): void { + if (facts.length === 0) springMessageProducerFacts.delete(filePath); + else springMessageProducerFacts.set(filePath, facts); +} + +export function getJavaSpringMessageProducerFacts( + filePath: string, +): readonly SpringMessageProducerFact[] { + return springMessageProducerFacts.get(filePath) ?? []; +} + /** Snapshot worker-local Java annotation facts for ParsedFile serialization. */ export function collectJavaCaptureSideChannel( filePath: string, @@ -125,7 +161,9 @@ export function collectJavaCaptureSideChannel( const configConsumers = springConfigConsumers.get(filePath) ?? []; const conditionFacts = springConditionalFacts.get(filePath) ?? []; const diFacts = springDiFacts.get(filePath) ?? []; + const dynamicLookupFacts = springDynamicLookupFacts.get(filePath) ?? []; const nonHttpHandlerFacts = springNonHttpHandlerFacts.get(filePath) ?? []; + const messageProducerFacts = springMessageProducerFacts.get(filePath) ?? []; const packageFact = getJavaPackageFact(filePath); if ( facts.length === 0 && @@ -133,7 +171,9 @@ export function collectJavaCaptureSideChannel( configConsumers.length === 0 && conditionFacts.length === 0 && diFacts.length === 0 && + dynamicLookupFacts.length === 0 && nonHttpHandlerFacts.length === 0 && + messageProducerFacts.length === 0 && packageFact === undefined ) { return undefined; @@ -146,7 +186,11 @@ export function collectJavaCaptureSideChannel( ...(configConsumers.length > 0 ? { springConfigConsumers: configConsumers } : {}), ...(conditionFacts.length > 0 ? { springConditionalFacts: conditionFacts } : {}), ...(diFacts.length > 0 ? { springDiFacts: diFacts } : {}), + ...(dynamicLookupFacts.length > 0 ? { springDynamicLookupFacts: dynamicLookupFacts } : {}), ...(nonHttpHandlerFacts.length > 0 ? { springNonHttpHandlerFacts: nonHttpHandlerFacts } : {}), + ...(messageProducerFacts.length > 0 + ? { springMessageProducerFacts: messageProducerFacts } + : {}), }; } @@ -169,7 +213,9 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void { setJavaSpringConfigConsumerFacts(parsed.filePath, []); setJavaSpringConditionalFacts(parsed.filePath, []); setJavaSpringDiFacts(parsed.filePath, []); + setJavaSpringDynamicLookupFacts(parsed.filePath, []); setJavaSpringNonHttpHandlerFacts(parsed.filePath, []); + setJavaSpringMessageProducerFacts(parsed.filePath, []); setJavaPackageFact(parsed.filePath, UNKNOWN_JVM_PACKAGE_FACT); return; } @@ -190,10 +236,18 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void { parsed.filePath, Array.isArray(data.springDiFacts) ? data.springDiFacts : [], ); + setJavaSpringDynamicLookupFacts( + parsed.filePath, + Array.isArray(data.springDynamicLookupFacts) ? data.springDynamicLookupFacts : [], + ); setJavaSpringNonHttpHandlerFacts( parsed.filePath, Array.isArray(data.springNonHttpHandlerFacts) ? data.springNonHttpHandlerFacts : [], ); + setJavaSpringMessageProducerFacts( + parsed.filePath, + Array.isArray(data.springMessageProducerFacts) ? data.springMessageProducerFacts : [], + ); setJavaPackageFact( parsed.filePath, isJvmPackageFact(data.packageFact) ? data.packageFact : UNKNOWN_JVM_PACKAGE_FACT, diff --git a/gitnexus/src/core/ingestion/languages/java/captures.ts b/gitnexus/src/core/ingestion/languages/java/captures.ts index ce5ed4b93..5d31aed4a 100644 --- a/gitnexus/src/core/ingestion/languages/java/captures.ts +++ b/gitnexus/src/core/ingestion/languages/java/captures.ts @@ -39,12 +39,18 @@ import { setJavaSpringConfigConsumerFacts, setJavaSpringConditionalFacts, setJavaSpringDiFacts, + setJavaSpringDynamicLookupFacts, + setJavaSpringMessageProducerFacts, setJavaSpringNonHttpHandlerFacts, } from './capture-side-channel.js'; import { captureJavaPackageFact } from './package-facts.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; import { captureJavaSpringConfigConsumerFacts } from './spring-config-bindings.js'; import { captureJavaSpringDiClassFact, type JavaSpringDiClassFact } from './spring-di.js'; +import type { SpringDynamicLookupFact } from '../../frameworks/spring/dynamic-lookups.js'; +import { captureJavaSpringDynamicLookupFact } from './spring-dynamic-lookup.js'; +import type { SpringMessageProducerFact } from '../../frameworks/spring/message-producers.js'; +import { captureJavaSpringMessageProducerFact } from './spring-message-producers.js'; import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js'; import { captureJavaSpringAopFacts, type JavaSpringAopFact } from './spring-aop.js'; import { @@ -56,6 +62,7 @@ import { type JavaSpringNonHttpHandlerFact, } from './spring-non-http-handlers.js'; import { synthesizeJavaRecordComponentAccessorCaptures } from './record-components.js'; +import { synthesizeLombokAccessorCaptures } from './lombok-synthesizer.js'; /** Declaration anchors that carry function-like arity metadata. */ const FUNCTION_DECL_TAGS = ['@declaration.method', '@declaration.constructor'] as const; @@ -146,6 +153,9 @@ export function emitJavaScopeCaptures( const springDiFacts: JavaSpringDiClassFact[] = []; const springNonHttpHandlerFacts: JavaSpringNonHttpHandlerFact[] = []; const springDiClassNodeIds = new Set(); + const springDynamicLookupFacts: SpringDynamicLookupFact[] = []; + const springMessageProducerFacts: SpringMessageProducerFact[] = []; + const springMemberCallNodeIds = new Set(); for (const m of rawMatches) { const grouped: Record = {}; @@ -165,6 +175,17 @@ export function emitJavaScopeCaptures( } if (Object.keys(grouped).length === 0) continue; + // One visit per member call node: the same invocation can back several + // query matches, and both Spring call-shape captures must see it once. + const memberCallNode = nodeIfType(nodeMap['@reference.call.member'], 'method_invocation'); + if (memberCallNode !== null && !springMemberCallNodeIds.has(memberCallNode.id)) { + springMemberCallNodeIds.add(memberCallNode.id); + const lookupFact = captureJavaSpringDynamicLookupFact(memberCallNode, filePath); + if (lookupFact !== null) springDynamicLookupFacts.push(lookupFact); + const producerFact = captureJavaSpringMessageProducerFact(memberCallNode, filePath); + if (producerFact !== null) springMessageProducerFacts.push(producerFact); + } + const springAopTypeNode = [ nodeIfType(nodeMap['@scope.class'], 'class_declaration'), nodeIfType(nodeMap['@scope.class'], 'interface_declaration'), @@ -401,7 +422,9 @@ export function emitJavaScopeCaptures( setJavaSpringAopFacts(filePath, springAopFacts); setJavaSpringConditionalFacts(filePath, springConditionalFacts); setJavaSpringDiFacts(filePath, springDiFacts); + setJavaSpringDynamicLookupFacts(filePath, springDynamicLookupFacts); setJavaSpringNonHttpHandlerFacts(filePath, springNonHttpHandlerFacts); + setJavaSpringMessageProducerFacts(filePath, springMessageProducerFacts); return [ ...resolveVarTypeBindings(out), @@ -409,6 +432,7 @@ export function emitJavaScopeCaptures( ...synthesizeJavaExplicitConstructorReferences(tree.rootNode), ...synthesizeJavaAnonymousClassDeclarations(tree.rootNode), ...synthesizeJavaRecordComponentAccessorCaptures(tree.rootNode), + ...synthesizeLombokAccessorCaptures(tree.rootNode), ...synthesizeCallableFlowCaptures(tree.rootNode, JAVA_CALLABLE_CAPTURE_OPTIONS), ]; } diff --git a/gitnexus/src/core/ingestion/languages/java/lombok-synthesizer.ts b/gitnexus/src/core/ingestion/languages/java/lombok-synthesizer.ts new file mode 100644 index 000000000..3f2eeab9a --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/java/lombok-synthesizer.ts @@ -0,0 +1,539 @@ +/** + * Lombok accessor synthesizer for Java. + * + * Lombok generates getters/setters at compile time. They are absent from the + * AST, so calls like `obj.getOrderId()` on a `@Data` class would otherwise + * leave unresolved CALLS edges. This module walks the tree-sitter Java AST + * and synthesizes Method graph members for the accessors Lombok would emit + * under the supported subset. + * + * ## Supported subset (v1) + * - Proven `lombok.Data` / `lombok.Getter` / `lombok.Setter` (FQN or import). + * - Class- or field-level enable; `AccessLevel.NONE` disables. + * - Default JavaBeans naming; primitive `boolean isX` → `isX` / `setX`. + * - Access levels PUBLIC/PROTECTED/PRIVATE/PACKAGE. + * - `@Accessors(chain=true)` modeled as setter return = declaring type. + * - `@Accessors(fluent=true)` / `prefix=…`: omit affected accessors (names + * cannot be proven without full Lombok config). + * - External `lombok.config`: unsupported (may change semantics invisibly). + * + * ## Identity + * Owner lookup uses in-memory AST node ids only. Method ids are derived from + * the stable declaring-owner graph key (the Class node id's name segment), + * never from persisted tree-sitter node ids. + */ + +import type Parser from 'tree-sitter'; +import type { CaptureMatch } from 'gitnexus-shared'; +import { jvmGetterName, jvmSetterName } from '../jvm/beanspec.js'; +import { + createExistingMethodIndex, + createJvmAccessorSynthesis, + hasExistingMethod, + rememberExistingMethodRange, + type ExistingMethodIndex, + type PlannedJvmAccessor, + type PlannedJvmAccessorOwner, + type SyntheticAccessorResult, + type SyntheticVisibility, +} from '../jvm/accessor-synthesis.js'; + +const JAVA_TYPE_DECLS = new Set([ + 'class_declaration', + 'enum_declaration', + 'interface_declaration', + 'record_declaration', +]); + +// ── Public result types (ParsedSymbol / ParsedNode compatible) ──────────── + +export type LombokVisibility = SyntheticVisibility; +export type SyntheticSymbol = SyntheticAccessorResult['symbols'][number]; +export type SyntheticNode = SyntheticAccessorResult['nodes'][number]; +export type SyntheticRelationship = SyntheticAccessorResult['relationships'][number]; +export type LombokSynthesisResult = SyntheticAccessorResult; +export type PlannedLombokAccessor = PlannedJvmAccessor; + +export interface AccessorConfig { + enabled: boolean; + visibility: LombokVisibility; +} + +interface AccessorsOptions { + /** When true, JavaBeans get/set/is prefixes are not used — omit (unsupported). */ + fluent: boolean; + /** When true, field prefixes alter base names — omit (unsupported). */ + hasPrefix: boolean; + /** When true, setters return the declaring type instead of void. */ + chain: boolean; +} + +interface LombokField { + name: string; + type: string; + isStatic: boolean; + isFinal: boolean; + startLine: number; + endLine: number; + declaratorNode: Parser.SyntaxNode; + fieldGetter: AccessorConfig | null; + fieldSetter: AccessorConfig | null; + accessors: AccessorsOptions; + accessorsPresent: boolean; +} + +interface LombokClass { + node: Parser.SyntaxNode; + name: string; + classGetter: AccessorConfig | null; + classSetter: AccessorConfig | null; + classAccessors: AccessorsOptions; + fields: LombokField[]; + existingMethods: ExistingMethodIndex; +} + +const LOMBOK_ANNOTATION_PACKAGE = new Map([ + ['Data', 'lombok'], + ['Getter', 'lombok'], + ['Setter', 'lombok'], + ['Accessors', 'lombok.experimental'], + ['Tolerate', 'lombok.experimental'], +]); + +export function getterName(fieldName: string, fieldType: string): string { + return jvmGetterName(fieldName, fieldType === 'boolean'); +} + +export function setterName(fieldName: string, fieldType: string): string { + return jvmSetterName(fieldName, fieldType === 'boolean'); +} + +// ── Provenance / imports ────────────────────────────────────────────────── + +function annotationSimpleName(nameText: string): string { + return nameText.split('.').pop() ?? nameText; +} + +interface LombokImportIndex { + bySimple: Map; + starPackages: Set; + shadowedSimpleNames: Set; +} + +/** + * Compilation-unit imports only — Java `import` is never nested in a type body. + */ +function collectLombokImports(root: Parser.SyntaxNode): LombokImportIndex { + const bySimple = new Map(); + const starPackages = new Set(); + const shadowedSimpleNames = new Set(); + for (const child of root.children) { + if (!JAVA_TYPE_DECLS.has(child.type) && child.type !== 'annotation_type_declaration') continue; + const name = child.childForFieldName('name')?.text; + if (name) shadowedSimpleNames.add(name); + } + for (const child of root.children) { + if (child.type !== 'import_declaration') continue; + if (/^import\s+static\b/.test(child.text)) continue; + const text = child.text + .replace(/^import\s+/, '') + .replace(/;\s*$/, '') + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/\s+/g, '') + .trim(); + if (text === 'lombok.*') { + starPackages.add('lombok'); + } else if (text === 'lombok.experimental.*') { + starPackages.add('lombok.experimental'); + } else if (!text.endsWith('.*')) { + bySimple.set(annotationSimpleName(text), text); + } + } + return { bySimple, starPackages, shadowedSimpleNames }; +} + +function isProvenLombokAnnotation(nameText: string, imports: LombokImportIndex): boolean { + const simple = annotationSimpleName(nameText); + const packageName = LOMBOK_ANNOTATION_PACKAGE.get(simple); + if (packageName === undefined) return false; + if (nameText.includes('.')) return nameText === `${packageName}.${simple}`; + const imported = imports.bySimple.get(simple); + if (imported !== undefined) return imported === `${packageName}.${simple}`; + if (imports.shadowedSimpleNames.has(simple)) return false; + return imports.starPackages.has(packageName); +} + +// ── AccessLevel / Accessors structural parse ────────────────────────────── + +function parseAccessLevelToken(text: string): LombokVisibility | 'none' | null { + const simple = annotationSimpleName(text.trim()); + switch (simple) { + case 'PUBLIC': + return 'public'; + case 'PROTECTED': + return 'protected'; + case 'PRIVATE': + return 'private'; + case 'PACKAGE': + case 'MODULE': // treated as package-private for graph metadata + return 'package'; + case 'NONE': + return 'none'; + default: + return null; + } +} + +function findAccessLevelInAnnotation(ann: Parser.SyntaxNode): LombokVisibility | 'none' | null { + // Positional: @Getter(AccessLevel.PROTECTED) or @Getter(lombok.AccessLevel.NONE) + // Named: @Getter(value = AccessLevel.PRIVATE) + const stack: Parser.SyntaxNode[] = [...ann.children]; + while (stack.length > 0) { + const n = stack.pop(); + if (!n) break; + if (n.type === 'field_access' || n.type === 'identifier') { + const level = parseAccessLevelToken(n.text); + if (level !== null) return level; + } + for (const c of n.children) stack.push(c); + } + return null; +} + +function defaultAccessors(): AccessorsOptions { + return { fluent: false, hasPrefix: false, chain: false }; +} + +function parseAccessorsAnnotation(ann: Parser.SyntaxNode): AccessorsOptions { + const opts = defaultAccessors(); + const stack: Parser.SyntaxNode[] = [...ann.children]; + while (stack.length > 0) { + const n = stack.pop(); + if (!n) break; + if (n.type === 'element_value_pair') { + const key = + n.childForFieldName('key')?.text ?? n.children.find((c) => c.type === 'identifier')?.text; + const valueNode = + n.childForFieldName('value') ?? + n.children.find( + (c) => + c.type === 'true' || c.type === 'false' || c.type === 'element_value_array_initializer', + ); + if (key === 'fluent' && (valueNode?.type === 'true' || valueNode?.type === 'false')) { + opts.fluent = valueNode.type === 'true'; + } + if (key === 'chain' && (valueNode?.type === 'true' || valueNode?.type === 'false')) { + opts.chain = valueNode.type === 'true'; + } + if (key === 'prefix') opts.hasPrefix = true; + } + for (const c of n.children) stack.push(c); + } + const text = ann.text; + if (/\bprefix\s*=/.test(text)) opts.hasPrefix = true; + if (/\bfluent\s*=\s*true\b/.test(text)) opts.fluent = true; + if (/\bfluent\s*=\s*false\b/.test(text)) opts.fluent = false; + if (/\bchain\s*=\s*true\b/.test(text)) opts.chain = true; + if (/\bchain\s*=\s*false\b/.test(text)) opts.chain = false; + return opts; +} + +interface ParsedAnnotations { + getter: AccessorConfig | null; + setter: AccessorConfig | null; + accessors: AccessorsOptions; + accessorsPresent: boolean; + tolerate: boolean; +} + +function parseModifierAnnotations( + modifiersNode: Parser.SyntaxNode | null, + imports: LombokImportIndex, +): ParsedAnnotations { + const result: ParsedAnnotations = { + getter: null, + setter: null, + accessors: defaultAccessors(), + accessorsPresent: false, + tolerate: false, + }; + if (!modifiersNode) return result; + + for (const child of modifiersNode.children) { + if (child.type !== 'marker_annotation' && child.type !== 'annotation') continue; + const nameNode = child.childForFieldName('name'); + const nameText = nameNode?.text ?? ''; + if (!isProvenLombokAnnotation(nameText, imports)) continue; + const simple = annotationSimpleName(nameText); + + if (simple === 'Tolerate') { + result.tolerate = true; + continue; + } + if (simple === 'Accessors') { + result.accessors = parseAccessorsAnnotation(child); + result.accessorsPresent = true; + continue; + } + if (simple === 'Data') { + result.getter ??= { enabled: true, visibility: 'public' }; + result.setter ??= { enabled: true, visibility: 'public' }; + continue; + } + if (simple === 'Getter' || simple === 'Setter') { + const level = child.type === 'annotation' ? findAccessLevelInAnnotation(child) : null; + const cfg: AccessorConfig = + level === 'none' + ? { enabled: false, visibility: 'public' } + : { enabled: true, visibility: level ?? 'public' }; + if (simple === 'Getter') result.getter = cfg; + else result.setter = cfg; + } + } + return result; +} + +function mergeAccessors( + classOpts: AccessorsOptions, + fieldOpts: AccessorsOptions, + fieldAccessorsPresent: boolean, +): AccessorsOptions { + return fieldAccessorsPresent ? fieldOpts : classOpts; +} + +function effectiveAccessor( + classCfg: AccessorConfig | null, + fieldCfg: AccessorConfig | null, +): AccessorConfig | null { + if (fieldCfg !== null) return fieldCfg; + return classCfg; +} + +// ── Field / method collection ───────────────────────────────────────────── + +function parseFieldDeclaration( + fieldNode: Parser.SyntaxNode, + imports: LombokImportIndex, +): LombokField[] { + const typeNode = fieldNode.childForFieldName('type'); + const fieldType = typeNode?.text ?? 'Object'; + const modifiers = fieldNode.children.find((c) => c.type === 'modifiers') ?? null; + let isStatic = false; + let isFinal = false; + if (modifiers) { + for (const mod of modifiers.children) { + if (mod.text === 'static') isStatic = true; + else if (mod.text === 'final') isFinal = true; + } + } + const fieldAnn = parseModifierAnnotations(modifiers, imports); + + const declarators: Parser.SyntaxNode[] = []; + const declaratorField = fieldNode.childForFieldName('declarator'); + if (declaratorField) declarators.push(declaratorField); + for (const child of fieldNode.children) { + if (child.type === 'variable_declarator' && child !== declaratorField) { + declarators.push(child); + } + } + + const startLine = fieldNode.startPosition.row + 1; + const endLine = fieldNode.endPosition.row + 1; + const out: LombokField[] = []; + for (const declaratorNode of declarators) { + const nameNode = declaratorNode.childForFieldName('name'); + if (!nameNode) continue; + out.push({ + name: nameNode.text, + type: fieldType, + isStatic, + isFinal, + startLine, + endLine, + declaratorNode, + fieldGetter: fieldAnn.getter, + fieldSetter: fieldAnn.setter, + accessors: fieldAnn.accessors, + accessorsPresent: fieldAnn.accessorsPresent, + }); + } + return out; +} + +function methodArityRange(methodNode: Parser.SyntaxNode): { min: number; max: number } { + const params = methodNode.childForFieldName('parameters'); + if (!params) return { min: 0, max: 0 }; + let count = 0; + for (const child of params.namedChildren) { + if (child.type === 'spread_parameter') return { min: count, max: Number.POSITIVE_INFINITY }; + if (child.type === 'formal_parameter') count += 1; + } + return { min: count, max: count }; +} + +function collectExistingMethods( + classBody: Parser.SyntaxNode | null, + imports: LombokImportIndex, +): ExistingMethodIndex { + const index = createExistingMethodIndex('case-folded'); + if (!classBody) return index; + const scan = (container: Parser.SyntaxNode): void => { + for (const child of container.children) { + if (child.type === 'enum_body_declarations') { + scan(child); + continue; + } + if (child.type !== 'method_declaration') continue; + const mods = child.children.find((c) => c.type === 'modifiers') ?? null; + const ann = parseModifierAnnotations(mods, imports); + if (ann.tolerate) continue; + const nameNode = child.childForFieldName('name'); + if (!nameNode) continue; + const arity = methodArityRange(child); + rememberExistingMethodRange(index, nameNode.text, arity.min, arity.max); + } + }; + scan(classBody); + return index; +} + +const TYPE_BODIES = new Set(['class_body', 'enum_body']); + +function findTypeBody(node: Parser.SyntaxNode): Parser.SyntaxNode | null { + return node.children.find((c) => TYPE_BODIES.has(c.type)) ?? null; +} + +function findLombokClasses(root: Parser.SyntaxNode, imports: LombokImportIndex): LombokClass[] { + const classes: LombokClass[] = []; + + function walk(node: Parser.SyntaxNode): void { + if (node.type === 'class_declaration' || node.type === 'enum_declaration') { + const modifiers = node.children.find((c) => c.type === 'modifiers') ?? null; + const classAnn = parseModifierAnnotations(modifiers, imports); + const nameNode = node.childForFieldName('name'); + const className = nameNode?.text ?? ''; + if (className) { + const body = findTypeBody(node); + const fields: LombokField[] = []; + if (body) { + const collectFields = (container: Parser.SyntaxNode): void => { + for (const child of container.children) { + if (child.type === 'field_declaration') { + for (const f of parseFieldDeclaration(child, imports)) { + if (f.isStatic) continue; + fields.push(f); + } + } else if (child.type === 'enum_body_declarations') { + collectFields(child); + } + } + }; + collectFields(body); + } + + const anyFieldEnable = fields.some( + (f) => f.fieldGetter?.enabled === true || f.fieldSetter?.enabled === true, + ); + const classEnable = classAnn.getter?.enabled === true || classAnn.setter?.enabled === true; + + // Class-level NONE alone is not enable — getter/setter configs may be disabled + if (classEnable || anyFieldEnable) { + classes.push({ + node, + name: className, + classGetter: classAnn.getter, + classSetter: classAnn.setter, + classAccessors: classAnn.accessors, + fields, + existingMethods: collectExistingMethods(body, imports), + }); + } + } + } + for (const child of node.children) walk(child); + } + + walk(root); + return classes; +} + +function planAccessors(cls: LombokClass): PlannedLombokAccessor[] { + const planned: PlannedLombokAccessor[] = []; + for (const field of cls.fields) { + const accessors = mergeAccessors(cls.classAccessors, field.accessors, field.accessorsPresent); + // fluent/prefix change names — omit rather than invent wrong names + if (accessors.fluent || accessors.hasPrefix) continue; + + const getterCfg = effectiveAccessor(cls.classGetter, field.fieldGetter); + const setterCfg = effectiveAccessor(cls.classSetter, field.fieldSetter); + + if (getterCfg?.enabled) { + const gName = getterName(field.name, field.type); + if (!hasExistingMethod(cls.existingMethods, gName, 0)) { + planned.push({ + kind: 'getter', + name: gName, + returnType: field.type, + parameterTypes: [], + visibility: getterCfg.visibility, + isStatic: false, + isAbstract: false, + startLine: field.startLine, + endLine: field.endLine, + declaratorNode: field.declaratorNode, + }); + } + } + + if (setterCfg?.enabled && !field.isFinal) { + const sName = setterName(field.name, field.type); + if (!hasExistingMethod(cls.existingMethods, sName, 1)) { + // chain=true → setter returns declaring type; never emit void in that case + const returnType = accessors.chain ? cls.name : 'void'; + planned.push({ + kind: 'setter', + name: sName, + returnType, + parameterTypes: [field.type], + visibility: setterCfg.visibility, + isStatic: false, + isAbstract: false, + startLine: field.startLine, + endLine: field.endLine, + declaratorNode: field.declaratorNode, + }); + } + } + } + return planned; +} + +function planLombokAccessorOwners(root: Parser.SyntaxNode): PlannedJvmAccessorOwner[] { + const imports = collectLombokImports(root); + return findLombokClasses(root, imports).map((cls) => ({ + node: cls.node, + name: cls.name, + accessors: planAccessors(cls), + })); +} + +const lombokAccessorSynthesis = createJvmAccessorSynthesis({ + language: 'java', + synthetic: 'lombok', + planOwners: planLombokAccessorOwners, +}); + +// ── Main API ────────────────────────────────────────────────────────────── + +export function synthesizeLombokAccessors( + tree: Parser.Tree, + filePath: string, + classOwnersById: ReadonlyMap, +): LombokSynthesisResult { + return lombokAccessorSynthesis.synthesize(tree, filePath, classOwnersById); +} + +/** Scope captures for Lombok accessors (dual-path parity with record components). */ +export function synthesizeLombokAccessorCaptures(rootNode: Parser.SyntaxNode): CaptureMatch[] { + return lombokAccessorSynthesis.captures(rootNode); +} diff --git a/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts index 0410baa6c..3f23040a6 100644 --- a/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/java/scope-resolver.ts @@ -35,6 +35,7 @@ import { attachJavaSpringConfigBindings } from './spring-config-bindings.js'; import { attachJavaSpringConditionalMetadata } from './spring-conditionals.js'; import { attachJavaSpringDiMetadata } from './spring-di.js'; import { attachJavaSpringNonHttpHandlerMetadata } from './spring-non-http-handlers.js'; +import { attachJavaSpringDynamicLookup } from './spring-dynamic-lookup.js'; import { applyJavaCaptureSideChannel, clearJavaClassAnnotationFacts, @@ -97,6 +98,7 @@ const javaScopeResolver: ScopeResolver = { attachJavaSpringDiMetadata(graph, parsedFiles, nodeLookup, indexes); attachJavaSpringNonHttpHandlerMetadata(graph, parsedFiles, nodeLookup, indexes); attachJavaSpringConfigBindings(graph, parsedFiles, nodeLookup, indexes, ctx); + attachJavaSpringDynamicLookup(graph, parsedFiles, nodeLookup, indexes); }, }; diff --git a/gitnexus/src/core/ingestion/languages/java/spring-actuator.ts b/gitnexus/src/core/ingestion/languages/java/spring-actuator.ts new file mode 100644 index 000000000..dd3627315 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/java/spring-actuator.ts @@ -0,0 +1,72 @@ +import type { GraphNode } from 'gitnexus-shared'; +import type { RuntimeCallableIdentity, RuntimeSymbolStrategy } from '../../language-provider.js'; + +const JVM_PRIMITIVES: Readonly> = { + B: 'byte', + C: 'char', + D: 'double', + F: 'float', + I: 'int', + J: 'long', + S: 'short', + Z: 'boolean', +}; + +function normalizedType(value: string, runtime: boolean): string { + let erased = value.trim(); + let arrayDimensions = 0; + if (erased.endsWith('...')) { + arrayDimensions++; + erased = erased.slice(0, -3); + } + while (erased.endsWith('[]')) { + arrayDimensions++; + erased = erased.slice(0, -2); + } + erased = erased.replace(/<.*>$/, '').replaceAll('$', '.').replaceAll('/', '.'); + const simple = erased.slice(erased.lastIndexOf('.') + 1); + const base = runtime ? (JVM_PRIMITIVES[simple] ?? simple) : simple; + return `${base}${'[]'.repeat(arrayDimensions)}`; +} + +function sourceTypeIsUnknown(value: string): boolean { + const type = normalizedType(value, false).replace(/(?:\[\])+$/, ''); + return type === '?' || /^[A-Z]$/.test(type); +} + +function matchesJavaCallable(node: GraphNode, runtime: RuntimeCallableIdentity): boolean { + if (node.label !== 'Method' || node.properties.name !== runtime.name) return false; + + const descriptorTypes = runtime.descriptorParameterTypes; + if (descriptorTypes === undefined) return true; + + const parameterCount = node.properties.parameterCount; + if (typeof parameterCount === 'number' && parameterCount !== descriptorTypes.length) return false; + + const sourceTypes = node.properties.parameterTypes; + if ( + !Array.isArray(sourceTypes) || + sourceTypes.length !== descriptorTypes.length || + !sourceTypes.every((type): type is string => typeof type === 'string') + ) { + return true; + } + + return sourceTypes.every((sourceType, index) => { + if (sourceTypeIsUnknown(sourceType)) return true; + const source = normalizedType(sourceType, false); + const descriptor = normalizedType(descriptorTypes[index] ?? '', true); + if (source === descriptor) return true; + // Java parser metadata currently drops the ellipsis from varargs and also + // leaves parameterCount open-ended. Only in that shape may T match JVM T[]. + return ( + typeof parameterCount !== 'number' && + descriptor.endsWith('[]') && + source === descriptor.slice(0, -2) + ); + }); +} + +export const javaRuntimeSymbolStrategy: RuntimeSymbolStrategy = { + matchesCallable: matchesJavaCallable, +}; diff --git a/gitnexus/src/core/ingestion/languages/java/spring-di.ts b/gitnexus/src/core/ingestion/languages/java/spring-di.ts index c6dcbe261..121cd1e3b 100644 --- a/gitnexus/src/core/ingestion/languages/java/spring-di.ts +++ b/gitnexus/src/core/ingestion/languages/java/spring-di.ts @@ -12,13 +12,72 @@ import { hasSpringBeanFactorySyntax, type SpringBeanFactoryMethodFact, } from '../../frameworks/spring/bean-factories.js'; +import { + normalizeSpringFactText, + type SpringArgumentFact, +} from '../../frameworks/spring/argument-facts.js'; import { parseSpringInjectionType } from '../../di-extractors/spring.js'; -import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { hasRecoveredSyntax, nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; import { isJavaPackageSiblingVisibilityIncomplete } from './package-siblings.js'; import { getJavaSpringDiFacts } from './capture-side-channel.js'; export interface JavaAnnotationSyntaxFact extends SpringDiAnnotationFact { readonly line: number; + /** Present only for callers that opt in via `javaSpringAnnotationFacts`. */ + readonly args?: readonly SpringArgumentFact[]; +} + +/** + * Options for `javaSpringAnnotationFacts`. + * + * The STRUCTURED arguments are opt-in because DI captures every annotated + * field, constructor, and method in the repository, and none of its consumers + * reads them. Note what this does and does not save: every fact already carries + * `text`, the annotation's full source, so the argument TEXT crosses the worker + * boundary either way. What the opt-in avoids is a second, parsed copy of that + * same text on facts that would never look at it. + */ +export interface JavaSpringAnnotationFactOptions { + readonly includeArguments?: boolean; +} + +const JAVA_COMMENT_NODE_TYPES = new Set(['line_comment', 'block_comment']); + +/** + * Annotation arguments as written, or `undefined` for a marker annotation. + * + * `@Scheduled` yields `undefined` (no argument list in the syntax) while + * `@Scheduled()` yields `[]` (an empty list was written). Named arguments keep + * their key, single-element ones stay positional, and array initializers are + * kept as one raw `{...}` text — splitting or dereferencing them would be + * resolution, which does not belong at capture time. + * + * An argument list that did not parse also yields `undefined`. Error recovery + * fills gaps with invented nodes — `@KafkaListener(topics = "orders", groupId =` + * hands back a `groupId` whose value is a `{}` that nobody wrote — and there is + * no fourth state here for "unreadable". Collapsing it into the marker case is + * deliberate: both tell a consumer there is nothing here to resolve, which is + * true, whereas a fabricated value would send it somewhere real and wrong. + */ +function javaAnnotationArgumentFacts(annotation: SyntaxNode): SpringArgumentFact[] | undefined { + const argumentList = annotation.childForFieldName('arguments'); + if (argumentList === null || hasRecoveredSyntax(argumentList)) return undefined; + const args: SpringArgumentFact[] = []; + for (const child of argumentList.namedChildren) { + if (JAVA_COMMENT_NODE_TYPES.has(child.type)) continue; + if (child.type === 'element_value_pair') { + const key = child.childForFieldName('key'); + const value = child.childForFieldName('value'); + if (key === null || value === null) { + args.push({ text: normalizeSpringFactText(child.text) }); + continue; + } + args.push({ name: key.text.trim(), text: normalizeSpringFactText(value.text) }); + continue; + } + args.push({ text: normalizeSpringFactText(child.text) }); + } + return args; } export type JavaSpringDependencyFact = SpringDiDependencyFact; @@ -36,7 +95,10 @@ export type JavaSpringDiClassFact = SpringDiClassFact< >; type JavaSpringBeanFactoryMethodFact = SpringBeanFactoryMethodFact; -export function javaSpringAnnotationFacts(node: SyntaxNode): JavaAnnotationSyntaxFact[] { +export function javaSpringAnnotationFacts( + node: SyntaxNode, + options: JavaSpringAnnotationFactOptions = {}, +): JavaAnnotationSyntaxFact[] { const facts: JavaAnnotationSyntaxFact[] = []; for (const child of node.namedChildren) { if (child.type !== 'modifiers') continue; @@ -44,10 +106,13 @@ export function javaSpringAnnotationFacts(node: SyntaxNode): JavaAnnotationSynta if (modifier.type !== 'marker_annotation' && modifier.type !== 'annotation') continue; const nameNode = modifier.childForFieldName('name') ?? modifier.firstNamedChild; if (nameNode === null) continue; + const args = + options.includeArguments === true ? javaAnnotationArgumentFacts(modifier) : undefined; facts.push({ name: nameNode.text.trim(), text: modifier.text.trim(), line: modifier.startPosition.row + 1, + ...(args === undefined ? {} : { args }), }); } } diff --git a/gitnexus/src/core/ingestion/languages/java/spring-dynamic-lookup.ts b/gitnexus/src/core/ingestion/languages/java/spring-dynamic-lookup.ts new file mode 100644 index 000000000..922264bf8 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/java/spring-dynamic-lookup.ts @@ -0,0 +1,77 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { + createSpringDynamicLookupMetadataAttacher, + springDynamicLookupCardinality, + type SpringDynamicLookupFact, +} from '../../frameworks/spring/dynamic-lookups.js'; +import { + findAncestorBeforeBoundary, + nodeToCapture, + type SyntaxNode, +} from '../../utils/ast-helpers.js'; +import { getJavaSpringDynamicLookupFacts } from './capture-side-channel.js'; + +const CALLABLE_NODE_TYPES = new Set([ + 'method_declaration', + 'constructor_declaration', + 'compact_constructor_declaration', +]); +const NO_CALLABLE_BOUNDARIES = new Set(); + +function classLiteralTypeName(argument: SyntaxNode): string | null { + if (argument.type !== 'class_literal' || argument.namedChildCount !== 1) return null; + return argument.namedChild(0)?.text.trim() ?? null; +} + +/** Capture real Java method invocations; comments and literals are never visited as calls. */ +export function captureJavaSpringDynamicLookupFact( + node: SyntaxNode, + filePath: string, +): SpringDynamicLookupFact | null { + if (node.type !== 'method_invocation') return null; + const receiverName = node.childForFieldName('object')?.text.trim(); + const methodName = node.childForFieldName('name')?.text.trim(); + const argumentsNode = node.childForFieldName('arguments'); + if (receiverName === undefined || methodName === undefined || argumentsNode === null) return null; + if (springDynamicLookupCardinality(receiverName, methodName) === null) return null; + + const argumentsWithoutComments = argumentsNode.namedChildren.filter( + (child) => child.type !== 'line_comment' && child.type !== 'block_comment', + ); + if (argumentsWithoutComments.length !== 1) return null; + const argument = argumentsWithoutComments[0]; + if (argument === undefined) return null; + const targetTypeName = classLiteralTypeName(argument); + if (targetTypeName === null) return null; + + const owner = findAncestorBeforeBoundary(node, CALLABLE_NODE_TYPES, NO_CALLABLE_BOUNDARIES); + if (owner === null) return null; + const ownerCapture = nodeToCapture('@spring-dynamic-lookup.owner', owner); + return { + ownerScopeId: makeScopeId({ + filePath, + range: ownerCapture.range, + kind: 'Function', + }), + ownerRange: ownerCapture.range, + receiverName, + methodName, + targetTypeName, + }; +} + +/** Standalone extractor for focused tests; production reuses scope-query call nodes. */ +export function captureJavaSpringDynamicLookupFacts( + rootNode: SyntaxNode, + filePath: string, +): SpringDynamicLookupFact[] { + return rootNode + .descendantsOfType('method_invocation') + .map((node) => captureJavaSpringDynamicLookupFact(node, filePath)) + .filter((fact): fact is SpringDynamicLookupFact => fact !== null); +} + +/** Attach Java lookup facts for later resolution by the shared DI phase. */ +export const attachJavaSpringDynamicLookup = createSpringDynamicLookupMetadataAttacher({ + getFacts: getJavaSpringDynamicLookupFacts, +}); diff --git a/gitnexus/src/core/ingestion/languages/java/spring-message-producers.ts b/gitnexus/src/core/ingestion/languages/java/spring-message-producers.ts new file mode 100644 index 000000000..ce59015ba --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/java/spring-message-producers.ts @@ -0,0 +1,100 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { + normalizeSpringFactText, + type SpringArgumentFact, +} from '../../frameworks/spring/argument-facts.js'; +import { + isSpringMessageProducerMethod, + springMessageProducerTemplateOf, + type SpringMessageProducerFact, +} from '../../frameworks/spring/message-producers.js'; +import { + findAncestorBeforeBoundary, + hasRecoveredSyntax, + nodeToCapture, + type SyntaxNode, +} from '../../utils/ast-helpers.js'; + +const CALLABLE_NODE_TYPES = new Set([ + 'method_declaration', + 'constructor_declaration', + 'compact_constructor_declaration', +]); +/** + * A type body ends the search for the publishing callable. + * + * Without it the ancestor walk passes THROUGH the body of a class declared + * inside a method, so a publish in that class's field initializer is attributed + * to the enclosing method, which may never run it. The identical construct at + * the top level of a class already yields no fact — there is no enclosing + * callable to find — and the rule has to read the same at every depth. + */ +const TYPE_BODY_BOUNDARIES = new Set([ + 'class_body', + 'interface_body', + 'enum_body', + 'enum_body_declarations', + 'annotation_type_body', +]); +const COMMENT_NODE_TYPES = new Set(['line_comment', 'block_comment']); + +/** Java has no named call arguments, so every argument is captured positionally. */ +function javaCallArgumentFacts(argumentList: SyntaxNode): SpringArgumentFact[] { + return argumentList.namedChildren + .filter((child) => !COMMENT_NODE_TYPES.has(child.type)) + .map((child) => ({ text: normalizeSpringFactText(child.text) })); +} + +/** + * Capture one messaging-template publish from a Java call already surfaced by + * the scope query, without resolving the destination it names. + * + * The destination argument may be a literal, a reference to a constant that + * lives in another file, or a `${...}` placeholder resolved from configuration; + * all three are recorded as written and left to a later phase. + * + * A call whose argument list did not parse yields NO fact. The fact exists to + * carry a destination, and error recovery invents argument boundaries — an + * unterminated `send(TOPIC,` absorbs the next declaration's source and offers + * it as an argument. There is no state on this fact that means "published + * somewhere unreadable", so the choice is between silence and a plausible lie, + * and silence is recoverable: the file is re-captured when it parses. + */ +export function captureJavaSpringMessageProducerFact( + node: SyntaxNode, + filePath: string, +): SpringMessageProducerFact | null { + if (node.type !== 'method_invocation') return null; + const methodName = node.childForFieldName('name')?.text.trim(); + if (methodName === undefined || !isSpringMessageProducerMethod(methodName)) return null; + const receiverText = node.childForFieldName('object')?.text; + if (receiverText === undefined) return null; + const receiverName = normalizeSpringFactText(receiverText); + const template = springMessageProducerTemplateOf(receiverName, methodName); + if (template === null) return null; + + const argumentList = node.childForFieldName('arguments'); + if (argumentList !== null && hasRecoveredSyntax(argumentList)) return null; + const owner = findAncestorBeforeBoundary(node, CALLABLE_NODE_TYPES, TYPE_BODY_BOUNDARIES); + if (owner === null) return null; + const ownerCapture = nodeToCapture('@spring-message-producer.owner', owner); + return { + ownerScopeId: makeScopeId({ filePath, range: ownerCapture.range, kind: 'Function' }), + ownerRange: ownerCapture.range, + template, + receiverName, + methodName, + ...(argumentList === null ? {} : { args: javaCallArgumentFacts(argumentList) }), + }; +} + +/** Standalone extractor for focused tests; production reuses scope-query call nodes. */ +export function captureJavaSpringMessageProducerFacts( + rootNode: SyntaxNode, + filePath: string, +): SpringMessageProducerFact[] { + return rootNode + .descendantsOfType('method_invocation') + .map((node) => captureJavaSpringMessageProducerFact(node, filePath)) + .filter((fact): fact is SpringMessageProducerFact => fact !== null); +} diff --git a/gitnexus/src/core/ingestion/languages/java/spring-non-http-handlers.ts b/gitnexus/src/core/ingestion/languages/java/spring-non-http-handlers.ts index 259dace3a..a9e7c867e 100644 --- a/gitnexus/src/core/ingestion/languages/java/spring-non-http-handlers.ts +++ b/gitnexus/src/core/ingestion/languages/java/spring-non-http-handlers.ts @@ -11,7 +11,17 @@ import { javaSpringAnnotationFacts, type JavaAnnotationSyntaxFact } from './spri export type JavaSpringNonHttpHandlerFact = SpringNonHttpHandlerFact; -/** Capture callable syntax while the Java class AST is already in hand. */ +/** + * Capture callable syntax while the Java class AST is already in hand. + * + * Annotation arguments are read in a second pass, only for callables that + * already carry a handler annotation, so the destination-bearing arguments + * (`topics`, `queues`, `destination`, `cron`) reach the fact without adding + * structured argument text to every annotation in the repository. Java can + * decide that on the simple name alone; Kotlin runs the same two passes but + * widens the first one with the file's import aliases, because a Kotlin handler + * annotation may be written under a name no list can contain. + */ export function captureJavaSpringNonHttpHandlerFacts( classNode: SyntaxNode, filePath: string, @@ -21,8 +31,8 @@ export function captureJavaSpringNonHttpHandlerFacts( if (body === null) return facts; for (const member of body.namedChildren) { if (member.type !== 'method_declaration') continue; - const annotations = javaSpringAnnotationFacts(member); - if (!hasSpringNonHttpHandlerRelevantAnnotation(annotations)) continue; + if (!hasSpringNonHttpHandlerRelevantAnnotation(javaSpringAnnotationFacts(member))) continue; + const annotations = javaSpringAnnotationFacts(member, { includeArguments: true }); const ownerRange = nodeToCapture('@spring-non-http-handler.owner', member).range; facts.push({ ownerScopeId: makeScopeId({ filePath, range: ownerRange, kind: 'Function' }), diff --git a/gitnexus/src/core/ingestion/languages/jvm/accessor-synthesis.ts b/gitnexus/src/core/ingestion/languages/jvm/accessor-synthesis.ts new file mode 100644 index 000000000..e1cb89253 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/jvm/accessor-synthesis.ts @@ -0,0 +1,316 @@ +/** + * Shared planning orchestration and emission for synthetic JVM accessors. + * + * Language adapters discover accessor plans. This module owns method-collision + * policy, graph emission, and scope captures without naming any language. + */ +import type Parser from 'tree-sitter'; +import type { Capture, CaptureMatch } from 'gitnexus-shared'; +import { toZeroBasedLine } from '../../utils/line-base.js'; + +export type SyntheticVisibility = 'public' | 'protected' | 'private' | 'package'; +export type MethodNameMatching = 'exact' | 'case-folded'; + +export interface ExistingMethodIndex { + readonly matching: MethodNameMatching; + readonly aritiesByName: Map>; + readonly arityRangesByName: Map>; +} + +export function createExistingMethodIndex(matching: MethodNameMatching): ExistingMethodIndex { + return { matching, aritiesByName: new Map(), arityRangesByName: new Map() }; +} + +function methodKey(index: ExistingMethodIndex, name: string): string { + return index.matching === 'case-folded' ? name.toLowerCase() : name; +} + +export function rememberExistingMethod( + index: ExistingMethodIndex, + name: string, + arity: number, +): void { + const key = methodKey(index, name); + let arities = index.aritiesByName.get(key); + if (!arities) { + arities = new Set(); + index.aritiesByName.set(key, arities); + } + arities.add(arity); +} + +export function rememberExistingMethodRange( + index: ExistingMethodIndex, + name: string, + min: number, + max: number, +): void { + if (min === max) { + rememberExistingMethod(index, name, min); + return; + } + const key = methodKey(index, name); + const ranges = index.arityRangesByName.get(key) ?? []; + ranges.push({ min, max }); + index.arityRangesByName.set(key, ranges); +} + +export function hasExistingMethod( + index: ExistingMethodIndex, + name: string, + arity: number, +): boolean { + const key = methodKey(index, name); + if (index.aritiesByName.get(key)?.has(arity) === true) return true; + return ( + index.arityRangesByName.get(key)?.some((range) => range.min <= arity && arity <= range.max) === + true + ); +} + +export interface SyntheticAccessorSymbol { + filePath: string; + name: string; + nodeId: string; + type: 'Method'; + ownerId: string; + qualifiedName: string; + parameterCount: number; + requiredParameterCount: number; + parameterTypes: string[]; + returnType: string; + visibility: SyntheticVisibility; + isStatic: boolean; + isAbstract: boolean; + isFinal: boolean; +} + +export interface SyntheticAccessorNode { + id: string; + label: 'Method'; + properties: { + name: string; + filePath: string; + startLine: number; + endLine: number; + language: string; + isExported: boolean; + synthetic: string; + visibility: SyntheticVisibility; + isStatic: boolean; + returnType: string; + parameterTypes: string[]; + parameterCount: number; + qualifiedName: string; + }; +} + +export interface SyntheticAccessorRelationship { + id: string; + sourceId: string; + targetId: string; + type: 'HAS_METHOD'; + confidence: number; + reason: string; +} + +export interface SyntheticAccessorResult { + symbols: SyntheticAccessorSymbol[]; + nodes: SyntheticAccessorNode[]; + relationships: SyntheticAccessorRelationship[]; +} + +export interface PlannedJvmAccessor { + kind: 'getter' | 'setter'; + name: string; + returnType: string; + parameterTypes: string[]; + visibility: SyntheticVisibility; + isStatic: boolean; + isAbstract: boolean; + startLine: number; + endLine: number; + declaratorNode: Parser.SyntaxNode; +} + +export interface PlannedJvmAccessorOwner { + node: Parser.SyntaxNode; + name: string; + accessors: readonly PlannedJvmAccessor[]; +} + +interface JvmAccessorSynthesisConfig { + language: string; + synthetic: string; + planOwners(rootNode: Parser.SyntaxNode): readonly PlannedJvmAccessorOwner[]; +} + +export interface JvmAccessorSynthesis { + synthesize( + tree: Parser.Tree, + filePath: string, + classOwnersById: ReadonlyMap, + ): SyntheticAccessorResult; + captures(rootNode: Parser.SyntaxNode): CaptureMatch[]; +} + +export function createJvmAccessorSynthesis( + config: JvmAccessorSynthesisConfig, +): JvmAccessorSynthesis { + return { + synthesize(tree, filePath, classOwnersById) { + const result = emptySyntheticAccessorResult(); + for (const owner of config.planOwners(tree.rootNode)) { + const ownerId = classOwnersById.get(owner.node.id); + if (!ownerId) continue; + emitPlannedAccessors({ + planned: owner.accessors, + filePath, + ownerId, + idPrefix: ownerIdNamePrefix(ownerId, filePath, owner.name), + language: config.language, + synthetic: config.synthetic, + result, + }); + } + return result; + }, + captures(rootNode) { + return capturesForPlannedAccessors(config.planOwners(rootNode)); + }, + }; +} + +function emptySyntheticAccessorResult(): SyntheticAccessorResult { + return { symbols: [], nodes: [], relationships: [] }; +} + +function ownerIdNamePrefix(ownerId: string, filePath: string, fallback: string): string { + const needle = `Class:${filePath}:`; + if (ownerId.startsWith(needle)) return ownerId.slice(needle.length); + const enumNeedle = `Enum:${filePath}:`; + if (ownerId.startsWith(enumNeedle)) return ownerId.slice(enumNeedle.length); + const ifaceNeedle = `Interface:${filePath}:`; + if (ownerId.startsWith(ifaceNeedle)) return ownerId.slice(ifaceNeedle.length); + return fallback; +} + +export function jvmTypeSimpleName(node: Parser.SyntaxNode): string | undefined { + const named = node.childForFieldName('name')?.text; + if (named) return named; + for (const child of node.namedChildren) { + if (child.type === 'type_identifier' || child.type === 'simple_identifier') return child.text; + } + return undefined; +} + +function emitPlannedAccessors(args: { + planned: readonly PlannedJvmAccessor[]; + filePath: string; + ownerId: string; + idPrefix: string; + language: string; + synthetic: string; + result: SyntheticAccessorResult; +}): void { + const emittedIds = new Set(); + for (const acc of args.planned) { + const arity = acc.parameterTypes.length; + const qualifiedName = `${args.idPrefix}.${acc.name}`; + const nodeId = `Method:${args.filePath}:${qualifiedName}#${arity}`; + if (emittedIds.has(nodeId)) continue; + emittedIds.add(nodeId); + args.result.nodes.push({ + id: nodeId, + label: 'Method', + properties: { + name: acc.name, + filePath: args.filePath, + startLine: toZeroBasedLine(acc.startLine), + endLine: toZeroBasedLine(acc.endLine), + language: args.language, + isExported: false, + synthetic: args.synthetic, + visibility: acc.visibility, + isStatic: acc.isStatic, + returnType: acc.returnType, + parameterTypes: acc.parameterTypes, + parameterCount: arity, + qualifiedName, + }, + }); + args.result.symbols.push({ + filePath: args.filePath, + name: acc.name, + nodeId, + type: 'Method', + ownerId: args.ownerId, + qualifiedName, + parameterCount: arity, + requiredParameterCount: arity, + parameterTypes: acc.parameterTypes, + returnType: acc.returnType, + visibility: acc.visibility, + isStatic: acc.isStatic, + isAbstract: acc.isAbstract, + isFinal: false, + }); + args.result.relationships.push({ + id: `HAS_METHOD:${args.ownerId}->${nodeId}`, + sourceId: args.ownerId, + targetId: nodeId, + type: 'HAS_METHOD', + confidence: 1.0, + reason: acc.kind === 'getter' ? `${args.synthetic}-getter` : `${args.synthetic}-setter`, + }); + } +} + +function accessorCapture(name: string, acc: PlannedJvmAccessor, text: string): Capture { + const node = acc.declaratorNode; + const startLine = node.startPosition.row + 1; + const startCol = node.startPosition.column; + const endLine = node.endPosition.row + 1; + const endCol = acc.kind === 'getter' ? node.endPosition.column : startCol; + return { name, range: { startLine, startCol, endLine, endCol }, text }; +} + +function capturesForPlannedAccessors(owners: readonly PlannedJvmAccessorOwner[]): CaptureMatch[] { + const captures: CaptureMatch[] = []; + for (const owner of owners) { + const enclosing = owner.name; + const emitted = new Set(); + for (const acc of owner.accessors) { + const arity = String(acc.parameterTypes.length); + const qualifiedName = `${enclosing}.${acc.name}`; + const identity = `${qualifiedName}#${arity}`; + if (emitted.has(identity)) continue; + emitted.add(identity); + captures.push({ + '@scope.function': accessorCapture('@scope.function', acc, acc.name), + }); + captures.push({ + '@declaration.method': accessorCapture('@declaration.method', acc, acc.name), + '@declaration.name': accessorCapture('@declaration.name', acc, acc.name), + '@declaration.qualified_name': accessorCapture( + '@declaration.qualified_name', + acc, + qualifiedName, + ), + '@declaration.parameter-count': accessorCapture('@declaration.parameter-count', acc, arity), + '@declaration.required-parameter-count': accessorCapture( + '@declaration.required-parameter-count', + acc, + arity, + ), + '@declaration.return-type': accessorCapture( + '@declaration.return-type', + acc, + acc.returnType, + ), + '@declaration.is-synthetic': accessorCapture('@declaration.is-synthetic', acc, 'true'), + }); + } + } + return captures; +} diff --git a/gitnexus/src/core/ingestion/languages/jvm/beanspec.ts b/gitnexus/src/core/ingestion/languages/jvm/beanspec.ts new file mode 100644 index 000000000..0a14209f5 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/jvm/beanspec.ts @@ -0,0 +1,49 @@ +/** + * Language-neutral JVM JavaBeans naming primitives. + * + * Language adapters choose whether to invent/preserve an `is` prefix and + * which single-character capitalization policy their compiler uses. + */ + +export function capitalizeBeanName(s: string): string { + if (s.length === 0) return s; + const first = s.charAt(0); + const upper = first.toUpperCase(); + // Java Character case conversion is one UTF-16 code unit. JavaScript + // full-case conversion may expand one unit (`ß` → `SS`), which would invent + // a method name no JVM compiler emits. + return (upper.length === 1 ? upper : first) + s.slice(1); +} + +/** + * Primitive-boolean / Kotlin `is`-prefix fields whose name already starts with + * `is` plus a non-lowercase character keep that name for the getter and drop + * the `is` prefix for the setter base (`isEnabled` → `isEnabled()` / + * `setEnabled(...)`, `is1` → `is1()` / `set1(...)`). Digits and punctuation + * count as non-lowercase, matching Lombok `!Character.isLowerCase` and kotlinc. + */ +export function booleanIsPrefixBase(fieldName: string, useIsPrefix: boolean): string | null { + if (!useIsPrefix || !fieldName.startsWith('is') || fieldName.length < 3) return null; + const third = fieldName.charAt(2); + return third === third.toUpperCase() ? fieldName.slice(2) : null; +} + +export function jvmGetterName( + fieldName: string, + useIsPrefix: boolean, + capitalize: (name: string) => string = capitalizeBeanName, +): string { + if (booleanIsPrefixBase(fieldName, useIsPrefix) !== null) return fieldName; + if (useIsPrefix) return `is${capitalize(fieldName)}`; + return `get${capitalize(fieldName)}`; +} + +export function jvmSetterName( + fieldName: string, + useIsPrefix: boolean, + capitalize: (name: string) => string = capitalizeBeanName, +): string { + const stripped = booleanIsPrefixBase(fieldName, useIsPrefix); + if (stripped !== null) return `set${stripped}`; + return `set${capitalize(fieldName)}`; +} diff --git a/gitnexus/src/core/ingestion/languages/kotlin.ts b/gitnexus/src/core/ingestion/languages/kotlin.ts index 18d4fd9c1..b04a58427 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin.ts @@ -24,6 +24,10 @@ import type { SyntaxNode } from '../utils/ast-helpers.js'; import { createCallExtractor } from '../call-extractors/generic.js'; import { kotlinCallConfig } from '../call-extractors/configs/jvm.js'; import { createKotlinCfgVisitor } from '../cfg/visitors/kotlin.js'; +import { + getKotlinSpringMessageProducerFacts, + getKotlinSpringNonHttpHandlerFacts, +} from './kotlin/capture-side-channel.js'; import { createFieldExtractor } from '../field-extractors/generic.js'; import { kotlinConfig } from '../field-extractors/configs/jvm.js'; import { createMethodExtractor } from '../method-extractors/generic.js'; @@ -41,6 +45,16 @@ import { kotlinMergeBindings, kotlinReceiverBinding, } from './kotlin/index.js'; +import { synthesizeLombokAccessors } from './kotlin/lombok-synthesizer.js'; +import { + extractKotlinRuntimeSymbolProperties, + kotlinRuntimeSymbolStrategy, +} from './kotlin/spring-actuator.js'; +import { extractKotlinSpringRoutes } from '../route-extractors/kotlin-spring.js'; +import { + extractKotlinModuleConstants, + foldKotlinOperands, +} from '../route-extractors/kotlin-const-resolver.js'; /** Check if a Kotlin function_declaration capture is inside a class_body (i.e., a method). * Kotlin grammar uses function_declaration for both top-level functions and class methods. @@ -174,6 +188,8 @@ export const kotlinProvider = defineLanguage({ // ── KDoc → description (issue #2270) ── descriptionExtractor: createLeadingDocDescriptionExtractor(), + definitionPropertiesExtractor: extractKotlinRuntimeSymbolProperties, + runtimeSymbolStrategy: kotlinRuntimeSymbolStrategy, labelOverride: (functionNode, defaultLabel) => { if (defaultLabel !== 'Function') return defaultLabel; @@ -202,4 +218,23 @@ export const kotlinProvider = defineLanguage({ mergeBindings: (_scope, bindings) => kotlinMergeBindings(bindings), receiverBinding: kotlinReceiverBinding, arityCompatibility: kotlinArityCompatibility, + synthesizeStructureMembers: synthesizeLombokAccessors, + + // ── Spring decorator routes + composed path constants (#3130) ── + extractDecoratorRoutes: extractKotlinSpringRoutes, + extractModuleConstants: extractKotlinModuleConstants, + foldRoutePathOperands: foldKotlinOperands, + + // Async messaging facts for the `springDestinations` phase. Both stores are + // repopulated on the main thread by `applyKotlinCaptureSideChannel`, so this + // answers for cache hits and misses alike. + getSpringMessagingFacts: (filePath) => ({ + handlers: getKotlinSpringNonHttpHandlerFacts(filePath), + producers: getKotlinSpringMessageProducerFacts(filePath), + }), + // Kotlin string literals interpolate: `"orders-$env"` and `"orders-${env}"` + // are string templates, and a Spring property placeholder has to escape the + // dollar (`"\${app.topic}"`). Destination resolution needs this to keep a + // runtime template out of the address namespace. + interpolatesStringLiterals: true, }); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts index 6ea9480a8..6983e55ec 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts @@ -49,16 +49,22 @@ import { } from '../jvm/package-facts.js'; import { getCompanionScopesForFile, markCompanionScope } from './companion-scopes.js'; import { getKotlinPackageFact, setKotlinPackageFact } from './package-facts.js'; +import type { SpringDynamicLookupFact } from '../../frameworks/spring/dynamic-lookups.js'; +import type { SpringMessageProducerFact } from '../../frameworks/spring/message-producers.js'; import type { KotlinSpringAopFact } from './spring-aop.js'; import type { KotlinSpringConditionalFact } from './spring-conditionals.js'; import type { KotlinSpringDiClassFact } from './spring-di.js'; import type { KotlinSpringNonHttpHandlerFact } from './spring-non-http-handlers.js'; +import type { KotlinSpringConfigConsumerFact } from './spring-config-bindings.js'; const classAnnotations = createClassAnnotationFactStore(); const springAopFacts = new Map(); const springConditionalFacts = new Map(); const springDiFacts = new Map(); +const springDynamicLookupFacts = new Map(); const springNonHttpHandlerFacts = new Map(); +const springConfigConsumerFacts = new Map(); +const springMessageProducerFacts = new Map(); /** * Plain JSON-serializable snapshot of the per-file Kotlin capture-time @@ -80,8 +86,14 @@ export interface KotlinCaptureSideChannel { readonly springConditionalFacts?: readonly KotlinSpringConditionalFact[]; /** Constructor, property, and method injection syntax captured per class. */ readonly springDiFacts?: readonly KotlinSpringDiClassFact[]; + /** Programmatic Spring bean lookups captured per callable. */ + readonly springDynamicLookupFacts?: readonly SpringDynamicLookupFact[]; /** Scheduled, event, messaging, and managed-job handler syntax captured per callable. */ readonly springNonHttpHandlerFacts?: readonly KotlinSpringNonHttpHandlerFact[]; + /** `@Value` / `@ConfigurationProperties` syntax captured per owner. */ + readonly springConfigConsumerFacts?: readonly KotlinSpringConfigConsumerFact[]; + /** Messaging-template publish syntax captured per callable. */ + readonly springMessageProducerFacts?: readonly SpringMessageProducerFact[]; } export function clearKotlinClassAnnotationFacts(): void { @@ -89,7 +101,10 @@ export function clearKotlinClassAnnotationFacts(): void { springAopFacts.clear(); springConditionalFacts.clear(); springDiFacts.clear(); + springDynamicLookupFacts.clear(); springNonHttpHandlerFacts.clear(); + springConfigConsumerFacts.clear(); + springMessageProducerFacts.clear(); } export function setKotlinSpringAopFacts( @@ -141,6 +156,20 @@ export function getKotlinSpringDiFacts(filePath: string): readonly KotlinSpringD return springDiFacts.get(filePath) ?? []; } +export function setKotlinSpringDynamicLookupFacts( + filePath: string, + facts: readonly SpringDynamicLookupFact[], +): void { + if (facts.length === 0) springDynamicLookupFacts.delete(filePath); + else springDynamicLookupFacts.set(filePath, facts); +} + +export function getKotlinSpringDynamicLookupFacts( + filePath: string, +): readonly SpringDynamicLookupFact[] { + return springDynamicLookupFacts.get(filePath) ?? []; +} + export function setKotlinSpringNonHttpHandlerFacts( filePath: string, facts: readonly KotlinSpringNonHttpHandlerFact[], @@ -155,6 +184,34 @@ export function getKotlinSpringNonHttpHandlerFacts( return springNonHttpHandlerFacts.get(filePath) ?? []; } +export function setKotlinSpringConfigConsumerFacts( + filePath: string, + facts: readonly KotlinSpringConfigConsumerFact[], +): void { + if (facts.length === 0) springConfigConsumerFacts.delete(filePath); + else springConfigConsumerFacts.set(filePath, facts); +} + +export function getKotlinSpringConfigConsumerFacts( + filePath: string, +): readonly KotlinSpringConfigConsumerFact[] { + return springConfigConsumerFacts.get(filePath) ?? []; +} + +export function setKotlinSpringMessageProducerFacts( + filePath: string, + facts: readonly SpringMessageProducerFact[], +): void { + if (facts.length === 0) springMessageProducerFacts.delete(filePath); + else springMessageProducerFacts.set(filePath, facts); +} + +export function getKotlinSpringMessageProducerFacts( + filePath: string, +): readonly SpringMessageProducerFact[] { + return springMessageProducerFacts.get(filePath) ?? []; +} + /** * `LanguageProvider.collectCaptureSideChannel` implementation for Kotlin. * Returns `undefined` when this file recorded no side-channel state at all, so @@ -168,7 +225,10 @@ export function collectKotlinCaptureSideChannel( const aopFacts = springAopFacts.get(filePath) ?? []; const conditionFacts = springConditionalFacts.get(filePath) ?? []; const diFacts = springDiFacts.get(filePath) ?? []; + const dynamicLookupFacts = springDynamicLookupFacts.get(filePath) ?? []; const nonHttpHandlerFacts = springNonHttpHandlerFacts.get(filePath) ?? []; + const configConsumerFacts = springConfigConsumerFacts.get(filePath) ?? []; + const messageProducerFacts = springMessageProducerFacts.get(filePath) ?? []; const packageFact = getKotlinPackageFact(filePath); if ( companionScopes.length === 0 && @@ -176,7 +236,10 @@ export function collectKotlinCaptureSideChannel( aopFacts.length === 0 && conditionFacts.length === 0 && diFacts.length === 0 && + dynamicLookupFacts.length === 0 && nonHttpHandlerFacts.length === 0 && + configConsumerFacts.length === 0 && + messageProducerFacts.length === 0 && packageFact === undefined ) { return undefined; @@ -189,7 +252,12 @@ export function collectKotlinCaptureSideChannel( ...(aopFacts.length > 0 ? { springAopFacts: aopFacts } : {}), ...(conditionFacts.length > 0 ? { springConditionalFacts: conditionFacts } : {}), ...(diFacts.length > 0 ? { springDiFacts: diFacts } : {}), + ...(dynamicLookupFacts.length > 0 ? { springDynamicLookupFacts: dynamicLookupFacts } : {}), ...(nonHttpHandlerFacts.length > 0 ? { springNonHttpHandlerFacts: nonHttpHandlerFacts } : {}), + ...(configConsumerFacts.length > 0 ? { springConfigConsumerFacts: configConsumerFacts } : {}), + ...(messageProducerFacts.length > 0 + ? { springMessageProducerFacts: messageProducerFacts } + : {}), }; } @@ -215,7 +283,10 @@ export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { setKotlinSpringAopFacts(parsed.filePath, []); setKotlinSpringConditionalFacts(parsed.filePath, []); setKotlinSpringDiFacts(parsed.filePath, []); + setKotlinSpringDynamicLookupFacts(parsed.filePath, []); setKotlinSpringNonHttpHandlerFacts(parsed.filePath, []); + setKotlinSpringConfigConsumerFacts(parsed.filePath, []); + setKotlinSpringMessageProducerFacts(parsed.filePath, []); setKotlinPackageFact(parsed.filePath, UNKNOWN_JVM_PACKAGE_FACT); return; } @@ -235,10 +306,22 @@ export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { parsed.filePath, Array.isArray(data.springDiFacts) ? data.springDiFacts : [], ); + setKotlinSpringDynamicLookupFacts( + parsed.filePath, + Array.isArray(data.springDynamicLookupFacts) ? data.springDynamicLookupFacts : [], + ); setKotlinSpringNonHttpHandlerFacts( parsed.filePath, Array.isArray(data.springNonHttpHandlerFacts) ? data.springNonHttpHandlerFacts : [], ); + setKotlinSpringConfigConsumerFacts( + parsed.filePath, + Array.isArray(data.springConfigConsumerFacts) ? data.springConfigConsumerFacts : [], + ); + setKotlinSpringMessageProducerFacts( + parsed.filePath, + Array.isArray(data.springMessageProducerFacts) ? data.springMessageProducerFacts : [], + ); setKotlinPackageFact( parsed.filePath, isJvmPackageFact(data.packageFact) ? data.packageFact : UNKNOWN_JVM_PACKAGE_FACT, diff --git a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts index 84ec8dc4d..a24073324 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts @@ -23,11 +23,20 @@ import { setKotlinSpringAopFacts, setKotlinSpringConditionalFacts, setKotlinSpringDiFacts, + setKotlinSpringDynamicLookupFacts, + setKotlinSpringMessageProducerFacts, setKotlinSpringNonHttpHandlerFacts, + setKotlinSpringConfigConsumerFacts, } from './capture-side-channel.js'; import { captureKotlinPackageFact } from './package-facts.js'; import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js'; +import { synthesizeLombokAccessorCaptures } from './lombok-synthesizer.js'; import { captureKotlinSpringDiClassFact, type KotlinSpringDiClassFact } from './spring-di.js'; +import { captureKotlinSpringConfigConsumerFacts } from './spring-config-bindings.js'; +import type { SpringDynamicLookupFact } from '../../frameworks/spring/dynamic-lookups.js'; +import { captureKotlinSpringDynamicLookupFact } from './spring-dynamic-lookup.js'; +import type { SpringMessageProducerFact } from '../../frameworks/spring/message-producers.js'; +import { captureKotlinSpringMessageProducerFact } from './spring-message-producers.js'; import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js'; import { captureKotlinSpringAopFacts, type KotlinSpringAopFact } from './spring-aop.js'; import { @@ -107,6 +116,9 @@ export function emitKotlinScopeCaptures( const springNonHttpHandlerFacts: KotlinSpringNonHttpHandlerFact[] = []; const springNonHttpHandlerTypeNodeIds = new Set(); const springDiClassNodeIds = new Set(); + const springDynamicLookupFacts: SpringDynamicLookupFact[] = []; + const springMessageProducerFacts: SpringMessageProducerFact[] = []; + const springMemberCallNodeIds = new Set(); const returnTypes = collectKotlinReturnTypeTexts(tree.rootNode); out.push(...synthesizeKotlinLocalAssignmentBindings(tree.rootNode, returnTypes)); out.push(...synthesizeKotlinLoopBindings(tree.rootNode, returnTypes)); @@ -130,6 +142,17 @@ export function emitKotlinScopeCaptures( } if (Object.keys(grouped).length === 0) continue; + // One visit per member call node: the same invocation can back several + // query matches, and both Spring call-shape captures must see it once. + const memberCallNode = nodeIfType(groupedNodes['@reference.call.member'], 'call_expression'); + if (memberCallNode !== null && !springMemberCallNodeIds.has(memberCallNode.id)) { + springMemberCallNodeIds.add(memberCallNode.id); + const lookupFact = captureKotlinSpringDynamicLookupFact(memberCallNode, filePath); + if (lookupFact !== null) springDynamicLookupFacts.push(lookupFact); + const producerFact = captureKotlinSpringMessageProducerFact(memberCallNode, filePath); + if (producerFact !== null) springMessageProducerFacts.push(producerFact); + } + // tree-sitter-kotlin represents both classes and interfaces with // `class_declaration`; `object_declaration` is the separate object form. const springAopTypeNode = [ @@ -357,7 +380,14 @@ export function emitKotlinScopeCaptures( setKotlinSpringAopFacts(filePath, springAopFacts); setKotlinSpringConditionalFacts(filePath, springConditionalFacts); setKotlinSpringDiFacts(filePath, springDiFacts); + setKotlinSpringDynamicLookupFacts(filePath, springDynamicLookupFacts); setKotlinSpringNonHttpHandlerFacts(filePath, springNonHttpHandlerFacts); + setKotlinSpringConfigConsumerFacts( + filePath, + captureKotlinSpringConfigConsumerFacts(tree.rootNode, filePath), + ); + setKotlinSpringMessageProducerFacts(filePath, springMessageProducerFacts); + out.push(...synthesizeLombokAccessorCaptures(tree.rootNode)); out.push(...synthesizeCallableFlowCaptures(tree.rootNode, KOTLIN_CALLABLE_CAPTURE_OPTIONS)); return out; } diff --git a/gitnexus/src/core/ingestion/languages/kotlin/lombok-synthesizer.ts b/gitnexus/src/core/ingestion/languages/kotlin/lombok-synthesizer.ts new file mode 100644 index 000000000..bb0c3d5b7 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/lombok-synthesizer.ts @@ -0,0 +1,512 @@ +/** + * Kotlin accessor synthesizer (same provider-hook role as Java Lombok). + * + * kotlinc emits JavaBeans getters/setters for `val`/`var` properties. Those + * methods are absent from the tree-sitter AST, so Java (and Kotlin) calls + * like `user.getName()` miss CALLS edges. Planning is Kotlin-specific; + * naming and Method emission share `jvm/beanspec` + `jvm/accessor-synthesis`. + * + * ## Supported subset (v1) + * - Class / data class / object / companion / interface `val`/`var` properties + * (interface accessors without a custom body are abstract JVM methods). + * - Primary-constructor `val`/`var` class parameters. + * - Names beginning with `is` + a non-lowercase character keep that getter name; all other + * properties, including `Boolean`, use `get`. + * - Custom `get()`/`set()` bodies still emit their JVM accessor Methods. + * - Explicit `fun getX` / `@JvmField` / `const` skip synthesis. + * - `@JvmName`-renamed accessors are suppressed until custom-name emission lands. + * Unsupported: `@JvmStatic` renaming, file-facade top-level properties. + */ +import type Parser from 'tree-sitter'; +import type { CaptureMatch } from 'gitnexus-shared'; +import { booleanIsPrefixBase, jvmGetterName, jvmSetterName } from '../jvm/beanspec.js'; +import { + createExistingMethodIndex, + createJvmAccessorSynthesis, + hasExistingMethod, + jvmTypeSimpleName, + rememberExistingMethod, + type ExistingMethodIndex, + type PlannedJvmAccessor, + type PlannedJvmAccessorOwner, + type SyntheticAccessorResult, + type SyntheticVisibility, +} from '../jvm/accessor-synthesis.js'; + +const KOTLIN_TYPE_DECLS = new Set(['class_declaration', 'object_declaration', 'companion_object']); + +function capitalizeAscii(name: string): string { + const first = name.charAt(0); + return first >= 'a' && first <= 'z' + ? String.fromCharCode(first.charCodeAt(0) - 32) + name.slice(1) + : name; +} + +export function kotlinGetterName(propertyName: string): string { + return jvmGetterName( + propertyName, + booleanIsPrefixBase(propertyName, true) !== null, + capitalizeAscii, + ); +} + +export function kotlinSetterName(propertyName: string): string { + return jvmSetterName(propertyName, true, capitalizeAscii); +} + +interface KtProperty { + name: string; + type: string; + isVar: boolean; + skipGetter: boolean; + skipSetter: boolean; + getterVisibility: SyntheticVisibility; + setterVisibility: SyntheticVisibility; + startLine: number; + endLine: number; + propertyNode: Parser.SyntaxNode; + declaratorNode: Parser.SyntaxNode; +} + +interface KtClass { + node: Parser.SyntaxNode; + name: string; + isStatic: boolean; + isInterface: boolean; + wasHoisted: boolean; + properties: KtProperty[]; + existingMethods: ExistingMethodIndex; +} + +interface KotlinImportIndex { + byLocalName: Map; + shadowedSimpleNames: Set; +} + +function collectKotlinImports(root: Parser.SyntaxNode): KotlinImportIndex { + const byLocalName = new Map(); + const shadowedSimpleNames = new Set(); + for (const child of root.children) { + if (child.type !== 'class_declaration') continue; + const name = jvmTypeSimpleName(child); + if (name) shadowedSimpleNames.add(name); + } + const importList = root.children.find((child) => child.type === 'import_list'); + for (const child of importList?.children ?? []) { + if (child.type !== 'import_header') continue; + const text = child.text + .replace(/^import\s+/, '') + .replace(/\/\*[\s\S]*?\*\//g, '') + .trim(); + const [pathText, aliasText] = text.split(/\s+as\s+/, 2); + const importPath = pathText?.replace(/\s+/g, ''); + if (!importPath || importPath.endsWith('.*')) continue; + const localName = aliasText?.trim() || importPath.split('.').pop(); + if (localName) byLocalName.set(localName, importPath); + } + return { byLocalName, shadowedSimpleNames }; +} + +function annotationUserTypeText(annotation: Parser.SyntaxNode): string { + const constructor = annotation.namedChildren.find((c) => c.type === 'constructor_invocation'); + const userType = + constructor?.namedChildren.find((c) => c.type === 'user_type') ?? + annotation.namedChildren.find((c) => c.type === 'user_type'); + return userType?.text ?? ''; +} + +function isKotlinJvmAnnotation( + annotation: Parser.SyntaxNode, + name: string, + imports: KotlinImportIndex, +): boolean { + const typeText = annotationUserTypeText(annotation); + const canonical = `kotlin.jvm.${name}`; + if (typeText.includes('.')) return typeText === canonical; + const imported = imports.byLocalName.get(typeText); + if (imported !== undefined) return imported === canonical; + if (imports.shadowedSimpleNames.has(typeText)) return false; + return typeText === name; +} + +function kotlinVisibility(modifiers: Parser.SyntaxNode | undefined): SyntheticVisibility { + if (!modifiers) return 'public'; + for (const child of modifiers.namedChildren) { + if (child.type !== 'visibility_modifier') continue; + if (child.text === 'private') return 'private'; + if (child.text === 'protected') return 'protected'; + if (child.text === 'internal') return 'package'; + } + return 'public'; +} + +function hasJvmField(node: Parser.SyntaxNode, imports: KotlinImportIndex): boolean { + const mods = node.children.find((c) => c.type === 'modifiers'); + return ( + mods?.namedChildren.some( + (child) => child.type === 'annotation' && isKotlinJvmAnnotation(child, 'JvmField', imports), + ) === true + ); +} + +function hasConst(node: Parser.SyntaxNode): boolean { + const mods = node.children.find((c) => c.type === 'modifiers'); + if ( + mods?.namedChildren.some( + (child) => child.type === 'property_modifier' && child.text === 'const', + ) + ) { + return true; + } + return node.namedChildren.some((child) => child.type === 'const'); +} + +function isVarBinding(node: Parser.SyntaxNode): boolean | null { + const kind = node.children.find((c) => c.type === 'binding_pattern_kind'); + const text = kind?.text; + if (text === 'var') return true; + if (text === 'val') return false; + return null; +} + +function inferredInitializerType(node: Parser.SyntaxNode): string | undefined { + switch (node.type) { + case 'string_literal': + case 'line_string_literal': + case 'multi_line_string_literal': + return 'String'; + case 'character_literal': + return 'Char'; + case 'boolean_literal': + case 'true': + case 'false': + return 'Boolean'; + case 'long_literal': + return 'Long'; + case 'unsigned_literal': + return /l$/i.test(node.text) ? 'ULong' : 'UInt'; + case 'integer_literal': + case 'decimal_integer_literal': + case 'hex_integer_literal': + case 'octal_integer_literal': + case 'binary_integer_literal': + return 'Int'; + case 'real_literal': + case 'decimal_floating_point_literal': + return /f$/i.test(node.text) ? 'Float' : 'Double'; + case 'prefix_expression': { + const operand = node.namedChildren.at(-1); + return operand ? inferredInitializerType(operand) : undefined; + } + case 'call_expression': { + const callee = node.namedChildren.find((child) => child.type === 'simple_identifier'); + if (!callee) return undefined; + const first = callee.text.charAt(0); + return first !== '' && first === first.toUpperCase() ? callee.text : undefined; + } + default: + return undefined; + } +} + +function propertyTypeText(node: Parser.SyntaxNode): string { + const declarator = + node.type === 'class_parameter' + ? node + : (node.children.find((c) => c.type === 'variable_declaration') ?? node); + const colon = declarator.children.find((c) => c.type === ':'); + let typeNode = colon?.nextNamedSibling ?? null; + while (typeNode?.type === 'type_modifiers') typeNode = typeNode.nextNamedSibling; + if (typeNode) return typeNode.text; + const initializer = node.namedChildren.find( + (child) => + child.id !== declarator.id && + child.type !== 'binding_pattern_kind' && + child.type !== 'modifiers', + ); + return initializer ? (inferredInitializerType(initializer) ?? 'unknown') : 'unknown'; +} + +function propertyNameNode(node: Parser.SyntaxNode): Parser.SyntaxNode | null { + if (node.type === 'class_parameter') { + return node.children.find((c) => c.type === 'simple_identifier') ?? null; + } + const decl = node.children.find((c) => c.type === 'variable_declaration'); + if (decl) { + return decl.children.find((c) => c.type === 'simple_identifier') ?? null; + } + return node.children.find((c) => c.type === 'simple_identifier') ?? null; +} + +function accessorMetadata( + prop: Parser.SyntaxNode, + propertyVisibility: SyntheticVisibility, + imports: KotlinImportIndex, +): { + getterVisibility: SyntheticVisibility; + setterVisibility: SyntheticVisibility; + skipGetter: boolean; + skipSetter: boolean; +} { + let getter = propertyVisibility; + let setter = propertyVisibility; + let skipGetter = false; + let skipSetter = false; + const propertyModifiers = prop.children.find((c) => c.type === 'modifiers'); + for (const annotation of propertyModifiers?.namedChildren ?? []) { + if ( + annotation.type !== 'annotation' || + !isKotlinJvmAnnotation(annotation, 'JvmName', imports) + ) { + continue; + } + const target = annotation.children.find((c) => c.type === 'use_site_target')?.text; + if (target === 'get:') skipGetter = true; + if (target === 'set:') skipSetter = true; + } + const apply = (node: Parser.SyntaxNode): void => { + const modifiers = node.children.find((c) => c.type === 'modifiers'); + if (!modifiers) return; + if (node.type === 'getter') getter = kotlinVisibility(modifiers); + if (node.type === 'setter') setter = kotlinVisibility(modifiers); + if ( + modifiers.namedChildren.some((annotation) => + isKotlinJvmAnnotation(annotation, 'JvmName', imports), + ) + ) { + if (node.type === 'getter') skipGetter = true; + if (node.type === 'setter') skipSetter = true; + } + }; + for (const child of prop.children) { + if (child.type === 'getter' || child.type === 'setter') apply(child); + } + let sib: Parser.SyntaxNode | null = prop.nextNamedSibling; + while (sib && (sib.type === 'getter' || sib.type === 'setter')) { + apply(sib); + sib = sib.nextNamedSibling; + } + return { + getterVisibility: getter, + setterVisibility: setter, + skipGetter, + skipSetter, + }; +} + +function hasKotlinAccessorBody(prop: Parser.SyntaxNode, kind: 'getter' | 'setter'): boolean { + const hasBody = (node: Parser.SyntaxNode): boolean => + node.type === kind && node.children.some((child) => child.type === 'function_body'); + if (prop.children.some(hasBody)) return true; + let sib: Parser.SyntaxNode | null = prop.nextNamedSibling; + while (sib && (sib.type === 'getter' || sib.type === 'setter')) { + if (hasBody(sib)) return true; + sib = sib.nextNamedSibling; + } + return false; +} + +function functionName(node: Parser.SyntaxNode): string | undefined { + return node.children.find((c) => c.type === 'simple_identifier')?.text; +} + +function functionArity(node: Parser.SyntaxNode): number { + const params = node.children.find((c) => c.type === 'function_value_parameters'); + let arity = + node.childForFieldName('receiver') !== null || + node.namedChildren.some((child) => child.type === 'receiver_type') + ? 1 + : 0; + const modifiers = node.children.find((child) => child.type === 'modifiers'); + if ( + modifiers?.namedChildren.some( + (child) => child.type === 'function_modifier' && child.text === 'suspend', + ) + ) { + arity += 1; + } + for (const child of params?.namedChildren ?? []) { + if (child.type === 'parameter' || child.type === 'parameter_with_optional_type') arity += 1; + } + return arity; +} + +function collectExistingMethods(...bodies: Array): ExistingMethodIndex { + const index = createExistingMethodIndex('exact'); + for (const body of bodies) { + if (!body) continue; + for (const child of body.children) { + if (child.type !== 'function_declaration') continue; + const name = functionName(child); + if (!name) continue; + rememberExistingMethod(index, name, functionArity(child)); + } + } + return index; +} + +function toKtProperty(child: Parser.SyntaxNode, imports: KotlinImportIndex): KtProperty | null { + const isVar = isVarBinding(child); + if (isVar === null) return null; + if (hasJvmField(child, imports) || hasConst(child)) return null; + const nameNode = propertyNameNode(child); + if (!nameNode) return null; + const mods = child.children.find((c) => c.type === 'modifiers'); + const visibility = kotlinVisibility(mods); + const accessor = accessorMetadata(child, visibility, imports); + return { + name: nameNode.text, + type: propertyTypeText(child), + isVar, + skipGetter: accessor.skipGetter, + skipSetter: accessor.skipSetter, + getterVisibility: accessor.getterVisibility, + setterVisibility: accessor.setterVisibility, + startLine: child.startPosition.row + 1, + endLine: child.endPosition.row + 1, + propertyNode: child, + declaratorNode: nameNode, + }; +} + +function collectTypedProperties( + parent: Parser.SyntaxNode | null, + type: 'class_parameter' | 'property_declaration', + imports: KotlinImportIndex, +): KtProperty[] { + if (!parent) return []; + const out: KtProperty[] = []; + for (const child of parent.namedChildren) { + if (child.type !== type) continue; + const prop = toKtProperty(child, imports); + if (prop) out.push(prop); + } + return out; +} + +function findKtClasses(root: Parser.SyntaxNode, imports: KotlinImportIndex): KtClass[] { + const classes: KtClass[] = []; + const graphOwnerNode = (node: Parser.SyntaxNode): Parser.SyntaxNode => { + if (node.type !== 'companion_object') return node; + if (jvmTypeSimpleName(node)) return node; + let current = node.parent; + while (current && !KOTLIN_TYPE_DECLS.has(current.type)) current = current.parent; + return current ?? node; + }; + const walk = (node: Parser.SyntaxNode): void => { + if (KOTLIN_TYPE_DECLS.has(node.type)) { + const ownerNode = graphOwnerNode(node); + const name = jvmTypeSimpleName(ownerNode) ?? ''; + const ctor = node.children.find((c) => c.type === 'primary_constructor') ?? null; + const body = node.children.find((c) => c.type === 'class_body') ?? null; + if (name) { + const properties = [ + ...collectTypedProperties(ctor, 'class_parameter', imports), + ...collectTypedProperties(body, 'property_declaration', imports), + ]; + if (properties.length > 0) { + const ownerBody = + ownerNode.id === node.id + ? null + : (ownerNode.children.find((child) => child.type === 'class_body') ?? null); + classes.push({ + node: ownerNode, + name, + isStatic: node.type === 'companion_object', + isInterface: node.children.some((child) => child.type === 'interface'), + wasHoisted: ownerNode.id !== node.id, + properties, + existingMethods: collectExistingMethods(body, ownerBody), + }); + } + } + if (body) { + for (const child of body.namedChildren) { + if (KOTLIN_TYPE_DECLS.has(child.type)) walk(child); + } + } + return; + } + for (const child of node.namedChildren) walk(child); + }; + walk(root); + return classes; +} + +function planAccessors(cls: KtClass): PlannedJvmAccessor[] { + const planned: PlannedJvmAccessor[] = []; + for (const prop of cls.properties) { + const gName = kotlinGetterName(prop.name); + if (!prop.skipGetter && !hasExistingMethod(cls.existingMethods, gName, 0)) { + planned.push({ + kind: 'getter', + name: gName, + returnType: prop.type, + parameterTypes: [], + visibility: prop.getterVisibility, + isStatic: cls.isStatic, + isAbstract: cls.isInterface && !hasKotlinAccessorBody(prop.propertyNode, 'getter'), + startLine: prop.startLine, + endLine: prop.endLine, + declaratorNode: prop.declaratorNode, + }); + } + if (prop.isVar && !prop.skipSetter) { + const sName = kotlinSetterName(prop.name); + if (!hasExistingMethod(cls.existingMethods, sName, 1)) { + planned.push({ + kind: 'setter', + name: sName, + returnType: 'void', + parameterTypes: [prop.type], + visibility: prop.setterVisibility, + isStatic: cls.isStatic, + isAbstract: cls.isInterface && !hasKotlinAccessorBody(prop.propertyNode, 'setter'), + startLine: prop.startLine, + endLine: prop.endLine, + declaratorNode: prop.declaratorNode, + }); + } + } + } + return planned; +} + +function planKotlinAccessorOwners(rootNode: Parser.SyntaxNode): PlannedJvmAccessorOwner[] { + const owners: PlannedJvmAccessorOwner[] = []; + const imports = collectKotlinImports(rootNode); + for (const cls of findKtClasses(rootNode, imports)) { + const accessors = planAccessors(cls); + const existingIndex = cls.wasHoisted + ? owners.findIndex((owner) => owner.node.id === cls.node.id) + : -1; + const existing = existingIndex >= 0 ? owners[existingIndex] : undefined; + if (existing) { + owners[existingIndex] = { + ...existing, + accessors: [...existing.accessors, ...accessors], + }; + } else { + owners.push({ node: cls.node, name: cls.name, accessors }); + } + } + return owners; +} + +const lombokAccessorSynthesis = createJvmAccessorSynthesis({ + language: 'kotlin', + synthetic: 'kotlin-jvm', + planOwners: planKotlinAccessorOwners, +}); + +export function synthesizeLombokAccessors( + tree: Parser.Tree, + filePath: string, + classOwnersById: ReadonlyMap, +): SyntheticAccessorResult { + return lombokAccessorSynthesis.synthesize(tree, filePath, classOwnersById); +} + +export function synthesizeLombokAccessorCaptures(rootNode: Parser.SyntaxNode): CaptureMatch[] { + return lombokAccessorSynthesis.captures(rootNode); +} diff --git a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts index 0fffeb60a..983f181b8 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts @@ -27,6 +27,8 @@ import { clearKotlinPackageFacts } from './package-facts.js'; import { attachKotlinSpringDiMetadata } from './spring-di.js'; import { attachKotlinSpringConditionalMetadata } from './spring-conditionals.js'; import { attachKotlinSpringNonHttpHandlerMetadata } from './spring-non-http-handlers.js'; +import { attachKotlinSpringConfigBindings } from './spring-config-bindings.js'; +import { attachKotlinSpringDynamicLookup } from './spring-dynamic-lookup.js'; /** * Kotlin scope resolver for RFC #909 Ring 3. @@ -148,6 +150,8 @@ export const kotlinScopeResolver: ScopeResolver = { attachKotlinSpringConditionalMetadata(graph, parsedFiles, nodeLookup, indexes); attachKotlinSpringDiMetadata(graph, parsedFiles, nodeLookup, indexes); attachKotlinSpringNonHttpHandlerMetadata(graph, parsedFiles, nodeLookup, indexes); + attachKotlinSpringDynamicLookup(graph, parsedFiles, nodeLookup, indexes); + attachKotlinSpringConfigBindings(graph, parsedFiles, nodeLookup, indexes); }, }; diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-actuator.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-actuator.ts new file mode 100644 index 000000000..5acb118b8 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-actuator.ts @@ -0,0 +1,229 @@ +import path from 'node:path'; +import type { GraphNode, ParsedImport } from 'gitnexus-shared'; +import type { + DefinitionPropertiesContext, + RuntimeCallableIdentity, + RuntimeSymbolStrategy, +} from '../../language-provider.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; + +const RUNTIME_OWNER_ALIASES = 'runtimeOwnerAliases'; +const RUNTIME_CALLABLE_ALIASES = 'runtimeCallableAliases'; +const KOTLIN_SUSPEND = 'kotlinSuspend'; +const fileFacadeMetadataCache = new WeakMap< + SyntaxNode, + { readonly packageName: string; readonly customFacade: string | undefined } +>(); + +function rootNode(node: SyntaxNode): SyntaxNode { + let current = node; + while (current.parent) current = current.parent; + return current; +} + +function packageName(root: SyntaxNode): string { + const header = root.namedChildren.find((child) => child.type === 'package_header'); + return header?.text.replace(/^package\s+/, '').trim() ?? ''; +} + +function qualify(packageNameValue: string, simpleName: string): string { + return packageNameValue.length === 0 ? simpleName : `${packageNameValue}.${simpleName}`; +} + +function standardFacadeName(filePath: string): string { + const stem = path.basename(filePath).replace(/\.(?:kt|kts)$/i, ''); + return `${stem.charAt(0).toUpperCase()}${stem.slice(1)}Kt`; +} + +function jvmNameIdentifiers( + imports: readonly ParsedImport[], + allowUnqualified: boolean, +): readonly string[] { + const names = new Set(['kotlin.jvm.JvmName']); + if (allowUnqualified) names.add('JvmName'); + for (const parsedImport of imports) { + if (parsedImport.kind !== 'named' && parsedImport.kind !== 'alias') continue; + if (parsedImport.importedName !== 'JvmName') continue; + const target = parsedImport.targetRaw.replace(/\\/g, '/'); + if (target === 'kotlin.jvm' || target === 'kotlin.jvm.JvmName') { + names.add(parsedImport.localName); + } + } + return [...names]; +} + +function annotationJvmName( + source: string, + target = '', + imports: readonly ParsedImport[] = [], + allowUnqualified = true, +): string | undefined { + const names = jvmNameIdentifiers(imports, allowUnqualified); + if (names.length === 0) return undefined; + const escapedTarget = target.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const prefix = target.length === 0 ? '' : `${escapedTarget}:`; + const namePattern = [...names] + .map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) + .join('|'); + return new RegExp(`@${prefix}(?:${namePattern})\\s*\\(\\s*["']([^"']+)["']\\s*\\)`).exec( + source, + )?.[1]; +} + +function fileFacadeMetadata( + root: SyntaxNode, + imports: readonly ParsedImport[], + allowUnqualified: boolean, +): { + readonly packageName: string; + readonly customFacade: string | undefined; +} { + const cached = fileFacadeMetadataCache.get(root); + if (cached !== undefined) return cached; + const metadata = { + packageName: packageName(root), + customFacade: annotationJvmName(root.text, 'file', imports, allowUnqualified), + }; + fileFacadeMetadataCache.set(root, metadata); + return metadata; +} + +function hasEnclosingType(node: SyntaxNode): boolean { + let current = node.parent; + while (current) { + if ( + current.type === 'class_declaration' || + current.type === 'object_declaration' || + current.type === 'companion_object' + ) { + return true; + } + current = current.parent; + } + return false; +} + +/** Transient graph metadata used only by the same analysis run's runtime import. */ +export function extractKotlinRuntimeSymbolProperties( + context: DefinitionPropertiesContext, +): Readonly> | undefined { + const properties: Record = {}; + const source = context.definitionNode.text; + const root = rootNode(context.definitionNode); + const allowUnqualifiedJvmName = !/\bannotation\s+class\s+JvmName\b/.test(root.text); + + if ( + (context.nodeLabel === 'Function' || context.nodeLabel === 'Method') && + context.definitionNode.type === 'function_declaration' + ) { + if (/\bsuspend\b/.test(source.slice(0, source.indexOf('fun') + 3))) { + properties[KOTLIN_SUSPEND] = true; + } + const callableJvmName = annotationJvmName( + source, + '', + context.parsedImports, + allowUnqualifiedJvmName, + ); + if (callableJvmName !== undefined) { + properties[RUNTIME_CALLABLE_ALIASES] = [callableJvmName]; + } + if (!hasEnclosingType(context.definitionNode)) { + const facade = fileFacadeMetadata(root, context.parsedImports, allowUnqualifiedJvmName); + properties[RUNTIME_OWNER_ALIASES] = [ + qualify(facade.packageName, facade.customFacade ?? standardFacadeName(context.filePath)), + ]; + } + } else if (context.nodeLabel === 'Property') { + const getterJvmName = annotationJvmName( + source, + 'get', + context.parsedImports, + allowUnqualifiedJvmName, + ); + if (getterJvmName !== undefined) { + properties[RUNTIME_CALLABLE_ALIASES] = [getterJvmName]; + } + if (!hasEnclosingType(context.definitionNode)) { + const facade = fileFacadeMetadata(root, context.parsedImports, allowUnqualifiedJvmName); + properties[RUNTIME_OWNER_ALIASES] = [ + qualify(facade.packageName, facade.customFacade ?? standardFacadeName(context.filePath)), + ]; + } + } + + return Object.keys(properties).length === 0 ? undefined : properties; +} + +function stringArrayProperty(node: GraphNode, property: string): readonly string[] { + const value = node.properties[property]; + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === 'string') + : []; +} + +function callableNames(node: GraphNode): readonly string[] { + return [String(node.properties.name), ...stringArrayProperty(node, RUNTIME_CALLABLE_ALIASES)]; +} + +function propertyGetterNames(node: GraphNode): readonly string[] { + const name = String(node.properties.name); + const capitalized = `${name.charAt(0).toUpperCase()}${name.slice(1)}`; + return [ + name.startsWith('is') && name.length > 2 && /[A-Z]/.test(name.charAt(2)) + ? name + : `get${capitalized}`, + ...stringArrayProperty(node, RUNTIME_CALLABLE_ALIASES), + ]; +} + +function sourceCallableName(runtimeName: string): string { + return runtimeName.endsWith('$default') ? runtimeName.slice(0, -'$default'.length) : runtimeName; +} + +function matchesKotlinCallable(node: GraphNode, runtime: RuntimeCallableIdentity): boolean { + // Kotlin property declarations and their synthesized JVM accessor Methods + // coexist in the graph. Bind runtime getters to the source Property so the + // synthetic accessor cannot turn an otherwise exact match into ambiguity. + if (node.properties.synthetic === 'kotlin-jvm') return false; + + const runtimeName = sourceCallableName(runtime.name); + if (node.label === 'Property') { + const names = propertyGetterNames(node); + if (!names.includes(runtime.name) && !names.includes(runtimeName)) return false; + } else if (!callableNames(node).includes(runtimeName)) { + return false; + } + + const parameterCount = node.properties.parameterCount; + const descriptorTypes = runtime.descriptorParameterTypes; + if ( + typeof parameterCount !== 'number' || + descriptorTypes === undefined || + runtime.name.endsWith('$default') + ) { + return true; + } + if (parameterCount === descriptorTypes.length) return true; + return ( + node.properties[KOTLIN_SUSPEND] === true && + parameterCount + 1 === descriptorTypes.length && + descriptorTypes.at(-1) === 'kotlin/coroutines/Continuation' + ); +} + +export const kotlinRuntimeSymbolStrategy: RuntimeSymbolStrategy = { + callableOwnerAliases(node, owner) { + const aliases = [...stringArrayProperty(node, RUNTIME_OWNER_ALIASES)]; + const ownerName = owner?.properties.qualifiedName; + if (typeof ownerName === 'string') { + aliases.push(ownerName); + if (node.properties.isStatic === true && !ownerName.endsWith('.Companion')) { + aliases.push(`${ownerName}.Companion`); + } + } + return aliases; + }, + + matchesCallable: matchesKotlinCallable, +}; diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-config-bindings.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-config-bindings.ts new file mode 100644 index 000000000..91aa3f9c0 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-config-bindings.ts @@ -0,0 +1,467 @@ +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { makeScopeId, type ParsedFile, type ScopeId } from 'gitnexus-shared'; +import { + bindSpringConfigConsumers, + type SpringConfigConsumer, +} from '../../frameworks/spring/config-bindings.js'; +import { createSpringAnnotationNameResolver } from '../../frameworks/spring/bean-candidates.js'; +import { + parseSpringAnnotationArguments, + parseStaticStringLiteral, +} from '../../frameworks/spring/annotation-arguments.js'; +import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; +import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { getKotlinParser } from './query.js'; +import { getKotlinSpringConfigConsumerFacts } from './capture-side-channel.js'; +import { isKotlinPackageSiblingVisibilityIncomplete } from './package-siblings.js'; + +const VALUE_ANNOTATION = 'org.springframework.beans.factory.annotation.Value'; +const CONFIGURATION_PROPERTIES_ANNOTATION = + 'org.springframework.boot.context.properties.ConfigurationProperties'; + +const VALUE_SIMPLE = 'Value'; +const CONFIGURATION_PROPERTIES_SIMPLE = 'ConfigurationProperties'; +const SKIP_USE_SITES = new Set(['get', 'property', 'file']); +const BIND_USE_SITES = new Set(['field', 'set', 'param']); +const OWNER_TYPES = new Set(['class_declaration', 'object_declaration', 'companion_object']); +const INTERPOLATION_TYPES = new Set([ + 'interpolated_identifier', + 'interpolated_expression', + 'interpolation_expression_start', +]); +const STRING_LITERAL_TYPES = new Set(['string_literal', 'character_literal']); + +export interface KotlinSpringConfigConsumerFact { + readonly consumer: SpringConfigConsumer; + readonly annotationName: string; + readonly classScopeId: ScopeId; +} + +interface KotlinAnnotation { + readonly name: string; + readonly node: SyntaxNode; + readonly useSiteTarget?: string; +} + +interface KotlinImports { + readonly exact: ReadonlyMap; + readonly wildcard: ReadonlySet; + readonly localTypes: ReadonlyMap; +} + +function firstDescendantOfType(node: SyntaxNode, type: string): SyntaxNode | undefined { + const stack = [...node.namedChildren].reverse(); + while (stack.length > 0) { + const current = stack.pop(); + if (current === undefined) continue; + if (current.type === type) return current; + for (let index = current.namedChildren.length - 1; index >= 0; index--) { + const child = current.namedChildren[index]; + if (child !== undefined) stack.push(child); + } + } + return undefined; +} + +function ownerName(declaration: SyntaxNode): string | undefined { + if (declaration.type === 'companion_object') { + const named = declaration.namedChildren.find((child) => child.type === 'type_identifier'); + return named?.text.trim() || 'Companion'; + } + return ( + declaration.namedChildren.find((child) => child.type === 'type_identifier')?.text.trim() ?? + declaration.namedChildren.find((child) => child.type === 'simple_identifier')?.text.trim() + ); +} + +function enclosingOwner(node: SyntaxNode): SyntaxNode | undefined { + let current = node.parent; + while (current !== null) { + if (OWNER_TYPES.has(current.type)) return current; + current = current.parent; + } + return undefined; +} + +function classScopeId(filePath: string, declaration: SyntaxNode): ScopeId { + return makeScopeId({ + filePath, + range: nodeToCapture('@scope.class', declaration).range, + kind: 'Class', + }); +} + +function collectKotlinImports(root: SyntaxNode): KotlinImports { + const exact = new Map(); + const wildcard = new Set(); + const localTypes = new Map(); + + for (const header of root.descendantsOfType('import_header')) { + const text = header.text.replace(/^import\s+/, '').trim(); + const aliasMatch = text.match(/^([\w.]+)\s+as\s+(\w+)\s*$/); + if (aliasMatch !== null) { + exact.set(aliasMatch[2], aliasMatch[1]); + continue; + } + if (text.endsWith('.*')) wildcard.add(text.slice(0, -2)); + else { + const simple = text.slice(text.lastIndexOf('.') + 1); + if (simple.length > 0) exact.set(simple, text); + } + } + + for (const type of ['class_declaration', 'object_declaration']) { + for (const declaration of root.descendantsOfType(type)) { + const name = ownerName(declaration); + if (name) { + const declarations = localTypes.get(name) ?? []; + declarations.push(declaration); + localTypes.set(name, declarations); + } + } + } + return { exact, wildcard, localTypes }; +} + +function annotationFromNode(annotation: SyntaxNode): KotlinAnnotation | null { + const nameNode = + firstDescendantOfType(annotation, 'user_type') ?? + firstDescendantOfType(annotation, 'type_identifier') ?? + firstDescendantOfType(annotation, 'simple_identifier'); + if (nameNode === undefined) return null; + const useSiteTarget = annotation.namedChildren + .find((child) => child.type === 'use_site_target') + ?.text.replace(/:\s*$/, '') + .trim(); + return { + name: nameNode.text.trim(), + node: annotation, + ...(useSiteTarget === undefined || useSiteTarget.length === 0 ? {} : { useSiteTarget }), + }; +} + +function annotationsOn(node: SyntaxNode): KotlinAnnotation[] { + const annotations: KotlinAnnotation[] = []; + for (const child of node.namedChildren) { + if (child.type === 'annotation') { + const fact = annotationFromNode(child); + if (fact !== null) annotations.push(fact); + continue; + } + if (child.type !== 'modifiers' && child.type !== 'parameter_modifiers') continue; + for (const nested of child.namedChildren) { + if (nested.type !== 'annotation') continue; + const fact = annotationFromNode(nested); + if (fact !== null) annotations.push(fact); + } + } + return annotations; +} + +function simpleName(rawName: string): string { + const parts = rawName.split('.'); + return parts[parts.length - 1] ?? rawName; +} + +function importedAs( + imports: KotlinImports, + simple: string, + fqn: string, + wildcardPackage: string, +): boolean { + if (imports.exact.get(simple) === fqn) return true; + // An explicit import wins over a star import in Kotlin, so a conflicting + // binding for the same simple name rules the Spring annotation out even when + // its package is wildcard-imported. + return imports.exact.get(simple) === undefined && imports.wildcard.has(wildcardPackage); +} + +function hasVisibleLocalType( + imports: KotlinImports, + simple: string, + annotation: SyntaxNode, +): boolean { + for (const declaration of imports.localTypes.get(simple) ?? []) { + const declarationOwner = enclosingOwner(declaration); + if (declarationOwner === undefined) return true; + let current: SyntaxNode | null = annotation; + while (current !== null) { + if (current.id === declarationOwner.id) return true; + current = current.parent; + } + } + return false; +} + +const SIMPLE_CONFIG_ANNOTATIONS = [ + { + simple: VALUE_SIMPLE, + kind: 'value', + fqn: VALUE_ANNOTATION, + wildcardPackage: 'org.springframework.beans.factory.annotation', + }, + { + simple: CONFIGURATION_PROPERTIES_SIMPLE, + kind: 'configuration-properties', + fqn: CONFIGURATION_PROPERTIES_ANNOTATION, + wildcardPackage: 'org.springframework.boot.context.properties', + }, +] as const; + +function configAnnotationKind( + annotation: KotlinAnnotation, + imports: KotlinImports, +): 'value' | 'configuration-properties' | null { + const rawName = annotation.name; + if (rawName === VALUE_ANNOTATION) return 'value'; + if (rawName === CONFIGURATION_PROPERTIES_ANNOTATION) return 'configuration-properties'; + const simple = simpleName(rawName); + const aliased = imports.exact.get(simple); + if (aliased === VALUE_ANNOTATION) return 'value'; + if (aliased === CONFIGURATION_PROPERTIES_ANNOTATION) return 'configuration-properties'; + for (const candidate of SIMPLE_CONFIG_ANNOTATIONS) { + if (simple !== candidate.simple) continue; + if (hasVisibleLocalType(imports, simple, annotation.node) && !imports.exact.has(simple)) { + return null; + } + return importedAs(imports, simple, candidate.fqn, candidate.wildcardPackage) + ? candidate.kind + : null; + } + return null; +} + +function hasInterpolation(annotation: SyntaxNode): boolean { + const stack: SyntaxNode[] = [...annotation.namedChildren]; + while (stack.length > 0) { + const current = stack.pop(); + if (current === undefined) continue; + if (INTERPOLATION_TYPES.has(current.type)) return true; + stack.push(...current.namedChildren); + } + return false; +} + +function decodeKotlinStringLiteral(literal: string): string | null { + const raw = literal.startsWith('"""') && literal.endsWith('"""'); + const delimiterLength = raw ? 3 : 1; + if (literal.length < delimiterLength * 2) return null; + const body = literal.slice(delimiterLength, -delimiterLength); + if (!raw && /(? + String.fromCharCode(Number.parseInt(hex, 16)), + ) + .replace(/\\(["'\\$btnfr])/g, (_match, escaped: string) => { + const controls: Record = { + b: '\b', + t: '\t', + n: '\n', + f: '\f', + r: '\r', + $: '$', + }; + return controls[escaped] ?? escaped; + }); +} + +function kotlinStringLiterals(annotation: SyntaxNode): string[] { + const literals: string[] = []; + const stack: SyntaxNode[] = [...annotation.namedChildren]; + while (stack.length > 0) { + const current = stack.pop(); + if (current === undefined) continue; + if (STRING_LITERAL_TYPES.has(current.type)) { + const decoded = decodeKotlinStringLiteral(current.text); + if (decoded !== null) literals.push(decoded); + continue; + } + stack.push(...current.namedChildren); + } + return literals; +} + +function parseValuePlaceholderKeys(annotation: SyntaxNode): string[] { + if (hasInterpolation(annotation)) return []; + const keys = new Set(); + for (const literal of kotlinStringLiterals(annotation)) { + for (const match of literal.matchAll(/\$\{([^{}]+)\}/g)) { + const key = match[1].split(':', 1)[0].trim(); + if (/^[A-Za-z0-9_.-]+$/.test(key)) keys.add(key); + } + } + return [...keys]; +} + +function parseConfigurationPropertiesPrefix(annotation: SyntaxNode): string | null { + if (hasInterpolation(annotation)) return null; + const argumentsList = parseSpringAnnotationArguments(annotation.text); + if (argumentsList !== null) { + const named = argumentsList.filter( + (argument) => argument.name === 'prefix' || argument.name === 'value', + ); + const positional = argumentsList.filter((argument) => argument.name === undefined); + const chosen = named.length === 1 ? named[0] : named.length === 0 ? positional[0] : undefined; + if (chosen !== undefined) { + const decoded = parseStaticStringLiteral(chosen.value); + if (decoded === null) return null; + const prefix = decoded.replace(/^\.+|\.+$/g, ''); + if (/^[A-Za-z0-9_.-]+$/.test(prefix)) return prefix; + return null; + } + if (argumentsList.length > 0) return null; + } + const literals = kotlinStringLiterals(annotation); + if (literals.length !== 1) return null; + const prefix = literals[0].trim().replace(/^\.+|\.+$/g, ''); + return /^[A-Za-z0-9_.-]+$/.test(prefix) ? prefix : null; +} + +function allowedUseSite(useSiteTarget: string | undefined): boolean { + if (useSiteTarget === undefined) return true; + if (SKIP_USE_SITES.has(useSiteTarget)) return false; + return BIND_USE_SITES.has(useSiteTarget); +} + +function hasBindingPattern(parameter: SyntaxNode): boolean { + return parameter.namedChildren.some((child) => child.type === 'binding_pattern_kind'); +} + +function propertyName(node: SyntaxNode): string | undefined { + if (node.type === 'class_parameter') { + return node.namedChildren.find((child) => child.type === 'simple_identifier')?.text.trim(); + } + const variable = node.namedChildren.find((child) => child.type === 'variable_declaration'); + return variable?.namedChildren.find((child) => child.type === 'simple_identifier')?.text.trim(); +} + +function underFileAnnotation(node: SyntaxNode): boolean { + let current: SyntaxNode | null = node; + while (current !== null) { + if (current.type === 'file_annotation') return true; + current = current.parent; + } + return false; +} + +function pushValueFacts( + facts: KotlinSpringConfigConsumerFact[], + member: SyntaxNode, + filePath: string, + imports: KotlinImports, +): void { + if (underFileAnnotation(member)) return; + const owner = enclosingOwner(member); + if (owner === undefined) return; + const fieldName = propertyName(member); + if (fieldName === undefined) return; + for (const annotation of annotationsOn(member)) { + if (!allowedUseSite(annotation.useSiteTarget)) continue; + if (configAnnotationKind(annotation, imports) !== 'value') continue; + const keys = parseValuePlaceholderKeys(annotation.node); + if (keys.length === 0) continue; + facts.push({ + consumer: { + kind: 'value', + fieldName, + line: member.startPosition.row + 1, + keys, + }, + annotationName: annotation.name, + classScopeId: classScopeId(filePath, owner), + }); + } +} + +/** Collect config facts from the Kotlin parser's existing AST (no reparse). */ +export function captureKotlinSpringConfigConsumerFacts( + root: SyntaxNode, + filePath: string, +): KotlinSpringConfigConsumerFact[] { + const imports = collectKotlinImports(root); + const facts: KotlinSpringConfigConsumerFact[] = []; + + for (const property of root.descendantsOfType('property_declaration')) { + pushValueFacts(facts, property, filePath, imports); + } + + for (const parameter of root.descendantsOfType('class_parameter')) { + if (!hasBindingPattern(parameter)) continue; + pushValueFacts(facts, parameter, filePath, imports); + } + + for (const type of ['class_declaration', 'object_declaration']) { + for (const declaration of root.descendantsOfType(type)) { + const className = ownerName(declaration); + if (className === undefined) continue; + for (const annotation of annotationsOn(declaration)) { + if (configAnnotationKind(annotation, imports) !== 'configuration-properties') { + continue; + } + const prefix = parseConfigurationPropertiesPrefix(annotation.node); + if (prefix === null) continue; + facts.push({ + consumer: { + kind: 'configuration-properties', + className, + line: declaration.startPosition.row + 1, + prefix, + }, + annotationName: annotation.name, + classScopeId: classScopeId(filePath, declaration), + }); + } + } + } + return facts; +} + +/** Parse Kotlin consumers for focused unit tests; production reuses the worker AST. */ +export function extractKotlinSpringConfigConsumers(source: string): SpringConfigConsumer[] { + const tree = parseSourceSafe(getKotlinParser(), source); + return captureKotlinSpringConfigConsumerFacts(tree.rootNode, '').map( + (fact) => fact.consumer, + ); +} + +export function extractKotlinSpringConfigConsumerFacts( + source: string, +): KotlinSpringConfigConsumerFact[] { + const tree = parseSourceSafe(getKotlinParser(), source); + return captureKotlinSpringConfigConsumerFacts(tree.rootNode, ''); +} + +/** Kotlin ScopeResolver post-resolution hook for Spring configuration consumers. */ +export function attachKotlinSpringConfigBindings( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + _nodeLookup: GraphNodeLookup, + indexes: ScopeResolutionIndexes, +): void { + const resolveAnnotation = createSpringAnnotationNameResolver(indexes); + const recognizedAnnotations = new Set([VALUE_ANNOTATION, CONFIGURATION_PROPERTIES_ANNOTATION]); + const batches: Array<{ filePath: string; consumers: SpringConfigConsumer[] }> = []; + for (const parsed of parsedFiles) { + const consumers: SpringConfigConsumer[] = []; + for (const fact of getKotlinSpringConfigConsumerFacts(parsed.filePath)) { + const classScope = indexes.scopeTree.getScope(fact.classScopeId); + if (classScope === undefined || classScope.kind !== 'Class') continue; + const expectedAnnotation = + fact.consumer.kind === 'value' ? VALUE_ANNOTATION : CONFIGURATION_PROPERTIES_ANNOTATION; + const enclosingScope = fact.consumer.kind === 'value' ? classScope.id : classScope.parent; + const resolved = resolveAnnotation( + fact.annotationName, + parsed, + enclosingScope, + recognizedAnnotations, + isKotlinPackageSiblingVisibilityIncomplete(parsed.filePath), + ); + if (resolved === expectedAnnotation) consumers.push(fact.consumer); + } + if (consumers.length > 0) batches.push({ filePath: parsed.filePath, consumers }); + } + bindSpringConfigConsumers(graph, batches); +} diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts index acb38360e..af0ba2cec 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-di.ts @@ -1,5 +1,9 @@ import { makeScopeId } from 'gitnexus-shared'; import { parseSpringInjectionType } from '../../di-extractors/spring.js'; +import { + normalizeSpringFactText, + type SpringArgumentFact, +} from '../../frameworks/spring/argument-facts.js'; import { createSpringDiMetadataAttacher, hasSpringDiRelevantAnnotation, @@ -13,13 +17,29 @@ import { hasSpringBeanFactorySyntax, type SpringBeanFactoryMethodFact, } from '../../frameworks/spring/bean-factories.js'; -import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; +import { hasRecoveredSyntax, nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; import { getKotlinSpringDiFacts } from './capture-side-channel.js'; import { isKotlinPackageSiblingVisibilityIncomplete } from './package-siblings.js'; export interface KotlinAnnotationSyntaxFact extends SpringDiAnnotationFact { readonly useSiteTarget?: string; readonly line: number; + /** Present only for callers that opt in via `kotlinSpringAnnotationFacts`. */ + readonly args?: readonly SpringArgumentFact[]; +} + +/** + * Options for `kotlinSpringAnnotationFacts`. + * + * The STRUCTURED arguments are opt-in because DI captures every annotated + * constructor parameter, property, and function in the repository, and none of + * its consumers reads them. Note what this does and does not save: every fact + * already carries `text`, the annotation's full source, so the argument TEXT + * crosses the worker boundary either way. What the opt-in avoids is a second, + * parsed copy of that same text on facts that would never look at it. + */ +export interface KotlinSpringAnnotationFactOptions { + readonly includeArguments?: boolean; } export type KotlinSpringDependencyFact = SpringDiDependencyFact; @@ -53,36 +73,128 @@ function firstDescendantOfType(node: SyntaxNode, type: string): SyntaxNode | und return undefined; } -function annotationFact(annotation: SyntaxNode): KotlinAnnotationSyntaxFact | null { +const KOTLIN_COMMENT_NODE_TYPES = new Set(['line_comment', 'multiline_comment']); + +/** + * Kotlin writes annotation arguments and call arguments with the same + * `value_arguments` node, so one reader serves `@KafkaListener(topics = [...])` + * and `kafkaTemplate.send(topic, payload)`. + * + * A named argument keeps its key; everything else — positional values, spreads, + * collection literals, and interpolated strings — is kept as raw text, because + * evaluating it would be resolution. + * + * Returns `null` for a list tree-sitter had to recover, and the callers decide + * what that means: a producer call drops the whole fact, since it has no state + * for "published somewhere unreadable", while an annotation reports no + * arguments and collapses into the marker form. Both answers say "nothing here + * to resolve", which is true; a fabricated value would send a consumer + * somewhere real and wrong. + * + * The check lives HERE, not only in the callers. This function is exported and + * already has a caller in another module, so a guard that every future caller + * has to remember is the same fragility this change set exists to remove — + * `null` makes the decision unavoidable at the type level. Per-argument + * re-checks are still pointless: `hasError` propagates from any argument up to + * the list, so a branch behind this one could never fire. + * + * A named argument is identified by the `=` TOKEN, and the two-child shape is + * only a corroborating detail. Today nothing well formed reaches two children + * without an `=`: an annotated positional argument such as + * `@Suppress("UNCHECKED_CAST") "orders"` arrives as ONE `prefix_expression`, not + * as two children, so the token test is currently redundant. It is kept as the + * leading condition anyway, because the failure it prevents is asymmetric — + * dropping it would let any future two-child positional shape be reported under + * an argument key the source never wrote, which is the failure mode this whole + * change set is about. + */ +export function kotlinValueArgumentFacts(valueArguments: SyntaxNode): SpringArgumentFact[] | null { + if (hasRecoveredSyntax(valueArguments)) return null; + const args: SpringArgumentFact[] = []; + for (const argument of valueArguments.namedChildren) { + if (argument.type !== 'value_argument') continue; + const parts = argument.namedChildren.filter( + (child) => !KOTLIN_COMMENT_NODE_TYPES.has(child.type), + ); + const named = argument.children.some((child) => child.type === '='); + const name = parts[0]; + const value = parts[1]; + if (named && parts.length === 2 && name !== undefined && value !== undefined) { + args.push({ name: name.text.trim(), text: normalizeSpringFactText(value.text) }); + continue; + } + args.push({ text: normalizeSpringFactText(argument.text) }); + } + return args; +} + +/** + * Arguments of one annotation, or `undefined` when it was written without an + * argument list (`@Scheduled`); `@Scheduled()` yields `[]` instead. + * + * Only the annotation's FIRST `user_type` / `constructor_invocation` child is + * read, which is the same element `annotationFact` names. That matters for the + * multi-annotation form `@field:[Alpha Beta("x")]`, where naively taking the + * first constructor invocation would hand Beta's arguments to Alpha. + * + * An argument list that did not parse also yields `undefined`, collapsing into + * the marker-annotation case on purpose: both say there is nothing readable to + * resolve, while the recovered tree would offer values nobody wrote. + */ +function kotlinAnnotationArgumentFacts(annotation: SyntaxNode): SpringArgumentFact[] | undefined { + const named = annotation.namedChildren.find( + (child) => child.type === 'user_type' || child.type === 'constructor_invocation', + ); + if (named === undefined || named.type !== 'constructor_invocation') return undefined; + const valueArguments = named.namedChildren.find((child) => child.type === 'value_arguments'); + if (valueArguments === undefined) return undefined; + // `null` here means recovered syntax; an annotation answers that by reporting + // no arguments at all, which is the marker-annotation form. + return kotlinValueArgumentFacts(valueArguments) ?? undefined; +} + +function annotationFact( + annotation: SyntaxNode, + options: KotlinSpringAnnotationFactOptions, +): KotlinAnnotationSyntaxFact | null { const nameNode = firstDescendantOfType(annotation, 'user_type'); if (nameNode === undefined) return null; const useSiteTarget = annotation.namedChildren .find((child) => child.type === 'use_site_target') ?.text.replace(/:\s*$/, '') .trim(); + const args = + options.includeArguments === true ? kotlinAnnotationArgumentFacts(annotation) : undefined; return { name: nameNode.text.trim(), text: annotation.text.trim(), line: annotation.startPosition.row + 1, ...(useSiteTarget === undefined || useSiteTarget.length === 0 ? {} : { useSiteTarget }), + ...(args === undefined ? {} : { args }), }; } -function annotationsFromModifierContainer(node: SyntaxNode): KotlinAnnotationSyntaxFact[] { +function annotationsFromModifierContainer( + node: SyntaxNode, + options: KotlinSpringAnnotationFactOptions = {}, +): KotlinAnnotationSyntaxFact[] { const facts: KotlinAnnotationSyntaxFact[] = []; for (const child of node.namedChildren) { if (child.type !== 'annotation') continue; - const fact = annotationFact(child); + const fact = annotationFact(child, options); if (fact !== null) facts.push(fact); } return facts; } -export function kotlinSpringAnnotationFacts(node: SyntaxNode): KotlinAnnotationSyntaxFact[] { +export function kotlinSpringAnnotationFacts( + node: SyntaxNode, + options: KotlinSpringAnnotationFactOptions = {}, +): KotlinAnnotationSyntaxFact[] { const facts: KotlinAnnotationSyntaxFact[] = []; for (const child of node.namedChildren) { if (child.type !== 'modifiers' && child.type !== 'parameter_modifiers') continue; - facts.push(...annotationsFromModifierContainer(child)); + facts.push(...annotationsFromModifierContainer(child, options)); } return facts; } diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-dynamic-lookup.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-dynamic-lookup.ts new file mode 100644 index 000000000..184048e07 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-dynamic-lookup.ts @@ -0,0 +1,90 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { + createSpringDynamicLookupMetadataAttacher, + springDynamicLookupCardinality, + type SpringDynamicLookupFact, +} from '../../frameworks/spring/dynamic-lookups.js'; +import { + findAncestorBeforeBoundary, + nodeToCapture, + type SyntaxNode, +} from '../../utils/ast-helpers.js'; +import { getKotlinSpringDynamicLookupFacts } from './capture-side-channel.js'; + +// Kotlin emits graph callables for functions and secondary constructors. +// `init {}` / primary-constructor bodies have no independent callable node, so +// attributing their lookups to the enclosing Class would violate graph semantics. +const CALLABLE_NODE_TYPES = new Set(['function_declaration', 'secondary_constructor']); +const NO_CALLABLE_BOUNDARIES = new Set(); +const KOTLIN_CLASS_LITERAL = + /^([A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)*)::class(?:\.java)?$/; + +function navigationParts(node: SyntaxNode): { receiverName: string; methodName: string } | null { + if (node.type !== 'navigation_expression') return null; + const text = node.text.trim(); + const separator = text.lastIndexOf('.'); + if (separator <= 0 || separator === text.length - 1) return null; + return { + receiverName: text.slice(0, separator), + methodName: text.slice(separator + 1), + }; +} + +function singleClassLiteralArgument(node: SyntaxNode): string | null { + const suffix = node.namedChildren.find((child) => child.type === 'call_suffix'); + const argumentsNode = suffix?.namedChildren.find((child) => child.type === 'value_arguments'); + if (argumentsNode === undefined) return null; + const argumentsWithoutComments = argumentsNode.namedChildren.filter( + (child) => child.type !== 'line_comment' && child.type !== 'multiline_comment', + ); + if (argumentsWithoutComments.length !== 1) return null; + const value = argumentsWithoutComments[0]; + if (value?.type !== 'value_argument' || value.namedChildCount !== 1) return null; + return value.namedChild(0)?.text.trim().match(KOTLIN_CLASS_LITERAL)?.[1] ?? null; +} + +/** Capture real Kotlin calls using `Type::class` or `Type::class.java`. */ +export function captureKotlinSpringDynamicLookupFact( + node: SyntaxNode, + filePath: string, +): SpringDynamicLookupFact | null { + if (node.type !== 'call_expression') return null; + const callee = node.namedChildren.find((child) => child.type === 'navigation_expression'); + if (callee === undefined) return null; + const parts = navigationParts(callee); + if (parts === null) return null; + if (springDynamicLookupCardinality(parts.receiverName, parts.methodName) === null) return null; + const targetTypeName = singleClassLiteralArgument(node); + if (targetTypeName === null) return null; + + const owner = findAncestorBeforeBoundary(node, CALLABLE_NODE_TYPES, NO_CALLABLE_BOUNDARIES); + if (owner === null) return null; + const ownerCapture = nodeToCapture('@spring-dynamic-lookup.owner', owner); + return { + ownerScopeId: makeScopeId({ + filePath, + range: ownerCapture.range, + kind: 'Function', + }), + ownerRange: ownerCapture.range, + receiverName: parts.receiverName, + methodName: parts.methodName, + targetTypeName, + }; +} + +/** Standalone extractor for focused tests; production reuses scope-query call nodes. */ +export function captureKotlinSpringDynamicLookupFacts( + rootNode: SyntaxNode, + filePath: string, +): SpringDynamicLookupFact[] { + return rootNode + .descendantsOfType('call_expression') + .map((node) => captureKotlinSpringDynamicLookupFact(node, filePath)) + .filter((fact): fact is SpringDynamicLookupFact => fact !== null); +} + +/** Attach Kotlin lookup facts for later resolution by the shared DI phase. */ +export const attachKotlinSpringDynamicLookup = createSpringDynamicLookupMetadataAttacher({ + getFacts: getKotlinSpringDynamicLookupFacts, +}); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-message-producers.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-message-producers.ts new file mode 100644 index 000000000..7f32709c8 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-message-producers.ts @@ -0,0 +1,132 @@ +import { makeScopeId } from 'gitnexus-shared'; +import { normalizeSpringFactText } from '../../frameworks/spring/argument-facts.js'; +import { + isSpringMessageProducerMethod, + springMessageProducerTemplateOf, + type SpringMessageProducerFact, +} from '../../frameworks/spring/message-producers.js'; +import { + findAncestorBeforeBoundary, + nodeToCapture, + type SyntaxNode, +} from '../../utils/ast-helpers.js'; +import { kotlinValueArgumentFacts } from './spring-di.js'; + +// Kotlin emits graph callables for functions and secondary constructors. +// `init {}` / primary-constructor bodies have no independent callable node, so +// attributing their publishes to the enclosing Class would violate graph +// semantics. +const CALLABLE_NODE_TYPES = new Set(['function_declaration', 'secondary_constructor']); +/** + * A class body ends the search, so the rule above holds at every depth. + * + * Without it the walk passes THROUGH the body of a class or object declared + * inside a function, and the publish in that body's property initializer — which + * likewise has no callable of its own — is attributed to the enclosing function + * instead of being dropped the way its top-level twin is. + */ +const TYPE_BODY_BOUNDARIES = new Set(['class_body', 'enum_class_body']); + +/** + * Strip null-assertion operators from a receiver. + * + * `?.` carries its marker on the navigation suffix, which the structural split + * already discards, but `!!` wraps the receiver in a `postfix_expression` whose + * text ends in the operator — enough to make `kafkaTemplate!!` fail the + * receiver-name check and lose a publish. Unwrapping is limited to `!!` + * because `counter++` produces the same node shape and is not a receiver name. + */ +function withoutNullAssertions(receiver: SyntaxNode): SyntaxNode { + let current = receiver; + while (current.type === 'postfix_expression') { + const operand = current.namedChildren[0]; + if (operand === undefined) return current; + const onlyNullAssertions = current.children.every( + (child) => child.id === operand.id || child.type === '!!', + ); + if (!onlyNullAssertions) return current; + current = operand; + } + return current; +} + +/** + * Split `receiver.method` structurally rather than by text. + * + * Text splitting would leave the safe-call marker on the receiver + * (`kafkaTemplate?` for `kafkaTemplate?.send(...)`). + */ +function navigationParts(callee: SyntaxNode): { receiverName: string; methodName: string } | null { + if (callee.type !== 'navigation_expression') return null; + const suffix = callee.namedChildren.find((child) => child.type === 'navigation_suffix'); + const receiver = callee.namedChildren.find((child) => child.type !== 'navigation_suffix'); + if (suffix === undefined || receiver === undefined) return null; + const methodName = suffix.namedChildren + .find((child) => child.type === 'simple_identifier') + ?.text.trim(); + if (methodName === undefined) return null; + return { + receiverName: normalizeSpringFactText(withoutNullAssertions(receiver).text), + methodName, + }; +} + +/** + * Capture one messaging-template publish from a Kotlin call already surfaced by + * the scope query, without resolving the destination it names. + * + * The destination argument may be a literal, a reference to a constant that + * lives in another file, or a `${...}` placeholder resolved from configuration; + * all three are recorded as written and left to a later phase. + * + * A call whose argument list did not parse yields NO fact, for the reason given + * on the Java side: error recovery guesses argument boundaries, and this fact + * has no way to say "published somewhere unreadable". + */ +export function captureKotlinSpringMessageProducerFact( + node: SyntaxNode, + filePath: string, +): SpringMessageProducerFact | null { + if (node.type !== 'call_expression') return null; + const callee = node.namedChildren[0]; + if (callee === undefined) return null; + const parts = navigationParts(callee); + if (parts === null || !isSpringMessageProducerMethod(parts.methodName)) return null; + const template = springMessageProducerTemplateOf(parts.receiverName, parts.methodName); + if (template === null) return null; + + const callSuffix = node.namedChildren.find((child) => child.type === 'call_suffix'); + // A trailing-lambda call (`send { ... }`) has no argument list at all, which + // is a different fact from an empty one (`send()`). + const valueArguments = callSuffix?.namedChildren.find( + (child) => child.type === 'value_arguments', + ); + // `null` from the reader means tree-sitter had to recover the list. A publish + // fact exists to carry a destination and has no state for "published + // somewhere unreadable", so the whole fact is withheld rather than reported + // with arguments the source never wrote. + const args = valueArguments === undefined ? undefined : kotlinValueArgumentFacts(valueArguments); + if (args === null) return null; + const owner = findAncestorBeforeBoundary(node, CALLABLE_NODE_TYPES, TYPE_BODY_BOUNDARIES); + if (owner === null) return null; + const ownerCapture = nodeToCapture('@spring-message-producer.owner', owner); + return { + ownerScopeId: makeScopeId({ filePath, range: ownerCapture.range, kind: 'Function' }), + ownerRange: ownerCapture.range, + template, + receiverName: parts.receiverName, + methodName: parts.methodName, + ...(args === undefined ? {} : { args }), + }; +} + +/** Standalone extractor for focused tests; production reuses scope-query call nodes. */ +export function captureKotlinSpringMessageProducerFacts( + rootNode: SyntaxNode, + filePath: string, +): SpringMessageProducerFact[] { + return rootNode + .descendantsOfType('call_expression') + .map((node) => captureKotlinSpringMessageProducerFact(node, filePath)) + .filter((fact): fact is SpringMessageProducerFact => fact !== null); +} diff --git a/gitnexus/src/core/ingestion/languages/kotlin/spring-non-http-handlers.ts b/gitnexus/src/core/ingestion/languages/kotlin/spring-non-http-handlers.ts index b46132073..a7b21ffd4 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/spring-non-http-handlers.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/spring-non-http-handlers.ts @@ -1,6 +1,7 @@ import { makeScopeId } from 'gitnexus-shared'; import { createSpringNonHttpHandlerMetadataAttacher, + hasSpringNonHttpHandlerRelevantAnnotation, type SpringNonHttpHandlerAnnotationFact, type SpringNonHttpHandlerFact, } from '../../frameworks/spring/non-http-handlers.js'; @@ -12,10 +13,68 @@ import { kotlinSpringAnnotationFacts } from './spring-di.js'; export type KotlinSpringNonHttpHandlerFact = SpringNonHttpHandlerFact; +/** + * Local names that reach a handler annotation only through an import alias. + * + * `import ...event.EventListener as SpringEvent` makes `@SpringEvent` a handler + * annotation whose simple name matches nothing, which is why the CALLABLE + * capture below has no name prefilter. The alias is not a mystery at capture + * time, though: the import header states both the local name and the FQN it + * stands for, so the same relevance predicate that Java uses on the annotation + * name can be applied to the IMPORTED name and the answer carried back to the + * alias. That recovers a name-based decision without discarding aliases. + * + * Only aliases are collected. A plain or wildcard import leaves the annotation + * written under its own simple name, which the direct check already sees. + */ +function aliasedHandlerAnnotationNames(classNode: SyntaxNode): ReadonlySet { + let root: SyntaxNode = classNode; + while (root.parent !== null) root = root.parent; + + const headers: SyntaxNode[] = []; + for (const child of root.namedChildren) { + if (child.type === 'import_header') headers.push(child); + else if (child.type === 'import_list') { + for (const header of child.namedChildren) { + if (header.type === 'import_header') headers.push(header); + } + } + } + + const aliases = new Set(); + for (const header of headers) { + const alias = header.namedChildren + .find((child) => child.type === 'import_alias') + ?.namedChildren.find((child) => child.type === 'type_identifier') + ?.text.trim(); + if (alias === undefined || alias.length === 0) continue; + const imported = header.namedChildren.find((child) => child.type === 'identifier')?.text.trim(); + if (imported === undefined || imported.length === 0) continue; + if (hasSpringNonHttpHandlerRelevantAnnotation([{ name: imported }])) aliases.add(alias); + } + return aliases; +} + /** * Capture annotated callables conservatively. A simple-name prefilter would * discard Kotlin aliases (for example, `EventListener as SpringEvent`) before * the post-import resolver can map the local name back to its annotation FQN. + * + * That conservatism applies to the CALLABLE — every annotated function still + * produces a fact, whatever its annotations are named. It does NOT have to + * apply to the arguments: reading them unconditionally charged every + * `@Transactional` and `@Deprecated` in a repository for data no consumer + * reads, and unlike the callable itself an argument list can be fetched on + * evidence. Arguments are therefore read in a second pass, for callables that + * either carry a handler annotation under its own name or use a local name this + * file aliased to one — the same two-pass shape as Java, with the alias set + * standing in for the name prefilter Kotlin cannot use. + * + * Measured on 200 annotated NON-handler functions in one file: the side-channel + * payload was 41069 bytes before arguments existed, 78797 with them read + * unconditionally, and 41069 again with this pass — byte for byte what it cost + * before the feature. The 200-handler equivalent pays 58649, which is the + * argument text the consumer asked for. */ export function captureKotlinSpringNonHttpHandlerFacts( classNode: SyntaxNode, @@ -26,9 +85,23 @@ export function captureKotlinSpringNonHttpHandlerFacts( (child) => child.type === 'class_body' || child.type === 'enum_class_body', ); if (body === undefined) return facts; + // Read the import headers at most once per class, and only when some callable + // actually fails the direct name check. + let aliasedHandlerNames: ReadonlySet | undefined; for (const member of body.namedChildren) { if (member.type !== 'function_declaration') continue; - const annotations = kotlinSpringAnnotationFacts(member); + const named = kotlinSpringAnnotationFacts(member); + if (named.length === 0) continue; + let readArguments = hasSpringNonHttpHandlerRelevantAnnotation(named); + if (!readArguments) { + aliasedHandlerNames ??= aliasedHandlerAnnotationNames(classNode); + readArguments = named.some( + (annotation) => aliasedHandlerNames?.has(annotation.name) === true, + ); + } + const annotations = readArguments + ? kotlinSpringAnnotationFacts(member, { includeArguments: true }) + : named; if (annotations.length === 0) continue; const ownerRange = nodeToCapture('@spring-non-http-handler.owner', member).range; facts.push({ @@ -40,6 +113,7 @@ export function captureKotlinSpringNonHttpHandlerFacts( ...(annotation.useSiteTarget === undefined ? {} : { useSiteTarget: annotation.useSiteTarget }), + ...(annotation.args === undefined ? {} : { args: annotation.args }), })), }); } diff --git a/gitnexus/src/core/ingestion/languages/php/import-target.ts b/gitnexus/src/core/ingestion/languages/php/import-target.ts index 96711ea02..9c9d14aa1 100644 --- a/gitnexus/src/core/ingestion/languages/php/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/php/import-target.ts @@ -21,9 +21,13 @@ import { resolvePhpImportInternal } from '../../import-resolvers/php.js'; import type { SuffixIndex } from '../../import-resolvers/utils.js'; import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { getWorkspaceFileIndex } from '../../import-resolvers/workspace-file-index.js'; -import type { ComposerConfig } from '../../language-config.js'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { + mergeComposerConfigs, + parseComposerConfig, + type ComposerConfig, +} from '../../language-config.js'; +import { readdirSync, readFileSync, type Dirent } from 'node:fs'; +import { dirname, join, relative } from 'node:path'; export interface PhpResolveContext { readonly fromFile: string; @@ -48,19 +52,18 @@ function namespaceDirectories( if (composerConfig === null) return [...directories]; - const normalizedTarget = normalizePhpPath(targetRaw); + const normalizedTarget = normalizePhpPath(targetRaw).replace(/^\/+/, ''); const mappings = [...composerConfig.psr4.entries()].sort((left, right) => { const lengthDifference = right[0].length - left[0].length; return lengthDifference !== 0 ? lengthDifference : left[0].localeCompare(right[0]); }); for (const [namespacePrefix, directoryPrefix] of mappings) { const normalizedPrefix = normalizePhpPath(namespacePrefix); - if ( - normalizedTarget !== normalizedPrefix && - !normalizedTarget.startsWith(`${normalizedPrefix}/`) - ) { - continue; - } + const matchesNamespace = + normalizedPrefix === '' || + normalizedTarget === normalizedPrefix || + normalizedTarget.startsWith(`${normalizedPrefix}/`); + if (!matchesNamespace) continue; const remainder = normalizedTarget.slice(normalizedPrefix.length).replace(/^\//, ''); const separator = remainder.lastIndexOf('/'); @@ -82,21 +85,11 @@ function parentDirectory(filePath: string): string { } function directoryAliases(filePath: string): string[] { - const normalizedPath = normalizePhpPath(filePath); - const separator = normalizedPath.lastIndexOf('/'); - if (separator < 0) return ['']; - - const parent = normalizedPath.slice(0, separator); - const aliases = new Set([parent]); - const segments = parent.split('/').filter(Boolean); - for (let index = 0; index < segments.length; index++) { - aliases.add(segments.slice(index).join('/')); - } - return [...aliases]; + return [parentDirectory(filePath)]; } /** - * Directory alias → the files under it, built once per pass. + * Exact repository-relative directory → the files under it, built once per pass. * * A scope-resolution pass shares one stable `parsedFiles` array across imports, * so the array identity is the memo key — see `perFileSet`. @@ -302,42 +295,67 @@ const getPhpWorkspaceIndex = perFileSet((allFilePaths: ReadonlySet): Php // ─── loadResolutionConfig ────────────────────────────────────────────────── /** - * Load and parse `composer.json` from the repo root. Returns a - * `ComposerConfig` object (PSR-4 namespace → directory mappings) or - * `null` when no `composer.json` is present or it cannot be parsed. + * Load and parse repository and package-local `composer.json` manifests. + * Package mappings are rebased to repository-relative paths before merging. * * The result is threaded into each `resolvePhpImportInternal` call as * the `composerConfig` argument. */ export function loadPhpComposerConfig(repoPath: string): ComposerConfig | null { - try { - const composerPath = join(repoPath, 'composer.json'); - const raw = readFileSync(composerPath, 'utf8'); - const parsed = JSON.parse(raw) as unknown; - if (typeof parsed !== 'object' || parsed === null) return null; + const skipDirectories = new Set([ + '.git', + '.gitnexus', + 'node_modules', + 'vendor', + 'dist', + 'build', + 'coverage', + ]); + const pending = [repoPath]; + const manifests: string[] = []; + let incomplete = false; + let visitedDirectories = 0; - const composer = parsed as Record; - const autoload = composer['autoload'] as Record | undefined; - if (autoload === undefined) return null; - - const psr4Raw = (autoload['psr-4'] ?? {}) as Record; - const psr4 = new Map(); - - for (const [ns, dirs] of Object.entries(psr4Raw)) { - // namespace prefix ends with `\` — keep as-is; resolver strips it - const normalizedNs = ns.replace(/\\$/, ''); - const dir = Array.isArray(dirs) ? dirs[0] : dirs; - if (typeof dir === 'string') { - // Normalize directory path (strip trailing slash) - const normalizedDir = dir.replace(/\/+$/, ''); - psr4.set(normalizedNs, normalizedDir); + while (pending.length > 0) { + const directory = pending.pop(); + if (directory === undefined) break; + if (++visitedDirectories > 20_000) { + incomplete = true; + break; + } + let entries: Dirent[]; + try { + entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => + left.name.localeCompare(right.name), + ); + } catch { + incomplete = true; + continue; + } + for (const entry of entries) { + if (entry.isFile() && entry.name === 'composer.json') { + manifests.push(join(directory, entry.name)); + } else if (entry.isDirectory() && !skipDirectories.has(entry.name)) { + pending.push(join(directory, entry.name)); } } - - return { psr4 }; - } catch { - return null; } + + const configs: ComposerConfig[] = []; + for (const manifest of manifests.sort()) { + try { + const baseDir = normalizePhpPath(relative(repoPath, dirname(manifest))); + const config = parseComposerConfig(JSON.parse(readFileSync(manifest, 'utf8')), baseDir); + if (config !== null) configs.push(config); + } catch { + incomplete = true; + } + } + + const merged = mergeComposerConfigs(configs); + if (merged === null) return null; + if (incomplete) merged.hasUnmodeledAutoload = true; + return merged; } // ─── resolvePhpImportTarget ──────────────────────────────────────────────── @@ -434,11 +452,7 @@ export function resolvePhpImportTargetInternal( ...new Set( directories.flatMap((directory) => { const files = directoryIndex.get(normalizePhpPath(directory)) ?? []; - // A suffix alias can match directories under different roots (for - // example app/Models and vendor/pkg/app/Models). Picking either root - // would be a guess, so fail closed to the composer resolution instead. - const distinctParents = new Set(files.map((file) => parentDirectory(file.filePath))); - return distinctParents.size > 1 ? [] : files; + return files; }), ), ]; diff --git a/gitnexus/src/core/ingestion/languages/python.ts b/gitnexus/src/core/ingestion/languages/python.ts index f9c345b4b..ce8d0fa90 100644 --- a/gitnexus/src/core/ingestion/languages/python.ts +++ b/gitnexus/src/core/ingestion/languages/python.ts @@ -44,6 +44,8 @@ import { } from './python/index.js'; import { extractDjangoRoutes } from '../route-extractors/django.js'; import { discoverDjangoRootUrls } from '../route-extractors/django-root-discovery.js'; +import { extractPythonModuleConstants } from '../route-extractors/python-const-resolver.js'; +import { pythonDecoratorRouteHandlerName } from '../route-extractors/python-decorator-handler.js'; const BUILT_INS: ReadonlySet = new Set([ 'print', @@ -142,6 +144,7 @@ export const pythonProvider = defineLanguage({ discoverDjangoRootUrls(files, contentMap, reader), extractRoutes: (tree, filePath, reader, parser) => parser ? extractDjangoRoutes(tree, filePath, parser, reader) : [], + decoratorRouteHandlerName: pythonDecoratorRouteHandlerName, labelOverride: pythonFunctionDefinitionLabel, // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── @@ -158,4 +161,17 @@ export const pythonProvider = defineLanguage({ receiverBinding: pythonReceiverBinding, arityCompatibility: pythonArityCompatibility, resolveImportTarget: resolvePythonImportTarget, + + // ── #2391 constant harvest, provider-hook form (#2980): module-level string + // constants + from-imports for non-literal decorator route paths. Bare-name + // refs fold through the shared resolver (no foldRoutePathOperands needed). + // No `moduleConstantHeuristic`: Python harvests unconditionally, exactly as + // #2391 shipped it. A content gate was tried here and removed on review — it + // required `NAME` immediately followed by `=`, so it silently dropped the two + // idiomatic typed-FastAPI shapes (`API: str = "/api"`, + // `API: Final[str] = "/api"`) and every composed constant whose RHS starts + // with an identifier (`USERS = BASE + "/users"`), i.e. it REGRESSED routes + // that already resolve on main. The worker treats a missing heuristic as + // default-open; only Java opts into a gate, where the cost actually bites. + extractModuleConstants: extractPythonModuleConstants, }); diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 40b91cd8d..4cf95f9c9 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -126,10 +126,13 @@ import { } from './javascript/index.js'; import { extractDispatchGuardRoutes } from '../route-extractors/dispatch-guard.js'; import { extractDataRouteTableRoutes } from '../route-extractors/data-route-table.js'; +import { extractNestRoutes } from '../route-extractors/nest.js'; +import { extractConvexEndpointProperties } from './typescript/convex-endpoint-metadata.js'; const extractJsTsRoutes = (...args: Parameters) => [ ...extractDispatchGuardRoutes(...args), ...extractDataRouteTableRoutes(...args), + ...extractNestRoutes(...args), ]; /** @@ -418,6 +421,7 @@ export const typescriptProvider = defineLanguage({ extractFunctionName: tsExtractFunctionName, }), variableExtractor: createVariableExtractor(typescriptVariableConfig), + definitionPropertiesExtractor: extractConvexEndpointProperties, classExtractor: createClassExtractor(typescriptClassConfig), // ── JSDoc → description (issue #2270). An exported decl is captured as the // inner declaration; its JSDoc precedes the wrapping `export_statement`. ── @@ -505,6 +509,7 @@ export const javascriptProvider = defineLanguage({ extractFunctionName: tsExtractFunctionName, }), variableExtractor: createVariableExtractor(javascriptVariableConfig), + definitionPropertiesExtractor: extractConvexEndpointProperties, classExtractor: createClassExtractor(javascriptClassConfig), // ── JSDoc → description (issue #2270). An exported decl is captured as the // inner declaration; its JSDoc precedes the wrapping `export_statement`. ── diff --git a/gitnexus/src/core/ingestion/languages/typescript/convex-endpoint-metadata.ts b/gitnexus/src/core/ingestion/languages/typescript/convex-endpoint-metadata.ts new file mode 100644 index 000000000..b31a079d1 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/typescript/convex-endpoint-metadata.ts @@ -0,0 +1,113 @@ +import type { ParsedImport } from 'gitnexus-shared'; +import type { DefinitionPropertiesContext } from '../../language-provider.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { assertCloneable } from '../../workers/clone-safety.js'; + +const GENERATED_ENDPOINT_FACTORIES: ReadonlySet = new Set([ + 'query', + 'mutation', + 'action', + 'internalQuery', + 'internalMutation', + 'internalAction', + 'httpAction', +]); + +const GENERIC_ENDPOINT_FACTORIES: ReadonlyMap = new Map( + [...GENERATED_ENDPOINT_FACTORIES].map((factory) => [`${factory}Generic`, factory]), +); + +const normalizeModuleTarget = (targetRaw: string): string => + targetRaw.replace(/\\/g, '/').replace(/\.(?:[cm]?[jt]s)$/, ''); + +const isGeneratedServerModule = (targetRaw: string): boolean => + /(?:^|\/)_generated\/server$/.test(normalizeModuleTarget(targetRaw)); + +function importedConvexFactory( + imports: readonly ParsedImport[], + localName: string, +): string | undefined { + for (const parsedImport of imports) { + if (parsedImport.kind !== 'named' && parsedImport.kind !== 'alias') continue; + if (parsedImport.localName !== localName) continue; + + const target = normalizeModuleTarget(parsedImport.targetRaw); + if (target === 'convex/server') { + return GENERIC_ENDPOINT_FACTORIES.get(parsedImport.importedName); + } + if (isGeneratedServerModule(target)) { + return GENERATED_ENDPOINT_FACTORIES.has(parsedImport.importedName) + ? parsedImport.importedName + : undefined; + } + } + return undefined; +} + +function matchingDeclarator(node: SyntaxNode, nodeName: string): SyntaxNode | undefined { + if (node.type === 'variable_declarator' && node.childForFieldName('name')?.text === nodeName) { + return node; + } + + if (node.type === 'export_statement') { + const declaration = node.childForFieldName('declaration'); + return declaration ? matchingDeclarator(declaration, nodeName) : undefined; + } + if (node.type !== 'lexical_declaration' && node.type !== 'variable_declaration') { + return undefined; + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if ( + child?.type === 'variable_declarator' && + child.childForFieldName('name')?.text === nodeName + ) { + return child; + } + } + return undefined; +} + +function findDeclarator(node: SyntaxNode, nodeName: string): SyntaxNode | undefined { + let current: SyntaxNode | null = node; + while (current) { + const declarator = matchingDeclarator(current, nodeName); + if (declarator) return declarator; + if (current.type === 'program' || current.type === 'statement_block') break; + current = current.parent; + } + return undefined; +} + +/** + * Stamp Convex runtime-dispatch metadata only when both the declaration shape + * and the factory import provenance are known. The MCP layer consumes the + * resulting property without reparsing lossy FTS text. + */ +export function extractConvexEndpointProperties( + context: DefinitionPropertiesContext, +): Readonly> | undefined { + if ((context.nodeLabel !== 'Const' && context.nodeLabel !== 'Function') || !context.isExported) { + return undefined; + } + + const declarator = findDeclarator(context.definitionNode, context.nodeName); + const value = declarator?.childForFieldName('value'); + if (!value || value.type !== 'call_expression') return undefined; + + const callee = value.childForFieldName('function'); + if (!callee || callee.type !== 'identifier') return undefined; + const factory = importedConvexFactory(context.parsedImports, callee.text); + if (factory === undefined) return undefined; + + const args = value.childForFieldName('arguments'); + if (!args || args.namedChildCount !== 1) return undefined; + const endpointDefinition = args.namedChild(0); + if ( + !endpointDefinition || + !['object', 'arrow_function', 'function_expression'].includes(endpointDefinition.type) + ) { + return undefined; + } + return assertCloneable({ convexEndpointFactory: factory }); +} diff --git a/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts b/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts index a111f5752..0e9871c64 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts @@ -23,7 +23,7 @@ import fs from 'fs/promises'; import path from 'path'; -import { isHardcodedIgnoredDirectory } from '../../../../config/ignore-service.js'; +import { isHardcodedIgnoredDirectoryAtPath } from '../../../../config/ignore-service.js'; import { logger } from '../../../logger.js'; /** One `paths` entry, pattern and targets kept in declaration order. */ @@ -291,9 +291,9 @@ async function findTsconfigFiles(repoRoot: string): Promise { } for (const entry of entries) { if (entry.isDirectory()) { - if (isHardcodedIgnoredDirectory(entry.name)) continue; - if (depth < SCAN_MAX_DEPTH) - queue.push({ dir: path.join(dir, entry.name), depth: depth + 1 }); + const childDir = path.join(dir, entry.name); + if (isHardcodedIgnoredDirectoryAtPath(repoRoot, childDir)) continue; + if (depth < SCAN_MAX_DEPTH) queue.push({ dir: childDir, depth: depth + 1 }); continue; } if (!entry.isFile()) continue; diff --git a/gitnexus/src/core/ingestion/model/field-registry.ts b/gitnexus/src/core/ingestion/model/field-registry.ts index c45fb9cb5..ce9df4982 100644 --- a/gitnexus/src/core/ingestion/model/field-registry.ts +++ b/gitnexus/src/core/ingestion/model/field-registry.ts @@ -2,9 +2,9 @@ * Field Registry * * Owner-scoped field/property index extracted from SymbolTable. - * Stores Property / Variable / Const / Static symbols keyed by - * `ownerNodeId\0fieldName` for O(1) lookup. Supports multiple defs - * under the same (owner, name) — e.g. legacy Property plus a + * Stores Property / Variable / Const / Static symbols in a nested + * `Map>` for O(1) lookup. Supports + * multiple defs under the same (owner, name) — e.g. legacy Property plus a * scope-resolution Variable reconciliation entry. */ @@ -49,13 +49,13 @@ export interface MutableFieldRegistry extends FieldRegistry { // --------------------------------------------------------------------------- export const createFieldRegistry = (): MutableFieldRegistry => { - const fieldByOwner = new Map(); + const fieldByOwner = new Map>(); const lookupAllByOwner = ( ownerNodeId: string, fieldName: string, ): readonly SymbolDefinition[] => { - return fieldByOwner.get(`${ownerNodeId}\0${fieldName}`) ?? EMPTY; + return fieldByOwner.get(ownerNodeId)?.get(fieldName) ?? EMPTY; }; const lookupFieldByOwner = ( @@ -67,12 +67,16 @@ export const createFieldRegistry = (): MutableFieldRegistry => { }; const register = (ownerNodeId: string, fieldName: string, def: SymbolDefinition): void => { - const key = `${ownerNodeId}\0${fieldName}`; - const existing = fieldByOwner.get(key); + let byName = fieldByOwner.get(ownerNodeId); + if (!byName) { + byName = new Map(); + fieldByOwner.set(ownerNodeId, byName); + } + const existing = byName.get(fieldName); if (existing) { existing.push(def); } else { - fieldByOwner.set(key, [def]); + byName.set(fieldName, [def]); } }; diff --git a/gitnexus/src/core/ingestion/model/method-registry.ts b/gitnexus/src/core/ingestion/model/method-registry.ts index 75f9834c2..f64a1d8c7 100644 --- a/gitnexus/src/core/ingestion/model/method-registry.ts +++ b/gitnexus/src/core/ingestion/model/method-registry.ts @@ -2,9 +2,9 @@ * Method Registry * * Owner-scoped method index extracted from SymbolTable. - * Stores Method/Constructor/Function-with-ownerId symbols keyed by - * `ownerNodeId\0methodName` for O(1) lookup. Supports overloads - * (array values) and arity-based filtering. + * Stores Method/Constructor/Function-with-ownerId symbols in a nested + * `Map>` for O(1) lookup. Supports + * overloads (array values) and arity-based filtering. */ import type { SymbolDefinition } from 'gitnexus-shared'; @@ -92,7 +92,7 @@ export interface MutableMethodRegistry extends MethodRegistry { // --------------------------------------------------------------------------- export const createMethodRegistry = (): MutableMethodRegistry => { - const methodByOwner = new Map(); + const methodByOwner = new Map>(); // Secondary flat-by-name index. Values are the SAME SymbolDefinition // references stored under `methodByOwner` — no copy, just a second key. // Populated in lockstep by `register()` and emptied by `clear()`. @@ -102,12 +102,15 @@ export const createMethodRegistry = (): MutableMethodRegistry => { // dedup fast-path. Monotonic: never unset except on `clear()`. let hasFunctionMethodsFlag = false; + const ownerDefs = (ownerNodeId: string, methodName: string): SymbolDefinition[] | undefined => + methodByOwner.get(ownerNodeId)?.get(methodName); + const lookupMethodByOwner = ( ownerNodeId: string, methodName: string, argCount?: number, ): SymbolDefinition | undefined => { - const defs = methodByOwner.get(`${ownerNodeId}\0${methodName}`); + const defs = ownerDefs(ownerNodeId, methodName); if (!defs || defs.length === 0) return undefined; // Arity narrowing: when an argCount is provided and there are multiple @@ -176,16 +179,20 @@ export const createMethodRegistry = (): MutableMethodRegistry => { ownerNodeId: string, methodName: string, ): readonly SymbolDefinition[] => { - return methodByOwner.get(`${ownerNodeId}\0${methodName}`) ?? EMPTY; + return ownerDefs(ownerNodeId, methodName) ?? EMPTY; }; const register = (ownerNodeId: string, methodName: string, def: SymbolDefinition): void => { - const key = `${ownerNodeId}\0${methodName}`; - const existing = methodByOwner.get(key); + let owned = methodByOwner.get(ownerNodeId); + if (!owned) { + owned = new Map(); + methodByOwner.set(ownerNodeId, owned); + } + const existing = owned.get(methodName); if (existing) { existing.push(def); } else { - methodByOwner.set(key, [def]); + owned.set(methodName, [def]); } const byName = methodsByName.get(methodName); if (byName) { diff --git a/gitnexus/src/core/ingestion/model/registration-table.ts b/gitnexus/src/core/ingestion/model/registration-table.ts index 2de59e966..e9e85eeb0 100644 --- a/gitnexus/src/core/ingestion/model/registration-table.ts +++ b/gitnexus/src/core/ingestion/model/registration-table.ts @@ -182,6 +182,9 @@ const LABEL_BEHAVIOR = { Section: 'inert', Route: 'inert', Tool: 'inert', + // A broker address, not a symbol: nothing in any language resolves a name to + // it, so it stays out of the dispatch and callable indexes like `Route`. + Destination: 'inert', // Taint/PDG substrate (issue #2080) — a control-flow node, never a // symbol-resolution target. Inert: file index only, no owner scope. BasicBlock: 'inert', diff --git a/gitnexus/src/core/ingestion/model/type-registry.ts b/gitnexus/src/core/ingestion/model/type-registry.ts index 4dc87e924..135e61dfd 100644 --- a/gitnexus/src/core/ingestion/model/type-registry.ts +++ b/gitnexus/src/core/ingestion/model/type-registry.ts @@ -70,7 +70,7 @@ export const createTypeRegistry = (): MutableTypeRegistry => { const classByName = new Map(); const classByQualifiedName = new Map(); const implByName = new Map(); - const nestedByOwner = new Map(); + const nestedByOwner = new Map>(); const lookupClassByName = (name: string): SymbolDefinition[] => { return classByName.get(name) ?? []; @@ -88,7 +88,7 @@ export const createTypeRegistry = (): MutableTypeRegistry => { ownerNodeId: string, simpleName: string, ): readonly SymbolDefinition[] => { - return nestedByOwner.get(`${ownerNodeId}\0${simpleName}`) ?? EMPTY; + return nestedByOwner.get(ownerNodeId)?.get(simpleName) ?? EMPTY; }; const registerClass = (name: string, qualifiedName: string, def: SymbolDefinition): void => { @@ -121,12 +121,16 @@ export const createTypeRegistry = (): MutableTypeRegistry => { simpleName: string, def: SymbolDefinition, ): void => { - const key = `${ownerNodeId}\0${simpleName}`; - const existing = nestedByOwner.get(key); + let byName = nestedByOwner.get(ownerNodeId); + if (!byName) { + byName = new Map(); + nestedByOwner.set(ownerNodeId, byName); + } + const existing = byName.get(simpleName); if (existing) { existing.push(def); } else { - nestedByOwner.set(key, [def]); + byName.set(simpleName, [def]); } }; diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index c91df2953..e4df7dae5 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -56,6 +56,8 @@ export interface WorkerExtractedData { * finalize-orchestrator. */ parsedFiles: ParsedFile[]; + /** Scope-extraction omissions represented by this worker/cache result. */ + scopeExtractionFailures: string[]; } type ParsedGraphNode = ParseWorkerResult['nodes'][number]; @@ -126,6 +128,7 @@ export const mergeChunkResults = ( const allORMQueries: ExtractedORMQuery[] = []; const fileScopeBindingsByFile: FileScopeBindings[] = []; const allParsedFiles: ParsedFile[] = []; + const scopeExtractionFailures: string[] = []; for (const result of chunkResults) { // Worker jobs and input files are already merged in stable start-index/path @@ -178,6 +181,9 @@ export const mergeChunkResults = ( if (result.fileScopeBindings) for (const item of result.fileScopeBindings) fileScopeBindingsByFile.push(item); if (result.parsedFiles) for (const item of result.parsedFiles) allParsedFiles.push(item); + for (const filePath of result.scopeExtractionFailures ?? []) { + scopeExtractionFailures.push(filePath); + } } return { @@ -195,6 +201,7 @@ export const mergeChunkResults = ( springTypes: allSpringTypes, fileScopeBindings: fileScopeBindingsByFile, parsedFiles: allParsedFiles, + scopeExtractionFailures, }; }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/di.ts b/gitnexus/src/core/ingestion/pipeline-phases/di.ts index 12ee5cb3e..b8bf69bd8 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/di.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/di.ts @@ -102,6 +102,10 @@ function providerCandidates( return recognized.length > 0 ? recognized : all; } +function isConcreteTypeNode(node: GraphNode | undefined): boolean { + return node?.label === 'Class' || node?.label === 'Record' || node?.label === 'Enum'; +} + export const diPhase: PipelinePhase = { name: 'di', deps: ['mro'], @@ -160,21 +164,27 @@ export const diPhase: PipelinePhase = { }; } - const interfaceToImplementers = new Map>(); + const directSubtypes = new Map>(); const directSupertypes = new Map>(); for (const rel of ctx.graph.iterRelationshipsByType('IMPLEMENTS')) { - const set = interfaceToImplementers.get(rel.targetId) ?? new Set(); - set.add(rel.sourceId); - interfaceToImplementers.set(rel.targetId, set); + const subtypes = directSubtypes.get(rel.targetId) ?? new Set(); + subtypes.add(rel.sourceId); + directSubtypes.set(rel.targetId, subtypes); const supertypes = directSupertypes.get(rel.sourceId) ?? new Set(); supertypes.add(rel.targetId); directSupertypes.set(rel.sourceId, supertypes); } for (const rel of ctx.graph.iterRelationshipsByType('EXTENDS')) { + const subtypes = directSubtypes.get(rel.targetId) ?? new Set(); + subtypes.add(rel.sourceId); + directSubtypes.set(rel.targetId, subtypes); const supertypes = directSupertypes.get(rel.sourceId) ?? new Set(); supertypes.add(rel.targetId); directSupertypes.set(rel.sourceId, supertypes); } + const orderedDirectSubtypes = new Map( + [...directSubtypes].map(([typeId, subtypes]) => [typeId, [...subtypes].sort().reverse()]), + ); const memberToClass = new Map(); for (const relationType of ['HAS_PROPERTY', 'HAS_METHOD'] as const) { @@ -187,16 +197,45 @@ export const diPhase: PipelinePhase = { const interfacesByLanguage = new Map(); const classesByLanguage = new Map(); ctx.graph.forEachNode((node) => { - if (node.label !== 'Class' && node.label !== 'Interface') return; + const concreteType = isConcreteTypeNode(node); + if (!concreteType && node.label !== 'Interface') return; const language = node.properties.language; if (typeof language !== 'string' || !candidateLanguages.has(language)) return; - const indexes = node.label === 'Class' ? classesByLanguage : interfacesByLanguage; + const indexes = concreteType ? classesByLanguage : interfacesByLanguage; const index = indexes.get(language) ?? emptyNameIndex(); addIndexedName(index, node); indexes.set(language, index); - if (node.label === 'Class') providerNodes.set(node.id, node); + if (concreteType) providerNodes.set(node.id, node); }); + const concreteSubtypesByRoot = new Map>(); + const concreteSubtypes = (rootTypeId: string, language: string): ReadonlySet => { + const cacheKey = `${language}\0${rootTypeId}`; + const cached = concreteSubtypesByRoot.get(cacheKey); + if (cached !== undefined) return cached; + + const concrete = new Set(); + const queue = [rootTypeId]; + const visited = new Set(); + while (queue.length > 0) { + const typeId = queue.pop(); + if (typeId === undefined || visited.has(typeId)) continue; + visited.add(typeId); + const typeNode = ctx.graph.getNode(typeId); + if ( + typeNode !== undefined && + isConcreteTypeNode(typeNode) && + typeNode.properties.language === language + ) { + concrete.add(typeId); + } + const children = orderedDirectSubtypes.get(typeId) ?? []; + queue.push(...children); + } + concreteSubtypesByRoot.set(cacheKey, concrete); + return concrete; + }; + // A declaration returning a concrete class is assignable to every class or // interface that type extends/implements. Expand once per language+type and // register the declaration under those ancestor names. This keeps named @@ -300,11 +339,15 @@ export const diPhase: PipelinePhase = { continue; } - const structural = new Set(); - if (typeof classEntry === 'string') structural.add(classEntry); - if (typeof interfaceEntry === 'string') { - for (const id of interfaceToImplementers.get(interfaceEntry) ?? []) structural.add(id); - } + const rootTypeId = + typeof classEntry === 'string' + ? classEntry + : typeof interfaceEntry === 'string' + ? interfaceEntry + : undefined; + const structural = new Set( + rootTypeId === undefined ? [] : concreteSubtypes(rootTypeId, candidate.language), + ); for (const id of providedTypes.get(candidate.language)?.get(candidate.targetTypeName) ?? []) { structural.add(id); } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/index.ts b/gitnexus/src/core/ingestion/pipeline-phases/index.ts index 359d80a65..99b7cc253 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/index.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/index.ts @@ -21,6 +21,7 @@ export { type ScopeResolutionOutput, } from '../scope-resolution/pipeline/phase.js'; export { springConfigPhase, type SpringConfigOutput } from './spring-config.js'; +export { springDestinationsPhase, type SpringDestinationsOutput } from './spring-destinations.js'; export { springAutoConfigurationPhase, type SpringAutoConfigurationOutput, diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 2a3d21612..f9cbbd47b 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -2,7 +2,7 @@ * Parse implementation — chunked parse + resolve loop. * * This is the core parsing engine of the ingestion pipeline. It reads - * source files in byte-budget chunks (~20MB each), parses via the worker + * source files in stable hash-bucket packs (~2MB each by default), parses via the worker * pool (the sole parse path — there is no sequential fallback), and emits * route CALLS edges. Import, * call, and inheritance resolution are owned by the scope-resolution @@ -27,6 +27,7 @@ import { loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, + packParseCacheChunks, } from '../../../storage/parse-cache.js'; import { clearParsedFileStore, @@ -35,7 +36,7 @@ import { getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, - restoreDurableParsedFileShard, + durableChunkHasShards, } from '../../../storage/parsedfile-store.js'; import type { ParseWorkerResult } from '../workers/parse-worker.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js'; @@ -60,7 +61,7 @@ import { createParserForLanguage, } from '../../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; -import { getProvider, providers } from '../languages/index.js'; +import { getProvider, getProviderForFile, providers } from '../languages/index.js'; import { SCOPE_RESOLVERS } from '../scope-resolution/pipeline/registry.js'; import { DATA_ROUTE_TABLE_SOURCE } from '../route-extractors/data-route-table.js'; import type Parser from 'tree-sitter'; @@ -88,10 +89,9 @@ import type { ExtractedRouterModuleAlias, } from '../route-extractors/fastapi-router-bindings.js'; import { normalizeExtractedRoutePath } from '../route-extractors/route-path.js'; -import { - resolveOperands, - type ModuleConstants, -} from '../route-extractors/python-const-resolver.js'; +import { resolveOperands } from '../route-extractors/python-const-resolver.js'; +import type { ModuleConstants } from '../route-extractors/constant-resolver.js'; +import { prepareRouteConstantsByProvider } from '../language-provider.js'; import { resolveInheritedSpringRoutes, type SharedSpringType, @@ -192,39 +192,19 @@ export function heapPressureRemedy(heapLimitBytes: number): string { ); } -/** Max bytes of source content to load per parse chunk. +/** Max bytes of source content to load per parse cache pack. * - * Memory bound for the worker pool dispatch + a granularity knob for - * the parse cache. A single file change invalidates only its enclosing - * chunk, so smaller budgets → finer-grained invalidation. - * - * Override via GITNEXUS_CHUNK_BYTE_BUDGET (bytes) — the default of 2MB - * gives a useful invalidation floor (~1/N chunks on a multi-MB repo) - * while keeping worker dispatch overhead under 5% on cold runs. - */ -/** - * Built-in chunk byte budget when neither `PipelineOptions.chunkByteBudget` - * nor `GITNEXUS_CHUNK_BYTE_BUDGET` is set. Tuned to give a useful - * cache-invalidation floor (~1/N chunks on a multi-MB repo) while keeping - * worker dispatch overhead under 5% on cold runs. Resolution happens at - * call time inside `runChunkedParseAndResolve` (U14 from PR #1693 review) - * — previously this was a module-load IIFE, which froze the env value at - * import time and meant per-call option threading silently no-op'd. + * Granularity knob for the parse cache: a single file change invalidates only + * its enclosing pack. Override via GITNEXUS_CHUNK_BYTE_BUDGET. Resolution + * happens at call time (U14 from PR #1693) — not at module load. */ const DEFAULT_CHUNK_BYTE_BUDGET = 2 * 1024 * 1024; /** - * Per-worker share of a chunk's byte budget when auto-scaling (#worker-idle). - * - * A chunk is a single `WorkerPool.dispatch` unit; the pool fans a chunk's files - * into sub-batch jobs and assigns them to idle workers (`wakeIdleSlots`). When - * the chunk budget (2 MB) was far below the 8 MB sub-batch cap, every chunk - * produced exactly ONE job → ONE busy worker while the other N-1 sat idle. To - * keep all workers fed, the auto chunk budget now scales as - * `poolSize × CHUNK_BYTES_PER_WORKER`, so each dispatch carries enough work to - * fan across the whole pool. Sequential / explicit-budget runs are unaffected. + * Byte unit for auto pool sizing (one worker per this much source). Same + * magnitude as the default cache pack, but not a membership input (#3088). */ -const CHUNK_BYTES_PER_WORKER = 2 * 1024 * 1024; +const CHUNK_BYTES_PER_WORKER = DEFAULT_CHUNK_BYTE_BUDGET; /** * Target jobs-per-worker per dispatch. More jobs than workers gives the pool's @@ -236,14 +216,12 @@ const TARGET_JOBS_PER_WORKER = 3; /** Floor for a derived sub-batch so jobs don't shrink to per-file IPC churn. */ const MIN_SUB_BATCH_BYTES = 256 * 1024; -function resolveChunkByteBudget(options?: PipelineOptions, effectivePoolSize = 1): number { +function resolveChunkByteBudget(options?: PipelineOptions): number { const opt = options?.chunkByteBudget; if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0) return opt; const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET); if (Number.isFinite(env) && env > 0) return env; - // Auto: size each chunk so a dispatch can fan across the whole pool. A - // single-worker (tiny-repo) run keeps the original 2 MB invalidation floor. - return Math.max(DEFAULT_CHUNK_BYTE_BUDGET, effectivePoolSize * CHUNK_BYTES_PER_WORKER); + return DEFAULT_CHUNK_BYTE_BUDGET; } // ── Main parse + resolve function ────────────────────────────────────────── @@ -477,11 +455,19 @@ export async function runChunkedParseAndResolve( * files. There is no sequential parser — the pool is the sole parse path * whenever a chunk misses the cache. */ usedWorkerPool: boolean; + /** Files dispatched to parser workers after parse-cache lookup. */ + reparsedFileCount: number; /** Worker-produced ParsedFile artifacts aggregated across chunks. * Threaded into scope-resolution as a re-extract cache so the warm- * cache analyze run can skip the dominant `extractParsedFile` cost * (otherwise ~58s on a 1000-file repo). */ parsedFiles: import('gitnexus-shared').ParsedFile[]; + /** Repo-wide harvested constants, already prepared per provider. See + * `ParseOutput.moduleConstants` for why this leaves the parse phase. */ + moduleConstants: ReadonlyMap; + scopeExtractionFailures: string[]; + /** Files excluded because their non-standalone language parser was unavailable. */ + unavailableScopeLanguageFiles: number; }> { const model = createSemanticModel(); const symbolTable = model.symbols; @@ -514,15 +500,10 @@ export async function runChunkedParseAndResolve( ); } } - - // Sort parseableScanned alphabetically for stable chunk membership - // across runs (Finding 4). Without this, filesystem-scan order can - // shift between runs (notably on macOS APFS where directory entry - // order can change after modifications) — different files in the - // same chunk → different chunk hash → cache miss even when no file - // content changed. The cache also becomes platform-specific: a - // Linux-built cache misses on macOS for the same repo. - parseableScanned.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + const unavailableScopeLanguageFiles = [...skippedByLang.values()].reduce( + (total, count) => total + count, + 0, + ); const totalParseable = parseableScanned.length; const totalBytes = parseableScanned.reduce((sum, f) => sum + f.size, 0); @@ -570,25 +551,25 @@ export async function runChunkedParseAndResolve( // runs. Resolving in the function body restores per-call configurability // and matches the pattern used by resolveAutoPoolSize and the U1 // parseChunkConcurrency resolver. - // Effective worker count, computed up-front so the chunk budget can scale to - // keep the whole pool busy (#worker-idle). The pool is ALWAYS used (sequential - // parsing was removed; the disabled channels threw above). Size it to the - // work: an explicit `--workers ` pins the size; otherwise the cores-based - // auto size is capped by the repo's worth of work (~one worker per - // CHUNK_BYTES_PER_WORKER of source) so a tiny repo spawns ~1 worker instead of - // a full pool, replacing the job the deleted small-repo threshold used to do. - // KTD-3 of the remove-sequential plan; the cap formula is intentionally coarse - // (tuning deferred). + // Effective worker count: explicit `--workers ` pins it; otherwise + // cores-based auto size is capped by source bytes / CHUNK_BYTES_PER_WORKER + // so a tiny repo does not spawn a full idle pool. Cache pack membership + // is independent of this number (#3088). const explicitPoolSize = options?.workerPoolSize; const workProportionalCap = Math.max(1, Math.ceil(totalBytes / CHUNK_BYTES_PER_WORKER)); const effectivePoolSize = explicitPoolSize && explicitPoolSize > 0 ? explicitPoolSize : Math.min(resolveAutoPoolSize(), workProportionalCap); - const chunkByteBudget = resolveChunkByteBudget(options, effectivePoolSize); - // Sub-batch size so each chunk fans into ~`TARGET_JOBS_PER_WORKER` jobs per - // worker, giving the pool's idle-slot assignment room to load-balance. An - // explicit `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` operator override wins. + // Cache packs: stable (language, hash(path) mod 128) buckets, then the + // per-call byte budget inside each bucket (#3088). Pool size is used only + // for worker count and sub-batch fan-out, not membership. + const chunkByteBudget = resolveChunkByteBudget(options); + // Sub-batch size so a 2 MiB pack fans into ~TARGET_JOBS_PER_WORKER jobs + // per worker, floored at MIN_SUB_BATCH_BYTES (256 KiB) so an 8-worker + // pool still gets ~8 jobs from one pack instead of one idle-heavy job + // (#worker-idle). Do not derive this from pool×2 MiB while dispatching a + // 2 MiB pack. An explicit GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES wins. const subBatchEnv = Number(process.env.GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES); const dispatchSubBatchMaxBytes = Number.isFinite(subBatchEnv) && subBatchEnv > 0 @@ -613,19 +594,14 @@ export async function runChunkedParseAndResolve( ); } - const chunks: string[][] = []; - let currentChunk: string[] = []; - let currentBytes = 0; - for (const file of parseableScanned) { - if (currentChunk.length > 0 && currentBytes + file.size > chunkByteBudget) { - chunks.push(currentChunk); - currentChunk = []; - currentBytes = 0; - } - currentChunk.push(file.path); - currentBytes += file.size; - } - if (currentChunk.length > 0) chunks.push(currentChunk); + const chunks: string[][] = packParseCacheChunks( + parseableScanned.map((file) => ({ + path: file.path, + size: file.size, + language: getLanguageFromFilename(file.path) ?? 'unknown', + })), + chunkByteBudget, + ); const numChunks = chunks.length; @@ -743,6 +719,7 @@ export async function runChunkedParseAndResolve( // the second-half of the parse-cache speedup since scope-resolution's // re-parse otherwise dominates the warm-cache wall-clock time. const allParsedFiles: import('gitnexus-shared').ParsedFile[] = []; + const scopeExtractionFailures = new Set(); // Incremental parse cache (Option B): chunk-level content-addressed. // When the chunk's (filePath, content-hash) signature matches a prior @@ -764,17 +741,18 @@ export async function runChunkedParseAndResolve( // a sibling of the run-scoped store, NOT cleared per run. Workers write a // shard per chunk hash; on a warm parse-cache hit we restore the chunk's // shards into the run-scoped store so scope-resolution streams them without - // re-parsing. `durableHitKeys` is the prior run's index, version-gated by - // PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk re-dispatches, which - // repopulates the durable store — never the main-thread extract fallback). + // re-parsing. `durableHitEntries` is the prior run's path-coverage index, + // version-gated by PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk + // re-dispatches, which repopulates the durable store). const durableParsedFileDir = parsedFileStorePath !== undefined ? getDurableParsedFileDir(parsedFileStorePath) : undefined; - const durableHitKeys = + const durableHitEntries = durableParsedFileDir !== undefined ? await loadDurableParsedFileIndex(durableParsedFileDir, PARSE_CACHE_VERSION) - : new Set(); + : new Map>(); let chunkCacheHits = 0; let chunkCacheMisses = 0; + let reparsedFileCount = 0; try { // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch @@ -844,13 +822,19 @@ export async function runChunkedParseAndResolve( chunkStartMs: number | null, ): Promise => { if (chunkWorkerData) { + for (const filePath of chunkWorkerData.scopeExtractionFailures) { + scopeExtractionFailures.add(filePath); + } if (chunkWorkerData.parsedFiles?.length) { if (parsedFileStorePath) { - await persistParsedFileChunk( + const wrote = await persistParsedFileChunk( parsedFileStorePath, `chunk-${chunkIdx}`, chunkWorkerData.parsedFiles, ); + if (!wrote) { + for (const item of chunkWorkerData.parsedFiles) allParsedFiles.push(item); + } } else { for (const item of chunkWorkerData.parsedFiles) allParsedFiles.push(item); } @@ -1040,8 +1024,16 @@ export async function runChunkedParseAndResolve( // store was introduced, or a pruned/version-stale shard — fall through to // a worker re-dispatch to repopulate them. NEVER let scope-resolution // re-extract on the main thread (the #1983 OOM the durable store closes). + const durableExpectedPaths = + chunkHash === null ? undefined : durableHitEntries.get(chunkHash); const durableHit = - chunkHash !== null && durableParsedFileDir !== undefined && durableHitKeys.has(chunkHash); + cachedRaw !== undefined && + cachedRaw.length > 0 && + chunkHash !== null && + durableParsedFileDir !== undefined && + parsedFileStorePath !== undefined && + durableExpectedPaths !== undefined && + (await durableChunkHasShards(parsedFileStorePath, chunkHash, durableExpectedPaths)); if (cachedRaw && cachedRaw.length > 0 && (durableHit || parsedFileStorePath === undefined)) { // Cache hit: replay cached worker output. Finalize any parked worker @@ -1074,27 +1066,14 @@ export async function runChunkedParseAndResolve( nodesCreated: graph.nodeCount, }, }); - // Restore the chunk's durable ParsedFile shards into the run-scoped - // store so scope-resolution finds full coverage with ZERO main-thread - // re-parse. A verbatim byte copy — byte-identical to a cold run. - if (durableHit && durableParsedFileDir && parsedFileStorePath && chunkHash) { - const restored = await restoreDurableParsedFileShard( - durableParsedFileDir, - parsedFileStorePath, - chunkHash, - ); - if (restored === 0) { - logger.warn( - `parsedfile-cache: durable shards missing for cached chunk ` + - `${chunkHash.slice(0, 8)} — scope-resolution will re-extract these files`, - ); - } - } + // The durable gate already snapshotted warm `.v8` shards into the + // run-scoped store for scope resolution. await applyChunkResults(chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs); } else { // Cache miss: dispatch to workers, capture the raw results, store // them under the chunk hash for the next run. chunkCacheMisses++; + reparsedFileCount += chunkFiles.length; if (durableParsedFileDir !== undefined && chunkHash !== null) { try { await prepareDurableParsedFileChunk(durableParsedFileDir, chunkHash); @@ -1291,11 +1270,27 @@ export async function runChunkedParseAndResolve( // carries `routePathExpr`/`routePathOperands` and an empty `routePath`; we fold // the operands against the repo-wide, file-path-keyed constant map. On failure // we DROP the route (KTD5 skip floor) rather than emit a phantom `POST /`. + // + // Built (and prepared) UNCONDITIONALLY when anything was harvested, because + // the map is also handed to downstream phases on `ParseOutput.moduleConstants` + // — `springDestinations` folds broker-address constants against exactly the + // same table. Preparation runs exactly once, here, on one map, before either + // consumer folds. Deferring it into each consumer instead would need + // `prepareRouteConstants` to be safe to call twice — it materializes deferred + // wildcard bindings IN PLACE — or would leave whichever consumer ran first + // folding against unprepared constants. Neither is worth the coupling; the + // cost here is one pass over the harvested constants of a repo that has some. + const repoConstants = new Map(); + for (const { filePath, constants } of allModuleConstants) { + repoConstants.set(filePath, constants); + } + if (repoConstants.size > 0) { + // Let each language prepare only its own constants slice before folding. + // This is where deferred wildcard bindings can be materialized once per + // provider without naming a language in the shared parse phase. + prepareRouteConstantsByProvider(repoConstants, getProviderForFile); + } if (allDecoratorRoutes.some((dr) => dr.routePathExpr !== undefined)) { - const repoConstants = new Map(); - for (const { filePath, constants } of allModuleConstants) { - repoConstants.set(filePath, constants); - } const resolvedRoutes: ExtractedDecoratorRoute[] = []; let skipped = 0; for (const dr of allDecoratorRoutes) { @@ -1303,8 +1298,15 @@ export async function runChunkedParseAndResolve( resolvedRoutes.push(dr); continue; } + // Provider-driven fold (#2980): languages with qualified-ref semantics + // (Java `ApiPaths.X` / `com.example.ApiPaths.X`) fold through their + // provider hook; everything else uses the shared language-agnostic + // operand fold. No language names in the shared layer. + const fold = getProviderForFile(dr.filePath)?.foldRoutePathOperands; const value = dr.routePathOperands - ? resolveOperands(dr.filePath, dr.routePathOperands, repoConstants) + ? fold + ? fold(dr.filePath, dr.routePathOperands, repoConstants) + : resolveOperands(dr.filePath, dr.routePathOperands, repoConstants) : null; if (value === null) { skipped++; @@ -1604,10 +1606,21 @@ export async function runChunkedParseAndResolve( // no pool was needed: a warm all-cache-hit run replays cached worker output // without spawning workers, or there were no parseable files. usedWorkerPool: workerPool !== undefined, + // Exact number of files sent through workers on parse-cache misses. A + // changed file can invalidate its whole content-addressed chunk, so this + // is intentionally measured at dispatch time rather than inferred from + // the git/hash diff. + reparsedFileCount, // Per-file ParsedFile artifacts produced by workers' calls to // `extractParsedFile`. Consumed by scope-resolution as a re-extraction // cache: when the file's ParsedFile is here, scope-resolution skips its own // `extractParsedFile` call. parsedFiles: allParsedFiles, + // Repo-wide, file-path-keyed constants, already through each provider's + // `prepareRouteConstants` hook. Empty when no provider harvests constants + // for the languages in this repo. + moduleConstants: repoConstants, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), + unavailableScopeLanguageFiles, }; } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 2484baa01..b47ceafdf 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -31,6 +31,7 @@ import type { } from '../workers/parse-worker.js'; import { runChunkedParseAndResolve } from './parse-impl.js'; import type { MutableSemanticModel } from '../model/index.js'; +import type { ModuleConstants } from '../route-extractors/constant-resolver.js'; export interface ParseOutput { /** @@ -71,6 +72,8 @@ export interface ParseOutput { * is no sequential parser; the pool is the sole parse path on a cache miss. */ readonly usedWorkerPool: boolean; + /** Files actually dispatched to parser workers after parse-cache lookup. */ + readonly reparsedFileCount: number; /** * Per-file `ParsedFile` artifacts produced by workers' calls to * `extractParsedFile`. Threaded through to `scopeResolutionPhase` @@ -80,6 +83,26 @@ export interface ParseOutput { * costing ~58s on a 1000-file repo). */ readonly parsedFiles: readonly ParsedFile[]; + /** + * Repo-wide string constants harvested by the providers that declare + * `extractModuleConstants`, keyed by file path and already through each + * provider's `prepareRouteConstants` hook. + * + * Exposed so a later phase can fold a constant reference the same way the + * decorator-route pass does — `springDestinations` resolves a broker address + * written as `Topics.ORDERS` against this table. It is a snapshot: parse does + * not mutate it after returning, and consumers must not either, because the + * preparation that made it foldable has already run. + * + * Empty when no provider in this repo harvests constants. A downstream fold + * against an empty table simply fails to resolve, which is a recorded refusal + * rather than a wrong answer. + */ + readonly moduleConstants: ReadonlyMap; + /** Files whose scope extraction failed while legacy parsing continued. */ + readonly scopeExtractionFailures: readonly string[]; + /** Files omitted because their non-standalone language parser was unavailable. */ + readonly unavailableScopeLanguageFiles: number; } export const parsePhase: PipelinePhase = { diff --git a/gitnexus/src/core/ingestion/pipeline-phases/processes.ts b/gitnexus/src/core/ingestion/pipeline-phases/processes.ts index 24117d532..e34e79d35 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/processes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/processes.ts @@ -4,7 +4,8 @@ * Detects execution flows (processes) and creates Process nodes + * STEP_IN_PROCESS edges. Also links Route/Tool nodes to processes. * - * @deps communities, routes, tools, pruneLocalSymbols, structure, parse + * @deps communities, routes, tools, springAutoConfiguration, + * pruneLocalSymbols, structure, parse * @reads graph (all nodes and relationships), communityResult, routeRegistry, * toolDefs, parse's allFetchCalls + allORMQueries (R3-6 sink sites) * @writes graph (Process nodes, STEP_IN_PROCESS edges, ENTRY_POINT_OF edges) @@ -52,7 +53,15 @@ export const processesPhase: PipelinePhase = { // sinks rather than failing the phase. `pruneLocalSymbols` is declared // explicitly so process extraction always reads the trimmed graph even if a // future option drops the intervening `mro`/`communities` phases. - deps: ['communities', 'routes', 'tools', 'pruneLocalSymbols', 'structure', 'parse'], + deps: [ + 'communities', + 'routes', + 'tools', + 'springAutoConfiguration', + 'pruneLocalSymbols', + 'structure', + 'parse', + ], async execute( ctx: PipelineContext, @@ -212,8 +221,38 @@ export const processesPhase: PipelinePhase = { }); }); + // The static registry is finalized before Spring runtime enrichment. Merge + // runtime-confirmed Route nodes from the graph after the explicit + // springAutoConfiguration dependency has completed, so Actuator-only + // mappings participate in the same process-linking path. + const processRouteRegistry = new Map(routeRegistry); + ctx.graph.forEachNode((node) => { + if ( + node.label !== 'Route' || + node.properties.runtimeSource !== 'spring-actuator' || + node.properties.runtimeConfirmed !== true + ) { + return; + } + const url = typeof node.properties.name === 'string' ? node.properties.name : undefined; + const filePath = + typeof node.properties.filePath === 'string' ? node.properties.filePath : undefined; + const method = + typeof node.properties.method === 'string' ? node.properties.method : undefined; + if (url === undefined || filePath === undefined) return; + const key = routeNodeKey(method, url); + if (!processRouteRegistry.has(key)) { + processRouteRegistry.set(key, { + filePath, + source: 'spring-actuator-runtime', + url, + ...(method === undefined ? {} : { method }), + }); + } + }); + // Link Route and Tool nodes to Processes - if (routeRegistry.size > 0 || toolDefs.length > 0) { + if (processRouteRegistry.size > 0 || toolDefs.length > 0) { // Two-tier route lookup, mirroring the tool tables 10 lines below. // Routes whose handler resolved key by `handlerSymbolId` (read from // the Route node's graph properties — routes.ts stamps it there) and @@ -228,7 +267,7 @@ export const processesPhase: PipelinePhase = { // routes phase stamps on the Route node was never consulted. const routesByHandlerId = new Map(); const routesWithoutHandlerByFile = new Map(); - for (const [, entry] of routeRegistry) { + for (const [, entry] of processRouteRegistry) { // Push the Route node identity (`routeNodeKey`), not the bare URL, so the // ENTRY_POINT_OF edge targets the same node id the routes phase created // (#2289: a same-URL GET/POST pair is two distinct Route nodes). diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index 8115f5b40..42bf639d2 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -195,6 +195,45 @@ export const routesPhase: PipelinePhase = { const allFetchCalls = [...parseFetchCalls]; const routeRegistry = new Map(); + /** + * Registry keys written straight from the file list below, never through + * `addRoute`. `resolveRouteHandlerSymbols` walks only `extractedRoutes` and + * `decoratorRoutes`, so it never sees these URLs and its `claimed` set never + * contains them — which means a handler stamped on one of these keys was + * resolved for a DIFFERENT route. + * + * That is reachable, and it fabricates rather than omits (#3049). A + * method-agnostic route (`@All`, a Django function view, a verb-less + * dispatch guard) keys by URL alone via `routeNodeKey`, so it collides with + * a file-convention route at the same URL. It claims the key unopposed in + * `claim()`, then loses first-writer-wins here in `addRoute` and is dropped + * as a duplicate — and without this guard the surviving file-convention node + * would read that handler and present another application's controller + * method as its own. `api_impact` is documented to be run BEFORE editing a + * route handler, so it would answer with a handler from the wrong app. + * + * Dropping the losing route is a separate and deliberate consequence of + * URL-only identity; this only stops the false attribution. + * + * Membership is recorded AT the pre-seeding `set`, mirroring `claim()` in + * call-processor.ts, which writes `claimed` and its result map together + * rather than re-deriving either by rescanning. Identifying pre-seeded + * entries by matching `entry.source` against a list of source strings would + * spell them a second time, away from the sites that produce them — and a + * fourth pre-seeded source added later would then reopen #3049 in silence. + * `addRoute` deliberately does NOT record here: its routes ARE + * handler-resolved, so suppressing them would widen the guard into a bug of + * its own. + * + * Each `add` below sits inside its own `!routeRegistry.has(key)` gate, as + * every `routeRegistry.set` in this phase does: the map is write-once per + * key, so a losing candidate cannot record a key it did not claim and no + * later writer can take a recorded key away. Key-membership is therefore + * equivalent to source-matching by construction — pre-seeded routes carry no + * verb and `routeNodeKey(undefined, url) === url` — which is why the + * two-candidates-one-URL case needs no fixture to settle it. + */ + const preSeededKeys = new Set(); // Detect Expo Router app/ roots vs Next.js app/ roots (monorepo-safe) const expoAppRoots = new Set(); @@ -217,32 +256,33 @@ export const routesPhase: PipelinePhase = { } } + // One writer for every pre-seeded route, so recording membership cannot be + // forgotten. Inlining `has` / `set` / `add` at each site made the invariant + // a convention three call sites had to remember — and a fourth source that + // forgot the `add` would reopen #3049 exactly as silently as the source-set + // it replaced. This is the shape `claim()` in call-processor.ts uses for the + // same reason: one helper writes the collection and its key set together. + const preSeed = (url: string, entry: Omit): boolean => { + if (routeRegistry.has(url)) return false; + routeRegistry.set(url, { ...entry, url }); + preSeededKeys.add(url); + return true; + }; + for (const p of allPaths) { if (expoAppPaths.has(p)) { const expoURL = expoFileToRouteURL(p); - if (expoURL && !routeRegistry.has(expoURL)) { - routeRegistry.set(expoURL, { - filePath: p, - source: 'expo-filesystem-route', - url: expoURL, - }); + if (expoURL && preSeed(expoURL, { filePath: p, source: 'expo-filesystem-route' })) { continue; } } const nextjsURL = nextjsFileToRouteURL(p); - if (nextjsURL && !routeRegistry.has(nextjsURL)) { - routeRegistry.set(nextjsURL, { - filePath: p, - source: 'nextjs-filesystem-route', - url: nextjsURL, - }); + if (nextjsURL && preSeed(nextjsURL, { filePath: p, source: 'nextjs-filesystem-route' })) { continue; } if (p.endsWith('.php')) { const phpURL = phpFileToRouteURL(p); - if (phpURL && !routeRegistry.has(phpURL)) { - routeRegistry.set(phpURL, { filePath: p, source: 'php-file-route', url: phpURL }); - } + if (phpURL) preSeed(phpURL, { filePath: p, source: 'php-file-route' }); } } @@ -296,28 +336,31 @@ export const routesPhase: PipelinePhase = { let handlerContents: Map | undefined; if (routeRegistry.size > 0) { - const handlerPathFor = (routeKey: string, entry: RouteEntry): string => { - if (entry.source !== DATA_ROUTE_TABLE_SOURCE) return entry.filePath; - const handlerSymbolId = routeHandlerSymbols.get(routeKey); - const resolvedPath = handlerSymbolId - ? ctx.graph.getNode(handlerSymbolId)?.properties.filePath - : undefined; - return typeof resolvedPath === 'string' ? resolvedPath : entry.filePath; - }; - const handlerPaths = [...routeRegistry].map(([key, entry]) => handlerPathFor(key, entry)); - handlerContents = await readFileContents(ctx.repoPath, handlerPaths); + // Resolve once so content attribution, the route stamp, and the edge use + // the same live graph node. Pre-seeded routes never own handler symbols. + const routes = [...routeRegistry].map(([routeKey, entry]) => { + const id = preSeededKeys.has(routeKey) ? undefined : routeHandlerSymbols.get(routeKey); + const node = id ? ctx.graph.getNode(id) : undefined; + const handlerSymbol = id && node ? { id, node } : undefined; + const resolvedPath = + entry.source === DATA_ROUTE_TABLE_SOURCE + ? handlerSymbol?.node.properties.filePath + : undefined; + const handlerPath = typeof resolvedPath === 'string' ? resolvedPath : entry.filePath; + return { routeKey, entry, handlerSymbol, handlerPath }; + }); + handlerContents = await readFileContents( + ctx.repoPath, + routes.map(({ handlerPath }) => handlerPath), + ); - for (const [routeKey, entry] of routeRegistry) { + for (const { routeKey, entry, handlerSymbol, handlerPath } of routes) { const { source: routeSource, method: routeMethod, url } = entry; - const handlerPath = handlerPathFor(routeKey, entry); const content = handlerContents.get(handlerPath); - const handlerSymbolId = routeHandlerSymbols.get(routeKey); + const handlerSymbolId = handlerSymbol?.id; const analysisContent = entry.source === DATA_ROUTE_TABLE_SOURCE && content - ? handlerSymbolContent( - content, - handlerSymbolId ? ctx.graph.getNode(handlerSymbolId) : undefined, - ) + ? handlerSymbolContent(content, handlerSymbol?.node) : content; const { responseKeys, errorKeys } = analysisContent @@ -353,6 +396,19 @@ export const routesPhase: PipelinePhase = { confidence: 1.0, reason: routeSource, }); + + // Keep the file edge for existing extractor queries; add the live + // definition edge for explicit handler-level traversal. + if (handlerSymbolId) { + ctx.graph.addRelationship({ + id: generateId('HANDLES_ROUTE', `${handlerSymbolId}->${routeNodeId}`), + sourceId: handlerSymbolId, + targetId: routeNodeId, + type: 'HANDLES_ROUTE', + confidence: 1.0, + reason: routeSource, + }); + } } if (isDev) { diff --git a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts index 0bfc45bd4..da8e4dd8f 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/runner.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/runner.ts @@ -16,23 +16,36 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; import { isDev } from '../utils/env.js'; import { logger } from '../../logger.js'; + +function assertUniquePhaseNames(phases: readonly PipelinePhase[]): void { + const seen = new Set(); + for (const phase of phases) { + if (seen.has(phase.name)) { + throw new Error(`Duplicate phase name: '${phase.name}'`); + } + seen.add(phase.name); + } +} + /** * Validate that the phases form a valid dependency graph (no cycles, all deps present). * Returns phases in topological execution order. + * + * `satisfied` names phases whose results are already available (a deferred + * follow-up run over the same context, #3016). Their edges are dropped rather + * than validated, because they are resolved by definition. */ -function topologicalSort(phases: readonly PipelinePhase[]): PipelinePhase[] { - const phaseMap = new Map(); - for (const phase of phases) { - if (phaseMap.has(phase.name)) { - throw new Error(`Duplicate phase name: '${phase.name}'`); - } - phaseMap.set(phase.name, phase); - } +function topologicalSort( + phases: readonly PipelinePhase[], + satisfied: ReadonlySet = new Set(), +): PipelinePhase[] { + assertUniquePhaseNames(phases); + const phaseMap = new Map(phases.map((p) => [p.name, p])); // Validate all deps exist for (const phase of phases) { for (const dep of phase.deps) { - if (!phaseMap.has(dep)) { + if (!phaseMap.has(dep) && !satisfied.has(dep)) { throw new Error(`Phase '${phase.name}' depends on '${dep}', which is not registered`); } } @@ -43,8 +56,9 @@ function topologicalSort(phases: readonly PipelinePhase[]): PipelinePhase[] { const reverseDeps = new Map(); for (const phase of phases) { - inDegree.set(phase.name, phase.deps.length); - for (const dep of phase.deps) { + const pendingDeps = phase.deps.filter((dep) => !satisfied.has(dep)); + inDegree.set(phase.name, pendingDeps.length); + for (const dep of pendingDeps) { let rev = reverseDeps.get(dep); if (!rev) { rev = []; @@ -143,15 +157,30 @@ function findCyclePath( * * @param phases All phases to execute (order doesn't matter — sorted internally) * @param ctx Shared pipeline context + * @param seed Results of phases that already ran against this same context, + * available to `phases` as dependencies (#3016 deferred derived + * phases). Included in the returned map. * @returns Map of phase name → PhaseResult (all completed phases) */ export async function runPipeline( phases: readonly PipelinePhase[], ctx: PipelineContext, + seed?: ReadonlyMap>, ): Promise>> { + // A seeded phase has already run against this context; re-running it would + // apply its graph writes a second time. "Already ran" is the whole meaning of + // the seed, so honour it here rather than making every caller pre-filter. + const satisfied = new Set(seed?.keys() ?? []); let sorted: PipelinePhase[]; try { - sorted = topologicalSort(phases); + // Duplicate names must be rejected on the caller-supplied list *before* + // seed-filtering. Filtering first would drop a seeded duplicate and let + // `topologicalSort` see a unique name (#3102). + assertUniquePhaseNames(phases); + sorted = topologicalSort( + phases.filter((p) => !satisfied.has(p.name)), + satisfied, + ); } catch (err) { // Emit a terminal 'error' progress event for graph-validation failures // (cycle detected, duplicate phase, missing dep) so CLI/MCP consumers see @@ -171,7 +200,7 @@ export async function runPipeline( } throw err; } - const results = new Map>(); + const results = new Map>(seed); for (const phase of sorted) { const start = Date.now(); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/scan.ts b/gitnexus/src/core/ingestion/pipeline-phases/scan.ts index 5a1353267..f41c4f84e 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/scan.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/scan.ts @@ -12,6 +12,8 @@ import type { PipelinePhase, PipelineContext } from './types.js'; import { walkRepositoryPaths } from '../filesystem-walker.js'; +import fs from 'node:fs/promises'; +import path from 'node:path'; export interface ScanOutput { scannedFiles: { path: string; size: number }[]; @@ -19,6 +21,104 @@ export interface ScanOutput { totalFiles: number; } +const SPRING_ACTUATOR_ENDPOINT_FILES = new Set([ + 'mappings.json', + 'beans.json', + 'conditions.json', + 'configprops.json', + 'env.json', +]); + +/** + * Runtime snapshots are external analysis inputs, not repository source. When + * a configured input lives below the repository root, exclude it before any + * downstream phase reads file contents. This is especially important for + * Actuator env/configprops payloads: their values must never become File-node + * content or enter FTS merely because the snapshot directory is in the repo. + */ +type CompiledActuatorExclusion = + | { kind: 'repo-root-endpoints' } + | { kind: 'dir'; resolved: string }; + +async function canonicalPath(filePath: string): Promise { + return fs.realpath(filePath).catch(() => path.resolve(filePath)); +} + +async function compileActuatorExclusions( + repoPath: string, + inputPaths: readonly string[], +): Promise<{ repo: string; exclusions: CompiledActuatorExclusion[] }> { + const repo = await canonicalPath(repoPath); + const lexicalRepo = path.resolve(repoPath); + const compiled: CompiledActuatorExclusion[] = []; + const seen = new Set(); + for (const inputPath of inputPaths) { + const lexicalInput = path.resolve(repoPath, inputPath); + const lexicalRelative = path.relative(lexicalRepo, lexicalInput); + const canonicalInput = await canonicalPath(lexicalInput); + const candidateInputs = new Set([canonicalInput]); + if ( + lexicalRelative === '' || + (lexicalRelative !== '..' && + !lexicalRelative.startsWith(`..${path.sep}`) && + !path.isAbsolute(lexicalRelative)) + ) { + // Preserve the configured in-repo alias as an exclusion even when its + // real target is outside the repository. + candidateInputs.add(path.resolve(repo, lexicalRelative)); + } + for (const input of candidateInputs) { + const inputRelativeToRepo = path.relative(repo, input); + if (inputRelativeToRepo === '') { + if (seen.has('')) continue; + seen.add(''); + compiled.push({ kind: 'repo-root-endpoints' }); + continue; + } + if ( + inputRelativeToRepo === '..' || + inputRelativeToRepo.startsWith(`..${path.sep}`) || + path.isAbsolute(inputRelativeToRepo) + ) { + continue; + } + if (seen.has(input)) continue; + seen.add(input); + compiled.push({ kind: 'dir', resolved: input }); + } + } + return { repo, exclusions: compiled }; +} + +function matchesActuatorExclusion( + canonicalRepoPath: string, + filePath: string, + exclusions: readonly CompiledActuatorExclusion[], +): boolean { + if (exclusions.length === 0) return false; + const candidate = path.resolve(canonicalRepoPath, filePath); + for (const exclusion of exclusions) { + if (exclusion.kind === 'repo-root-endpoints') { + const candidateRelativeToRepo = path.relative(canonicalRepoPath, candidate); + if ( + path.dirname(candidateRelativeToRepo) === '.' && + SPRING_ACTUATOR_ENDPOINT_FILES.has(path.basename(candidateRelativeToRepo).toLowerCase()) + ) { + return true; + } + continue; + } + const relative = path.relative(exclusion.resolved, candidate); + if ( + relative === '' || + (relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)) + ) { + return true; + } + } + return false; +} + export const scanPhase: PipelinePhase = { name: 'scan', deps: [], @@ -30,20 +130,47 @@ export const scanPhase: PipelinePhase = { message: 'Scanning repository...', }); - const scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => { - const scanProgress = Math.round((current / total) * 15); - ctx.onProgress({ - phase: 'extracting', - percent: scanProgress, - message: 'Scanning repository...', - detail: filePath, - stats: { - filesProcessed: current, - totalFiles: total, - nodesCreated: ctx.graph.nodeCount, - }, + const { repo: canonicalRepoPath, exclusions: actuatorExclusions } = + await compileActuatorExclusions(ctx.repoPath, [ + ...(ctx.options?.springActuatorPath === undefined ? [] : [ctx.options.springActuatorPath]), + ...(ctx.options?.springActuatorScanExclusions ?? []), + ]); + let scannedFiles; + try { + scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => { + const scanProgress = Math.round((current / total) * 15); + const isRuntimeInput = matchesActuatorExclusion( + canonicalRepoPath, + filePath, + actuatorExclusions, + ); + ctx.onProgress({ + phase: 'extracting', + percent: scanProgress, + message: 'Scanning repository...', + ...(isRuntimeInput ? {} : { detail: filePath }), + stats: { + filesProcessed: current, + totalFiles: total, + nodesCreated: ctx.graph.nodeCount, + }, + }); }); - }); + } catch (err) { + // Missing roots throw so status cannot treat an empty glob as "every + // covered file was deleted". The pipeline still reports an empty scan + // for a path that is not a directory, matching analyze of a bad cwd. + if (err instanceof Error && err.message.startsWith('walkRepositoryPaths:')) { + return { scannedFiles: [], allPaths: [], totalFiles: 0 }; + } + throw err; + } + + if (actuatorExclusions.length > 0) { + scannedFiles = scannedFiles.filter( + (file) => !matchesActuatorExclusion(canonicalRepoPath, file.path, actuatorExclusions), + ); + } const totalFiles = scannedFiles.length; const allPaths = scannedFiles.map((f) => f.path); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/spring-auto-configuration.ts b/gitnexus/src/core/ingestion/pipeline-phases/spring-auto-configuration.ts index a578bfa0d..ba0bad613 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/spring-auto-configuration.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/spring-auto-configuration.ts @@ -21,6 +21,11 @@ import { SPRING_AUTO_CONFIGURATION_IMPORT_REASON, SPRING_AUTO_CONFIGURATION_SYNTHETIC_DESCRIPTION, } from '../frameworks/spring/auto-configuration.js'; +import { + importSpringActuatorRuntime, + MAX_RUNTIME_RECORDS, + type SpringActuatorImportStats, +} from '../frameworks/spring/actuator-runtime.js'; import { isDev } from '../utils/env.js'; import type { StructureOutput } from './structure.js'; import type { PipelineContext, PipelinePhase, PhaseResult } from './types.js'; @@ -74,6 +79,7 @@ export interface SpringAutoConfigurationOutput { readonly metadataFiles: number; readonly autoConfigurations: number; readonly ambiguousAutoConfigurations: number; + readonly actuatorRuntime?: SpringActuatorImportStats; } export function classifySpringAutoConfigurationMetadata( @@ -305,10 +311,25 @@ export const springAutoConfigurationPhase: PipelinePhase 0) { + logger.warn( + `Spring Actuator runtime import reached the ${MAX_RUNTIME_RECORDS.toLocaleString('en-US')}-record limit for: ${actuatorRuntime.truncatedEndpoints.join(', ')}. Runtime evidence is incomplete.`, + ); + } + return { metadataFiles, autoConfigurations, ambiguousAutoConfigurations: ambiguousQualifiedNames.size, + ...(actuatorRuntime === undefined ? {} : { actuatorRuntime }), }; }, }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts b/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts new file mode 100644 index 000000000..0106eaa98 --- /dev/null +++ b/gitnexus/src/core/ingestion/pipeline-phases/spring-destinations.ts @@ -0,0 +1,565 @@ +/** + * Phase: springDestinations + * + * Materializes Spring async messaging as graph structure: a `Destination` node + * per broker address, `CONSUMES_FROM` from every `@KafkaListener`-family + * handler, and `PUBLISHES_TO` from every messaging-template publish. The + * inbound and outbound facts are captured during parse and survive the parse + * cache; until now nothing read them. + * + * Shaped after `Route` + `HANDLES_ROUTE` in `routes.ts` — a framework overlay + * node keyed by what it names, with the callable pointing at it, down to the + * detail that the key pairs the address with the one dimension that can make + * two same-named things different places: the method for a Route, the broker + * here. + * + * ── THE KEYING RULE, WHICH IS THE POINT OF THE PHASE ───────────────────── + * + * A `Destination` connects two services precisely because both sides mint the + * SAME node id from the SAME address on the SAME broker. That is the whole + * value, and it is also the whole hazard: an address that could not be resolved + * must never be allowed to key a node. + * + * resolved id = generateId('Destination', `
`) `address` present + * unresolved id = generateId('Destination', ) `address` ABSENT + * + * Two unrelated services that each merely write `@KafkaListener(topics = + * "${app.topic}")` have said nothing whatever about each other. Keyed on the + * placeholder text they would land on one node and READ AS CONNECTED, in a + * report, as a fact. A missing edge is visible as a gap; a false one is not. + * + * A status property would not have prevented this, and neither would a second + * label: the id is what merges the nodes, and both sides would still compute + * the same id. Only the KEY prevents it, so an unresolved destination is keyed + * by its source LOCATION — a value no second file can produce. + * + * The same rule governs the `address` PROPERTY, which is the join key a + * cross-repository pass would match on. It is written only when resolved. An + * absent property cannot match another absent property, so the structural + * guarantee survives being read back out of the database. The unresolved + * spelling is kept in `name`, for a human reading the node. + * + * The BROKER is part of the connecting key rather than a reason to withdraw + * one — see {@link destinationNodeKey}, which owns that argument and the + * evidence for it. Two brokers claiming one address is therefore an ordinary + * two-node situation here, exactly like `GET /x` and `POST /x`, and this phase + * needs no vocabulary for it: nothing is being taken away, so there is nothing + * to diagnose. + * + * `name` must never be used to join two destinations, and nothing does — but + * the stronger claim that nothing reads it at all would be false. `Destination` + * is in `VALID_NODE_LABELS`, and `mcp/local/local-backend.ts` resolves a + * symbol with an unlabeled `WHERE n.name = $symName`, so a destination can be + * returned by name like any other node. That is a lookup, not a join: it + * matches a caller-supplied string against one node, never one destination + * against another, so it cannot manufacture the connection this phase exists to + * prevent. + * + * @deps parse, scopeResolution, springConfig + * @reads Spring messaging capture facts, Method/Function nodes, Property nodes + * @writes Destination nodes; CONSUMES_FROM / PUBLISHES_TO / USES edges + */ + +import type { GraphNode, Range } from 'gitnexus-shared'; +import { generateId } from '../../../lib/utils.js'; +import { logger } from '../../logger.js'; +import type { KnowledgeGraph } from '../../graph/types.js'; +import { SPRING_CONFIG_DESCRIPTION } from '../frameworks/spring/config-bindings.js'; +import { + parseSpringStringLiteral, + resolveSpringDestination, + selectConsumerDestinationArguments, + selectProducerDestinationArguments, + type SpringDestinationCandidate, + type SpringDestinationRefusal, + type SpringDestinationResolution, + type SpringDestinationSelection, +} from '../frameworks/spring/destinations.js'; +import { destinationNodeKey } from '../destination-key.js'; +import { getProviderForFile } from '../languages/index.js'; +import { isDev } from '../utils/env.js'; +import type { ModuleConstants } from '../route-extractors/constant-resolver.js'; +import type { ParseOutput } from './parse.js'; +import type { PipelineContext, PipelinePhase, PhaseResult } from './types.js'; +import { getPhaseOutput } from './types.js'; + +export interface SpringDestinationsOutput { + /** Destination nodes keyed by `(broker, address)`, and which therefore + * connect to every other site that named the same address on the same + * broker. */ + readonly resolvedDestinations: number; + /** Destination nodes keyed by source location, and therefore unable to + * connect: the address did not resolve. */ + readonly unresolvedDestinations: number; + /** CONSUMES_FROM + PUBLISHES_TO edges emitted. */ + readonly edges: number; + /** + * Every refusal, counted by reason. This is the phase's real measure: the + * feature is judged on the unresolved FRACTION, and a silent skip would hide + * exactly the number that says whether it works. + */ + readonly refusalsByReason: Readonly>; + /** Destination -> Property provenance edges for `${key}` placeholders. */ + readonly configKeyLinks: number; +} + +/** + * Exact-range index of callable nodes, mirroring the bridge in + * `non-http-handlers.ts`. + * + * A duplicate range maps to `null` rather than to one of its nodes: two + * callables sharing a span means the index cannot say which one publishes, and + * attributing the publish to an arbitrary one of them is the failure this + * phase is least able to detect afterwards. The File-level edge below is the + * fallback, so a `null` here costs precision, not the fact. + */ +function callableOwnersByRange(graph: KnowledgeGraph): ReadonlyMap { + const owners = new Map(); + for (const node of graph.iterNodes()) { + if ( + (node.label !== 'Method' && node.label !== 'Function') || + typeof node.properties.filePath !== 'string' + ) { + continue; + } + const key = `${node.properties.filePath}\0${node.properties.startLine}\0${node.properties.endLine}`; + owners.set(key, owners.has(key) ? null : node); + } + return owners; +} + +/** Capture ranges are 1-based; graph nodes carry 0-based lines. */ +function ownerKey(filePath: string, range: Range): string { + return `${filePath}\0${range.startLine - 1}\0${range.endLine - 1}`; +} + +/** + * Spring configuration `Property` nodes grouped by KEY. + * + * Deliberately a multimap. `spring-config.ts` keys a Property node per FILE + * (`spring-config::`), so one key declared in `application.yml` and + * again in `application-prod.yml` is TWO nodes. Linking to only the first would + * silently pin a destination to an arbitrary profile. An EMPTY match set is + * normal, not an error — a key may be supplied by an environment variable or a + * config server and never appear in a checked-in file at all. + */ +function springConfigPropertiesByKey(graph: KnowledgeGraph): ReadonlyMap { + const byKey = new Map(); + for (const node of graph.iterNodes()) { + if (node.label !== 'Property') continue; + const description = node.properties.description; + if (typeof description !== 'string' || !description.startsWith(SPRING_CONFIG_DESCRIPTION)) { + continue; + } + const key = node.properties.name; + if (typeof key !== 'string' || key === '') continue; + const existing = byKey.get(key); + if (existing === undefined) byKey.set(key, [node.id]); + else existing.push(node.id); + } + return byKey; +} + +/** + * Fold a constant reference against the harvested repo constants, using the + * owning provider's own fold when it declares one. + * + * Only providers that declare `extractModuleConstants` contribute to the table, + * so a language that harvests nothing simply resolves nothing here and the + * cascade records `unresolved-constant`. That is a countable gap, not a wrong + * answer. + */ +function makeConstantResolver( + filePath: string, + repo: ReadonlyMap, +): ((name: string) => string | null) | undefined { + if (repo.size === 0) return undefined; + const fold = getProviderForFile(filePath)?.foldRoutePathOperands; + if (fold === undefined) return undefined; + return (name: string): string | null => fold(filePath, [{ kind: 'ref', name }], repo); +} + +interface DestinationSite { + readonly filePath: string; + /** Owner callable's capture range, when the fact carried one. */ + readonly ownerRange?: Range; + /** Owner callable's scope id. Unique per callable even when the range is + * absent, which is the only reason an unresolved key is site-unique on the + * handler side — see {@link destinationNodeId}. */ + readonly ownerScopeId: string; + readonly candidate: SpringDestinationCandidate; + readonly resolution: SpringDestinationResolution; +} + +/** + * Identity for a destination node. + * + * The connecting key is `(broker, address)` and nothing else — not the file, + * not the site. That is what lets a publisher in one module and a subscriber in + * another meet on one node, which is the entire point, and it is minted by the + * framework-neutral {@link destinationNodeKey} so a non-Spring producer can mint + * the same identity without importing anything Spring. + * + * The broker is IN that key rather than a reason to withhold one. The argument + * for it, including the inferred-broker objection and why the previous rule was + * worse, lives on `destinationNodeKey` — one copy, next to the code that + * decides it. + * + * The site key has to identify the site EXACTLY. It carries the file path, so + * no second file can ever produce it — that is the cross-repository guarantee, + * and nothing added below can weaken it. Everything else in the key is there to + * keep two sites inside ONE file apart, which is the same false identity at a + * smaller scale: + * + * - the owner SCOPE ID, because two callables can start on the same line and + * because `ownerRange` is optional on a handler fact — keyed on the line + * alone, a file whose handlers carried no range collapsed every consumer in + * it onto line 0; + * - the full owner RANGE, which separates two sites the scope id cannot (a + * scope id is stable, but two callables that share one are still two + * callables); + * - the raw TEXT plus argument and element position, because two publishes to + * two different placeholders inside one method share the owner entirely. + * + * The residual: two publishes with identical text at identical argument + * positions inside ONE callable share a node. They are indistinguishable to + * this phase — a producer fact carries the owner's range, not the call's — and + * merging two publishes of the same unreadable address from one method is the + * one collapse that asserts nothing false about anybody. + */ +function destinationNodeId(site: DestinationSite): string { + if (site.resolution.kind === 'resolved') { + return generateId( + 'Destination', + destinationNodeKey(site.candidate.broker, site.resolution.address), + ); + } + const { candidate, ownerRange } = site; + const position = + ownerRange === undefined + ? 'no-range' + : `${ownerRange.startLine}:${ownerRange.startCol}:${ownerRange.endLine}:${ownerRange.endCol}`; + return generateId( + 'Destination', + [ + site.filePath, + site.ownerScopeId, + position, + candidate.role, + candidate.source, + candidate.argIndex, + candidate.elementIndex, + candidate.rawText, + ].join(':'), + ); +} + +/** + * Display spelling for a destination's `name`. + * + * A resolved destination's `name` is its address, bare. An unresolved one keeps + * the source text — but unquoted, so the two are spelled the same way. Keeping + * the quotes on one and not the other made the same value read differently + * depending on whether it resolved, for no gain to the human the property + * exists for. + */ +function destinationDisplayName(rawText: string): string { + return parseSpringStringLiteral(rawText) ?? rawText.trim(); +} + +function edgeReason(candidate: SpringDestinationCandidate): string { + const argument = candidate.argName ?? `arg${candidate.argIndex}`; + const element = `${argument}[${candidate.elementIndex}]`; + const exchange = candidate.exchange === undefined ? '' : ` exchange=${candidate.exchange}`; + return `spring-${candidate.source}:${element}${exchange}`; +} + +export const springDestinationsPhase: PipelinePhase = { + name: 'springDestinations', + // `parse` supplies the file list and the harvested constants; `scopeResolution` + // must have run so the Method/Function nodes exist AND so each provider's + // `applyCaptureSideChannel` has restored the messaging facts onto the main + // thread; `springConfig` must have run so the Property nodes a `${key}` + // placeholder links to are already in the graph. + deps: ['parse', 'scopeResolution', 'springConfig'], + + async execute( + ctx: PipelineContext, + deps: ReadonlyMap>, + ): Promise { + // `allPaths`, NOT `parsedFiles`. On any run with a storage path — which is + // every run of the CLI — worker-produced ParsedFiles are flushed to a disk + // store and `ParseOutput.parsedFiles` comes back EMPTY, with + // scope-resolution streaming them back per language. Iterating it therefore + // found nothing in production while every in-process test passed, because a + // direct pipeline call has no storage path and keeps them in memory. + // + // The fact stores are keyed by file path and are populated by the same + // streaming pass, so the path list is the right cursor for them anyway: it + // does not care how the ParsedFile got to scope resolution. + const { allPaths, moduleConstants } = getPhaseOutput(deps, 'parse'); + const refusalsByReason: Record = {}; + const countRefusal = (reason: SpringDestinationRefusal): void => { + refusalsByReason[reason] = (refusalsByReason[reason] ?? 0) + 1; + }; + + // ── Gather: facts → candidates → resolutions ────────────────────────── + const sites: DestinationSite[] = []; + for (const filePath of allPaths) { + const provider = getProviderForFile(filePath); + const facts = provider?.getSpringMessagingFacts?.(filePath); + if (facts === undefined) continue; + if (facts.handlers.length === 0 && facts.producers.length === 0) continue; + + // Built lazily and reused for the whole file: the fold state behind it is + // per-call, but resolving the provider and checking the table is not free + // and every candidate in the file wants the same closure. + const constant = makeConstantResolver(filePath, moduleConstants); + // The owning language's capability, not its name. In an interpolating + // language `"orders-$env"` is a runtime template rather than an address + // and `"${app.topic}"` is a template rather than a Spring placeholder; + // the resolver needs to know which regime it is in, and this phase must + // not learn which language that is (AGENTS.md — shared ingestion code + // plugs language behaviour in through provider hooks). + const interpolatesStringLiterals = provider?.interpolatesStringLiterals === true; + const record = ( + selection: SpringDestinationSelection, + ownerScopeId: string, + ownerRange: Range | undefined, + ): void => { + for (const refusal of selection.refusals) countRefusal(refusal.reason); + for (const candidate of selection.candidates) { + const resolution = resolveSpringDestination(candidate, { + constant, + interpolatesStringLiterals, + }); + if (resolution.kind === 'unresolved') countRefusal(resolution.reason); + sites.push({ + filePath, + ...(ownerRange === undefined ? {} : { ownerRange }), + ownerScopeId, + candidate, + resolution, + }); + } + }; + + for (const handler of facts.handlers) { + for (const annotation of handler.annotations) { + // A Kotlin use-site target describes a generated property element, not + // the callable, so its arguments are not this handler's. + if (annotation.useSiteTarget !== undefined) continue; + const selection = selectConsumerDestinationArguments(annotation.name, annotation.args); + if (selection === null) continue; + record(selection, String(handler.ownerScopeId), handler.ownerRange); + } + } + for (const producer of facts.producers) { + record( + selectProducerDestinationArguments(producer), + String(producer.ownerScopeId), + producer.ownerRange, + ); + } + } + if (sites.length === 0) { + return { + resolvedDestinations: 0, + unresolvedDestinations: 0, + edges: 0, + refusalsByReason, + configKeyLinks: 0, + }; + } + + // ── Emit ────────────────────────────────────────────────────────────── + // + // A single pass. There used to be a preliminary one that looked for an + // address claimed by two brokers so the emit could withdraw it from both — + // the disagreement had to be known before the first node was minted, + // because re-keying a node after it has grown an edge is the kind of work + // that is easy to get half-right. With the broker in the key there is + // nothing to decide up front: each site's identity is a function of that + // site alone, so no other site can change it and no lookahead is needed. + const owners = callableOwnersByRange(ctx.graph); + const configProperties = springConfigPropertiesByKey(ctx.graph); + const linkedConfigKeys = new Set(); + let resolvedDestinations = 0; + let unresolvedDestinations = 0; + let edges = 0; + let configKeyLinks = 0; + + for (const site of sites) { + const { candidate, resolution } = site; + // The one predicate the rest of the loop is written against: may this + // site's node be keyed by its address, and therefore meet another site on + // it? Exactly when the address resolved. Otherwise the node gets its + // location-based key, no `address` property, and its own file. + const connects = resolution.kind === 'resolved'; + const nodeId = destinationNodeId(site); + const isNew = ctx.graph.getNode(nodeId) === undefined; + if (isNew) { + if (connects) resolvedDestinations += 1; + else unresolvedDestinations += 1; + ctx.graph.addNode({ + id: nodeId, + label: 'Destination', + properties: { + // For a resolved address this equals `address`. For an unresolved + // one it is the UNRESOLVED SPELLING, unquoted, kept so a human + // reading the node sees what the source actually said. Either way + // it is kept out of `address` unless the node connects, so nothing + // joins on it. + // + // Note that `name` is the ADDRESS, not the node key: two nodes on + // one spelling over two brokers share a `name` and differ by id. + // That is deliberate — `name` is for a human, and telling them the + // topic is called `kafka orders` would be a lie. + name: + resolution.kind === 'resolved' + ? resolution.address + : destinationDisplayName(candidate.rawText), + // A CONNECTING destination carries NO location, and that is load + // bearing rather than cosmetic. + // + // It is shared by every site that names the address, so no single + // file identifies it — but more importantly, the incremental + // writeback deletes by location: `deleteNodesForFiles` issues + // `MATCH (n:) WHERE n.filePath IN [...] DETACH DELETE n` for + // every changed file. Stamping the first-seen file here would make + // a shared destination collateral damage whenever THAT file + // changed, and DETACH DELETE would take its edges from every OTHER + // file with it. Those files are not in the write set, so their + // edges would never be rebuilt: a publisher and a subscriber that + // genuinely agree on an address would silently stop being + // connected, depending on which of them the indexer happened to + // walk first. Omitting the property makes the `IN` predicate unable + // to match, so the node survives the writeback and every referrer + // keeps its edge. + // + // The cost used to be the opposite error — a destination whose + // last referrer was deleted lingered as an edgeless orphan until a + // full rebuild — and that is no longer paid. Because the per-file + // predicate can neither remove such a node nor admit a newly + // introduced one, the whole layer is instead delete-alled + // (`deleteAllDestinations`) and re-included graph-wide + // (`isGraphWideNode`) on every incremental writeback. Both halves + // move together: the delete without the re-include drops the layer, + // and the re-include without the delete duplicates every edge. + // + // A NON-CONNECTING destination is the opposite case — it belongs to + // exactly one site, its id already says so, and it SHOULD be + // deleted and re-created with its file. + // `''`, not absent: `NodeProperties.filePath` is required, and the + // empty string is the established spelling for a node with no file + // (`pipeline-phases/communities.ts` does the same). It is equally + // unmatchable by the `IN` predicate and the CSV writes it as an + // empty field, which COPY loads as NULL. + ...(connects + ? { filePath: '' } + : { + filePath: site.filePath, + ...(site.ownerRange === undefined + ? {} + : { + startLine: site.ownerRange.startLine - 1, + endLine: site.ownerRange.endLine - 1, + }), + }), + // `address` IS THE JOIN KEY and is written ONLY when the node + // connects. See the module header: an absent property cannot match + // another absent property, so the structural guarantee survives + // being read back out of the database. + // + // `resolution` always says how the node got here — the provenance + // of a real address, or the named refusal that stopped one. Every + // value in the column now comes from the resolver's own closed + // vocabulary, because the phase no longer has a verdict of its own + // to record: nothing is withdrawn here. + ...(resolution.kind === 'resolved' + ? { address: resolution.address, resolution: resolution.via } + : { resolution: resolution.reason }), + ...(resolution.kind === 'unresolved' && resolution.configKey !== undefined + ? { configKey: resolution.configKey } + : {}), + // The `${key:default}` default text. Kept because the source wrote + // it and throwing it away would make an overridable default + // indistinguishable from a bare `${key}`; NOT an address and never + // part of the id, because configuration can override it and this + // graph cannot see whether it did. + ...(resolution.kind === 'unresolved' && resolution.configDefault !== undefined + ? { configDefault: resolution.configDefault } + : {}), + broker: candidate.broker, + }, + }); + } + + // Link a placeholder's KEY to the configuration entries that could supply + // it — `${key}` and `${key:default}` alike, since a default changes + // nothing about where the real value comes from. This is PROVENANCE, not + // resolution: the node stays unresolved and keeps its location-based id + // even when Property nodes are found, because the VALUE is still not in + // the graph and letting a Property sighting upgrade the node would + // reintroduce the false connection the keying rule exists to prevent. + // `${}` names no key at all and reports none, so nothing is ever looked + // up under the empty string. + if (resolution.kind === 'unresolved' && resolution.configKey !== undefined) { + for (const propertyId of configProperties.get(resolution.configKey) ?? []) { + const linkId = `${nodeId}->${propertyId}`; + if (linkedConfigKeys.has(linkId)) continue; + linkedConfigKeys.add(linkId); + ctx.graph.addRelationship({ + id: generateId('USES', linkId), + sourceId: nodeId, + targetId: propertyId, + type: 'USES', + confidence: 1.0, + reason: `spring-destination:config-key:${resolution.configKey}`, + }); + configKeyLinks += 1; + } + } + + // One edge per address. An array-valued `topics` really does subscribe to + // several places, and each gets its own edge rather than a group node, so + // "who reads from `a`" stays one hop. `reason` carries which argument and + // which element it came from. + const type = candidate.role === 'consumer' ? 'CONSUMES_FROM' : 'PUBLISHES_TO'; + const owner = + site.ownerRange === undefined + ? undefined + : (owners.get(ownerKey(site.filePath, site.ownerRange)) ?? undefined); + // Prefer the callable; fall back to its File when the owner is unknown or + // ambiguous. Unlike `routes.ts` this does NOT emit both — there is no + // legacy File-level consumer to keep working here, and a second edge per + // publish would double the async surface of the graph for no query. + const sourceId = owner?.id ?? generateId('File', site.filePath); + if (owner === undefined && ctx.graph.getNode(sourceId) === undefined) continue; + const reason = edgeReason(candidate); + ctx.graph.addRelationship({ + id: generateId(type, `${sourceId}->${nodeId}:${reason}`), + sourceId, + targetId: nodeId, + type, + confidence: 1.0, + reason, + }); + edges += 1; + } + + if (isDev) { + logger.info( + `📮 Spring destinations: ${resolvedDestinations} resolved, ${unresolvedDestinations} unresolved, ${edges} edges`, + ); + } + + return { + resolvedDestinations, + unresolvedDestinations, + edges, + refusalsByReason, + configKeyLinks, + }; + }, +}; diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 34ecfc111..00892d07c 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -35,6 +35,7 @@ import { springConfigPhase, springAutoConfigurationPhase, springAopPhase, + springDestinationsPhase, springAopInheritancePhase, pruneLocalSymbolsPhase, taintSummariesPhase, @@ -46,6 +47,7 @@ import { PhaseRegistry, type ScopeResolutionOutput, type PipelinePhase, + type PipelineContext, type CommunitiesOutput, type ProcessesOutput, } from './pipeline-phases/index.js'; @@ -58,6 +60,20 @@ export interface PipelineOptions { * to retain those nodes under `skipGraphPhases`. */ skipGraphPhases?: boolean; + /** + * Skip only Leiden community detection and process/flow extraction (#3016). + * MRO/DI still run. Used on warm incremental analyze so persisted + * Community/Process rows can be kept instead of wipe+rewrite. + */ + skipDerivedGraphPhases?: boolean; + /** + * Explicit local Spring Boot Actuator snapshot input. Accepts a directory + * containing endpoint-named JSON files or a JSON bundle keyed by endpoint. + * Undefined keeps runtime enrichment completely disabled. + */ + springActuatorPath?: string; + /** Repo-relative Actuator inputs retained only for a cleanup scan. */ + springActuatorScanExclusions?: readonly string[]; /** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */ springAopMaxCandidateInspectionsPerAdvice?: number; /** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */ @@ -272,7 +288,8 @@ export interface PipelineOptions { * Phase dependency graph: * * scan → structure → [springConfig, markdown, cobol] → parse → [routes, tools, orm] - * → crossFile → scopeResolution → [springAutoConfiguration, springAop] → pruneLocalSymbols + * → crossFile → scopeResolution → [springAutoConfiguration, springAop, + * springDestinations] → pruneLocalSymbols * → mro → springAopInheritance → di → communities → processes * * To add a new phase: create a file in pipeline-phases/, export the phase @@ -301,6 +318,11 @@ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { .register(scopeResolutionPhase) .register(springAutoConfigurationPhase) .register(springAopPhase) + // Async messaging overlay. Must follow scopeResolution twice over: the + // owner Method/Function nodes have to exist, and each provider's + // `applyCaptureSideChannel` has to have restored the messaging facts onto + // the main thread. It also reads the Property nodes springConfig emits. + .register(springDestinationsPhase) .register(pruneLocalSymbolsPhase) // M4 (#2084): interprocedural taint fixpoint — the first real opt-in // pdg-gated phase. Off ⇒ absent ⇒ byte-identical graph. No always-on @@ -310,8 +332,12 @@ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { .register(mroPhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(springAopInheritancePhase, { enabledWhen: (o) => !o.skipGraphPhases }) .register(diPhase, { enabledWhen: (o) => !o.skipGraphPhases }) - .register(communitiesPhase, { enabledWhen: (o) => !o.skipGraphPhases }) - .register(processesPhase, { enabledWhen: (o) => !o.skipGraphPhases }) + .register(communitiesPhase, { + enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true, + }) + .register(processesPhase, { + enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true, + }) // Normalize a missing options object once here so phase predicates above // take a required PipelineOptions and need no `?.` guard (#2080 review S1). .build(options ?? {}) @@ -351,18 +377,19 @@ export const runPipelineFromRepo = async ( } const phases = buildPhaseList(options); + const ctx: PipelineContext = { + repoPath, + graph: graphEmitSink ?? graph, + onProgress, + options, + pipelineStart, + graphEmit: graphEmitSink, + }; let graphEmitManifest: GraphEmitManifest | undefined; let results; try { - results = await runPipeline(phases, { - repoPath, - graph: graphEmitSink ?? graph, - onProgress, - options, - pipelineStart, - graphEmit: graphEmitSink, - }); + results = await runPipeline(phases, ctx); graphEmitManifest = graphEmitSink?.finalize(); } finally { // Release per-pair fds when the pipeline threw before finalize ran. @@ -370,14 +397,18 @@ export const runPipelineFromRepo = async ( } // Extract final results for the PipelineResult contract - const { totalFiles, usedWorkerPool } = getPhaseOutput<{ - totalFiles: number; - usedWorkerPool: boolean; - }>(results, 'parse'); + const { totalFiles, usedWorkerPool, reparsedFileCount, unavailableScopeLanguageFiles } = + getPhaseOutput<{ + totalFiles: number; + usedWorkerPool: boolean; + reparsedFileCount: number; + unavailableScopeLanguageFiles: number; + }>(results, 'parse'); let communityResult: CommunitiesOutput['communityResult'] | undefined; let processResult: ProcessesOutput['processResult'] | undefined; const scopeResolutionOutput = getPhaseOutput(results, 'scopeResolution'); + const scopeExtractionFailures = scopeResolutionOutput.scopeExtractionFailures; const resolutionOutcomes = scopeResolutionOutput.resolutionOutcomes; const undecidedSatisfaction = scopeResolutionOutput.undecidedSatisfaction; // Streamed PDG-emit manifest (#2202): present only when streaming was on. @@ -408,7 +439,7 @@ export const runPipelineFromRepo = async ( }, }); - return { + const result: PipelineResult = { // The RAW graph, deliberately — NOT `graphEmitSink`. Phases above received // the sink so their reads are complete, but `loadGraphToLbug` feeds this to // `streamAllCSVsToDisk`, and the sink's complete iterator would then emit @@ -424,7 +455,45 @@ export const runPipelineFromRepo = async ( resolutionOutcomes, undecidedSatisfaction, usedWorkerPool, + reparsedFileCount, + scopeExtractionFailures, + unavailableScopeLanguageFiles, pdgEmitManifest, propertyInference, }; + + // #3016: hand back a way to run the derived phases `skipDerivedGraphPhases` + // held back. Which phases those are is answered by re-asking the registry + // with only that flag cleared — the one form of the question that stays + // correct when a different predicate (`skipGraphPhases`) also disables them, + // since then they are absent for a reason a deferred run cannot fix and the + // filter yields nothing. The sink guard mirrors the `graph` note above: a + // streaming run is a full rebuild, which never sets the skip flag, so an + // active sink here means the two got combined by mistake — and deferred + // phases writing into a finalized sink would emit past its manifest. + const deferredDerivedPhases = + options?.skipDerivedGraphPhases === true && graphEmitSink === undefined + ? buildPhaseList({ ...options, skipDerivedGraphPhases: false }).filter( + (p) => (p.name === 'communities' || p.name === 'processes') && !results.has(p.name), + ) + : []; + + if (deferredDerivedPhases.length > 0) { + result.runDeferredDerivedPhases = async () => { + const derived = await runPipeline(deferredDerivedPhases, ctx, results); + // Presence-checked for the same reason as the block above: a phase the + // registry filtered out is absent, and `getPhaseOutput` throws on absent. + if (derived.has('communities')) { + result.communityResult = getPhaseOutput( + derived, + 'communities', + ).communityResult; + } + if (derived.has('processes')) { + result.processResult = getPhaseOutput(derived, 'processes').processResult; + } + }; + } + + return result; }; diff --git a/gitnexus/src/core/ingestion/route-extractors/constant-resolver.ts b/gitnexus/src/core/ingestion/route-extractors/constant-resolver.ts index 509d1de99..8605f5a02 100644 --- a/gitnexus/src/core/ingestion/route-extractors/constant-resolver.ts +++ b/gitnexus/src/core/ingestion/route-extractors/constant-resolver.ts @@ -33,7 +33,7 @@ const MAX_RESOLVE_DEPTH = 8; * whose true value is genuinely huge — building it risks a `RangeError`/heap OOM, * so we floor to `null` (skip) instead (#2393). The depth cap bounds recursion but * NOT output size, which grows multiplicatively; this bounds the output. */ -const MAX_FOLD_LENGTH = 8192; +export const MAX_FOLD_LENGTH = 8192; /** * One term of a constant's right-hand side. A `+`-concatenation @@ -66,6 +66,30 @@ export interface ModuleConstants { readonly literals: Map; readonly exprs: Map; readonly imports: Map; + /** + * On-demand (wildcard) import specifiers whose bound member names could not + * be enumerated at extract time — Java `import static a.b.C.*;`, Python + * `from m import *`. The agnostic fold never reads this (it has no way to + * enumerate a target module's exports); a language binding materializes the + * promised bindings from a repo-wide map after extraction — see + * `expandJavaWildcardStaticImports` in the Java binding — so they resolve + * through the plain `imports` path with no special cases in the fold. + */ + readonly wildcardImports?: readonly string[]; +} + +const NO_UNFOLDABLE_DECLARATIONS: ReadonlySet = new Set(); + +/** + * Declaration keys a language extractor found but could not fold. Java and + * Kotlin both use this metadata to keep lower-priority imports from replacing + * a real local declaration; other producers simply return the empty set. + */ +export function unfoldableDeclarationsOf(mc: ModuleConstants | undefined): ReadonlySet { + const declarations = ( + mc as (ModuleConstants & { readonly unfoldableDeclarations?: unknown }) | undefined + )?.unfoldableDeclarations; + return declarations instanceof Set ? declarations : NO_UNFOLDABLE_DECLARATIONS; } /** Repo-wide map: unique file key (e.g. `app/constants.py`) → that file's @@ -175,10 +199,18 @@ function computeFold( return null; } -function newState(repo: RepoConstants, resolveImport: ImportResolver): ResolveState { +function newState( + repo: RepoConstants, + resolveImport: ImportResolver, + repoKeys?: ReadonlySet, +): ResolveState { return { repo, - repoKeys: new Set(repo.keys()), + // Materializing the key set here is O(files), and this runs once per fold — + // which is once per import hop, not once per scan. A binding that already + // holds the set (every one of them does; it is a projection of the same map + // it builds `repo` from) passes it in and skips the copy entirely. + repoKeys: repoKeys ?? new Set(repo.keys()), resolveImport, visited: new Set(), memo: new Map(), @@ -195,8 +227,9 @@ export function resolveConstant( name: string, repo: RepoConstants, resolveImport: ImportResolver, + repoKeys?: ReadonlySet, ): string | null { - return foldName(fileKey, name, newState(repo, resolveImport), 0); + return foldName(fileKey, name, newState(repo, resolveImport, repoKeys), 0); } /** @@ -209,6 +242,7 @@ export function resolveOperands( operands: readonly Operand[], repo: RepoConstants, resolveImport: ImportResolver, + repoKeys?: ReadonlySet, ): string | null { - return foldExpr(fileKey, operands, newState(repo, resolveImport), 0); + return foldExpr(fileKey, operands, newState(repo, resolveImport, repoKeys), 0); } diff --git a/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts b/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts index 053d207fe..bcc879e3c 100644 --- a/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts +++ b/gitnexus/src/core/ingestion/route-extractors/data-route-table.ts @@ -116,7 +116,14 @@ function decodeJavaScriptStringLiteral(raw: string): string | null { return decoded; } -function plainString(node: SyntaxNode): string | null { +/** + * A `string`/`template_string` node's decoded value, or `null` when it is not a + * readable literal (an interpolated template, an unterminated escape). Shared + * with the NestJS extractor so both agree on what a readable literal is — + * notably that escapes must be DECODED, not dropped, because tree-sitter splits + * a literal around every `escape_sequence`. + */ +export function plainString(node: SyntaxNode): string | null { if (node.type === 'string') return decodeJavaScriptStringLiteral(node.text); if ( node.type === 'template_string' && @@ -129,7 +136,12 @@ function plainString(node: SyntaxNode): string | null { return null; } -function propertyName(node: SyntaxNode): string | null { +/** + * A property key's name, for the spellings that carry one — `{ path: … }` and + * `{ 'path': … }`. A computed key (`{ [KEY]: … }`) has none. Shared with the + * NestJS extractor, which reads `@Controller({ path: … })` the same way. + */ +export function propertyName(node: SyntaxNode): string | null { if (node.type === 'identifier' || node.type === 'property_identifier') return node.text; if (node.type === 'string') return plainString(node); return null; diff --git a/gitnexus/src/core/ingestion/route-extractors/java-const-resolver.ts b/gitnexus/src/core/ingestion/route-extractors/java-const-resolver.ts new file mode 100644 index 000000000..95f9f4710 --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/java-const-resolver.ts @@ -0,0 +1,791 @@ +/** + * Java binding for the language-agnostic constant resolver (#2391 core). + * + * Supplies the two Java-specific pieces the shared fold in + * `constant-resolver.ts` needs — {@link resolveJavaImport} (import-specifier → + * file, honoring JVM package/classpath rules) and + * {@link extractJavaModuleConstants} (tree → {@link ModuleConstants}) — plus a + * pre-bound {@link resolveJavaConstant} wrapper so callers stay + * language-oblivious. The reusable fold, the cycle guard, and the depth cap + * all live in the agnostic core. + * + * Java constant shape (one per type declaration; nested classes flatten into + * the same file-level namespace, mirroring how `Outer.CONST` and a top-level + * `CONST` are indistinguishable at the fold layer): + * + * public class ApiPathConstants { + * public static final String DIAGNOSIS_SAVE_V1 = "/api/v1/diagnosis/add"; + * public static final String API_CIS_SAVE_SUMMARY = API_CIS_V1 + "summary/save"; + * } + * + * Reference shapes at annotation sites this binding resolves: + * @PostMapping(ApiPathConstants.DIAGNOSIS_SAVE_V1) // qualified + * @PostMapping(com.winning.opt.X.ApiPathConstants.Y) // FQN-qualified + * @PostMapping(DIAGNOSIS_SAVE_V1) // static-imported + * @PostMapping(API_CIS_V1 + "summary/save") // inline concat + * + * Which ANNOTATIONS count as routes is a separate question this module has no + * say in: `spring-shared.ts` holds an exact-name map, so a vendor alias like + * `@WinPostMapping` yields no route on this base regardless of how its value + * folds (#2883). Folding and alias recognition compose; neither implies the + * other. + * + * Import shapes consumed: + * import com.winning.opt.diagnosis.api.constants.ApiPathConstants; + * import static com.winning.opt.diagnosis.api.constants.ApiPathConstants.API_CIS_V1; + * + * Keying (KTD4 parity with the Python binding): the repo map is keyed by + * unique POSIX file path. A Java import `com.a.b.CONSTS` resolves to the file + * whose path ends with `com/a/b/CONSTS.java`; when 2+ files share that suffix + * the import is ambiguous and returns null (skip floor), never a wrong path. + */ + +import type Parser from 'tree-sitter'; +import { unquoteSpringLiteral } from './spring-shared.js'; +import { + MAX_FOLD_LENGTH, + type ImportBinding, + type ImportResolver, + type ModuleConstants, + type Operand, + type RepoConstants, + unfoldableDeclarationsOf, +} from './constant-resolver.js'; + +export type { + ImportBinding, + ModuleConstants, + Operand, + RepoConstants, +} from './constant-resolver.js'; + +export interface JavaModuleConstants extends ModuleConstants { + /** Declaration keys whose initializer exists but cannot be folded. */ + readonly unfoldableDeclarations: ReadonlySet; +} + +/** + * Cheap content gate: can this Java file DEFINE a string constant that a route + * annotation might reference? + * + * Exported so BOTH sides of the pipeline use the same predicate and cannot + * disagree about which files carry constants — the ingestion provider + * (`languages/java.ts`, as `moduleConstantHeuristic`) and the group extractor's + * `prepareRepo` pre-pass (`group/extractors/http-patterns/java.ts`). They used + * to spell it differently, and the two spellings disagreed on a constant + * INTERFACE: the group admitted it and published a provider contract at the + * folded path, while ingestion rejected the file and emitted no Route node for + * it — an R4 parity break in the losing direction, since ingestion is the side + * that drives the graph and `api_impact`. + * + * Arms: + * - a `static` … `String NAME =` declaration, with the modifier run matched as + * a span so every legal order works (`static public final String`, + * `public final static String`) and so `java.lang.String` — which the + * extractor accepts — is admitted too. + * - an `interface` declaration carrying a String assignment — interface fields + * are implicitly `public static final` (JLS 9.3), so a pure constant + * interface has neither keyword and no import. The assignment conjunct keeps + * a file whose PROSE merely mentions "interface " from costing a parse. + */ +// `static` … `String NAME =` on one declaration. The modifier run is matched as +// a span rather than as the adjacent pair `static final`, because the extractor +// scans modifiers INDEPENDENTLY (`isStaticFinal`) and Java lets them appear in +// any order — `static public final String`, `public final static String` — and +// because the type may be written out as `java.lang.String`, which the +// extractor also accepts. A gate narrower than the extractor it feeds is the +// same defect class as the ingestion/group divergence this predicate exists to +// prevent, just one layer down. +// +// The span excludes `;{}()` so it cannot jump a statement or block boundary: a +// local `String s = "x"` inside `static void f() { … }` is not matched, because +// reaching it from `static` crosses `(`, `)` and `{`. `final` is not required +// even though the extractor requires it — the gate may be wider than the +// extractor, never narrower. +const STATIC_STRING_CONSTANT_RE = /\bstatic\b[^;{}()]{0,80}\bString\s+\w+\s*=/; +const INTERFACE_DECL_RE = /\binterface\s+\w/; +const STRING_ASSIGNMENT_RE = /\bString\s+\w+\s*=/; + +export function isJavaConstantFile(source: string): boolean { + if (STATIC_STRING_CONSTANT_RE.test(source)) return true; + // The interface arm is a bare word match, so on its own it admits any file + // whose PROSE mentions "interface " — and every admitted file costs the group + // side a full extra parse. Requiring a String assignment as well keeps every + // shape `extractJavaModuleConstants` accepts in an interface body (bare + // `String`, `java.lang.String`, no space before `=`, multi-declarator) while + // dropping the comment-only matches. + return INTERFACE_DECL_RE.test(source) && STRING_ASSIGNMENT_RE.test(source); +} + +/** + * The Java {@link ImportResolver}: map a fully-qualified import specifier to + * the unique file key it refers to, or null when it cannot be pinned to + * exactly one file. + * + * `com.winning.opt.X.ApiPathConstants` → the file key ending in + * `com/winning/opt/X/ApiPathConstants.java`. Because the repo map is + * file-path-keyed and Maven multi-module trees repeat package roots across + * modules (`winning-opt-a/.../api/constants/ApiPathConstants.java` and + * `winning-opt-b/.../api/constants/ApiPathConstants.java`), suffix matching + * stays UNIQUE-suffix: an import whose full package+class path matches N files + * in N different modules cannot be pinned, so it returns null — the skip floor + * this module promises, never a wrong path. + * + * A nearest-shared-directory tie-break was tried here and removed on review: + * javac resolves duplicate FQNs by CLASSPATH ORDER, not directory proximity, so + * a `src/test` fixture copy or a module that merely sits closer in the tree can + * outrank the real dependency and yield a silently wrong literal. In a resolver + * whose whole contract is skip-or-correct, a plausible guess is the one answer + * that cannot be allowed. + */ +export const resolveJavaImport: ImportResolver = (_importingFileKey, moduleSpec, repoKeys) => { + // A static import `a.b.C.CONST` names the class as all-but-last segment; + // a plain import `a.b.C` names the class as last segment. Both resolve to + // a file ending `a/b/C.java`; treating the whole spec as a path and + // trimming the last segment when the direct hit fails covers both shapes. + const asPath = moduleSpec.replace(/\./g, '/'); + const classFile = `${asPath}.java`; + + // Compare in POSIX space: on Windows the repo keys can carry backslash + // separators, which would otherwise never match a '/'-joined class file + // (observed as 675 calls with zero hits on a backslash-keyed repo). + const toPosix = (p: string): string => p.replace(/\\/g, '/'); + + // Exact package-path suffix match, unique or nothing. + let hit: string | null = null; + for (const key of repoKeys) { + const posixKey = toPosix(key); + if (posixKey === classFile || posixKey.endsWith(`/${classFile}`)) { + if (hit !== null) return null; // 2+ modules carry this FQN — unresolvable + hit = key; + } + } + return hit; +}; + +/** + * Is `node` a Java string literal (`"..."`), and if so what value does the + * route layer give it? + * + * tree-sitter-java splits a `string_literal` AROUND its `escape_sequence` + * children, so joining `string_fragment`s alone silently DELETES every escape: + * `"/user/{id:\\d+}"` — the standard Spring path-variable regex constraint — + * folded to `/user/{id:d+}`, and a pure-escape literal (`"\\t"`) folded to the + * empty string. Slicing the quotes off the raw text keeps the source spelling, + * which is precisely what the LITERAL path does + * ({@link unquoteSpringLiteral}) — so `@GetMapping(ApiPaths.USER_REGEX)` and + * `@GetMapping("/user/{id:\\d+}")` now emit the same path for the same Java + * source instead of two spellings the graph cannot reconcile. Same + * `string_fragment`-join trap as the NestJS one in #3017. + */ +function stringLiteralValue(node: Parser.SyntaxNode): string | null { + if (node.type !== 'string_literal') return null; + // A Java text block is also a `string_literal` here, and `unquoteSpringLiteral` + // has a `"""` arm that would hand back the raw block — leading newline and + // incidental indentation included, both of which Java strips. Nothing + // downstream normalizes that, so it would publish a Route at a path like + // "\n /api/v1/x\n ". The old fragment-join returned '' here, which + // floored to skip; keep that floor rather than trade it for a wrong path. + if (node.text.startsWith('"""')) return null; + return unquoteSpringLiteral(node.text); +} + +/** + * Flatten a qualified-name expression (`ApiPaths`, `com.example.ApiPaths`) to + * its dotted text, or null when any segment is not a plain identifier (calls, + * `this`, array access, generics — not a static constant shape). + */ +function flattenQualifiedIdentifier(node: Parser.SyntaxNode): string | null { + if (node.type === 'identifier') return node.text; + if (node.type === 'field_access') { + const object = node.childForFieldName('object'); + const field = node.childForFieldName('field'); + if (object && field) { + const head = flattenQualifiedIdentifier(object); + return head === null ? null : `${head}.${field.text}`; + } + } + return null; +} + +/** + * Parse a Java constant initializer into an operand list, or null when it is + * not a foldable string expression. Handles a bare string literal, a bare + * identifier (`X = Y`), qualified/static-import-free references + * (`X = CONSTS.Y` — recorded as ONE ref named `CONSTS.Y`), and + * left-associative `+` chains of the three. Everything else — numbers, calls, + * ternaries, method refs, `String.format`, enum constants — returns null, + * which makes the constant unresolvable (→ skip floor), never a wrong value. + */ +export function parseJavaConstOperands( + node: Parser.SyntaxNode | null | undefined, + depth = 0, +): Operand[] | null { + if (!node) return null; + if (depth > 64) return null; + if (node.type === 'string_literal') { + const value = stringLiteralValue(node); + return value === null ? null : [{ kind: 'literal', value }]; + } + if (node.type === 'identifier') { + return [{ kind: 'ref', name: node.text }]; + } + // `CONSTS.FIELD` — field_access in tree-sitter-java for expressions. The + // object side may itself be a chain (`com.example.ApiPaths` parses as + // nested field_access), so flatten recursively: every segment must be a + // plain identifier/keyword to qualify (a call `f().X`, `this.X`, or an + // array access object side is not a constant shape → null, skip floor). + if (node.type === 'field_access') { + const object = node.childForFieldName('object'); + const field = node.childForFieldName('field'); + if (object && field) { + const objectName = flattenQualifiedIdentifier(object); + if (objectName !== null) return [{ kind: 'ref', name: `${objectName}.${field.text}` }]; + } + return null; + } + if (node.type === 'binary_expression') { + const isPlus = (node.children ?? []).some((c) => c.type === '+'); + if (!isPlus) return null; + const left = parseJavaConstOperands(node.childForFieldName('left'), depth + 1); + const right = parseJavaConstOperands(node.childForFieldName('right'), depth + 1); + if (left === null || right === null) return null; + return [...left, ...right]; + } + return null; +} + +/** + * Extract the file-level string constants and import bindings of one parsed + * Java file into the {@link ModuleConstants} shape the resolver consumes. + * + * Constants: every `static final String NAME = …` field of every type + * declaration in the file (nested classes included — their simple names + * would collide at the fold layer, but qualified refs carry the class name + * so nesting only matters for same-name fields, which flatten last-wins). + * Interface constants (`String NAME = "…"`) are implicitly static final and + * are collected too. + * + * References to OTHER constants via qualified names (`ApiPathConstants.X`) + * are stored as refs named `ApiPathConstants.X`; at the fold layer such a ref + * resolves through the import map (`ApiPathConstants` → module) followed by + * field lookup in the target file's OWN class-name-qualified namespace. To + * support that, constant names are ALSO recorded under + * `.` (both spellings share one entry). + * + * Last-wins in source order; a non-foldable rebind (`X = compute()`) drops X + * to unresolvable rather than keeping a stale literal. + */ +export function extractJavaModuleConstants(tree: Parser.Tree): JavaModuleConstants { + const literals = new Map(); + const exprs = new Map(); + const imports = new Map(); + const unfoldableDeclarations = new Set(); + + // On-demand static imports (`import static a.b.C.*`) — expanded post-map + // by expandJavaWildcardStaticImports below. + const wildcardImports: string[] = []; + + // Pass 1: imports (both shapes). + const walkImports = (node: Parser.SyntaxNode): void => { + if (node.type === 'import_declaration') { + // import a.b.C; | import static a.b.C; | import static a.b.C.F; + // import static a.b.C.*; — asterisk is a sibling of scoped_identifier + // (tree-sitter-java), not the last path segment. Same detection as + // import-decomposer.ts (`static-wildcard`). + const isStatic = node.children.some((c) => c.type === 'static' && c.text === 'static'); + const isWildcard = node.children.some((c) => c.type === 'asterisk'); + const scoped = + node.children.find((c) => c.type === 'scoped_identifier') ?? + node.children.find((c) => c.type === 'identifier'); + if (scoped) { + const text = scoped.text; + if (isStatic && isWildcard) { + // Class FQN only — members are materialized post-map. + if (text.length > 0 && !wildcardImports.includes(text)) { + wildcardImports.push(text); + } + } else { + const lastDot = text.lastIndexOf('.'); + const fqn = text.slice(0, lastDot); + const name = text.slice(lastDot + 1); + if (isStatic) { + // Preserve the declaring type in the target lookup. Constants from + // multiple types share one file-level map, so a bare `F` could + // otherwise resolve to a sibling type's flattened field. + const declaringClass = fqn.slice(fqn.lastIndexOf('.') + 1); + imports.set(name, { + module: fqn, + originalName: `${declaringClass}.${name}`, + }); + } else { + // import a.b.C → module IS the class FQN; originalName is the class + // simple name. resolveJavaImport maps `a.b.C` → `a/b/C.java`. + imports.set(name, { module: text, originalName: name }); + } + } + } + } + for (const child of node.children ?? []) walkImports(child); + }; + walkImports(tree.rootNode); + + // Pass 2: constants. A field declaration is a constant when it is + // `static final` (explicit) or inside an interface (implicit). + const isStaticFinal = (modifiers: Parser.SyntaxNode | null | undefined): boolean => { + if (!modifiers) return false; + let sawStatic = false; + let sawFinal = false; + for (const m of modifiers.children ?? []) { + if (m.type === 'static') sawStatic = true; + if (m.type === 'final') sawFinal = true; + } + return sawStatic && sawFinal; + }; + + const collectFieldConstants = ( + classBody: Parser.SyntaxNode, + insideInterface: boolean, + declaringClass: string | null, + ): void => { + for (const member of classBody.children ?? []) { + // tree-sitter-java: interface fields are `constant_declaration`, class + // fields are `field_declaration`. Both carry `variable_declarator`s. + if (member.type !== 'field_declaration' && member.type !== 'constant_declaration') continue; + const mods = member.children.find((c) => c.type === 'modifiers'); + if (!insideInterface && !isStaticFinal(mods)) continue; + // Type must be String (java.lang.String is implicit-imported). + const typeNode = member.childForFieldName('type'); + if (!typeNode) continue; + const typeText = typeNode.text; + if (typeText !== 'String' && typeText !== 'java.lang.String') continue; + + const declarators = member.children.filter((c) => c.type === 'variable_declarator'); + for (const decl of declarators) { + const nameNode = decl.childForFieldName('name'); + const valueNode = decl.childForFieldName('value'); + if (!nameNode) continue; + const name = nameNode.text; + const operands = parseJavaConstOperands(valueNode); + // Same-name shadowing across nested types (legal Java, unlike + // same-class redeclaration): a later binding must REPLACE the earlier + // flattened simple-name entry — including dropping it to unresolvable + // when the new initializer is not foldable (`X = compute()`) — rather + // than leave the stale outer literal resolvable. Skip floor, mirroring + // Python #2391's rebind-drop. Qualified `Class.FIELD` aliases are + // per-type-keyed but same-named nested types can still collide, so + // they get the same replace/drop treatment. + const qname = declaringClass ? `${declaringClass}.${name}` : null; + if (operands === null) { + literals.delete(name); + exprs.delete(name); + unfoldableDeclarations.add(name); + // …and the static IMPORT of the same simple name. A local + // `static final String` shadows `import static a.b.C.PATH` inside + // that class (JLS 6.4.1), so the correct answer for a non-foldable + // rebind is "unresolvable" — leaving the import alive makes the fold + // fall through it (computeFold: literals → exprs → imports) and + // return the IMPORTED value, i.e. a wrong path where the skip floor + // is owed. #2393's Python defect, reproduced for Java. + // + // The delete is file-scoped because these maps are (see the header: + // nested types flatten into one file-level namespace). So a SIBLING + // top-level class in the same file that legitimately uses the import + // loses it too and floors to skip, where javac would resolve it. + // That direction is the acceptable one — a missing route, not a wrong + // one — and the shape (two top-level classes, one shadowing a static + // import with a non-foldable initializer) is vanishingly rare next to + // the wrong-value it prevents. + imports.delete(name); + if (qname) { + literals.delete(qname); + exprs.delete(qname); + unfoldableDeclarations.add(qname); + } + continue; + } + unfoldableDeclarations.delete(name); + if (qname) unfoldableDeclarations.delete(qname); + const literalValue = + operands.length === 1 && operands[0].kind === 'literal' + ? (operands[0] as { value: string }).value + : null; + if (literalValue !== null) { + literals.set(name, literalValue); + exprs.delete(name); + } else { + exprs.set(name, operands); + literals.delete(name); + } + // Qualified alias: `CONSTS.X` refs (folded refs carry the class name). + if (qname) { + if (literalValue !== null) { + literals.set(qname, literalValue); + exprs.delete(qname); + } else { + exprs.set(qname, operands); + literals.delete(qname); + } + } + } + } + }; + + const walkTypes = (node: Parser.SyntaxNode, insideInterface: boolean): void => { + for (const child of node.children ?? []) { + const isInterface = child.type === 'interface_declaration'; + // Enums and records are ordinary type declarations for constant + // purposes — their fields need an explicit `static final` (JLS 8.9/8.10), + // unlike an interface's implicitly-constant ones. They used to be only + // RECURSED into, never collected, so a `static final String` declared + // directly in an enum or record was silently absent from the map. + const isTypeDecl = + isInterface || + child.type === 'class_declaration' || + child.type === 'enum_declaration' || + child.type === 'record_declaration'; + if (!isTypeDecl) { + walkTypes(child, insideInterface); + continue; + } + const className = child.childForFieldName('name')?.text ?? null; + const body = child.children.find( + (c) => c.type === 'class_body' || c.type === 'interface_body' || c.type === 'enum_body', + ); + if (!body) continue; + // An enum's members hang one level deeper, under `enum_body_declarations` + // (the `enum_body` itself holds only the enum constants). + const memberBody = body.children.find((c) => c.type === 'enum_body_declarations') ?? body; + // Recompute implicit interface semantics at each type boundary: a + // class nested in an interface is a normal class whose fields need + // explicit `static final` (JLS 9.5 — only the interface's own fields + // are implicitly public static final). Propagating the outer + // `insideInterface` flag in would harvest mutable nested fields as + // constants and let a same-name nested field shadow a real interface + // constant with a stale value. + if (className) collectFieldConstants(memberBody, isInterface, className); + // Recurse over the WHOLE body, not just `memberBody`: an enum's constants + // are siblings of `enum_body_declarations`, so narrowing here dropped any + // type nested inside an enum-constant body whenever the enum also had + // member declarations. For a class/interface/record the two are the same + // node; for an enum `body` is a strict superset, and the extra visit to + // `enum_body_declarations` collects nothing twice (collectFieldConstants + // is still called on `memberBody` alone). + walkTypes(body, isInterface); + } + }; + walkTypes(tree.rootNode, false); + + return { + literals, + exprs, + imports: imports as Map, + wildcardImports, + unfoldableDeclarations, + }; +} + +/** + * Per-fold state. Mirrors the guards the agnostic core carries in `foldName`, + * which this binding stopped delegating to once it had to resolve qualified + * operands itself: + * + * - `memo` caches SUCCESSES only and is never popped. Without it a + * shared-descendant DAG (`X_k = X_{k+1} + X_{k+1}`) re-folds each child once + * per reference — O(2^depth) — and {@link MAX_FOLD_LENGTH} cannot save it, + * because a chain whose intermediate values are the empty string never + * accumulates any output. Measured before this state existed: one route over + * a 31-line constants file took 2.7 s at 26 levels and 11 s at 28, on the + * main thread, per file. A `null` may be transient (a name that cycles on one + * branch can resolve on another), so caching it would be unsound. + * - `visited` is the ACTIVE resolution stack, popped on unwind, so diamonds + * fold instead of false-cycling while true cycles still terminate. + * - `constantKeys` is the candidate set import ambiguity is measured over: + * files that actually DEFINE a constant. Handing `resolveJavaImport` every + * repo key made the two subsystems disagree — ingestion's map also holds + * import-only files (its gate has an import arm), so a duplicate FQN that + * defines nothing was invisible to the group and made ingestion alone floor + * to skip. Hoisting it also stops rebuilding the set on every qualified ref. + */ +interface JavaFoldState { + readonly repo: RepoConstants; + readonly constantKeys: ReadonlySet; + readonly visited: Set; + readonly memo: Map; +} + +function newFoldState(repo: RepoConstants): JavaFoldState { + const constantKeys = new Set(); + for (const [key, mc] of repo) { + if (mc.literals.size > 0 || mc.exprs.size > 0) constantKeys.add(key); + } + return { repo, constantKeys, visited: new Set(), memo: new Map() }; +} + +/** + * Resolve a single Java constant referenced in `fileKey` to its literal string + * value, folding `+` concatenation and following import chains via + * {@link resolveJavaImport}, or null when it cannot be fully folded. + * + * `name` may be simple (`DIAGNOSIS_SAVE_V1`, resolved via static import or + * same-file constant) or qualified (`ApiPathConstants.DIAGNOSIS_SAVE_V1`, + * resolved via the class import + the target file's qualified alias). + */ +export function resolveJavaConstant( + fileKey: string, + name: string, + repo: RepoConstants, + depth = 0, +): string | null { + return resolveWithState(fileKey, name, newFoldState(repo), depth); +} + +function resolveWithState( + fileKey: string, + name: string, + state: JavaFoldState, + depth: number, +): string | null { + if (depth > 32) return null; + const guard = `${fileKey}::${name}`; + const memoized = state.memo.get(guard); + if (memoized !== undefined) return memoized; + if (state.visited.has(guard)) return null; // cycle: `name` is on the active stack + state.visited.add(guard); + try { + const result = computeJavaFold(fileKey, name, state, depth); + if (result !== null) state.memo.set(guard, result); + return result; + } finally { + state.visited.delete(guard); + } +} + +function computeJavaFold( + fileKey: string, + name: string, + state: JavaFoldState, + depth: number, +): string | null { + const { repo, constantKeys } = state; + // Qualified ref (`ApiPathConstants.FIELD`): constants and imports are keyed by + // their IN-FILE name, so a dotted name never hits directly. Split head.tail: + // resolve the head through the importing file's class import, then look the + // tail up in the target file — first as the class-qualified alias `Head.TAIL` + // (what extractJavaModuleConstants records), then as a bare `TAIL` (same-file + // nested/interface constant). + const dot = name.indexOf('.'); + if (dot > 0) { + const head = name.slice(0, dot); + const tail = name.slice(dot + 1); + const imp = repo.get(fileKey)?.imports.get(head); + if (imp) { + const targetFile = resolveJavaImport(fileKey, imp.module, constantKeys); + if (targetFile !== null) { + const qualified = resolveWithState(targetFile, `${head}.${tail}`, state, depth + 1); + if (qualified !== null) return qualified; + const bare = resolveWithState(targetFile, tail, state, depth + 1); + if (bare !== null) return bare; + } + return null; + } + // Un-imported qualified name (FQN form `com.a.b.C.FIELD`): try resolving + // the longest dotted prefix as a class import target. + const parts = name.split('.'); + for (let cut = parts.length - 2; cut >= 1; cut--) { + const fqn = parts.slice(0, cut + 1).join('.'); + const targetFile = resolveJavaImport(fileKey, fqn, constantKeys); + if (targetFile !== null) { + const field = parts.slice(cut + 1).join('.'); + const declaring = parts[cut]; + const qualified = resolveWithState(targetFile, `${declaring}.${field}`, state, depth + 1); + if (qualified !== null) return qualified; + return resolveWithState(targetFile, field, state, depth + 1); + } + } + // No import bound the head and no FQN prefix resolved — fall through. A + // dotted name is ALSO a valid key in this file's own maps: + // `extractJavaModuleConstants` records every constant under + // `.` as well as its simple name, so a same-file + // qualified reference (`ApiPaths.X` inside ApiPaths.java) resolves below. + } + + // Name lookup: literals, then same-file expressions, then the import chase. + // Reached for a bare name and for a dotted name that named no import. + // Expressions are folded HERE rather than handed to the agnostic core because + // an operand of a Java initializer may itself be a QUALIFIED ref + // (`X = BConsts.Y + "/tail"`) and the core only knows bare names: it looks + // `BConsts.Y` up in maps keyed by simple name, misses, and floors the whole + // chain to null. Recursing through this function gives every operand the same + // qualified treatment the entry-point name got. + const mc = repo.get(fileKey); + if (!mc) return null; + const literal = mc.literals.get(name); + if (literal !== undefined) return literal; + const expr = mc.exprs.get(name); + if (expr !== undefined) return foldOperands(fileKey, expr, state, depth + 1); + if (unfoldableDeclarationsOf(mc).has(name)) return null; + const imp = mc.imports.get(name); + if (imp !== undefined) { + const targetFile = resolveJavaImport(fileKey, imp.module, constantKeys); + if (targetFile === null) return null; + return resolveWithState(targetFile, imp.originalName, state, depth + 1); + } + return null; +} + +/** + * Concatenate an operand list, resolving each `ref` through the qualified-aware + * walk so `Class.CONST` works at every position, not just at the entry point. + * + * Bounded by {@link MAX_FOLD_LENGTH}: the depth cap bounds RECURSION but not + * OUTPUT, which grows multiplicatively (`X = A + A; A = B + B; …`), so a + * pathological chain would build a gigabyte-scale string before any cap fired. + * Overrun floors to null (#2393). + */ +function foldOperands( + fileKey: string, + operands: readonly Operand[], + state: JavaFoldState, + depth: number, +): string | null { + let out = ''; + for (const op of operands) { + if (op.kind === 'literal') { + out += op.value; + } else { + const piece = resolveWithState(fileKey, op.name, state, depth); + if (piece === null) return null; + out += piece; + } + if (out.length > MAX_FOLD_LENGTH) return null; + } + return out; +} + +/** + * Fold an inline operand list (e.g. `API_CIS_V1 + "summary/save"`) against + * `fileKey`, or null when any piece is unresolvable (skip floor). + */ +export function foldJavaOperands( + fileKey: string, + operands: readonly Operand[], + repo: RepoConstants, +): string | null { + const out = foldOperands(fileKey, operands, newFoldState(repo), 0); + return out === '' ? null : out; +} + +/** + * Constant-defining file keys used by Java import resolution. + * + * Build once per repo pass. Recomputing this set for every wildcard-importing + * controller makes expansion quadratic in controller count. + */ +export function buildJavaConstantKeys(repo: RepoConstants): ReadonlySet { + const repoKeys = new Set(); + for (const [key, target] of repo) { + if (target.literals.size > 0 || target.exprs.size > 0) repoKeys.add(key); + } + return repoKeys; +} + +/** Direct static members owned by `classSimple`, excluding nested-type members. */ +function directJavaMembers(target: ModuleConstants, classSimple: string): Set { + const members = new Set(); + const prefix = `${classSimple}.`; + for (const map of [target.literals, target.exprs]) { + for (const key of map.keys()) { + if (!key.startsWith(prefix)) continue; + const member = key.slice(prefix.length); + if (member.length > 0 && !member.includes('.')) members.add(member); + } + } + return members; +} + +export interface JavaConstantIndex { + readonly keys: ReadonlySet; + /** Every resolvable path suffix as a dotted module name; null means ambiguous. */ + readonly byModule: ReadonlyMap; + /** Direct members by constant-defining file, built once for all importers. */ + readonly membersByFile: ReadonlyMap>; +} + +/** + * Build all Java import suffixes once, turning repeated wildcard target lookup + * from O(importers × constant files) into O(path segments + importers). + */ +export function buildJavaConstantIndex(repo: RepoConstants): JavaConstantIndex { + const keys = buildJavaConstantKeys(repo); + const byModule = new Map(); + const membersByFile = new Map>(); + for (const key of keys) { + const normalized = key.replace(/\\/g, '/').replace(/^\.\//, ''); + if (!normalized.endsWith('.java')) continue; + const segments = normalized.slice(0, -'.java'.length).split('/'); + const classSimple = segments[segments.length - 1]; + const constants = repo.get(key); + if (constants) membersByFile.set(key, directJavaMembers(constants, classSimple)); + for (let start = 0; start < segments.length; start++) { + const moduleName = segments.slice(start).join('.'); + const existing = byModule.get(moduleName); + if (existing === undefined) byModule.set(moduleName, key); + else if (existing !== key) byModule.set(moduleName, null); + } + } + return { keys, byModule, membersByFile }; +} + +export function expandJavaWildcardStaticImports( + mc: ModuleConstants, + _fileKey: string, + repo: RepoConstants, + index: JavaConstantIndex = buildJavaConstantIndex(repo), +): ModuleConstants { + const wildcards = mc.wildcardImports; + if (!wildcards || wildcards.length === 0) return mc; + // Resolve targets against constant-DEFINING files only. Ingestion's harvest + // also admits import-only files; measuring uniqueness over every key made + // a duplicate empty FQN floor ingestion to skip while group still folded + // (#2980 R4). + const explicitImports = new Set(mc.imports.keys()); + const pending = new Map(); + for (const fqn of wildcards) { + const targetKey = index.byModule.get(fqn) ?? null; + if (targetKey === null) continue; + const classSimple = fqn.slice(fqn.lastIndexOf('.') + 1); + const members = index.membersByFile.get(targetKey); + if (!members) continue; + for (const name of members) { + // Same-file declarations and explicit imports have higher precedence + // than on-demand imports. An unfoldable declaration must remain a skip, + // not be resurrected from a wildcard target. + if ( + mc.literals.has(name) || + mc.exprs.has(name) || + unfoldableDeclarationsOf(mc).has(name) || + explicitImports.has(name) + ) { + continue; + } + const binding = { module: fqn, originalName: `${classSimple}.${name}` }; + const previous = pending.get(name); + if (previous === undefined) pending.set(name, binding); + else if (previous !== null && previous.module !== fqn) pending.set(name, null); + } + } + for (const [name, binding] of pending) { + if (binding !== null) mc.imports.set(name, binding); + } + return mc; +} + +/** Prepare every Java constants entry with one shared suffix index. */ +export function prepareJavaRouteConstants(repo: RepoConstants): JavaConstantIndex { + const index = buildJavaConstantIndex(repo); + for (const [fileKey, mc] of repo) { + expandJavaWildcardStaticImports(mc, fileKey, repo, index); + } + return index; +} diff --git a/gitnexus/src/core/ingestion/route-extractors/js-const-resolver.ts b/gitnexus/src/core/ingestion/route-extractors/js-const-resolver.ts new file mode 100644 index 000000000..131055deb --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/js-const-resolver.ts @@ -0,0 +1,1213 @@ +/** + * JavaScript/TypeScript binding for the language-agnostic constant resolver. + * + * Supplies the two JS-specific pieces the shared fold in `constant-resolver.ts` + * needs — {@link resolveJsImport} (import specifier → file key, honoring + * relative paths, extensionless imports, directory `index` files and bare + * alias-style specifiers) and {@link extractJsModuleFacts} (tree → + * {@link ModuleConstants} plus the export/HTTP-client facts below) — mirroring + * how `python-const-resolver.ts` binds the same core for Python (#2391). + * + * Two JS-shaped facts the Python binding has no analogue for: + * + * 1. **Object-literal path tables.** Python route constants are module-level + * scalars (`API_V1 = "/v1"`); the JS convention is one frozen table — + * `export const API_ROUTE_PATH = { LINKS: "/links", … } as const` — read at + * the call site as `API_ROUTE_PATH.LINKS`. The extractor flattens such a + * table into DOTTED literal keys (`API_ROUTE_PATH.LINKS` → `/links`) so the + * agnostic fold, which does a plain `literals.get(name)`, resolves a member + * reference with no changes to the core. + * + * 2. **Export aliasing.** `export default routeApiClient` and + * `export { a as b }` mean the name an importer writes is often not the + * name the defining file bound. {@link JsModuleFacts.exports} maps the + * EXPORTED name (including `default`) to the local one so a cross-file + * chase lands on the right binding. + * + * Both stay in this binding — the shared core keeps knowing nothing about any + * language. + * + * Keying matches the Python binding: the repo map is keyed by unique POSIX file + * path, and an import that cannot be pinned to exactly one file resolves to + * `null` (skip) rather than an arbitrary winner. An unresolved path is a + * missing contract; a wrongly-resolved one is a false cross-repo link, which is + * strictly worse. + */ + +import type Parser from 'tree-sitter'; +import { + MAX_FOLD_LENGTH, + resolveConstant as foldConstant, + type ImportResolver, + type ModuleConstants, + type Operand, + type RepoConstants, +} from './constant-resolver.js'; + +export type { + ImportBinding, + ModuleConstants, + Operand, + RepoConstants, +} from './constant-resolver.js'; + +/** Extensions an extensionless JS/TS import may resolve to, in resolution order. */ +const JS_EXTENSIONS = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts'] as const; + +/** + * Bound on the re-export chase in {@link resolveJsMemberPath}. Mirrors the + * fold's own `MAX_RESOLVE_DEPTH`: a barrel that re-exports through more hops + * than this floors to `null` (skip), never to a guess. + */ +const MAX_REEXPORT_HOPS = 8; + +/** The synthetic local name a bare `export default ` binds to. */ +const DEFAULT_LOCAL = '__default__'; + +/** + * Per-file facts beyond the agnostic {@link ModuleConstants}: which exported + * name maps to which local binding, and which local bindings hold an HTTP + * client instance. + */ +export interface JsModuleFacts { + /** String constants, dotted table members, `+`-expressions and imports. */ + readonly constants: ModuleConstants; + /** Exported name (incl. `default`) → local binding name in this file. */ + readonly exports: Map; + /** + * Module specifiers this file re-exports wholesale (`export * from './m'`). + * A directory barrel is built almost entirely out of these, and a barrel is + * what application code imports — so without following them, every name + * reached through one resolves to nothing. + */ + readonly starExports: string[]; + /** + * Local names proven to hold an HTTP client INSTANCE — bound directly to + * `axios.create(...)`, or to another local name that is one. Cross-file + * chains are followed at query time by {@link isHttpClientRef}, not here. + */ + readonly clients: Set; + /** + * True when this file declares its own top-level binding named `axios` that + * is NOT the axios module. + * + * The bare spelling `axios` is trusted without proof — it predates this + * binding and is what the original query matched on. That is right for + * `import axios from 'axios'` and for `const axios = require('axios')`, and + * wrong for `const axios = fakeFactory`, where the spelling is the only + * evidence and it is false. One flag, because the shortcut only ever applies + * to this one name. + */ + readonly axiosShadowed: boolean; +} + +/** + * Repo-wide facts, with everything the shared fold needs precomputed. + * + * `constants`, `keys` and `resolveImport` are derived from `byFile` and built + * ONCE by {@link buildJsRepoFacts}, never per lookup: materializing a key set + * at each call site makes every resolution O(files) and the whole scan + * quadratic in a repo's file count. That was only half true before — + * `resolveConstant` rebuilt its own key set on every fold regardless, so the + * mitigation this comment describes was not in force for any resolution that + * went through the shared core. It now takes `keys` as an argument. + */ +export interface JsRepoFacts { + readonly byFile: ReadonlyMap; + readonly constants: RepoConstants; + readonly keys: ReadonlySet; + /** + * {@link resolveJsImport} bound to a prebuilt basename index and memoized for + * the lifetime of the facts. Every resolution inside this module goes through + * it rather than the bare export: the widened consumer query matches every + * `.(…)` call in the repo, so an unindexed lookup ran once + * per call site over every repo key. + */ + readonly resolveImport: ImportResolver; +} + +/** + * Repo keys bucketed by final path segment. + * + * A tail lookup only ever matches keys whose last segment equals the + * candidate's last segment, so the bucket is the entire search space — turning + * an O(files) sweep per candidate into one map hit. `import-resolvers/utils.ts` + * already ships `buildSuffixIndex` for the same job, but it keeps only a first + * winner per suffix; this index has to SEE a collision to refuse it (below), so + * it keeps the whole bucket. + */ +type BasenameIndex = ReadonlyMap; + +function buildBasenameIndex(repoKeys: ReadonlySet): BasenameIndex { + const index = new Map(); + for (const key of repoKeys) { + const base = key.slice(key.lastIndexOf('/') + 1); + const bucket = index.get(base); + if (bucket) bucket.push(key); + else index.set(base, [key]); + } + return index; +} + +/** Build the {@link JsRepoFacts} projections from per-file facts. */ +export function buildJsRepoFacts(byFile: ReadonlyMap): JsRepoFacts { + const constants = new Map(); + for (const [key, value] of byFile) constants.set(key, value.constants); + const keys = new Set(byFile.keys()); + const index = buildBasenameIndex(keys); + const memo = new Map(); + const resolveImport: ImportResolver = (importingFileKey, moduleSpec, repoKeys) => { + // Only the relative arm reads `importingFileKey`, but keying on both is a + // string concat and keeps the memo correct if that ever stops being true. + // + // `repoKeys` is deliberately NOT part of the key: every caller inside this + // module passes `facts.keys`, which is fixed for the lifetime of these + // facts and is the set `index` was built from. A caller passing a different + // set would get an answer computed against `facts.keys` — so don't. + const memoKey = `${importingFileKey}\u0000${moduleSpec}`; + const cached = memo.get(memoKey); + if (cached !== undefined) return cached; + const resolved = resolveImportWith(index, importingFileKey, moduleSpec, repoKeys); + memo.set(memoKey, resolved); + return resolved; + }; + return { byFile, constants, keys, resolveImport }; +} + +function dirOf(fileKey: string): string { + const slash = fileKey.lastIndexOf('/'); + return slash >= 0 ? fileKey.slice(0, slash) : ''; +} + +/** Collapse `a/b/../c` and `./` segments in a POSIX-ish path. */ +function normalizePosix(path: string): string { + const out: string[] = []; + for (const seg of path.split('/')) { + if (seg === '' || seg === '.') continue; + if (seg === '..') { + if (out.length > 0 && out[out.length - 1] !== '..') out.pop(); + else out.push('..'); + } else { + out.push(seg); + } + } + return out.join('/'); +} + +/** + * Candidate file keys for a module path with no extension: the path itself + * (already-suffixed imports), each known extension, and the directory-`index` + * forms. Order matters only for the relative case, where the first existing + * candidate wins — matching bundler/`tsc` resolution order closely enough that + * a repo with both `x.ts` and `x.js` picks the TypeScript source. + */ +function candidatesFor(modPath: string): string[] { + const out = [modPath]; + for (const ext of JS_EXTENSIONS) out.push(`${modPath}${ext}`); + for (const ext of JS_EXTENSIONS) out.push(`${modPath}/index${ext}`); + return out; +} + +/** + * The MODULE a repo key denotes: the key without its extension, and without a + * trailing `/index`. + * + * `x/routes.ts` and `x/routes/index.ts` are two spellings of the same module + * `x/routes` — Node and `tsc` both pick the file over the directory, so a tail + * matching both is not ambiguous, it just has a precedence order. Two + * DIFFERENT identities sharing one tail is the real ambiguity, and that is what + * {@link resolveImportWith} refuses. + */ +function moduleIdentityOf(key: string): string { + for (const ext of JS_EXTENSIONS) { + if (!key.endsWith(ext)) continue; + const withoutExt = key.slice(0, -ext.length); + return withoutExt.endsWith('/index') ? withoutExt.slice(0, -'/index'.length) : withoutExt; + } + return key; +} + +/** + * The JS/TS {@link ImportResolver}. + * + * Relative specifiers (`./api-routes`, `../shared/api-routes`) resolve against + * the importing file's directory and must hit an existing key exactly. + * + * An alias-style specifier (`@/api-modules/shared/api-routes`, `~/x/y`) or a + * multi-segment bare one is matched by UNIQUE PATH SUFFIX, the same strategy + * the Python binding uses for absolute imports. This deliberately resolves + * aliases without reading `tsconfig.json`: an alias prefix is arbitrary (`@/`, + * `~/`, `#app/`, any `paths` key), but the segments AFTER it are a real path + * tail, and matching that tail against the indexed file set answers the + * question directly. + * + * Two rules keep that from inventing resolutions: + * + * - **A tail claimed by two distinct modules returns `null`**, checked across + * EVERY candidate extension rather than within one. Returning on the first + * extension that matched let precedence pre-empt the guard, so a `.ts`/`.tsx` + * or `.ts`/`.js` collision — every Next.js repo — picked an arbitrary winner + * while this docstring promised a skip. + * - **A single-segment bare specifier never matches a repo file.** `axios`, + * `lodash` and the Node builtin `http` are npm/runtime modules, not ours to + * resolve; without this, a repo holding `src/lib/http.ts` "proved" that + * `import http from 'http'` was an axios client. An alias tail always has a + * sigil or a `/`, so this costs the feature nothing. + */ +function resolveImportWith( + index: BasenameIndex, + importingFileKey: string, + moduleSpec: string, + repoKeys: ReadonlySet, +): string | null { + if (moduleSpec === '') return null; + + if (moduleSpec.startsWith('./') || moduleSpec.startsWith('../')) { + const base = dirOf(importingFileKey); + const joined = normalizePosix(`${base}/${moduleSpec}`); + // A `../` chain that climbs above the repo root leaves a leading `..` + // segment; that import escapes the indexed tree and cannot be pinned. + if (joined === '' || joined.startsWith('..')) return null; + for (const candidate of candidatesFor(joined)) { + if (repoKeys.has(candidate)) return candidate; + } + return null; + } + + // Strip a leading alias sigil so `@/a/b` and `~/a/b` reduce to the tail + // `a/b`. A scoped package (`@scope/pkg`) keeps its `@` and simply fails to + // match any repo file below, which is the desired outcome. + const aliased = /^[@~#]\//.test(moduleSpec); + const tail = aliased ? moduleSpec.slice(2) : moduleSpec; + if (tail === '' || tail.startsWith('.')) return null; + if (!aliased && !tail.includes('/')) return null; // bare npm package / Node builtin + + let winner: string | null = null; + let winnerRank = Number.POSITIVE_INFINITY; + let identity: string | null = null; + const candidates = candidatesFor(tail); + for (let rank = 0; rank < candidates.length; rank++) { + const candidate = candidates[rank]; + const bucket = index.get(candidate.slice(candidate.lastIndexOf('/') + 1)); + if (bucket === undefined) continue; + for (const key of bucket) { + if (key !== candidate && !key.endsWith(`/${candidate}`)) continue; + const keyIdentity = moduleIdentityOf(key); + if (identity === null) identity = keyIdentity; + else if (identity !== keyIdentity) return null; // two modules share this tail + if (rank < winnerRank) { + winner = key; + winnerRank = rank; + } + } + } + return winner; +} + +/** + * Standalone {@link ImportResolver} — the same rules as {@link resolveImportWith} + * with the basename index built on the spot. + * + * Production goes through `JsRepoFacts.resolveImport`, which holds one index + * for the whole repo and memoizes; this export exists so the resolution rules + * can be exercised directly against a key set. + */ +export const resolveJsImport: ImportResolver = (importingFileKey, moduleSpec, repoKeys) => + resolveImportWith(buildBasenameIndex(repoKeys), importingFileKey, moduleSpec, repoKeys); + +/** Unwrap TS `x as const` / `x satisfies T` to the underlying expression. */ +function unwrapTsExpression(node: Parser.SyntaxNode): Parser.SyntaxNode { + let cur = node; + while (cur.type === 'as_expression' || cur.type === 'satisfies_expression') { + const inner = cur.namedChild(0); + if (!inner) break; + cur = inner; + } + return cur; +} + +/** + * The literal string a node denotes, or `null` when it is not a plain literal. + * A template string counts only when it has no `${…}` substitution — an + * interpolated one is an expression, handled by {@link parseJsConstOperands}. + */ +function literalStringOf(node: Parser.SyntaxNode): string | null { + const n = unwrapTsExpression(node); + if (n.type === 'string') { + const fragments = n.namedChildren.filter((c) => c.type === 'string_fragment'); + if (fragments.length === 0) return n.namedChildren.length === 0 ? '' : null; + return fragments.map((f) => f.text).join(''); + } + if (n.type === 'template_string') { + if (n.namedChildren.some((c) => c.type === 'template_substitution')) return null; + const fragments = n.namedChildren.filter((c) => c.type === 'string_fragment'); + return fragments.map((f) => f.text).join(''); + } + return null; +} + +/** The static key a property name node denotes (`FOO`, `'foo'`, `"foo"`). */ +function staticKeyOf(node: Parser.SyntaxNode): string | null { + if (node.type === 'property_identifier' || node.type === 'identifier') return node.text; + if (node.type === 'string') return literalStringOf(node); + return null; +} + +/** + * Flatten an object literal into dotted `prefix.KEY` → literal entries. + * Nested objects recurse (`API.USERS.ME`); a computed key, a spread, or a + * non-string value is skipped — the table's other entries stay usable. + */ +function flattenObjectLiteral( + obj: Parser.SyntaxNode, + prefix: string, + into: Map, + depth = 0, +): void { + if (depth > MAX_REEXPORT_HOPS) return; + for (const pair of obj.namedChildren) { + if (pair.type !== 'pair') continue; + const keyNode = pair.childForFieldName('key'); + const valueNode = pair.childForFieldName('value'); + if (!keyNode || !valueNode) continue; + const key = staticKeyOf(keyNode); + if (key === null) continue; + const value = unwrapTsExpression(valueNode); + const literal = literalStringOf(value); + if (literal !== null) { + into.set(`${prefix}.${key}`, literal); + } else if (value.type === 'object') { + flattenObjectLiteral(value, `${prefix}.${key}`, into, depth + 1); + } + } +} + +/** + * Parse a `+`-concatenation / template string into an operand list the shared + * fold can resolve, or `null` when a term is not a string literal or a + * resolvable name reference. + * + * Handles the two shapes a JS route path is built with: + * `BASE + "/users"` → [ref BASE, literal /users] + * `` `${BASE}/users/${id}` `` → [ref BASE, literal /users/, ref id] + * + * A member reference inside either (`${API_ROUTE_PATH.LISTS}`) becomes a + * dotted `ref`, which the flattened table above resolves directly. + */ +export function parseJsConstOperands(node: Parser.SyntaxNode, depth = 0): Operand[] | null { + if (depth > MAX_EXPR_DEPTH) return null; + const n = unwrapTsExpression(node); + + const literal = literalStringOf(n); + if (literal !== null) return [{ kind: 'literal', value: literal }]; + + if (n.type === 'identifier') return [{ kind: 'ref', name: n.text }]; + + if (n.type === 'member_expression') { + const dotted = dottedNameOf(n); + return dotted === null ? null : [{ kind: 'ref', name: dotted }]; + } + + if (n.type === 'binary_expression') { + const operator = n.childForFieldName('operator'); + if (operator?.text !== '+') return null; + const left = n.childForFieldName('left'); + const right = n.childForFieldName('right'); + if (!left || !right) return null; + const l = parseJsConstOperands(left, depth + 1); + const r = parseJsConstOperands(right, depth + 1); + return l === null || r === null ? null : [...l, ...r]; + } + + if (n.type === 'template_string') { + const out: Operand[] = []; + for (const child of n.namedChildren) { + if (child.type === 'string_fragment') { + out.push({ kind: 'literal', value: child.text }); + } else if (child.type === 'template_substitution') { + const inner = child.namedChild(0); + if (!inner) return null; + const parsed = parseJsConstOperands(inner, depth + 1); + if (parsed === null) return null; + out.push(...parsed); + } + } + return out; + } + + return null; +} + +/** + * The dotted name a member expression denotes (`A.B.C`), or `null` for a + * computed / non-identifier chain (`A[key]`, `fn().B`) that has no stable + * textual key. + */ +export function dottedNameOf(node: Parser.SyntaxNode): string | null { + const parts: string[] = []; + let cur: Parser.SyntaxNode | null = node; + while (cur && cur.type === 'member_expression') { + const property = cur.childForFieldName('property'); + if (!property || property.type !== 'property_identifier') return null; + parts.unshift(property.text); + cur = cur.childForFieldName('object'); + } + if (!cur || cur.type !== 'identifier') return null; + parts.unshift(cur.text); + return parts.join('.'); +} + +/** + * How many wrapping calls {@link bindsAxiosClient} will look through. A factory + * is one hop (`setupInterceptors(axios.create())`); a couple more costs nothing + * and bounds the walk. + */ +const MAX_CLIENT_WRAP_DEPTH = 4; + +/** True when a node is `axios.create(...)`, allowing an aliased axios import. */ +function isAxiosCreateCall( + node: Parser.SyntaxNode, + imports: ReadonlyMap, + axiosShadowed: boolean, +): boolean { + if (node.type !== 'call_expression') return false; + const fn = node.childForFieldName('function'); + if (!fn || fn.type !== 'member_expression') return false; + if (fn.childForFieldName('property')?.text !== 'create') return false; + const object = fn.childForFieldName('object'); + if (!object || object.type !== 'identifier') return false; + // `import axios from 'axios'` is the overwhelming convention, but the local + // name is the importer's choice (`import ax from 'axios'`), so trust the + // module specifier over the spelling whenever the file declares one. The + // bare spelling is the fallback, and it is only evidence while the file has + // not bound that name to something else. + if (imports.get(object.text)?.module === 'axios') return true; + return object.text === 'axios' && !axiosShadowed; +} + +/** The module a `require('…')` initializer names, or `null` if it is not one. */ +function requireSpecifierOf(node: Parser.SyntaxNode): string | null { + const n = unwrapTsExpression(node); + if (n.type !== 'call_expression') return null; + if (n.childForFieldName('function')?.text !== 'require') return null; + const args = n.childForFieldName('arguments'); + const first = args?.namedChild(0); + return first ? literalStringOf(first) : null; +} + +/** + * Whether an initializer BINDS an axios instance — i.e. the instance is the + * VALUE of the binding, not merely present somewhere inside it. + * + * A direct `const api = axios.create(...)` is the textbook form, but the shape + * real applications ship is a factory that decorates the instance and hands it + * back: + * + * const routeApiClient = setupClientInterceptors({ + * axiosInstance: axios.create({ baseURL: API_URL }), + * }); + * + * Requiring the call to be the whole initializer would reject that — and it is + * the single binding every call site in such an app goes through. So a wrapping + * CALL whose result is bound counts, and the instance may be one of its + * arguments or a property of a directly-passed object literal. + * + * What does NOT count is the instance being an INGREDIENT of the bound value. + * The premise "an expression that builds an axios instance and binds the result + * is an HTTP client" is only true when the instance is the result; a plain + * subtree scan also admitted + * + * const registry = { http: axios.create(), version: 'v1' }; // object literal + * const client = MOCK ? memoryStore : axios.create(); // ternary branch + * new Map([['api', axios.create()]]); // constructor arg + * new LRUCache({ fetchMethod: axios.create().get }); // constructor arg + * + * and `.get`/`.delete` are the two most common non-HTTP method names in JS, so + * every one of those made an ordinary cache or registry an HTTP consumer. Those + * node types are simply not walked here. + * + * A nested function body is still skipped wherever it appears — a callback that + * builds its own client does not vouch for the outer name. + */ +function bindsAxiosClient( + node: Parser.SyntaxNode, + imports: ReadonlyMap, + axiosShadowed: boolean, + depth = 0, +): boolean { + if (depth > MAX_CLIENT_WRAP_DEPTH) return false; + const n = unwrapTsExpression(node); + + if (isAxiosCreateCall(n, imports, axiosShadowed)) return true; + + // Transparent wrappers around the value itself. + if ( + n.type === 'await_expression' || + n.type === 'parenthesized_expression' || + n.type === 'non_null_expression' + ) { + const inner = n.namedChild(0); + return inner !== null && bindsAxiosClient(inner, imports, axiosShadowed, depth + 1); + } + + if (n.type !== 'call_expression') return false; + + const args = n.childForFieldName('arguments'); + if (!args) return false; + for (const arg of args.namedChildren) { + if (argumentHoldsAxiosClient(arg, imports, axiosShadowed, depth + 1)) return true; + } + return false; +} + +/** + * Whether a wrapping call's ARGUMENT carries the instance. + * + * Inside an argument the instance may sit in an options object at any nesting + * (`createClient({ transport: { instance: axios.create() } })`) or in a list of + * decorators (`compose([axios.create(), withAuth])`). That is safe because the + * bound value is still the call's RESULT. It is the mirror of what + * {@link bindsAxiosClient} refuses: an object, array, ternary or `new` as the + * bound value itself never reaches here. + */ +function argumentHoldsAxiosClient( + node: Parser.SyntaxNode, + imports: ReadonlyMap, + axiosShadowed: boolean, + depth: number, +): boolean { + if (depth > MAX_CLIENT_WRAP_DEPTH) return false; + const n = unwrapTsExpression(node); + if (bindsAxiosClient(n, imports, axiosShadowed, depth)) return true; + + if (n.type === 'object') { + for (const pair of n.namedChildren) { + if (pair.type !== 'pair') continue; + const value = pair.childForFieldName('value'); + if (value && argumentHoldsAxiosClient(value, imports, axiosShadowed, depth + 1)) return true; + } + return false; + } + + if (n.type === 'array') { + for (const element of n.namedChildren) { + if (argumentHoldsAxiosClient(element, imports, axiosShadowed, depth + 1)) return true; + } + } + return false; +} + +/** + * Record one `name = value` binding into the accumulating facts. + * Shared by plain declarations and their `export const` form. + */ +function recordBinding( + name: string, + valueNode: Parser.SyntaxNode, + literals: Map, + exprs: Map, + clients: Set, + imports: ReadonlyMap, + axiosShadowed: boolean, +): void { + const value = unwrapTsExpression(valueNode); + + if (bindsAxiosClient(value, imports, axiosShadowed)) { + clients.add(name); + return; + } + + // `const client = someOtherClient` — an alias. Recorded as a client-chase + // edge (below) and as a constant ref, since one of the two will resolve. + if (value.type === 'identifier') { + exprs.set(name, [{ kind: 'ref', name: value.text }]); + return; + } + + if (value.type === 'object') { + flattenObjectLiteral(value, name, literals); + return; + } + + const literal = literalStringOf(value); + if (literal !== null) { + literals.set(name, literal); + return; + } + + const operands = parseJsConstOperands(value); + if (operands !== null) exprs.set(name, operands); +} + +/** Record every `variable_declarator` in a declaration node. */ +function recordDeclaration( + decl: Parser.SyntaxNode, + literals: Map, + exprs: Map, + clients: Set, + imports: ReadonlyMap, + axiosShadowed: boolean, + exports: Map | null, +): void { + for (const declarator of decl.namedChildren) { + if (declarator.type !== 'variable_declarator') continue; + const nameNode = declarator.childForFieldName('name'); + const valueNode = declarator.childForFieldName('value'); + if (!nameNode || nameNode.type !== 'identifier' || !valueNode) continue; + recordBinding(nameNode.text, valueNode, literals, exprs, clients, imports, axiosShadowed); + exports?.set(nameNode.text, nameNode.text); + } +} + +/** Record one `import … from 'm'` statement's local bindings. */ +function recordImportStatement( + stmt: Parser.SyntaxNode, + imports: Map, +): void { + const source = stmt.childForFieldName('source'); + const moduleSpec = source ? literalStringOf(source) : null; + if (moduleSpec === null) return; + for (const clause of stmt.namedChildren) { + if (clause.type !== 'import_clause') continue; + for (const spec of clause.namedChildren) { + // `import Default from 'm'` + if (spec.type === 'identifier') { + imports.set(spec.text, { module: moduleSpec, originalName: 'default' }); + } else if (spec.type === 'namespace_import') { + const alias = spec.namedChild(0); + // `import * as NS from 'm'` — `NS.X` resolves to the target's `X`. + if (alias) imports.set(alias.text, { module: moduleSpec, originalName: '*' }); + } else if (spec.type === 'named_imports') { + for (const named of spec.namedChildren) { + if (named.type !== 'import_specifier') continue; + const nameNode = named.childForFieldName('name'); + const aliasNode = named.childForFieldName('alias'); + if (!nameNode) continue; + const local = (aliasNode ?? nameNode).text; + imports.set(local, { module: moduleSpec, originalName: nameNode.text }); + } + } + } + } +} + +/** + * Extract one file's {@link JsModuleFacts} from its parsed tree. + * + * Only TOP-LEVEL declarations are collected. A route table or an API client + * defined inside a function body is not a module constant, and treating it as + * one would let an unrelated same-named local shadow the real export. + */ +export function extractJsModuleFacts(tree: Parser.Tree): JsModuleFacts { + const literals = new Map(); + const exprs = new Map(); + const imports = new Map(); + const exports = new Map(); + const starExports: string[] = []; + const clients = new Set(); + + // Imports first. ES module bindings are hoisted — `const c = ax.create(…)` + // above `import ax from 'axios'` is legal and binds the same `ax` — but + // `bindsAxiosClient` consults `imports` as each declaration is recorded, so + // in source order an import declared later was simply not there yet and the + // client went unproven. + // + // CommonJS `const ax = require('axios')` is collected here too. It is the + // same binding by another spelling, and without it an aliased require + // resolved to nothing at all while the un-aliased one worked only because + // `axios` happens to be the name the spelling shortcut trusts. + let axiosShadowed = false; + for (const stmt of tree.rootNode.namedChildren) { + if (stmt.type === 'import_statement') { + recordImportStatement(stmt, imports); + continue; + } + const decl = stmt.type === 'export_statement' ? stmt.childForFieldName('declaration') : stmt; + if ( + decl === null || + (decl.type !== 'lexical_declaration' && decl.type !== 'variable_declaration') + ) { + continue; + } + for (const declarator of decl.namedChildren) { + if (declarator.type !== 'variable_declarator') continue; + const nameNode = declarator.childForFieldName('name'); + const valueNode = declarator.childForFieldName('value'); + if (!nameNode || nameNode.type !== 'identifier') continue; + const required = valueNode === null ? null : requireSpecifierOf(valueNode); + if (required !== null) { + imports.set(nameNode.text, { module: required, originalName: 'default' }); + } else if (nameNode.text === 'axios') { + axiosShadowed = true; + } + } + } + + for (const stmt of tree.rootNode.namedChildren) { + if (stmt.type === 'lexical_declaration' || stmt.type === 'variable_declaration') { + recordDeclaration(stmt, literals, exprs, clients, imports, axiosShadowed, null); + continue; + } + + if (stmt.type === 'import_statement') continue; // hoisted above + + if (stmt.type !== 'export_statement') continue; + + const source = stmt.childForFieldName('source'); + const reexportFrom = source ? literalStringOf(source) : null; + const declaration = stmt.childForFieldName('declaration'); + const value = stmt.childForFieldName('value'); + + // `export const X = …` / `export default ` + if (declaration) { + if ( + declaration.type === 'lexical_declaration' || + declaration.type === 'variable_declaration' + ) { + recordDeclaration(declaration, literals, exprs, clients, imports, axiosShadowed, exports); + } + continue; + } + + if (value) { + // `export default routeApiClient` / `export default axios.create(...)` + if (value.type === 'identifier') { + exports.set('default', value.text); + } else { + recordBinding(DEFAULT_LOCAL, value, literals, exprs, clients, imports, axiosShadowed); + exports.set('default', DEFAULT_LOCAL); + } + continue; + } + + // `export * from './m'` / `export * as NS from './m'`. Neither has an + // export_clause; the namespace form additionally binds a local alias. + if (reexportFrom !== null && !stmt.namedChildren.some((c) => c.type === 'export_clause')) { + const namespaceAlias = stmt.namedChildren.find((c) => c.type === 'namespace_export'); + const alias = namespaceAlias?.namedChild(0)?.text; + if (alias !== undefined) { + imports.set(alias, { module: reexportFrom, originalName: '*' }); + exports.set(alias, alias); + } else { + starExports.push(reexportFrom); + } + continue; + } + + // `export { a, b as c }` and `export { a } from './m'` + for (const clause of stmt.namedChildren) { + if (clause.type !== 'export_clause') continue; + for (const spec of clause.namedChildren) { + if (spec.type !== 'export_specifier') continue; + const nameNode = spec.childForFieldName('name'); + const aliasNode = spec.childForFieldName('alias'); + if (!nameNode) continue; + const exported = (aliasNode ?? nameNode).text; + if (reexportFrom !== null) { + imports.set(exported, { module: reexportFrom, originalName: nameNode.text }); + exports.set(exported, exported); + } else { + exports.set(exported, nameNode.text); + } + } + } + } + + return { constants: { literals, exprs, imports }, exports, starExports, clients, axiosShadowed }; +} + +/** + * Resolve a path reference at a call site to its literal string, or `null`. + * + * `ref` is the dotted name as written (`API_ROUTE_PATH.LINKS`, or a bare + * `BASE_PATH`). Resolution order: + * + * 1. The dotted name as a constant of the CURRENT file — hits when the table + * is declared in the same file (flattened to dotted literal keys). + * 2. The base name as an IMPORT of the current file — hop to the defining + * file and look the dotted name up there, re-hopping through barrels that + * re-export it, bounded by {@link MAX_REEXPORT_HOPS}. + * + * Returns `null` on anything it cannot fully fold, which leaves the call site + * exactly as unmatched as it is today — never a guessed path. + */ +export function resolveJsMemberPath( + fileKey: string, + ref: string, + facts: JsRepoFacts, +): string | null { + const direct = foldConstant(fileKey, ref, facts.constants, facts.resolveImport, facts.keys); + if (direct !== null) return direct; + + const dot = ref.indexOf('.'); + if (dot < 0) return null; + const base = ref.slice(0, dot); + const member = ref.slice(dot + 1); + + const binding = facts.byFile.get(fileKey)?.constants.imports.get(base); + if (!binding) return null; + const targetKey = facts.resolveImport(fileKey, binding.module, facts.keys); + if (targetKey === null) return null; + + // `import * as NS from 'm'` — `NS.TABLE.KEY` addresses the target's own + // `TABLE.KEY`, so the namespace alias drops out of the reference entirely. + if (binding.originalName === '*') { + const nextDot = member.indexOf('.'); + if (nextDot < 0) return null; + return resolveExportedMember( + targetKey, + member.slice(0, nextDot), + member.slice(nextDot + 1), + facts, + 0, + new Set(), + ); + } + + return resolveExportedMember(targetKey, binding.originalName, member, facts, 0, new Set()); +} + +/** + * Resolve `.` against a module's PUBLIC surface, following + * whatever indirection stands between the name and its definition. + * + * Three ways a module can expose a name, tried in order: + * 1. it defines it (possibly under a different local name — `export { a as b }`) + * 2. it re-exports it explicitly (`export { a } from './m'`) + * 3. it re-exports a whole module (`export * from './m'`) + * + * The third is the one that matters in practice: application code imports a + * DIRECTORY (`@/api-modules/shared`), whose `index.ts` is nothing but + * `export * from './api-routes'`. Stopping at the barrel resolves nothing at + * all, so the star edges have to be walked. `seen` makes mutually-importing + * barrels terminate instead of recursing forever. + */ +function resolveExportedMember( + fileKey: string, + exported: string, + member: string, + facts: JsRepoFacts, + depth: number, + seen: Set, +): string | null { + if (depth > MAX_REEXPORT_HOPS) return null; + const guard = `${fileKey}::${exported}.${member}`; + if (seen.has(guard)) return null; + seen.add(guard); + + const file = facts.byFile.get(fileKey); + if (!file) return null; + + const local = file.exports.get(exported) ?? exported; + const here = foldConstant( + fileKey, + `${local}.${member}`, + facts.constants, + facts.resolveImport, + facts.keys, + ); + if (here !== null) return here; + + const binding = file.constants.imports.get(exported); + if (binding) { + const targetKey = facts.resolveImport(fileKey, binding.module, facts.keys); + if (targetKey !== null) { + const viaImport = resolveExportedMember( + targetKey, + binding.originalName === '*' ? exported : binding.originalName, + member, + facts, + depth + 1, + seen, + ); + if (viaImport !== null) return viaImport; + } + } + + // Every star edge is walked, not just up to the first hit: two barrels + // re-exporting the same name is ambiguous in JS itself, so answering with + // whichever module happens to come first in `starExports` would be a guess + // dressed as a resolution. + let viaStar: string | null = null; + for (const spec of file.starExports) { + const targetKey = facts.resolveImport(fileKey, spec, facts.keys); + if (targetKey === null) continue; + const found = resolveExportedMember(targetKey, exported, member, facts, depth + 1, seen); + if (found === null) continue; + if (viaStar !== null && viaStar !== found) return null; + viaStar = found; + } + + return viaStar; +} + +/** The local binding an exported name refers to in `fileKey` (identity if unaliased). */ +function resolveExportLocal(facts: JsRepoFacts, fileKey: string, exported: string): string { + return facts.byFile.get(fileKey)?.exports.get(exported) ?? exported; +} + +/** + * Recursion ceiling for the path-expression fold, and the matching term cap for + * a `+` chain. + * + * Nothing on this path was bounded before: `flattenConcat` recursed once per + * term, mutually with {@link foldTermOrPlaceholder}, on the SCAN side — which + * `prepareRepo`'s `try/catch` does not cover and which `HttpLanguagePlugin.scan` + * contractually may not throw from. ~6 400 concat terms (38 KB of source) threw + * `RangeError: Maximum call stack size exceeded` out of `extract()`, and + * `sync.ts` turns that into an unexplained "missing repo" with every contract of + * every kind — HTTP, gRPC, topics, includes — dropped for that repo and nothing + * logged. A hand-written route path is a handful of terms. + */ +const MAX_EXPR_DEPTH = 64; +const MAX_CONCAT_TERMS = 256; + +/** One folded term, and whether its text is KNOWN rather than a placeholder. */ +interface FoldedTerm { + readonly text: string; + readonly concrete: boolean; +} + +/** + * A folded path expression, and whether its FIRST term was concrete. + * + * `anchored` is what separates a partially-folded path from a fabricated one. + * `${API_ROUTE_PATH.LISTS}/${eventId}/add` is anchored — its leading segment is + * a resolved route constant and the rest is honest `{param}`s. `${base}${suffix}` + * and `${BASE}/users` are not: nothing pins where the path starts, so consumer + * normalization squashes them to `/{param}{param}` and `/{param}/users`, which + * exact-match real provider routes and invent cross-repo links. The docstring + * on {@link resolveJsPathExpression} always claimed at least one literal segment + * was required; only now is it true. + */ +interface FoldedPath { + readonly text: string; + readonly anchored: boolean; +} + +/** + * Resolve one term of a partially-foldable path, re-emitting it as a + * `${…}` placeholder when it cannot be folded. + * + * The placeholder is deliberate, not a fallback wart: consumer-side path + * normalization rewrites `${…}` to `{param}`, which is exactly the right + * reading for a term that IS a runtime value (`${eventId}`). Re-emitting keeps + * a mixed path like `` `${API_ROUTE_PATH.LISTS}/${eventId}/add` `` resolvable to + * `/curator-lists/{param}/add` instead of collapsing its known prefix to + * `{param}/{param}/add`. + */ +function foldTermOrPlaceholder( + fileKey: string, + node: Parser.SyntaxNode, + facts: JsRepoFacts, + depth: number, +): FoldedTerm | null { + if (depth > MAX_EXPR_DEPTH) return null; + const n = unwrapTsExpression(node); + + const literal = literalStringOf(n); + if (literal !== null) return { text: literal, concrete: true }; + + // A template nested inside a substitution — `` `${BASE}${`/${id}/unlike`}` `` + // is a real shape. Recursing keeps its literal segments; emitting it verbatim + // would collapse the whole inner template to one `{param}` and lose them. + if (n.type === 'template_string' || n.type === 'binary_expression') { + const nested = foldPathExpression(fileKey, n, facts, depth + 1); + if (nested !== null) return { text: nested.text, concrete: nested.anchored }; + } + + const dotted = n.type === 'identifier' ? n.text : dottedNameOf(n); + if (dotted !== null) { + const resolved = resolveJsMemberPath(fileKey, dotted, facts); + if (resolved !== null) return { text: resolved, concrete: true }; + return { text: `\${${dotted}}`, concrete: false }; + } + + return { text: `\${${n.text}}`, concrete: false }; +} + +/** Flatten a left-nested `a + b + c` chain into its terms, or `null` if not all `+`. */ +function flattenConcat(node: Parser.SyntaxNode, depth: number): Parser.SyntaxNode[] | null { + if (depth > MAX_EXPR_DEPTH) return null; + const n = unwrapTsExpression(node); + if (n.type !== 'binary_expression') return [n]; + + // The LEFT spine is walked iteratively: `a + b + c + …` parses left-nested, + // so recursing once per term is one stack frame per term. Only a `+` on the + // right can still nest, and that recursion is depth-capped. + const reversed: Parser.SyntaxNode[] = []; + let cur: Parser.SyntaxNode = n; + for (;;) { + if (reversed.length > MAX_CONCAT_TERMS) return null; + if (cur.childForFieldName('operator')?.text !== '+') return null; + const left = cur.childForFieldName('left'); + const right = cur.childForFieldName('right'); + if (!left || !right) return null; + reversed.push(right); + const nextLeft = unwrapTsExpression(left); + if (nextLeft.type !== 'binary_expression') { + reversed.push(nextLeft); + break; + } + cur = nextLeft; + } + + const out: Parser.SyntaxNode[] = []; + for (let i = reversed.length - 1; i >= 0; i--) { + const term = reversed[i]; + if (unwrapTsExpression(term).type !== 'binary_expression') { + out.push(term); + continue; + } + const nested = flattenConcat(term, depth + 1); + if (nested === null) return null; + out.push(...nested); + if (out.length > MAX_CONCAT_TERMS) return null; + } + return out; +} + +/** + * The fold behind {@link resolveJsPathExpression}, carrying the recursion depth + * and reporting whether the result is anchored. + * + * `MAX_FOLD_LENGTH` is checked on the ACCUMULATED text, not per term. The + * shared core caps each folded constant at that length; joining an unbounded + * number of them made the cap a ~2048x amplifier instead of a ceiling (each + * `${A}` costs 4 source characters and can yield 8 192), and the result is not + * transient — it becomes `contractId` and `meta.path` in `contracts.json` and + * `bridge.lbug`. Measured 200 KB of source to 941 MB of heap before this. + */ +function foldPathExpression( + fileKey: string, + node: Parser.SyntaxNode, + facts: JsRepoFacts, + depth: number, +): FoldedPath | null { + if (depth > MAX_EXPR_DEPTH) return null; + const n = unwrapTsExpression(node); + + const literal = literalStringOf(n); + if (literal !== null) return { text: literal, anchored: true }; + + if (n.type === 'identifier' || n.type === 'member_expression') { + const dotted = n.type === 'identifier' ? n.text : dottedNameOf(n); + if (dotted === null) return null; + const resolved = resolveJsMemberPath(fileKey, dotted, facts); + return resolved === null ? null : { text: resolved, anchored: true }; + } + + const terms: Parser.SyntaxNode[] = []; + if (n.type === 'template_string') { + for (const child of n.namedChildren) { + if (child.type === 'string_fragment') { + terms.push(child); + } else if (child.type === 'template_substitution') { + const inner = child.namedChild(0); + if (inner === null) return null; + terms.push(inner); + } + } + } else if (n.type === 'binary_expression') { + const flattened = flattenConcat(n, depth); + if (flattened === null) return null; + terms.push(...flattened); + } else { + return null; + } + + let out = ''; + let anchored: boolean | null = null; + for (const term of terms) { + const folded = + term.type === 'string_fragment' + ? { text: term.text, concrete: true } + : foldTermOrPlaceholder(fileKey, term, facts, depth + 1); + if (folded === null) return null; + if (anchored === null) anchored = folded.concrete; + out += folded.text; + if (out.length > MAX_FOLD_LENGTH) return null; + } + return anchored === null ? null : { text: out, anchored }; +} + +/** + * Resolve the first argument of an HTTP call to a path string, or `null` when + * the expression is not a path shape this binding understands. + * + * Accepts a plain literal, a constant reference (`BASE_PATH`), a table member + * (`API_ROUTE_PATH.LINKS`), a template string, and a `+`-concatenation of any + * of those. Template and concat forms fold PARTIALLY — see + * {@link foldTermOrPlaceholder}. + * + * A reference that resolves to nothing returns `null` (skip), and so does a + * mixed expression whose leading term is unresolved — see {@link FoldedPath}. + */ +export function resolveJsPathExpression( + fileKey: string, + node: Parser.SyntaxNode, + facts: JsRepoFacts, +): string | null { + const folded = foldPathExpression(fileKey, node, facts, 0); + return folded !== null && folded.anchored ? folded.text : null; +} + +/** + * Whether `name`, as referenced in `fileKey`, holds an HTTP client instance. + * + * Chases local aliases and import/export hops so the common app shape — + * `axios.create()` in `lib/axios.config.ts`, `export default apiClient`, + * `import apiClient from '@/lib/axios.config'` at the call site — is proven + * rather than pattern-matched on the receiver's spelling. + * + * Deliberately conservative: an unproven receiver returns `false`, which keeps + * today's behavior for it. The alternative — trusting any identifier with an + * HTTP-verb method — would classify every Express `router.get('/x', handler)` + * provider as a consumer of itself. + */ +/** + * Whether `name`, as a receiver in `fileKey`, IS the axios module — as opposed + * to an instance built from it, which is {@link isHttpClientRef}'s question. + * + * Two ways to be it. The bare spelling `axios` predates the widened query — it + * is what the original `(#eq? @obj "axios")` pattern matched — so it stays + * trusted by default, and a file with no facts keeps exactly that behavior; it + * is withdrawn only where the file itself binds that name to something else. + * The other way is a declared import or `require` of `'axios'` under any local + * name, which is proof rather than convention and covers the aliased form the + * spelling rule cannot see. + */ +export function isAxiosNamespace(fileKey: string, name: string, facts: JsRepoFacts): boolean { + const file = facts.byFile.get(fileKey); + if (file === undefined) return name === 'axios'; + if (file.constants.imports.get(name)?.module === 'axios') return true; + return name === 'axios' && !file.axiosShadowed; +} + +export function isHttpClientRef(fileKey: string, name: string, facts: JsRepoFacts): boolean { + let currentKey = fileKey; + let currentName = name; + + for (let hop = 0; hop < MAX_REEXPORT_HOPS; hop++) { + const file = facts.byFile.get(currentKey); + if (!file) return false; + + if (file.clients.has(currentName)) return true; + + // Local alias: `const client = configuredClient`. + const expr = file.constants.exprs.get(currentName); + if (expr && expr.length === 1 && expr[0].kind === 'ref') { + currentName = expr[0].name; + continue; + } + + const binding = file.constants.imports.get(currentName); + if (!binding) return false; + const targetKey = facts.resolveImport(currentKey, binding.module, facts.keys); + if (targetKey === null) return false; + + currentKey = targetKey; + currentName = resolveExportLocal(facts, targetKey, binding.originalName); + } + return false; +} diff --git a/gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts b/gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts new file mode 100644 index 000000000..311612e2d --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/kotlin-const-resolver.ts @@ -0,0 +1,1616 @@ +/** + * Kotlin binding for the language-agnostic constant resolver (#2391 core). + * + * Supplies the two Kotlin-specific pieces — {@link resolveKotlinImport} (import + * specifier → file, honoring JVM package rules) and + * {@link extractKotlinModuleConstants} (tree → {@link ModuleConstants}) — plus + * the folding entry points {@link resolveKotlinConstant} and + * {@link foldKotlinOperands}, so callers stay language-oblivious. + * + * WHAT IS ACTUALLY SHARED WITH THE AGNOSTIC CORE. One value — + * {@link MAX_FOLD_LENGTH} — and five types. The core's own `resolveConstant` / + * `resolveOperands` are NOT called: the fold state machine below (cycle guard, + * success memo, depth caps, operand concatenation — roughly 200 of this file's + * lines) is a local fork, close enough to `java-const-resolver.ts`'s already + * forked copy that the two read as the same code with the language name + * swapped. + * + * That fork is a consequence, not an oversight. The core keys its maps by + * SIMPLE name, and a Kotlin operand can be a QUALIFIED reference at any + * position (`X = ApiPaths.Y + "/tail"`); handed to the core, `ApiPaths.Y` misses + * every map and floors the whole chain to null — see {@link computeKotlinFold}, + * which resolves operands through the qualified-aware walk for exactly this + * reason. The import chase is Kotlin-specific too: a member import is spelled + * identically to a type import, so {@link resolveImportedName} has to try both + * readings, and the core exposes no hook for that. Java forked first on the same + * grounds. Teaching the core qualified names, and retiring both copies against + * it, is the standing follow-up; until then the honest description of this file + * is "a second fork", not "a binding over a shared fold". + * + * Kotlin shares the JVM package/import model with Java, so this binding mirrors + * `java-const-resolver.ts` in structure, naming and skip-floor discipline. The + * four places Kotlin genuinely differs are handled explicitly, not translated: + * + * 1. **Where a constant can live.** Java has one carrier (`static final` on a + * type). Kotlin has three: a top-level `const val`/`val`, a member of an + * `object`, and a member of a `companion object` — the last is referenced + * through its ENCLOSING class (`Holder.NAME`), not through `Companion`. + * 2. **No `String` type gate.** Kotlin infers property types, so + * `const val ORDERS = "/orders"` carries no type node to check. The + * initializer decides: anything {@link parseKotlinConstOperands} cannot fold + * to a string (a number, a call, a template) drops the constant. + * 3. **File names and directories are free.** `object ApiPaths` may live in + * `Constants.kt`, and a file's `package` need not match the directory it + * sits in, so a `/.kt` PATH lookup is a convention and not a + * rule. The authority is each file's DECLARED `package`, which + * {@link extractKotlinModuleConstants} records and {@link resolveKotlinImport} + * requires an exact match on; the path is only a tie-break among files that + * already declare the right package. + * 4. **Member imports are unmarked.** Java spells them `import static a.b.C.F`; + * Kotlin writes `import a.b.C.F`, which is byte-identical to a type import + * of a class `F` in package `a.b.C`. Nothing in the syntax says which, so + * the fold tries both readings (see `resolveImportedName`) instead of + * guessing from casing. + * 5. **Any identifier may be backtick-quoted.** `` package com.example.`api` `` + * and `package com.example.api` are the SAME package to the compiler, and a + * keyword segment (`` com.example.`fun` ``) can only be spelled the quoted + * way. The grammar keeps the backticks in the node text, so every identifier is + * read through {@link unquoteKotlinIdentifier} before it becomes a map key + * or a lookup name — see that function for what a verbatim comparison cost. + * + * THREE PLACES THIS BINDING NO LONGER MIRRORS JAVA, each because the mirrored + * behavior was wrong rather than merely different, and each open as a Java + * follow-up rather than fixed here: + * + * * `java-const-resolver.ts` flattens nested types into one file-level + * namespace and argues the collision away — "qualified refs carry the class + * name, so nesting only matters for same-name fields, which flatten + * last-wins". The argument does not hold: the collision is one level BELOW + * the qualification, in the initializer, so a fully qualified `A.ROUTE` whose + * initializer names a bare sibling `BASE` still resolves through whichever + * same-named sibling was walked last. {@link extractKotlinModuleConstants} + * keys by Kotlin's own visibility instead. + * * Java's import resolution can lean on the `/.java` layout the + * language enforces. Kotlin's cannot, and inferring the package from the path + * lets a path-suffix twin outrank the real declaration — so + * {@link resolveKotlinImport} reads the declared `package` instead. + * * Java's fold entry points take a file and a name, because a Java `static + * final` reachable by simple name is reachable that way from anywhere in the + * file. A Kotlin COMPANION member is not: it is bound unqualified only inside + * its enclosing class body. {@link foldKotlinOperands} therefore also takes + * the enclosing type chain of the reference site, which is what lets the + * binding answer a bare reference by Kotlin's scoping rather than by "whoever + * was walked last" — see {@link qualifyKotlinRefInEnclosingTypes}. + * + * Constant shapes this binding harvests: + * + * const val TOP_LEVEL = "/api/v1" // file top level + * object ApiPaths { // object member + * const val BASE = "/api/v1" + * val ORDERS = BASE + "/orders" + * } + * class Holder { companion object { const val H = "/h" } } // → Holder.H + * + * Reference shapes at annotation sites this binding resolves: + * @PostMapping(ApiPaths.ORDERS) // qualified + * @PostMapping(com.example.app.api.ApiPaths.ORDERS) // FQN-qualified + * @PostMapping(ORDERS) // single-name import + * @PostMapping(ORDERS) after import com.example.api.* // package-star import + * @PostMapping(ApiPaths.BASE + "/orders") // inline concat + * + * Which ANNOTATIONS count as routes is a separate question this module has no + * say in — `spring-shared.ts` owns that map. Folding and annotation recognition + * compose; neither implies the other. + * + * Keying (parity with the Java and Python bindings): the repo map is keyed by + * unique POSIX file path, and an import that cannot be pinned to exactly one + * file returns null (skip floor), never a wrong path. A missing route is a + * missing fact; a wrongly folded one is a false edge in the graph. "Exactly one + * file" is decided from the DECLARED package, not from the path: a path is a + * repository-layout accident that any decoy directory can imitate, whereas the + * `package` header is the declaration the compiler itself resolves against. + * + * POSIX keys are a PRECONDITION this module cannot check cheaply, so it is + * enforced at the one boundary that produces them: `http-patterns/kotlin.ts` + * normalizes separators on both the write side (the `prepareRepo` map keys) and + * the read side (`scan`'s `fileRel`). It has to, because the orchestrator's file + * list comes from glob v13, which has no `posix: true` and joins with the + * platform separator — so on Windows the keys arrive backslashed and every + * `/.kt` test in {@link resolveKotlinImport} would miss, silently + * disabling cross-file folding on that platform alone. Normalizing INSIDE this + * module instead cannot work: the resolver returns the key it matched, and a + * normalized return value would then miss in a map that was never normalized. + * + * WHERE THIS IS WIRED. Java reaches its binding from BOTH layers: the group + * extractor (`group/extractors/http-patterns/java.ts`) and the ingestion + * provider (`languages/java.ts`). Kotlin now does the same: the group extractor + * (`group/extractors/http-patterns/kotlin.ts`) plus `languages/kotlin.ts` + * (`extractDecoratorRoutes`, `extractModuleConstants`, `foldRoutePathOperands`). + * The dedicated ingestion walker is `route-extractors/kotlin-spring.ts`; it + * does not reuse Java `spring.ts`. + */ + +import type Parser from 'tree-sitter'; +import { unquoteSpringLiteral } from './spring-shared.js'; +import { + MAX_FOLD_LENGTH, + unfoldableDeclarationsOf, + type ImportBinding, + type ModuleConstants, + type Operand, + type RepoConstants, +} from './constant-resolver.js'; + +export type { + ImportBinding, + ModuleConstants, + Operand, + RepoConstants, +} from './constant-resolver.js'; +export { unfoldableDeclarationsOf } from './constant-resolver.js'; + +/** + * What {@link extractKotlinModuleConstants} returns: the agnostic + * {@link ModuleConstants} plus the one piece of per-file metadata JVM import + * resolution cannot be honest without — the file's declared `package`. + * + * Deliberately a KOTLIN-LOCAL widening rather than a field on the shared type. + * `ModuleConstants` is consumed by the Java, JS and Python bindings too, and + * none of them needs this: Python resolves imports from the module path, and + * Java's `package` is already pinned by the `/.java` rule the + * language enforces. Adding a required field there would force three unrelated + * bindings to fill it in; adding an optional one would put a Kotlin-shaped hole + * in a type whose whole point is language neutrality. + * + * Read the metadata through {@link declaredPackageOf} and + * {@link unfoldableDeclarationsOf}, never by field access: a + * {@link RepoConstants} is typed over the agnostic shape, so an entry that some + * other producer put there carries no package and must be REJECTED as a + * candidate rather than silently treated as the default package. Missing + * unfoldable-declaration metadata instead means "none known", preserving the + * agnostic entry's existing behavior. + */ +export interface KotlinModuleConstants extends ModuleConstants { + /** The file's declared `package`, or `''` for the default package. */ + readonly packageName: string; + /** Declaration keys whose initializer cannot be folded. */ + readonly unfoldableDeclarations: ReadonlySet; + /** Top-level properties and types that shadow lower-priority star imports. */ + readonly topLevelDeclarations: ReadonlySet; +} + +/** + * The declared `package` of the file `mc` describes, or null when the entry did + * not come from {@link extractKotlinModuleConstants} and therefore cannot be + * matched against an import specifier. + */ +function declaredPackageOf(mc: ModuleConstants | undefined): string | null { + const declared = (mc as KotlinModuleConstants | undefined)?.packageName; + return typeof declared === 'string' ? declared : null; +} + +const NO_TOP_LEVEL_DECLARATIONS: ReadonlySet = new Set(); + +/** Kotlin top-level names known to shadow package-star imports. */ +function topLevelDeclarationsOf(mc: ModuleConstants | undefined): ReadonlySet { + const declarations = (mc as KotlinModuleConstants | undefined)?.topLevelDeclarations; + return declarations instanceof Set ? declarations : NO_TOP_LEVEL_DECLARATIONS; +} + +/** Source extensions a Kotlin declaration can live in. */ +const KOTLIN_EXTENSIONS = ['.kt', '.kts'] as const; + +/** + * The name a backtick-quoted Kotlin identifier denotes: `` `api` `` → `api`. + * + * Quotes are spelling, not part of the name. tree-sitter-kotlin keeps them in + * node text, so every identifier that becomes a map key or lookup is read + * through here. Applied per dot-separated segment — a quoted identifier cannot + * contain `.`. Both the declaration side ({@link declaredPackage}) and the + * import side ({@link resolveKotlinImport}) are normalized, because either may + * carry the quotes while the other spells the same name plainly. + */ +export function unquoteKotlinIdentifier(text: string): string { + return text.length >= 2 && text.startsWith('`') && text.endsWith('`') ? text.slice(1, -1) : text; +} + +/** {@link unquoteKotlinIdentifier} applied to every segment of a dotted name. */ +function unquoteKotlinDottedName(text: string): string { + return text.includes('`') ? text.split('.').map(unquoteKotlinIdentifier).join('.') : text; +} + +/** + * Recursion ceiling for {@link parseKotlinConstOperands}, counted in `+` links. + * + * This bounds SYNTAX depth, not resolution: `A + B + C` nests one + * `additive_expression` per link, so the cap is really "how long a concatenation + * may one initializer be". Deliberately loose — generated route tables do + * concatenate a dozen fragments, and overrunning costs a skipped route, so the + * cap is a guard against pathological input rather than a statement about + * reasonable code. + */ +const MAX_OPERAND_PARSE_DEPTH = 64; + +/** + * Recursion ceiling for the fold, counted in REFERENCE hops (`A = B`, `B = C`). + * + * Larger than the agnostic core's own `MAX_RESOLVE_DEPTH` (8), which is + * module-private in `constant-resolver.ts` and therefore cannot simply be + * reused, and equal to the value the Java binding spells inline. It backstops + * the cycle guard, which terminates loops but not a long acyclic chain; the + * memo makes reaching it cheap. Both caps floor to null, i.e. to a skipped + * route. + */ +const MAX_FOLD_DEPTH = 32; + +/** + * Cheap content gate: can this Kotlin file DEFINE a string constant that a route + * annotation might reference? + * + * Exported so every caller uses the same predicate and none can disagree with + * {@link extractKotlinModuleConstants} about which files carry constants — the + * defect class the Java binding's shared `isJavaConstantFile` exists to prevent. + * + * Arms, all intended to be WIDER than the extractor (a gate may over-admit — it + * only costs a parse — while rejecting a file the extractor would accept costs a + * fact): + * - `const val NAME [: T] =`. `const` is legal only at a file's top level or in + * an `object`/`companion object`, i.e. exactly the carriers the extractor + * harvests, so this arm needs no scope check. + * - an `object` (or `companion object`) declaration together with a `val NAME =` + * binding. A non-`const` `val` is the other half of the extractor's input and + * carries no keyword of its own; requiring an `object` nearby keeps a file + * whose only `val`s are function locals from costing a parse. It still admits + * a top-level `val` in a file that happens to declare an object elsewhere, + * which is the harmless direction. + * - a `val NAME =` binding at file scope. A small lexical walk tracks braces + * and parentheses while skipping comments and literals, admitting the + * top-level non-`const` property the extractor harvests without turning every + * function-local `val` or constructor property into an extra parse. + * + * Both name arms accept a BACKTICK-QUOTED identifier as well as a bare one, + * because the extractor does: `unquoteKotlinIdentifier` strips the quoting + * everywhere a name becomes a key, so `const val \`ORDERS\` = "/orders"` is a + * constant this module resolves. A gate that matched only `\w+` rejected the + * file outright and the reference floored to skip — a gate narrower than the + * extractor, which is the one direction the arms above are meant to exclude. + */ +const KOTLIN_NAME = String.raw`(?:\w+|\`[^\`\n]+\`)`; +const CONST_VAL_AT = new RegExp(String.raw`const\s+val\s+${KOTLIN_NAME}(?=\s|:|=|$)`, 'y'); +const VAL_DECLARATION_AT = new RegExp(String.raw`val\s+${KOTLIN_NAME}(?=\s|:|=|by\b|$)`, 'y'); + +/** Is `source[index...]` the keyword `word`, rather than part of an identifier? */ +function keywordAt(source: string, index: number, word: string): boolean { + if (!source.startsWith(word, index)) return false; + const before = index === 0 ? '' : source[index - 1]; + const after = source[index + word.length] ?? ''; + return !/[\w$]/.test(before) && !/[\w$]/.test(after); +} + +/** Test a sticky declaration pattern at one source offset without slicing. */ +function declarationAt(pattern: RegExp, source: string, index: number): boolean { + pattern.lastIndex = index; + return pattern.test(source); +} + +export function isKotlinConstantFile(source: string): boolean { + let braces = 0; + let parens = 0; + let blockCommentDepth = 0; + let sawObject = false; + + for (let i = 0; i < source.length; i++) { + if (blockCommentDepth > 0) { + if (source.startsWith('/*', i)) { + blockCommentDepth++; + i++; + } else if (source.startsWith('*/', i)) { + blockCommentDepth--; + i++; + } + continue; + } + + if (source.startsWith('//', i)) { + const newline = source.indexOf('\n', i + 2); + if (newline < 0) break; + i = newline; + continue; + } + if (source.startsWith('/*', i)) { + blockCommentDepth = 1; + i++; + continue; + } + + const quote = source[i]; + if (source.startsWith('"""', i)) { + const end = source.indexOf('"""', i + 3); + if (end < 0) break; + i = end + 2; + continue; + } + if (quote === '"' || quote === "'") { + for (i++; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === quote) break; + } + continue; + } + if (quote === '`') { + const end = source.indexOf('`', i + 1); + if (end < 0) break; + i = end; + continue; + } + + if (quote === '{') { + braces++; + continue; + } + if (quote === '}') { + braces = Math.max(0, braces - 1); + continue; + } + if (quote === '(') { + parens++; + continue; + } + if (quote === ')') { + parens = Math.max(0, parens - 1); + continue; + } + + if (keywordAt(source, i, 'object')) { + sawObject = true; + i += 'object'.length - 1; + continue; + } + if (keywordAt(source, i, 'const') && declarationAt(CONST_VAL_AT, source, i)) return true; + if (keywordAt(source, i, 'val') && declarationAt(VAL_DECLARATION_AT, source, i)) { + if (sawObject || (braces === 0 && parens === 0)) return true; + i += 'val'.length - 1; + } + } + return false; +} + +/** Does `key` name the file `.kt` / `.kts`? */ +function isFileNamedAfterDeclaration(key: string, asPath: string): boolean { + for (const ext of KOTLIN_EXTENSIONS) { + const candidate = `${asPath}${ext}`; + if (key === candidate || key.endsWith(`/${candidate}`)) return true; + } + return false; +} + +/** + * Does the file `mc` describes declare a top-level entity called `name` — an + * `object`/companion carrier whose members are keyed `name.`, or a + * top-level constant keyed `name` outright? + * + * A true result is strong enough to select a unique declaring file before path + * fallbacks: the tested key set is a superset of every local key the fold may + * subsequently read for that imported name. Two matching files are therefore + * ambiguous; one is authoritative. A miss falls back to the conservative path + * heuristics below, whose result is still verified by the actual map lookup. + */ +function declaresTopLevelName(mc: ModuleConstants, name: string): boolean { + const prefix = `${name}.`; + for (const map of [mc.literals, mc.exprs]) { + if (map.has(name)) return true; + for (const key of map.keys()) if (key.startsWith(prefix)) return true; + } + for (const key of unfoldableDeclarationsOf(mc)) { + if (key === name || key.startsWith(prefix)) return true; + } + return false; +} + +/** Files and declaration ownership for one exact Kotlin package. */ +export interface KotlinPackageConstants { + readonly files: readonly string[]; + /** Unique declaring file, or null when the package declares the name twice. */ + readonly declarers: ReadonlyMap; +} + +/** One exact package-qualified declaration and its in-file lookup key. */ +interface KotlinImportTarget { + readonly fileKey: string; + readonly localName: string; +} + +/** + * Repo-wide projections reused by every fold in one extraction run. + * + * `repo` includes importing files overlaid by `scan`; `constantKeys` and + * `byPackage` include only files that define a foldable or explicitly + * unfoldable declaration, preserving import ambiguity semantics. + */ +export interface KotlinConstantIndex { + readonly repo: RepoConstants; + readonly constantKeys: ReadonlySet; + readonly byPackage: ReadonlyMap; + /** Exact FQN → unique file/local key, or null when the FQN is duplicated. */ + readonly byFqn: ReadonlyMap; +} + +/** Does this file contribute declarations to Kotlin import ambiguity? */ +function contributesKotlinConstants(mc: ModuleConstants): boolean { + return ( + mc.literals.size > 0 || + mc.exprs.size > 0 || + unfoldableDeclarationsOf(mc).size > 0 || + topLevelDeclarationsOf(mc).size > 0 + ); +} + +/** Foldable or explicitly unfoldable constants that must live in index projections. */ +function hasIndexedConstants(mc: ModuleConstants): boolean { + return mc.literals.size > 0 || mc.exprs.size > 0 || unfoldableDeclarationsOf(mc).size > 0; +} + +/** Top-level names declared by one file (`Outer.X` contributes `Outer`). */ +function topLevelDeclarationNames(mc: ModuleConstants): Set { + const names = new Set(); + for (const map of [mc.literals, mc.exprs]) { + for (const key of map.keys()) { + const dot = key.indexOf('.'); + names.add(dot < 0 ? key : key.slice(0, dot)); + } + } + for (const key of unfoldableDeclarationsOf(mc)) { + const dot = key.indexOf('.'); + names.add(dot < 0 ? key : key.slice(0, dot)); + } + for (const name of topLevelDeclarationsOf(mc)) names.add(name); + return names; +} + +/** Every declaration key recorded for a file, foldable or not. */ +function declarationKeys(mc: ModuleConstants): Set { + return new Set([...mc.literals.keys(), ...mc.exprs.keys(), ...unfoldableDeclarationsOf(mc)]); +} + +/** Build the immutable import projections once for a repo constant map. */ +export function buildKotlinConstantIndex(repo: RepoConstants): KotlinConstantIndex { + const constantKeys = new Set(); + const byFqn = new Map(); + const mutablePackages = new Map< + string, + { files: string[]; declarers: Map } + >(); + + for (const [key, mc] of repo) { + if (!contributesKotlinConstants(mc)) continue; + constantKeys.add(key); + const packageName = declaredPackageOf(mc); + if (packageName === null) continue; + let bucket = mutablePackages.get(packageName); + if (!bucket) { + bucket = { files: [], declarers: new Map() }; + mutablePackages.set(packageName, bucket); + } + bucket.files.push(key); + for (const name of topLevelDeclarationNames(mc)) { + if (!bucket.declarers.has(name)) bucket.declarers.set(name, key); + else if (bucket.declarers.get(name) !== key) bucket.declarers.set(name, null); + } + for (const declaration of declarationKeys(mc)) { + const parts = declaration.split('.'); + // A member key `Outer.Inner.Q` proves the file declares both owner paths + // as well as the member itself. This lets imports of nested objects and + // their members retain the complete in-file lookup path. + for (let length = 1; length <= parts.length; length++) { + const localName = parts.slice(0, length).join('.'); + const fqn = packageName === '' ? localName : `${packageName}.${localName}`; + const existing = byFqn.get(fqn); + if (existing === undefined) byFqn.set(fqn, { fileKey: key, localName }); + else if (existing !== null && existing.fileKey !== key) byFqn.set(fqn, null); + } + } + } + + return { repo, constantKeys, byPackage: mutablePackages, byFqn }; +} + +/** Read-only one-entry overlay without copying the repo-wide constant map. */ +class KotlinConstantOverlay implements ReadonlyMap { + readonly [Symbol.toStringTag] = 'KotlinConstantOverlay'; + + constructor( + private readonly base: RepoConstants, + private readonly overlayKey: string, + private readonly overlayValue: ModuleConstants, + ) {} + + get size(): number { + return this.base.size + (this.base.has(this.overlayKey) ? 0 : 1); + } + + get(key: string): ModuleConstants | undefined { + return key === this.overlayKey ? this.overlayValue : this.base.get(key); + } + + has(key: string): boolean { + return key === this.overlayKey || this.base.has(key); + } + + *entries(): MapIterator<[string, ModuleConstants]> { + let replaced = false; + for (const [key, value] of this.base) { + if (key === this.overlayKey) { + replaced = true; + yield [key, this.overlayValue]; + } else { + yield [key, value]; + } + } + if (!replaced) yield [this.overlayKey, this.overlayValue]; + } + + *keys(): MapIterator { + for (const [key] of this.entries()) yield key; + } + + *values(): MapIterator { + for (const [, value] of this.entries()) yield value; + } + + [Symbol.iterator](): MapIterator<[string, ModuleConstants]> { + return this.entries(); + } + + forEach( + callbackfn: ( + value: ModuleConstants, + key: string, + map: ReadonlyMap, + ) => void, + thisArg?: unknown, + ): void { + for (const [key, value] of this.entries()) callbackfn.call(thisArg, value, key, this); + } +} + +/** + * Add one scan-time file without rebuilding the base index when it only imports + * constants. A newly discovered declaration is rare and rebuilds once for that + * file's scan, never once per route. + */ +export function overlayKotlinConstantIndex( + index: KotlinConstantIndex, + fileKey: string, + mc: ModuleConstants, +): KotlinConstantIndex { + // Same-file shadows are read from `mc` itself. Rebuild only when this key's + // constant projections would change; a controller with a class name but no + // foldable constants stays on the overlay so scan stays linear. + const existing = index.repo.get(fileKey); + if (!hasIndexedConstants(mc) && (!existing || !hasIndexedConstants(existing))) { + return { ...index, repo: new KotlinConstantOverlay(index.repo, fileKey, mc) }; + } + const repo = new Map(index.repo); + repo.set(fileKey, mc); + return buildKotlinConstantIndex(repo); +} + +/** + * Map a fully-qualified import specifier to the unique file key it refers to, or + * null when it cannot be pinned to exactly one file. + * + * A specifier is split at its last dot into the package it names and the + * declaration inside it (`com.example.app.api` + `ApiPaths`). Resolution then + * runs in three steps, all of them "unique or nothing": + * + * 0. **Declared package** — only files whose `package` header is EXACTLY the + * sought package can carry the declaration (compared after + * {@link unquoteKotlinIdentifier}, since backtick quoting is spelling and + * not identity). This is the authority, and it + * is checked first. Kotlin does not require a file's directory to match its + * package, so the reverse test — "does this path end with the package?" — + * answers a different question, one any decoy directory can satisfy: a file + * at `src/x/com/example/api/ApiPaths.kt` declaring `package x.com.example.api` + * is not `com.example.api.ApiPaths` and must never be folded as it, and a + * path-suffix test also lets a root-level `package data` be impersonated by + * `…/com/example/data/`. An entry with no recorded package is rejected, not + * assumed to be the default package. + * 1. **Declared name** — when exactly one file declares the sought name, use + * it. When two do, the FQN itself is duplicated in the repository and names + * no single declaration, so return null. This is the general form of the + * same-FQN check step 2 could only make for files that happen to follow the + * file-name convention, and it is what stops a `src/test/…` copy of a + * production constant from being folded into a production route. + * 2. **File named after the declaration** — when declaration metadata found no + * owner, try the package-matching file ending + * `com/example/app/api/ApiPaths.kt`. Kotlin does not require this (`object + * ApiPaths` may live in `Constants.kt`), so it is only a fallback candidate; + * the subsequent map lookup must still prove that it carries the value. + * 3. **Sole file in the package** — when declaration metadata cannot identify + * the name, use the unique package-matching candidate. The set passed in + * contains files with foldable or explicitly unfoldable declarations, so + * unrelated files cannot create ambiguity once step 1 identifies a unique + * declarer. With 2+ unidentified candidates it returns null. + * + * Steps 2 and 3 can still hand back a file that does not declare the wanted name + * (its package is right and it is the only candidate, but the name lives + * elsewhere or nowhere). That remains safe by construction: the fold looks the + * name up in that file's map, misses, and returns null. + * + * A "nearest shared directory" tie-break is deliberately NOT applied when a step + * has several candidates, for the reason the Java binding records: the JVM + * resolves duplicate FQNs by classpath order, not directory proximity, so a test + * fixture copy sitting closer in the tree can outrank the real dependency and + * yield a silently wrong literal. In a resolver whose whole contract is + * skip-or-correct, a plausible guess is the one answer that cannot be allowed. + * + * This can no longer be typed as the agnostic {@link ModuleConstants} consumer's + * `ImportResolver`, whose signature carries only file KEYS: deciding a candidate + * on its declared package needs the map those keys index. Nothing is lost — the + * core's own fold is not used here either (see the module header), and the + * alternative is a resolver that must guess from a path. + */ +export function resolveKotlinImport( + _importingFileKey: string, + rawModuleSpec: string, + candidateKeys: ReadonlySet, + repo: RepoConstants, +): string | null { + // Normalized here as well as at extraction, so the function answers the same + // question however a caller spells the specifier. + const moduleSpec = unquoteKotlinDottedName(rawModuleSpec); + const lastDot = moduleSpec.lastIndexOf('.'); + const packageName = lastDot < 0 ? '' : moduleSpec.slice(0, lastDot); + const simpleName = lastDot < 0 ? moduleSpec : moduleSpec.slice(lastDot + 1); + + // Step 0 + step 1 in one pass over the candidates. + const inPackage: string[] = []; + let declaring: string | null = null; + for (const key of candidateKeys) { + const mc = repo.get(key); + if (!mc || declaredPackageOf(mc) !== packageName) continue; + inPackage.push(key); + if (declaresTopLevelName(mc, simpleName)) { + if (declaring !== null) return null; // 2+ files declare this FQN + declaring = key; + } + } + if (inPackage.length === 0) return null; + if (declaring !== null) return declaring; + if (inPackage.length === 1) return inPackage[0]; // steps 2 and 3 agree + + // Step 2: the file-name convention, as a tie-break among valid candidates. + const asPath = moduleSpec.replace(/\./g, '/'); + let named: string | null = null; + for (const key of inPackage) { + if (!isFileNamedAfterDeclaration(key, asPath)) continue; + if (named !== null) return null; // 2+ files spell the convention + named = key; + } + // Step 3 is "the sole candidate", already returned above. + return named; +} + +/** Indexed equivalent of {@link resolveKotlinImport}, with identical fallbacks. */ +export function resolveKotlinImportWithIndex( + rawModuleSpec: string, + index: KotlinConstantIndex, +): string | null { + return resolveKotlinImportTarget(rawModuleSpec, index)?.fileKey ?? null; +} + +/** Resolve an import to both its file and complete in-file declaration path. */ +function resolveKotlinImportTarget( + rawModuleSpec: string, + index: KotlinConstantIndex, +): KotlinImportTarget | null { + const moduleSpec = unquoteKotlinDottedName(rawModuleSpec); + const lastDot = moduleSpec.lastIndexOf('.'); + const packageName = lastDot < 0 ? '' : moduleSpec.slice(0, lastDot); + const simpleName = lastDot < 0 ? moduleSpec : moduleSpec.slice(lastDot + 1); + const bucket = index.byPackage.get(packageName); + // Preserve the top-level interpretation whenever the exact declared package + // exists. A parent package may legally contain nested objects whose joined + // path spells the same FQN; letting that projection win would make a nested + // decoy override the real top-level declaration. + if (!bucket) return index.byFqn.get(moduleSpec) ?? null; + + if (bucket.declarers.has(simpleName)) { + const fileKey = bucket.declarers.get(simpleName); + return fileKey === null || fileKey === undefined ? null : { fileKey, localName: simpleName }; + } + if (bucket.files.length === 1) return { fileKey: bucket.files[0], localName: simpleName }; + + const asPath = moduleSpec.replace(/\./g, '/'); + let named: string | null = null; + for (const key of bucket.files) { + if (!isFileNamedAfterDeclaration(key, asPath)) continue; + if (named !== null) return null; + named = key; + } + return named === null ? null : { fileKey: named, localName: simpleName }; +} + +/** + * Is `node` a Kotlin string literal, and if so what value does the route layer + * give it? + * + * Two rejections, both floors rather than guesses: + * - **String templates.** `"$base/orders"` parses as a `string_literal` whose + * children include an interpolation alongside the `string_content` runs. + * Joining the content runs would silently DELETE the interpolated part and + * publish `/orders` — a path the application does not serve. Any named child + * that is not `string_content` means the value is not statically knowable, so + * the literal is refused. (The same test makes the function safe against a + * grammar that splits escape sequences into their own nodes: it would floor + * to skip, never to a de-escaped path.) + * - **Multi-line raw strings.** A single-line `"""/api"""` is exact — unlike a + * Java text block, a Kotlin raw string performs no escape processing and no + * incidental-indentation stripping, so it folds to precisely its content. A + * multi-line one carries newlines (and usually a `.trimIndent()` call this + * layer cannot fold), so it is refused. + * + * Otherwise the quotes are sliced off the RAW TEXT via + * {@link unquoteSpringLiteral} — the same function the literal path uses — so + * `@GetMapping(ApiPaths.USER_REGEX)` and `@GetMapping("/user/{id:\\d+}")` emit + * the same path for the same Kotlin source. + */ +function stringLiteralValue(node: Parser.SyntaxNode): string | null { + if (node.type !== 'string_literal') return null; + for (const child of node.namedChildren) { + if (child.type !== 'string_content') return null; + } + const raw = node.text; + if (raw.startsWith('"""') && raw.includes('\n')) return null; + return unquoteSpringLiteral(raw); +} + +/** + * Flatten a navigation expression (`ApiPaths`, `com.example.app.ApiPaths`) to + * its dotted text, or null when any segment is not a plain identifier (calls, + * `this`, indexing, safe navigation — not a constant shape). + */ +function flattenNavigation(node: Parser.SyntaxNode): string | null { + if (node.type === 'simple_identifier') return unquoteKotlinIdentifier(node.text); + if (node.type === 'navigation_expression') { + const target = node.namedChild(0); + const suffix = node.namedChildren.find((c) => c.type === 'navigation_suffix'); + const field = suffix?.namedChildren.find((c) => c.type === 'simple_identifier'); + if (target && field) { + const head = flattenNavigation(target); + return head === null ? null : `${head}.${unquoteKotlinIdentifier(field.text)}`; + } + } + return null; +} + +/** + * Parse a Kotlin constant initializer (or an inline annotation argument) into an + * operand list, or null when it is not a foldable string expression. Handles a + * bare string literal, a bare identifier (`X = Y`), a qualified reference + * (`X = ApiPaths.Y` — recorded as ONE ref named `ApiPaths.Y`), and + * left-associative `+` chains of the three. Everything else — numbers, calls, + * `when`/`if` expressions, templates, `buildString` — returns null, which makes + * the constant unresolvable (→ skip floor), never a wrong value. + * + * A chain nests: tree-sitter-kotlin parses `A + B + C` as + * `additive_expression(additive_expression(A, B), C)`, so every node here has + * exactly two operands and arbitrary-length chains fold by recursion. The same + * node type also carries `-`, which is not a string operation, so a `+` token + * must be present. + * + * A PARENTHESIZED operand (`(A + B) + "/c"`) is deliberately NOT unwrapped, + * matching `parseJavaConstOperands`, which has no parenthesis arm either. The + * shape is vanishingly rare in a route annotation and the cost of omitting it is + * a skipped route, not a wrong one; adding it to both bindings at once is the + * only way to keep them in parity, so it is left to a follow-up. + */ +export function parseKotlinConstOperands( + node: Parser.SyntaxNode | null | undefined, + depth = 0, +): Operand[] | null { + if (!node) return null; + if (depth > MAX_OPERAND_PARSE_DEPTH) return null; + if (node.type === 'string_literal') { + const value = stringLiteralValue(node); + return value === null ? null : [{ kind: 'literal', value }]; + } + if (node.type === 'simple_identifier') { + return [{ kind: 'ref', name: unquoteKotlinIdentifier(node.text) }]; + } + if (node.type === 'navigation_expression') { + const name = flattenNavigation(node); + return name === null ? null : [{ kind: 'ref', name }]; + } + // `additive_expression` covers both `+` and `-` in tree-sitter-kotlin; only a + // `+` chain concatenates strings. + if (node.type === 'additive_expression') { + if (!(node.children ?? []).some((c) => c.type === '+')) return null; + const operandNodes = node.namedChildren; + if (operandNodes.length !== 2) return null; + const left = parseKotlinConstOperands(operandNodes[0], depth + 1); + const right = parseKotlinConstOperands(operandNodes[1], depth + 1); + if (left === null || right === null) return null; + return [...left, ...right]; + } + return null; +} + +/** The `val`/`var` keyword a property declaration binds with, or null. */ +function bindingKind(property: Parser.SyntaxNode): string | null { + return property.children.find((c) => c.type === 'binding_pattern_kind')?.text ?? null; +} + +/** + * The initializer expression of a property declaration, or null when it has + * none. + * + * Reads the `=` that is a DIRECT child of the `property_declaration`, so a + * custom getter (`val X: String get() = "/g"`, whose `=` lives under `getter`) + * and a delegate (`val X by lazy { … }`, which has no `=` at all) both yield + * null. Both are computed at access time and are not constants. + */ +function initializerOf(property: Parser.SyntaxNode): Parser.SyntaxNode | null { + let equalsIndex = -1; + for (let i = 0; i < property.childCount; i++) { + if (property.child(i)?.type === '=') { + equalsIndex = i; + break; + } + } + if (equalsIndex < 0) return null; + for (let i = equalsIndex + 1; i < property.childCount; i++) { + const child = property.child(i); + if (child?.isNamed) return child; + } + return null; +} + +/** + * One `val` declaration, captured before anything is written to the file's + * namespace so that the DECLARING SCOPE of every initializer is known regardless + * of the order the declarations appear in. + */ +interface KotlinConstDeclaration { + /** The declaration's simple name. */ + readonly name: string; + /** `.`, or null for a top-level declaration. */ + readonly qualified: string | null; + /** + * The qualified-key prefixes in LEXICAL scope for this declaration's + * initializer, innermost first (`['Outer.Inner', 'Outer']`). Empty at file + * level. + */ + readonly scopes: readonly string[]; + /** + * Is the simple name a FILE-LEVEL binding — one any reference in the file can + * use unqualified? True only for a top-level `val`. FALSE for a member of a + * named `object` (which every caller outside that object's body must qualify) + * and FALSE for a companion member, whose unqualified binding exists only + * inside its enclosing class body and is reached through + * {@link qualifyKotlinRefInEnclosingTypes} instead. + */ + readonly fileLevelName: boolean; + /** + * Does an unfoldable initializer here take a same-named IMPORT down with it? + * + * True wherever the declaration binds the simple name for at least some of the + * file — a top-level `val` (everywhere) or a companion member (inside its + * class). Deliberately wider than {@link fileLevelName}: a companion's shadow + * is scoped, but this map is not, and over-deleting an import can only cost a + * route, whereas under-deleting one publishes the imported value at a + * reference the compiler resolves to the unfoldable member. An `object` member + * shadows nothing and is false. + */ + readonly shadowsImport: boolean; + /** The parsed initializer, or null when it is not a foldable string. */ + readonly operands: readonly Operand[] | null; +} + +/** + * The file's declared `package`, or `''` when it declares none (default + * package). Shaped exactly like the import walk below: `package_header` holds + * one `identifier` whose `simple_identifier` children are the dotted segments. + * + * Each segment is unquoted (see {@link unquoteKotlinIdentifier}), so a package + * declared `` com.example.`api` `` is recorded — and therefore matched — as the + * same package an import spells `com.example.api`. + */ +function declaredPackage(root: Parser.SyntaxNode): string { + const header = root.children.find((c) => c.type === 'package_header'); + const identifier = header?.children.find((c) => c.type === 'identifier'); + if (!identifier) return ''; + return identifier.namedChildren + .filter((c) => c.type === 'simple_identifier') + .map((c) => unquoteKotlinIdentifier(c.text)) + .join('.'); +} + +/** + * Extract the declared package, file-level string constants, named imports and + * package-star import scopes of one parsed Kotlin file into the + * {@link KotlinModuleConstants} shape the resolver consumes. + * + * Constants come from the three carriers Kotlin allows a caller to reach without + * an instance: file top level, `object` members, and `companion object` members. + * A `val` in a plain class or interface body is per-instance or abstract and is + * NOT collected — the Kotlin analogue of Java's `static final` requirement. `var` + * is rejected outright. + * + * KEYS FOLLOW KOTLIN'S OWN VISIBILITY, not a flattened namespace. Every constant + * is recorded under `.`, the spelling a qualified reference + * uses, with a companion member keyed under its ENCLOSING CLASS (`Holder.NAME`) + * because that is how Kotlin source refers to it — `Companion` never appears in + * a reference. The SIMPLE name is recorded only for a TOP-LEVEL `val`, the one + * carrier whose bare binding really does span the file. A member of a named + * `object` gets no bare key, because `BASE` alone does not name `A.BASE` from + * anywhere outside `object A`'s own body. Writing one anyway (as this binding + * and the Java one both used to) fabricates a binding the language does not + * have, and a fabricated key outranks the genuine `import com.example.api.Paths.ORDERS` + * that {@link computeKotlinFold} consults only after literals and expressions. + * + * A COMPANION member gets no bare key either: it is bound unqualified inside + * its enclosing class body and nowhere else. {@link qualifyKotlinRefInEnclosingTypes} + * rewrites a bare name to `.` when an enclosing type + * declares it, so the companion wins inside its own class and loses everywhere + * else. + * + * An initializer that names a SIBLING is resolved the same way, against its own + * scope chain, innermost first, before the file level: inside + * `object A { const val BASE = "/right"; const val ROUTE = BASE + "/m" }` the + * operand `BASE` is rewritten to `A.BASE`. Collecting every declaration before + * recording any keeps that independent of declaration order. + * + * A TOP-LEVEL initializer has an EMPTY scope chain, so its bare operands stay + * bare and resolve at file level — they must not pick up a companion key. + * + * A non-foldable rebind (`X = compute()`) DROPS X to unresolvable rather than + * leaving a stale literal — and drops a same-named import with it whenever the + * declaration shadows that import ANYWHERE (top level, or a companion inside its + * class). The import map has no scopes, so a companion's shadow is applied + * file-wide: the conservative direction, costing a route rather than publishing + * the imported value at a reference the compiler binds to the unfoldable member. + * An `object` member shadows nothing and must leave the import alone. + */ +export function extractKotlinModuleConstants(tree: Parser.Tree): KotlinModuleConstants { + const literals = new Map(); + const exprs = new Map(); + const imports = new Map(); + const wildcardImports: string[] = []; + const unfoldableDeclarations = new Set(); + const topLevelDeclarations = new Set(); + + // Pass 1: imports. + const walkImports = (node: Parser.SyntaxNode): void => { + if (node.type === 'import_header') { + const isWildcard = node.children.some((c) => c.type === 'wildcard_import'); + const identifier = node.children.find((c) => c.type === 'identifier'); + if (identifier) { + const segments = identifier.namedChildren + .filter((c) => c.type === 'simple_identifier') + .map((c) => unquoteKotlinIdentifier(c.text)); + if (isWildcard) { + // Package star (`pkg.*`) or classifier star (`Type.*`). Resolution + // decides which reading the specifier actually names. + const scope = segments.join('.'); + if (scope.length > 0 && !wildcardImports.includes(scope)) wildcardImports.push(scope); + } else if (segments.length >= 2) { + const spec = segments.join('.'); + const originalName = segments[segments.length - 1]; + const aliasNode = node.children + .find((c) => c.type === 'import_alias') + ?.namedChildren.find((c) => c.type === 'type_identifier'); + const alias = aliasNode ? unquoteKotlinIdentifier(aliasNode.text) : undefined; + // `module` is the specifier AS WRITTEN, complete. Kotlin does not mark + // member imports, so the fold — not the extractor — decides whether the + // trailing segment is a declaration or one of its members. + imports.set(alias ?? originalName, { module: spec, originalName }); + } + } + return; + } + for (const child of node.children ?? []) walkImports(child); + }; + walkImports(tree.rootNode); + + // Pass 2a: collect every declaration, writing nothing yet. Which member each + // unqualified operand means depends on the whole file, so no key can be + // written — and no operand rewritten — until the last declaration is in. + const declarations: KotlinConstDeclaration[] = []; + /** Declaring scope → the simple names it declares, foldable or not. */ + const membersByScope = new Map>(); + + const collectProperties = ( + body: Parser.SyntaxNode, + declaringType: string | null, + scopes: readonly string[], + fileLevelName: boolean, + shadowsImport: boolean, + ): void => { + for (const member of body.children ?? []) { + if (member.type !== 'property_declaration') continue; + if (bindingKind(member) !== 'val') continue; + const declaration = member.children.find((c) => c.type === 'variable_declaration'); + const nameNode = declaration?.namedChildren.find((c) => c.type === 'simple_identifier'); + if (!nameNode) continue; + const name = unquoteKotlinIdentifier(nameNode.text); + if (declaringType !== null) { + let members = membersByScope.get(declaringType); + if (!members) membersByScope.set(declaringType, (members = new Set())); + // Recorded even when the initializer does not fold: a sibling reference + // to an unfoldable member must resolve to that member and then MISS, + // not fall through to a same-named constant at file level. + members.add(name); + } + declarations.push({ + name, + qualified: declaringType === null ? null : `${declaringType}.${name}`, + scopes, + fileLevelName, + shadowsImport, + operands: parseKotlinConstOperands(initializerOf(member)), + }); + } + }; + + const bodyOf = (node: Parser.SyntaxNode): Parser.SyntaxNode | undefined => + node.children.find((c) => c.type === 'class_body'); + + /** The declared name of an `object_declaration` / `class_declaration`. */ + const typeNameOf = (node: Parser.SyntaxNode): string | null => { + const ident = node.children.find((c) => c.type === 'type_identifier'); + return ident ? unquoteKotlinIdentifier(ident.text) : null; + }; + + /** Append one simple type name to its enclosing qualified type path. */ + const nestedTypeName = (enclosingType: string | null, name: string | null): string | null => { + if (name === null) return enclosingType; + return enclosingType === null ? name : `${enclosingType}.${name}`; + }; + + /** Prepend a qualified scope unless it is already the innermost scope. */ + const withScope = (scope: string | null, scopes: readonly string[]): readonly string[] => + scope === null || scopes[0] === scope ? scopes : [scope, ...scopes]; + + // Package-star imports have lower priority than declarations in this file, + // including declarations the string-constant extractor intentionally does + // not harvest (a `var`, plain class, or unfoldable property). Record their + // names separately so a star import cannot turn one of those shadows into a + // false route constant. + for (const child of tree.rootNode.children ?? []) { + if (child.type === 'property_declaration') { + const declaration = child.children.find((c) => c.type === 'variable_declaration'); + const name = declaration?.namedChildren.find((c) => c.type === 'simple_identifier'); + if (name) topLevelDeclarations.add(unquoteKotlinIdentifier(name.text)); + continue; + } + if (child.type === 'object_declaration' || child.type === 'class_declaration') { + const name = child.children.find((c) => c.type === 'type_identifier'); + if (name) topLevelDeclarations.add(unquoteKotlinIdentifier(name.text)); + } + } + + const walkDeclarations = ( + node: Parser.SyntaxNode, + enclosingType: string | null, + scopes: readonly string[], + ): void => { + for (const child of node.children ?? []) { + if (child.type === 'object_declaration') { + const name = typeNameOf(child); + const body = bodyOf(child); + if (!body) continue; + // Carry the full path: a nested object member is `Outer.Inner.NAME`, not + // `Inner.NAME`. Inside the body a bare name searches that qualified + // scope first, then each enclosing type. + const declaredType = nestedTypeName(enclosingType, name); + const inner = withScope(declaredType, scopes); + collectProperties(body, declaredType, inner, false, false); + walkDeclarations(body, declaredType, inner); + continue; + } + if (child.type === 'companion_object') { + const body = bodyOf(child); + if (!body) continue; + // Referenced through the enclosing class (`Holder.NAME`), never through + // `Companion` — so the qualified alias is keyed on `enclosingType`. The + // simple name is bound inside that class body only, which is a SCOPE and + // not a file-level key: it is reached from the reference site by + // `qualifyKotlinRefInEnclosingTypes`, through this same `Holder.NAME`. + const inner = withScope(enclosingType, scopes); + collectProperties(body, enclosingType, inner, false, true); + walkDeclarations(body, enclosingType, inner); + continue; + } + if (child.type === 'class_declaration') { + // A class/interface body's own `val`s are per-instance or abstract, so + // only its nested objects and companion contribute constants. + const name = typeNameOf(child); + const body = bodyOf(child); + const declaredType = nestedTypeName(enclosingType, name); + if (body) walkDeclarations(body, declaredType, withScope(declaredType, scopes)); + continue; + } + walkDeclarations(child, enclosingType, scopes); + } + }; + + collectProperties(tree.rootNode, null, [], true, true); + walkDeclarations(tree.rootNode, null, []); + + // Pass 2b: rewrite each initializer's unqualified operands against the scope + // chain that encloses it, then record. Only a top-level declaration writes a + // bare key, so nothing here can collide across scopes; a companion's + // unqualified binding is applied at the reference site instead. + // A PARTIALLY qualified reference is resolved here too, not just a bare one: + // inside `object Outer`, the initializer `Inner.Q + "/m"` names `Outer.Inner.Q`, + // and taking a dotted name as already complete looked up a key nothing + // declares. Split at the last dot and prefix the scope onto the OWNER, so the + // bare case (`ownerSuffix === null`) stays exactly what it was. + const qualifyRef = (refName: string, scopes: readonly string[]): string => { + const lastDot = refName.lastIndexOf('.'); + const ownerSuffix = lastDot < 0 ? null : refName.slice(0, lastDot); + const member = lastDot < 0 ? refName : refName.slice(lastDot + 1); + for (const scope of scopes) { + const declaringType = ownerSuffix === null ? scope : `${scope}.${ownerSuffix}`; + if (membersByScope.get(declaringType)?.has(member)) return `${declaringType}.${member}`; + } + return refName; // file level, or unresolvable — the fold decides + }; + + for (const decl of declarations) { + const keys: string[] = []; + if (decl.fileLevelName) keys.push(decl.name); + if (decl.qualified !== null) keys.push(decl.qualified); + + if (decl.operands === null) { + for (const key of keys) { + literals.delete(key); + exprs.delete(key); + unfoldableDeclarations.add(key); + } + if (decl.shadowsImport) imports.delete(decl.name); + continue; + } + + const operands = decl.operands.map((op) => + op.kind === 'ref' ? { kind: 'ref' as const, name: qualifyRef(op.name, decl.scopes) } : op, + ); + const literalValue = + operands.length === 1 && operands[0].kind === 'literal' ? operands[0].value : null; + for (const key of keys) { + unfoldableDeclarations.delete(key); + if (literalValue !== null) { + literals.set(key, literalValue); + exprs.delete(key); + } else { + exprs.set(key, operands); + literals.delete(key); + } + } + } + + return { + literals, + exprs, + imports, + wildcardImports, + packageName: declaredPackage(tree.rootNode), + unfoldableDeclarations, + topLevelDeclarations, + }; +} + +/** + * Per-fold state. Mirrors {@link resolveJavaConstant}'s, for the same reasons: + * + * - `memo` caches SUCCESSES only and is never popped, so a shared-descendant + * DAG (`X_k = X_{k+1} + X_{k+1}`) folds in O(nodes) instead of O(2^depth). + * A `null` may be transient — a name that cycles on one branch can resolve on + * another — so caching it would be unsound. + * - `visited` is the ACTIVE resolution stack, popped on unwind, so diamonds fold + * instead of false-cycling while true cycles still terminate. + * - `index` carries the constant-defining key set and exact-package declaration + * buckets built by `prepareRepo`. A public one-shot call can still build it + * lazily, while production folds reuse it across every route in the scan. + * Import-only files stay out of the candidate set so they cannot manufacture + * ambiguity. + */ +interface KotlinFoldState { + readonly index: KotlinConstantIndex; + readonly visited: Set; + readonly memo: Map; +} + +function newFoldState(repo: RepoConstants, index?: KotlinConstantIndex): KotlinFoldState { + return { + index: index ?? buildKotlinConstantIndex(repo), + visited: new Set(), + memo: new Map(), + }; +} + +/** + * Resolve a single Kotlin constant referenced in `fileKey` to its literal string + * value, folding `+` concatenation and following import chains via + * {@link resolveKotlinImport}, or null when it cannot be fully folded. + * + * `name` may be simple (`ORDERS`, resolved via a single-name import or a + * same-file constant) or qualified (`ApiPaths.ORDERS`, resolved via the type + * import plus the target file's qualified alias). + */ +export function resolveKotlinConstant( + fileKey: string, + name: string, + repo: RepoConstants, + depth = 0, + index?: KotlinConstantIndex, +): string | null { + return resolveWithState(fileKey, name, newFoldState(repo, index), depth); +} + +function resolveWithState( + fileKey: string, + name: string, + state: KotlinFoldState, + depth: number, +): string | null { + if (depth > MAX_FOLD_DEPTH) return null; + const guard = `${fileKey}::${name}`; + const memoized = state.memo.get(guard); + if (memoized !== undefined) return memoized; + if (state.visited.has(guard)) return null; // cycle: `name` is on the active stack + state.visited.add(guard); + try { + const result = computeKotlinFold(fileKey, name, state, depth); + if (result !== null) state.memo.set(guard, result); + return result; + } finally { + state.visited.delete(guard); + } +} + +/** + * Resolve a name bound by an import, trying both readings of the specifier. + * + * Kotlin writes a member import exactly like a type import, so + * `import com.example.app.api.ApiPaths.ORDERS` is syntactically + * indistinguishable from a type import of `ORDERS` in package + * `com.example.app.api.ApiPaths`. Rather than guess from casing — a convention, + * not a rule, and one that quietly breaks on `object apiPaths` or `const val + * Orders` — both readings are attempted and the first that actually RESOLVES + * wins. A reading that resolves to no constant simply falls through. + */ +function resolveImportedName( + fileKey: string, + imp: ImportBinding, + state: KotlinFoldState, + depth: number, +): string | null { + // Reading A: the specifier names the declaration itself (a top-level + // `const val`, or a type whose file we then search). + const direct = resolveKotlinImportTarget(imp.module, state.index); + if (direct !== null) { + const value = resolveWithState(direct.fileKey, direct.localName, state, depth); + if (value !== null) return value; + } + // Reading B: the specifier names a MEMBER of the declaration one segment up + // (`…ApiPaths.ORDERS` → member `ORDERS` of `ApiPaths`). + const dot = imp.module.lastIndexOf('.'); + if (dot <= 0) return null; + const ownerSpec = imp.module.slice(0, dot); + const owner = resolveKotlinImportTarget(ownerSpec, state.index); + if (owner === null) return null; + return resolveWithState(owner.fileKey, `${owner.localName}.${imp.originalName}`, state, depth); +} + +/** + * Resolve one name contributed by Kotlin star imports. + * + * Star imports have lower priority than local declarations and explicit + * imports; callers enforce that ordering before reaching this helper. A name + * must identify one declaration across every imported scope. Two stars + * exporting the same name, a duplicated FQN, or a package and a classifier + * that disagree on the target, floor to null rather than guessing. + * + * A specifier is tried as a package (`import pkg.*`) and as a classifier + * (`import Type.*` — object, class, or companion members). Kotlin allows both. + */ +function resolveKotlinWildcardImportTarget( + mc: ModuleConstants, + name: string, + index: KotlinConstantIndex, +): KotlinImportTarget | null { + const scopes = mc.wildcardImports; + if (!scopes || scopes.length === 0) return null; + + let resolved: KotlinImportTarget | null = null; + for (const rawScope of scopes) { + const scope = unquoteKotlinDottedName(rawScope); + const candidates: KotlinImportTarget[] = []; + + const bucket = index.byPackage.get(scope); + if (bucket?.declarers.has(name)) { + const fileKey = bucket.declarers.get(name); + if (fileKey === null || fileKey === undefined) return null; + candidates.push({ fileKey, localName: name }); + } + + const owner = index.byFqn.get(scope); + if (owner === null) return null; + if (owner !== undefined) { + const localName = `${owner.localName}.${name}`; + const target = index.repo.get(owner.fileKey); + if ( + target && + (target.literals.has(localName) || + target.exprs.has(localName) || + unfoldableDeclarationsOf(target).has(localName)) + ) { + candidates.push({ fileKey: owner.fileKey, localName }); + } + } + + for (const candidate of candidates) { + if ( + resolved !== null && + (resolved.fileKey !== candidate.fileKey || resolved.localName !== candidate.localName) + ) { + return null; + } + resolved = candidate; + } + } + return resolved; +} + +/** + * Resolve a top-level name visible from the importing file's own package. + * `undefined` means the package does not declare the name; `null` means it is + * ambiguous and must floor rather than fall through to a star import. + */ +function resolveKotlinSamePackageTarget( + fileKey: string, + name: string, + index: KotlinConstantIndex, +): KotlinImportTarget | null | undefined { + const packageName = declaredPackageOf(index.repo.get(fileKey)); + if (packageName === null) return undefined; + const bucket = index.byPackage.get(packageName); + if (!bucket?.declarers.has(name)) return undefined; + const declaringFile = bucket.declarers.get(name); + if (declaringFile === null || declaringFile === undefined) return null; + // Same-file literals, expressions, and shadows are handled before this step. + if (declaringFile === fileKey) return undefined; + return { fileKey: declaringFile, localName: name }; +} + +function computeKotlinFold( + fileKey: string, + name: string, + state: KotlinFoldState, + depth: number, +): string | null { + const { repo } = state.index; + const mc = repo.get(fileKey); + if (!mc) return null; + // Qualified reference (`ApiPaths.ORDERS`): constants and imports are keyed by + // their IN-FILE name, so a dotted name never hits directly. Split head.tail, + // resolve the head through the importing file's type import, then look the + // member up in the target file under its declaring name. + // + // Unlike the Java binding there is NO bare-`tail` fallback: in Kotlin + // `Head.TAIL` means TAIL is a member of the object or companion `Head`, so a + // top-level `TAIL` in the target file is a different declaration and matching + // it would fabricate a value. + const dot = name.indexOf('.'); + if (dot > 0) { + const head = name.slice(0, dot); + const tail = name.slice(dot + 1); + const imp = repo.get(fileKey)?.imports.get(head); + if (imp) { + const target = resolveKotlinImportTarget(imp.module, state.index); + if (target === null) return null; + // `originalName` un-aliases `import … .ApiPaths as Paths`, so the lookup + // uses the declaring type's real name. + return resolveWithState(target.fileKey, `${target.localName}.${tail}`, state, depth + 1); + } + // A same-file top-level declaration outranks every star import. + if (!topLevelDeclarationsOf(mc).has(head)) { + const samePackageTarget = resolveKotlinSamePackageTarget(fileKey, head, state.index); + if (samePackageTarget === null) return null; + if (samePackageTarget !== undefined) { + return resolveWithState( + samePackageTarget.fileKey, + `${samePackageTarget.localName}.${tail}`, + state, + depth + 1, + ); + } + const wildcardTarget = resolveKotlinWildcardImportTarget(mc, head, state.index); + if (wildcardTarget !== null) { + return resolveWithState( + wildcardTarget.fileKey, + `${wildcardTarget.localName}.${tail}`, + state, + depth + 1, + ); + } + } + // Un-imported qualified name (FQN form `com.example.app.api.ApiPaths.ORDERS`): + // try the longest dotted prefix that resolves to a file. + const parts = name.split('.'); + for (let cut = parts.length - 2; cut >= 1; cut--) { + const fqn = parts.slice(0, cut + 1).join('.'); + const target = resolveKotlinImportTarget(fqn, state.index); + if (target !== null) { + const member = parts.slice(cut + 1).join('.'); + return resolveWithState(target.fileKey, `${target.localName}.${member}`, state, depth + 1); + } + } + // No import bound the head and no FQN prefix resolved — fall through. A + // dotted name is ALSO a valid key in this file's own maps, so a same-file + // qualified reference (`ApiPaths.ORDERS` inside the file declaring + // `object ApiPaths`) resolves below. + } + + // Name lookup: literals, then same-file expressions, then the import chase. + // Expressions are folded HERE rather than handed to the agnostic core because + // an operand may itself be a QUALIFIED reference (`X = ApiPaths.Y + "/tail"`) + // and the core only knows bare names: it would look `ApiPaths.Y` up in maps + // keyed by simple name, miss, and floor the whole chain to null. + const literal = mc.literals.get(name); + if (literal !== undefined) return literal; + const expr = mc.exprs.get(name); + if (expr !== undefined) return foldOperands(fileKey, expr, state, depth + 1); + const imp = mc.imports.get(name); + if (imp !== undefined) return resolveImportedName(fileKey, imp, state, depth + 1); + // Any local declaration still shadows a lower-priority package-star import, + // including a var/plain type that is absent from the constant maps. + if (topLevelDeclarationsOf(mc).has(name) || unfoldableDeclarationsOf(mc).has(name)) return null; + const samePackageTarget = resolveKotlinSamePackageTarget(fileKey, name, state.index); + if (samePackageTarget === null) return null; + if (samePackageTarget !== undefined) { + return resolveWithState( + samePackageTarget.fileKey, + samePackageTarget.localName, + state, + depth + 1, + ); + } + const wildcardTarget = resolveKotlinWildcardImportTarget(mc, name, state.index); + if (wildcardTarget !== null) { + return resolveWithState(wildcardTarget.fileKey, wildcardTarget.localName, state, depth + 1); + } + return null; +} + +/** + * Concatenate an operand list, resolving each `ref` through the qualified-aware + * walk so `ApiPaths.BASE` works at every position, not just at the entry point. + * + * Bounded by {@link MAX_FOLD_LENGTH}: the depth cap bounds RECURSION but not + * OUTPUT, which grows multiplicatively (`X = A + A; A = B + B; …`), so a + * pathological chain would build a gigabyte-scale string before any cap fired. + * Overrun floors to null. + */ +function foldOperands( + fileKey: string, + operands: readonly Operand[], + state: KotlinFoldState, + depth: number, +): string | null { + let out = ''; + for (const op of operands) { + if (op.kind === 'literal') { + out += op.value; + } else { + const piece = resolveWithState(fileKey, op.name, state, depth); + if (piece === null) return null; + out += piece; + } + if (out.length > MAX_FOLD_LENGTH) return null; + } + return out; +} + +/** + * Rewrite one BARE reference to the enclosing type that binds it, or leave it + * bare when none does — the reference-site twin of the `qualifyRef` that + * {@link extractKotlinModuleConstants} applies to sibling initializers. + * + * `enclosingTypes` is the chain of qualified type paths the reference sits + * inside, INNERMOST FIRST (`['Outer.Inner', 'Outer']`). A companion member is + * keyed `.` and is bound unqualified exactly within that + * class body — including its nested types, which is why the whole chain is + * walked and not just the innermost link. An `object`'s own members are in scope + * inside its body under the same `.` key, so the same walk covers + * both. + * + * Innermost-first, and BEFORE the file-level maps the fold consults next, is + * Kotlin's own order: a companion member shadows a same-named top-level + * declaration and a same-named import throughout its class. Outside that class + * the bare name never means the companion at all, which is precisely what an + * empty chain expresses. + */ +function qualifyKotlinRefInEnclosingTypes( + fileKey: string, + name: string, + repo: RepoConstants, + enclosingTypes: readonly string[], +): string { + // A dotted reference carries AN owner, not necessarily its OWN full one, so it + // is resolved against the enclosing scopes exactly like a bare name. Kotlin + // binds `Inner.Q` inside `object Outer` to `Outer.Inner.Q`, and returning it + // unchanged looked for a key nothing declares. Worse, when the partial owner + // also names a top-level declaration the unchanged form MATCHES it: with a + // top-level `object ApiPaths` beside a nested one, `@GetMapping(ApiPaths.ORDERS)` + // inside the class holding the nested object resolved to the top-level value — + // a path the application does not serve, where the compiler binds the nested + // one. The scopes are already qualified (`kotlinEnclosingTypeNames`), so + // prefixing them onto whatever the reference spells is the whole rule. + const mc = repo.get(fileKey); + if (!mc) return name; + const unfoldableDeclarations = unfoldableDeclarationsOf(mc); + for (const type of enclosingTypes) { + const key = `${type}.${name}`; + if (mc.literals.has(key) || mc.exprs.has(key) || unfoldableDeclarations.has(key)) { + return key; + } + } + return name; +} + +/** + * Fold an inline operand list (e.g. `ApiPaths.BASE + "/orders"`) against + * `fileKey`, or null when any piece is unresolvable (skip floor). + * + * `enclosingTypes` is the chain of type declarations the REFERENCE sits inside + * (innermost first), and it is applied to the entry operands only — everything + * deeper is either already qualified by + * {@link extractKotlinModuleConstants} against its own declaring scope, or lives + * in another file where this chain means nothing. Passing it empty answers + * "what does this name mean at file level", which is the right question for a + * reference outside any type and the only one a caller without position + * information can honestly ask. + * + * An empty result is a SUCCESS, not a skip. `const val ROOT = ""` folds to `""`, + * which `joinPath` then resolves against the class-level prefix exactly as it + * resolves the literal `@GetMapping("")` — both mean "the prefix itself", the + * Spring idiom for a collection root. Collapsing it into `null` would make a + * resolved-empty path indistinguishable from an unresolvable one — the skip + * floor is reserved for "could not fold", and nothing else in the resolver + * conflates the two: {@link resolveKotlinConstant} returns `''` for an empty + * constant, and `resolveOperands` in the shared core returns its fold + * unfiltered. Matches `foldJavaOperands`, so the two JVM bindings do not + * diverge on the same input. + */ +export function foldKotlinOperands( + fileKey: string, + operands: readonly Operand[], + repo: RepoConstants, + enclosingTypes: readonly string[] = [], + index?: KotlinConstantIndex, +): string | null { + // Allocation gate only: skip the map when there is nothing to qualify against + // or no reference to qualify. It must not restate the rule — a dotted operand + // is qualified too, so testing for a bare one here decided the result instead + // of merely avoiding an allocation, and did so per-OPERAND-LIST: the same + // `Inner.Q` folded or not depending on whether a SIBLING operand happened to + // be bare. + const needsQualify = enclosingTypes.length > 0 && operands.some((op) => op.kind === 'ref'); + const scoped = needsQualify + ? operands.map((op) => + op.kind === 'ref' + ? { + kind: 'ref' as const, + name: qualifyKotlinRefInEnclosingTypes(fileKey, op.name, repo, enclosingTypes), + } + : op, + ) + : operands; + return foldOperands(fileKey, scoped, newFoldState(repo, index), 0); +} diff --git a/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts b/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts new file mode 100644 index 000000000..1ac21e2f9 --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/kotlin-spring.ts @@ -0,0 +1,326 @@ +/** + * Kotlin Spring MVC route annotations for the ingestion pipeline (#3130). + * + * This module only walks a caller-provided tree. In particular, it does not + * import or load tree-sitter-kotlin at module initialization, so platforms + * without the optional grammar can still load the language-provider registry. + */ +import type Parser from 'tree-sitter'; +import type { ExtractedDecoratorRoute } from '../workers/parse-worker.js'; +import { + intersectSpringHttpMethods, + springAnnotationHttpMethods, + unquoteSpringLiteral, +} from './spring-shared.js'; +import { + extractKotlinModuleConstants, + parseKotlinConstOperands, + unfoldableDeclarationsOf, + unquoteKotlinIdentifier, + type KotlinModuleConstants, + type Operand, +} from './kotlin-const-resolver.js'; + +/** Direct declaration annotations in source order. */ +function declarationAnnotations(node: Parser.SyntaxNode): Parser.SyntaxNode[] { + const modifiers = node.namedChildren.find((child) => child.type === 'modifiers'); + return modifiers?.namedChildren.filter((child) => child.type === 'annotation') ?? []; +} + +/** `@Foo`, `@Foo(...)`, or `@a.b.Foo(...)` → `Foo`. */ +function annotationName(annotation: Parser.SyntaxNode): string | null { + const constructor = annotation.namedChildren.find( + (child) => child.type === 'constructor_invocation', + ); + const userType = + annotation.namedChildren.find((child) => child.type === 'user_type') ?? + constructor?.namedChildren.find((child) => child.type === 'user_type'); + const identifiers = + userType?.namedChildren.filter((child) => child.type === 'type_identifier') ?? []; + const identifier = identifiers.at(-1); + return identifier ? unquoteKotlinIdentifier(identifier.text) : null; +} + +function annotationArguments(annotation: Parser.SyntaxNode): Parser.SyntaxNode[] { + const constructor = annotation.namedChildren.find( + (child) => child.type === 'constructor_invocation', + ); + const values = constructor?.namedChildren.find((child) => child.type === 'value_arguments'); + return values?.namedChildren.filter((child) => child.type === 'value_argument') ?? []; +} + +interface AnnotationArgument { + readonly name?: string; + readonly expression: Parser.SyntaxNode; +} + +/** + * Kotlin represents positional and named annotation arguments with the same + * `value_argument` node. A direct `=` token distinguishes the named form. + */ +function readAnnotationArgument(argument: Parser.SyntaxNode): AnnotationArgument | null { + const named = argument.children.some((child) => child.type === '='); + if (!named) { + const expression = argument.namedChild(0); + return expression ? { expression } : null; + } + const key = argument.namedChild(0); + const expression = argument.namedChild(1); + if (key?.type !== 'simple_identifier' || !expression) return null; + return { name: unquoteKotlinIdentifier(key.text), expression }; +} + +function routeArguments(annotation: Parser.SyntaxNode): AnnotationArgument[] | null { + const out: AnnotationArgument[] = []; + for (const argument of annotationArguments(annotation)) { + const parsed = readAnnotationArgument(argument); + if (!parsed) return null; + if (parsed.name === undefined || parsed.name === 'path' || parsed.name === 'value') { + out.push(parsed); + } + } + return out; +} + +/** A fully static Kotlin string literal: no `$name` or `${expr}` children. */ +function isPlainStringLiteral(node: Parser.SyntaxNode): boolean { + return ( + node.type === 'string_literal' && + node.namedChildren.every((child) => child.type === 'string_content') + ); +} + +/** + * Empty `[]` / `arrayOf()` is Spring "no path", not an unresolvable prefix. + * tree-sitter-kotlin may put a zero-width recovery child inside `[]`. + */ +function isEmptyKotlinPathCollection(node: Parser.SyntaxNode): boolean { + if (node.type === 'collection_literal') { + return node.namedChildren.every((child) => child.text.length === 0); + } + if (node.type !== 'call_expression') return false; + const callee = node.namedChild(0); + if (callee?.type !== 'simple_identifier' || unquoteKotlinIdentifier(callee.text) !== 'arrayOf') { + return false; + } + const suffix = node.namedChildren.find((child) => child.type === 'call_suffix'); + const args = suffix?.namedChildren.find((child) => child.type === 'value_arguments'); + if (!args) return true; + return args.namedChildren.every((child) => child.type !== 'value_argument'); +} + +function isKotlinInterface(node: Parser.SyntaxNode): boolean { + return node.children.some((child) => child.type === 'interface'); +} + +function isAbstractOrSealedClass(node: Parser.SyntaxNode): boolean { + const modifiers = node.namedChildren.find((child) => child.type === 'modifiers'); + return ( + modifiers?.namedChildren.some((child) => { + if (child.type !== 'inheritance_modifier' && child.type !== 'class_modifier') { + return false; + } + const text = child.text.trim(); + return text === 'abstract' || text === 'sealed'; + }) === true + ); +} + +function directFunctions(node: Parser.SyntaxNode): Parser.SyntaxNode[] { + const body = node.namedChildren.find((child) => child.type === 'class_body'); + return body?.namedChildren.filter((child) => child.type === 'function_declaration') ?? []; +} + +function functionName(node: Parser.SyntaxNode): string | null { + const field = node.childForFieldName('name'); + const identifier = + field?.type === 'simple_identifier' + ? field + : node.namedChildren.find((child) => child.type === 'simple_identifier'); + return identifier ? unquoteKotlinIdentifier(identifier.text) : null; +} + +/** + * `springAnnotationHttpMethods` parses Java `{A, B}` collections. + * Translate only Kotlin `method = [A, B]` before delegating. + */ +function kotlinSpringHttpMethods(name: string, annotation: Parser.SyntaxNode): readonly string[] { + if (name !== 'RequestMapping') return springAnnotationHttpMethods(name, annotation.text); + const normalized = annotation.text.replace( + /(\bmethod\s*=\s*)\[([^\]]*)\]/gs, + (_match, assignment: string, values: string) => `${assignment}{${values}}`, + ); + return springAnnotationHttpMethods(name, normalized); +} + +function typeName(node: Parser.SyntaxNode): string | null { + const identifier = node.children.find((child) => child.type === 'type_identifier'); + return identifier ? unquoteKotlinIdentifier(identifier.text) : null; +} + +/** + * Qualified enclosing type paths, innermost first. Kotlin companion members + * are keyed through their enclosing class, so companion_object itself adds no + * segment. + */ +function enclosingTypeNames(node: Parser.SyntaxNode): string[] { + const simpleNames: string[] = []; + for (let current: Parser.SyntaxNode | null = node.parent; current; current = current.parent) { + if (current.type !== 'class_declaration' && current.type !== 'object_declaration') continue; + const name = typeName(current); + if (name) simpleNames.push(name); + } + return simpleNames.map((_, index) => simpleNames.slice(index).reverse().join('.')); +} + +function declarationExists(constants: KotlinModuleConstants, name: string): boolean { + return ( + constants.literals.has(name) || + constants.exprs.has(name) || + unfoldableDeclarationsOf(constants).has(name) + ); +} + +/** + * The provider fold hook has a language-neutral three-argument signature and + * cannot receive a Kotlin reference site's enclosing type chain. Qualify only + * names whose owner is proven by this same tree; unresolved/imported names are + * left untouched for the repo-wide resolver. + */ +function qualifySameFileOperands( + operands: readonly Operand[], + functionNode: Parser.SyntaxNode, + constants: KotlinModuleConstants, +): Operand[] { + const enclosingTypes = enclosingTypeNames(functionNode); + return operands.map((operand) => { + if (operand.kind === 'literal') return operand; + for (const owner of enclosingTypes) { + const qualified = `${owner}.${operand.name}`; + if (declarationExists(constants, qualified)) { + return { kind: 'ref', name: qualified }; + } + } + return operand; + }); +} + +interface ClassMapping { + readonly prefix: string; + readonly methods: readonly string[]; +} + +/** + * Read the one optional class-level RequestMapping. A present route member must + * be exactly one plain string literal, an empty `[]`/`arrayOf()` (no prefix), + * or absent; constants, interpolation, non-empty collections, duplicate + * mappings, and dynamic expressions fail closed for the whole class. + */ +function classMapping(annotations: readonly Parser.SyntaxNode[]): ClassMapping | null { + const mappings = annotations.filter( + (annotation) => annotationName(annotation) === 'RequestMapping', + ); + if (mappings.length === 0) return { prefix: '', methods: ['*'] }; + if (mappings.length !== 1) return null; + + const mapping = mappings[0]; + const paths = routeArguments(mapping); + if (paths === null || paths.length > 1) return null; + + let prefix = ''; + if (paths.length === 1) { + const path = paths[0].expression; + if (isEmptyKotlinPathCollection(path)) { + prefix = ''; + } else if (isPlainStringLiteral(path)) { + const literal = unquoteSpringLiteral(path.text); + if (literal === null) return null; + prefix = literal; + } else { + return null; + } + } + + const methods = kotlinSpringHttpMethods('RequestMapping', mapping); + return methods.length === 0 ? null : { prefix, methods }; +} + +/** + * Extract direct Spring handler methods from concrete Kotlin RestControllers. + */ +export function extractKotlinSpringRoutes( + tree: Parser.Tree, + filePath: string, + lineOffset = 0, +): ExtractedDecoratorRoute[] { + const routes: ExtractedDecoratorRoute[] = []; + let moduleConstants: KotlinModuleConstants | undefined; + + for (const classNode of tree.rootNode.descendantsOfType('class_declaration')) { + if (isKotlinInterface(classNode) || isAbstractOrSealedClass(classNode)) continue; + const annotations = declarationAnnotations(classNode); + const annotationNames = annotations.map(annotationName); + if (!annotationNames.includes('RestController')) continue; + if (annotationNames.includes('FeignClient')) continue; + + const ownerMapping = classMapping(annotations); + if (ownerMapping === null) continue; + + for (const functionNode of directFunctions(classNode)) { + const handlerName = functionName(functionNode); + if (!handlerName) continue; + + for (const annotation of declarationAnnotations(functionNode)) { + const decoratorName = annotationName(annotation); + if (!decoratorName) continue; + + const methodMethods = kotlinSpringHttpMethods(decoratorName, annotation); + const methods = intersectSpringHttpMethods(ownerMapping.methods, methodMethods); + if (methods.length === 0) continue; + + const paths = routeArguments(annotation); + if (paths === null || paths.length > 1) continue; + + let routePath = ''; + let routePathExpr: string | undefined; + let routePathOperands: Operand[] | undefined; + if (paths.length === 1) { + const expression = paths[0].expression; + if (isEmptyKotlinPathCollection(expression)) { + routePath = ''; + } else if (isPlainStringLiteral(expression)) { + const literal = unquoteSpringLiteral(expression.text); + if (literal === null) continue; + routePath = literal; + } else { + const operands = parseKotlinConstOperands(expression); + if (operands === null) continue; + moduleConstants ??= extractKotlinModuleConstants(tree); + routePathExpr = expression.text; + routePathOperands = qualifySameFileOperands(operands, functionNode, moduleConstants); + } + } + + for (const httpMethod of methods) { + routes.push({ + filePath, + routePath, + httpMethod, + decoratorName, + lineNumber: annotation.startPosition.row + lineOffset, + ...(ownerMapping.prefix ? { prefix: ownerMapping.prefix } : {}), + handlerName, + ...(routePathExpr === undefined + ? {} + : { + routePathExpr, + routePathOperands, + }), + }); + } + } + } + } + + return routes; +} diff --git a/gitnexus/src/core/ingestion/route-extractors/nest.ts b/gitnexus/src/core/ingestion/route-extractors/nest.ts new file mode 100644 index 000000000..136c6354a --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/nest.ts @@ -0,0 +1,548 @@ +/** + * NestJS decorator routes for the indexer. + * + * A NestJS endpoint is declared across two decorators: `@Controller('venues')` + * on the class supplies the prefix, and `@Get('search')` on a method supplies + * the verb and the remainder. Neither half is a route on its own, which is why + * a pattern that only looks at one of them finds nothing. + * + * Until this existed, TypeScript's `extractDecoratorRoutes` hook was dispatch + * guards plus static data route tables only, so a NestJS repo produced + * essentially no `Route` nodes. That is not a quiet gap: `route_map`, + * `api_impact` and `shape_check` all read `Route` nodes and answer "no routes + * matching …" when there are none — so `api_impact`, whose documented job is to + * be run BEFORE modifying a route handler, reported every live endpoint as + * non-existent, and a not-found reads as a safe change (#3009). + * + * The extraction mirrors `spring.ts`, which solves the identical shape for + * `@RequestMapping` + `@GetMapping`: collect class-level prefixes keyed by class + * node id, then walk method decorators and attach the prefix of their enclosing + * class. As there, the prefix travels on `ExtractedDecoratorRoute.prefix` and + * the routes phase performs the join via `normalizeExtractedRoutePath`, so + * NestJS routes are keyed identically to every other framework's. + * + * The multi-path form `@Get(['a', 'b'])` mounts the handler at BOTH paths, so + * it emits both routes: N paths is N elements of the returned + * `ExtractedDecoratorRoute[]`, which is already how this layer spells N routes + * — the same representation `spring.ts` reaches for `@GetMapping({"/a","/b"})`, + * and the reason neither needs a special case downstream. The CLASS-level array + * (`@Controller(['a', 'b'])`) is DECLINED rather than cross-multiplied over the + * class's methods, again matching `spring.ts`: there an array-form class prefix + * only ever suppresses the class, with the cross-product tracked in #2280. + * + * Known limitation: the URLs produced here are CONTROLLER-RELATIVE. A global + * prefix (`app.setGlobalPrefix('api')`) and URI versioning are applied by the + * bootstrap file, not by any decorator this file can see, so neither is + * reflected — a route served at `/api/v1/venues/search` is stored as + * `/venues/search`. The module's "drop rather than guess" floor is unavailable + * for it: the evidence lives in a different file, so honouring it would mean + * dropping every Nest route in every repo. `spring.ts` has the same hole for + * `server.servlet.context-path`; `ExtractedDecoratorRoute.prefix` is the + * channel a cross-file follow-up would use, the way FastAPI resolves its mount. + */ + +import type Parser from 'tree-sitter'; +import type { ExtractedDecoratorRoute } from '../workers/parse-worker.js'; +import { plainString, propertyName } from './data-route-table.js'; +import { isDev } from '../utils/env.js'; +import { logger } from '../../logger.js'; + +/** + * NestJS method decorators → HTTP verb. A Map rather than an object literal + * because the lookup key is an arbitrary decorator name read out of source: a + * plain object answers `@toString()` with `Object.prototype.toString`, which is + * truthy and would be emitted verbatim as the route's httpMethod. + */ +const NEST_METHOD_DECORATORS: ReadonlyMap = new Map([ + ['Get', 'GET'], + ['Post', 'POST'], + ['Put', 'PUT'], + ['Patch', 'PATCH'], + ['Delete', 'DELETE'], + ['Head', 'HEAD'], + ['Options', 'OPTIONS'], + ['All', '*'], + // `@Sse` mounts a real GET endpoint that streams; it is as much a route as + // `@Get`. `@Search` is deliberately absent — `normalizeRouteMethod` rejects + // SEARCH as non-standard and would key the route by URL alone, colliding + // with every other verb on that path. + ['Sse', 'GET'], +]); + +/** + * Class node types that can carry a `@Controller`. `export abstract class C` + * parses as `abstract_class_declaration`, a DIFFERENT node type — and a + * decorated abstract base sharing CRUD routes with its subclasses is ordinary + * Nest, so matching `class_declaration` alone silently drops the whole + * controller rather than one route. + */ +const CLASS_DECLARATION_TYPES: ReadonlySet = new Set([ + 'class_declaration', + 'abstract_class_declaration', +]); + +/** + * Cheap parse-free gate. Every JS/TS file in every repo reaches this hook, so + * skip the walk unless the file could plausibly declare a controller. A file + * without the substring cannot produce a route here, because a `@Controller` + * decorator is REQUIRED before any method decorator is believed (see below). + */ +const CONTROLLER_HINT = '@Controller'; + +/** The decorator's name — `Controller` for `@Controller('x')`, `Get` for `@Get()`. */ +function decoratorName(decorator: Parser.SyntaxNode): string | null { + const inner = decorator.namedChild(0); + if (!inner) return null; + // `@Get()` is a call_expression; a bare `@Injectable` is a plain identifier. + if (inner.type === 'identifier') return inner.text; + if (inner.type === 'call_expression') { + const fn = inner.childForFieldName('function'); + return fn?.type === 'identifier' ? fn.text : null; + } + return null; +} + +/** + * The literal path(s) a decorator call mounts, one entry per path — or `['']` + * when the decorator takes no argument (`@Controller()` / `@Get()` — both legal + * and both meaning "no path segment of my own"). + * + * A list rather than a single string because `@Get(['a', 'b'])` mounts the + * handler at two URLs, and two routes is what the caller's output contract + * already says that in: `ExtractedDecoratorRoute[]`. No new field, and no + * special case at the emit site — the same shape `spring.ts` gets for free from + * a query that matches one element at a time. + * + * Returns `null` when an argument IS present but is not a readable literal. + * That is deliberately distinct from `['']`: a computed prefix + * (`@Controller(ROUTES.VENUES)`) whose value we cannot read must drop the route + * rather than silently mount it at the wrong URL. `route_map` presents its + * output as fact, and a wrong path is worse than a missing one. `[]` is a third + * answer and means neither of those: `@Get([])` is legal, knowably mounts + * nothing, and so emits nothing — it must never be read as the unknowable case, + * which is the one that suppresses a whole controller. + * + * Reading one literal is delegated to `plainString`, the same judge the + * data-route-table extractor uses, so both agree on what is readable. Filtering + * `string_fragment` children and joining them looks equivalent and is not: + * tree-sitter SPLITS a literal around each `escape_sequence`, and the join then + * DELETES the escape rather than decoding it. `@Get(':id(\\d+)')` — the ordinary + * spelling of a Nest regex param, whose value is `:id(\d+)` — came out as + * `:id(d+)`, and `@Get('/v\u0069ews')` came out as `/vews`. Both are paths the + * app never serves, i.e. the wrong-URL outcome the paragraph above forbids. + */ +function decoratorLiteralPaths(decorator: Parser.SyntaxNode): readonly string[] | null { + const call = decorator.namedChild(0); + // A Nest route decorator is a FACTORY: `@Get()` invokes it and returns the + // decorator that registers the route. A BARE `@Get` is the factory itself, + // never applied, so Nest registers nothing — emitting a route for it would + // publish a URL the app does not serve. The same holds one level up for a + // bare `@Controller`, which registers no controller. + // + // `@Get()` with no ARGUMENT is different and still a real pathless route: + // what distinguishes them is the call, not the argument list. That case falls + // through to the `!first` branch below. + if (call?.type !== 'call_expression') return null; + const first = call.childForFieldName('arguments')?.namedChild(0); + if (!first) return ['']; + // The object form belongs to `@Controller` alone — a verb decorator takes + // `string | string[]`, so Nest mounts nothing for `@Get({ path: 'a' })`. + // Reading it as a route would mint a URL the app never serves, which is the + // invented fact this module refuses; an unreadable shape drops instead. + if (first.type === 'object' && decoratorName(decorator) !== 'Controller') return null; + return literalPaths(first); +} + +/** + * The paths carried by one decorator ARGUMENT node, split out from + * {@link decoratorLiteralPaths} only so the object form can re-enter it: Nest + * accepts an array inside `{ path: … }` as well, and reusing the same judge is + * what keeps `@Controller({ path: ['a', 'b'] })` from being read by a second, + * laxer set of rules that has drifted from this one. + */ +function literalPaths(node: Parser.SyntaxNode): readonly string[] | null { + // `@Controller({ path: 'cats', version: '1' })` is the documented form for + // URI/header versioning, and its path is a plain literal sitting right there. + // Worth reading rather than dropping, because the asymmetry is severe: an + // unreadable METHOD path costs one route, an unreadable PREFIX costs every + // route on the class. + if (node.type === 'object') { + // But a `path` pair only PROVES the mount when nothing else in the object + // can replace it, and the first match proves nothing on its own: + // `{ path: 'cats', ...options }` mounts wherever `options.path` says, and + // `{ path: 'cats', path: 'dogs' }` mounts at `dogs` — last write wins in + // both. Either one publishes `/cats`, a URL the app never serves, and it + // looks exactly like a correct one, which is the wrong-answer-dressed-as- + // fact this module refuses. So the object is read only when EVERY member is + // a named, non-repeated pair. That whole-entry fail-closed walk is the + // shape `routeFromObject` uses in `data-route-table.ts`. + const values = new Map(); + for (const child of node.namedChildren) { + // Skipped FIRST. A comment between two pairs is ordinary formatting; run + // through the not-a-pair test below it would refuse the object and cost + // the class every route it has, over a comment. + if (child.type === 'comment') continue; + // `spread_element` (`{ ...options }`), `shorthand_property_identifier` + // (`{ path }`) and `method_definition` (`{ getFoo() {} }`) all land here + // — probed and identical across the three grammars this extractor runs + // under. None offers a key/value this file can read, and the first can + // introduce or overwrite `path` from a value declared elsewhere. + if (child.type !== 'pair') return null; + const key = child.childForFieldName('key'); + const value = child.childForFieldName('value'); + if (key === null || value === null) return null; + // Compared through `propertyName`, the same judge used to READ the key — + // so `{ path: … }` and `{ 'path': … }` are one key and collide as + // duplicates. Comparing raw key text instead makes them two distinct + // keys, and `{ path: 'cats', 'path': 'dogs' }` silently mounts the loser. + const name = propertyName(key); + // No readable name means a computed key (`{ [dynamicKey]: 'b' }`), which + // could evaluate to `path` and take the mount with it — refused, not + // ignored. A repeated key is refused wherever it appears, not only on + // `path`: a duplicate anywhere is evidence the object is not the fixed + // literal it reads as, and cost is one controller against a wrong URL. + if (name === null || values.has(name)) return null; + values.set(name, value); + } + + // Deliberately NOT `containsExecutingExpression` (data-route-table.ts): that + // guards whole-entry declarativeness for a static route table, a different + // invariant. Here only `path` has to be provable, so a non-literal value on + // an unrelated key — `{ path: 'a', scope: Scope.REQUEST }`, ordinary Nest — + // stays benign and keeps its controller. + const path = values.get('path'); + // A missing `path` keeps the existing drop and must never read as `''`: + // `@Controller({ version: '1' })` mounts at a prefix this decorator does + // not state, and `''` would publish every one of its methods at the root. + return path === undefined ? null : literalPaths(path); + } + + // `array` is the node type in all three grammars this extractor runs under — + // tree-sitter-typescript's `typescript` and `tsx`, and tree-sitter-javascript + // — probed rather than assumed, because a name that differs in one of them + // would silently restore the old drop for that grammar alone. + if (node.type === 'array') { + const paths: string[] = []; + for (const element of node.namedChildren) { + const value = plainString(element); + // One unreadable element poisons the whole array. Emitting the readable + // ones would present a partial mapping as a complete one — the endpoint + // behind `ROUTES.ADMIN` would be missing from a controller that otherwise + // looks fully covered, which is the same wrong-answer-dressed-as-fact this + // module refuses above, only harder to notice. + if (value === null) return null; + paths.push(value); + } + return paths; + } + + const value = plainString(node); + return value === null ? null : [value]; +} + +/** + * Decorators that immediately precede `node` among its parent's named children. + * In tree-sitter-typescript a decorator is a SIBLING placed before the thing it + * decorates — at `export_statement`/`program` level for a class — and + * decorators stack. + * + * Walks the sibling chain rather than indexing into `parent.namedChildren`, + * which is the same uncached-getter trap {@link collectClassRoutes} documents: + * a class's parent is usually `program`, so reading the list marshals every + * top-level statement in the file, once per class. That is quadratic in + * top-level statements — measured 200ms for a file of 800 classes, against + * 0.9ms for this form. + */ +function precedingDecorators(node: Parser.SyntaxNode): Parser.SyntaxNode[] { + const out: Parser.SyntaxNode[] = []; + for (let sibling = node.previousNamedSibling; sibling; sibling = sibling.previousNamedSibling) { + // A comment between the decorators and the thing they decorate is ordinary + // (`@Post('x')` then a JSDoc block then the method) and must not terminate + // the stack — doing so makes the whole decorated route invisible. + if (sibling.type === 'comment') continue; + if (sibling.type !== 'decorator') break; + out.push(sibling); + } + return out; +} + +/** + * Leading `decorator` children of a node, stopping at the first child that is + * neither a decorator nor a comment. Comments are skipped for the same reason + * as in {@link precedingDecorators}: a doc block sitting between `@Controller` + * and the class must not hide the decorator. + */ +function leadingDecorators(node: Parser.SyntaxNode): Parser.SyntaxNode[] { + const out: Parser.SyntaxNode[] = []; + for (const child of node.namedChildren) { + if (child.type === 'comment') continue; + if (child.type !== 'decorator') break; + out.push(child); + } + return out; +} + +/** + * Cap on the decorator text quoted in the dropped-controller log. Long enough + * to identify the shape, short enough not to dump a wrapped multi-line + * decorator into the operator's terminal. + */ +const DROPPED_CONTROLLER_LOG_LIMIT = 160; + +/** + * Every decorator attached to a class, across the two shapes the grammar + * produces — which differ by whether the class is exported: + * + * `@Controller('a') class A {}` → decorator is a CHILD of class_declaration + * `@Controller('a') export class A {}` → decorator is a child of export_statement, + * i.e. a SIBLING of the class_declaration + * + * Checking only one of them silently drops half of all controllers, so collect + * from both, plus the sibling position for the class itself. There is no fourth + * source: both grammars fold a class's decorators INTO the `export_statement` + * production, so an `export_statement` never has one as a preceding sibling. + */ +function classDecorators(classNode: Parser.SyntaxNode): Parser.SyntaxNode[] { + const out = [...leadingDecorators(classNode), ...precedingDecorators(classNode)]; + const wrapper = classNode.parent; + if (wrapper?.type === 'export_statement') out.push(...leadingDecorators(wrapper)); + return out; +} + +/** + * The `@Controller(...)` prefix for a class, or undefined when it has none. + * One string, not a list: a class-level array (`@Controller(['a', 'b'])`) is + * DECLINED here, exactly as `spring.ts` declines an array-form + * `@RequestMapping` — it detects the shape only to suppress the class, leaving + * the prefix × method cross-product to #2280. Collapsing to `null` is that + * suppression, and this parity is deliberate, not an oversight: the two + * extractors solve the same shape and should not disagree about which half of + * it is supported. + */ +function controllerPrefix( + classNode: Parser.SyntaxNode, + filePath: string, +): string | null | undefined { + for (const decorator of classDecorators(classNode)) { + if (decoratorName(decorator) !== 'Controller') continue; + const paths = decoratorLiteralPaths(decorator); + // `@Controller([])` lands here too and needs no answer of its own: a + // controller mounted at no path serves no route, so "emit nothing for this + // class" is what both readings of it come to. + if (paths === null || paths.length !== 1) { + // The single funnel for EVERY whole-controller drop — an unreadable + // constant (`@Controller(ROUTES.VENUES)`), a multi-path array, an + // unreadable array element, and an options object whose `path` another + // member could override all return null here. Reporting at the refusal + // sites instead would make the rarest cause the loudest, and leave the + // motivating one from this module's own header silent. + // + // `isDev` at `info`, not `debug`: the logger's base level IS `info`, so + // an isDev-gated `debug` is gated twice and stays silent in exactly the + // dev run it exists for. Same shape the routes phase uses. + if (isDev) { + const shape = decorator.text.replace(/\s+/g, ' '); + logger.info( + `🗺️ NestJS: dropped @Controller in ${filePath} — its prefix is not provable: ${ + shape.length > DROPPED_CONTROLLER_LOG_LIMIT + ? `${shape.slice(0, DROPPED_CONTROLLER_LOG_LIMIT)}…` + : shape + }`, + ); + } + return null; + } + return paths[0]; + } + return undefined; +} + +/** + * Extract NestJS routes from one parsed TypeScript/JavaScript file. + * + * A method decorator is only believed when its enclosing class carries a + * `@Controller`. `@Get`/`@Post`/`@Delete` are common identifiers, and without + * that requirement any unrelated library using the same decorator names would + * mint phantom endpoints. + */ +export function extractNestRoutes( + tree: Parser.Tree, + filePath: string, + lineOffset = 0, +): ExtractedDecoratorRoute[] { + if (!tree.rootNode.text.includes(CONTROLLER_HINT)) return []; + + const out: ExtractedDecoratorRoute[] = []; + + const visit = (node: Parser.SyntaxNode): void => { + if (CLASS_DECLARATION_TYPES.has(node.type)) { + const prefix = controllerPrefix(node, filePath); + // `undefined` — not a controller at all. `null` — a controller whose + // prefix could not be read, so its routes' URLs are unknowable. + if (prefix !== undefined) { + if (prefix !== null) collectClassRoutes(node, prefix, filePath, lineOffset, out); + return; // a controller's methods are handled here; don't re-walk them + } + } + for (const child of node.namedChildren) visit(child); + }; + + visit(tree.rootNode); + return out; +} + +/** + * Modifiers that take a `method_definition` out of Nest's handler set. + * + * Nest's `RequestMapping` writes the handler onto the class PROTOTYPE's + * `descriptor.value`, and `RouterExplorer` scans prototype instance methods for + * that metadata. A `static` method lives on the constructor and is never + * scanned; an accessor's descriptor carries `get`/`set` and no `value` to + * register. A verb decorator on any of the three therefore mounts NOTHING, so a + * route minted from one is a URL the app does not serve — the invented fact + * this module refuses everywhere else. + */ +const NON_HANDLER_MODIFIERS: ReadonlySet = new Set(['static', 'get', 'set']); + +/** Longest entry above — the cheap gate that keeps `.trim()` off a method body. */ +const LONGEST_NON_HANDLER_MODIFIER = 6; + +/** + * Whether Nest could register this `method_definition` as a request handler. + * + * Reads `children`, NOT `namedChildren`, and that is the whole difficulty: + * `static`, `get` and `set` are ANONYMOUS tokens in all three grammars this + * extractor runs under, so they never appear among named children. A static + * method, a getter, a setter and a plain method expose the IDENTICAL + * `namedChildren` (`property_identifier`, `formal_parameters`, + * `statement_block`) — probed, not assumed — so the module's usual + * `namedChildren` idiom cannot see the modifier at all and every one of the + * three reads as an ordinary handler. + * + * Matches on child TEXT, not node type, and skips the `name` field — the same + * two rules `hasKeyword` in `field-extractors/configs/helpers.ts` applies, and + * that the TS/JS captures and method extractor already use for this question. + * The text rule is load-bearing: `static` reaches the tree as an anonymous + * token in some grammar versions and a keyword node in others, so a + * `child.type === 'static'` test silently stops firing on a grammar bump — here + * that would readmit exactly the phantom routes this function removes, with the + * suite still green. Skipping `name` is what keeps a method literally called + * `get()` or `static()` from reading as a modifier. + * + * Open-coded rather than calling `hasKeyword` three times, which was measured + * at 13.96us per method against 4.30us here: that helper takes ONE keyword, so + * three keywords is three full passes, and it calls `.text.trim()` on every + * child including `statement_block` — the whole method body. `.some()` does not + * rescue it, since a real handler matches nothing and pays all three. The + * length guard keeps `.trim()` off a multi-KB body; no modifier exceeds it. + * + * The scan is bounded by one method's own children (a handful), so it is not + * the uncached-getter trap {@link collectClassRoutes} documents — that one bites + * when a PARENT's child list is re-marshalled once per member. + */ +function isRequestHandler(member: Parser.SyntaxNode): boolean { + const nameNode = member.childForFieldName('name'); + for (const child of member.children) { + if (child === nameNode) continue; + const text = child.text; + if (text.length <= LONGEST_NON_HANDLER_MODIFIER && NON_HANDLER_MODIFIERS.has(text.trim())) { + return false; + } + } + return true; +} + +function collectClassRoutes( + classNode: Parser.SyntaxNode, + prefix: string, + filePath: string, + lineOffset: number, + out: ExtractedDecoratorRoute[], +): void { + const body = classNode.childForFieldName('body'); + if (!body) return; + + // ONE forward pass over the body, accumulating the decorator run and flushing + // it at each method. Calling `precedingDecorators` per method instead is + // quadratic in methods-per-controller for a reason that is invisible in the + // source: `namedChildren` is an UNCACHED getter in node-tree-sitter, so every + // call re-marshals the entire class body into fresh JS objects before the + // `findIndex`. Measured here, 800 methods cost 362ms (450us/method, up from + // 42us/method at 50); a single pass is flat. `spring.ts` never had this + // because a Java annotation is a child of the declaration it annotates. + const pending: Parser.SyntaxNode[] = []; + + for (const member of body.namedChildren) { + if (member.type === 'decorator') { + pending.push(member); + continue; + } + // Same reason as in `precedingDecorators`: a JSDoc block between a + // decorator stack and its method must not hide the route (a real + // controller shape, pinned by the suite). Known limitation of that skip: a + // decorator ORPHANED by a commented-out handler is then absorbed onto the + // NEXT method, minting a phantom route with the wrong handler. There is no + // AST fix — an orphan followed by a comment is indistinguishable from a + // stack whose method happens to be documented — and losing every + // documented route is the worse trade, so it is made deliberately. + if (member.type === 'comment') continue; + + // tree-sitter-javascript makes a method decorator a CHILD of the + // `method_definition`, not a preceding sibling as in tree-sitter-typescript + // — and this extractor is registered on the JavaScript provider too, which + // already advertises `framework: 'nestjs'`. Reading only siblings meant + // every `.js` Nest controller emitted nothing. On TypeScript the first + // named child is the method name, so `leadingDecorators` contributes + // nothing there and no route is collected twice. + // + // A static member, a getter and a setter are decorated exactly like a + // handler and registered as none, so they contribute no decorators (see + // `isRequestHandler`). They still fall THROUGH to the `pending.length = 0` + // below rather than `continue` past it: skipping the clear would hand their + // decorator run to the next method, trading a phantom route for a + // misattributed one — the strictly worse of the two, since it corrupts a + // route that is otherwise correct. + const decorators = + member.type === 'method_definition' && isRequestHandler(member) + ? [...pending, ...leadingDecorators(member)] + : []; + for (const decorator of decorators) { + const name = decoratorName(decorator); + if (name === null) continue; + const httpMethod = NEST_METHOD_DECORATORS.get(name); + if (httpMethod === undefined) continue; + + const routePaths = decoratorLiteralPaths(decorator); + if (routePaths === null) continue; // unreadable → skip + + const handlerName = member.childForFieldName('name')?.text; + + // One route per path. `@Get(['a', 'b'])` mounts the handler at both, and + // everything else about the two is identical — same verb, same handler, + // same line — so the loop is the whole of the multi-path support. An + // empty array falls out as zero iterations without a special case. + for (const routePath of routePaths) { + out.push({ + filePath, + // A pathless `@Get()` is the controller's index route and carries no + // segment of its own. Emit '/' rather than '': `claim()` in + // call-processor short-circuits on a falsy routePath, so an empty + // string would still produce the Route node but silently lose its + // handler symbol — the route would exist with nothing attached to it. + // Both spellings normalize to the same URL against the prefix. + routePath: routePath === '' ? '/' : routePath, + httpMethod, + decoratorName: name, + lineNumber: member.startPosition.row + 1 + lineOffset, + prefix: prefix === '' ? null : prefix, + ...(handlerName === undefined ? {} : { handlerName }), + }); + } + } + + // Anything that is not a decorator or a comment ends the run — including + // the method that just consumed it, so a decorated FIELD's stack is never + // absorbed onto the method after it. + pending.length = 0; + } +} diff --git a/gitnexus/src/core/ingestion/route-extractors/python-decorator-handler.ts b/gitnexus/src/core/ingestion/route-extractors/python-decorator-handler.ts new file mode 100644 index 000000000..fca06c857 --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/python-decorator-handler.ts @@ -0,0 +1,19 @@ +/** + * Return the function name attached to a Python decorator's immediate + * `decorated_definition`; reject every other shape rather than climbing. + */ + +import type { SyntaxNode } from '../utils/ast-helpers.js'; + +export function pythonDecoratorRouteHandlerName(decoratorNode: SyntaxNode): string | undefined { + const decorated = decoratorNode.parent; + if (decorated === null || decorated.type !== 'decorated_definition') return undefined; + + // `async def` is still a `function_definition` in tree-sitter-python (the + // `async` keyword is an anonymous child), so async handlers need no branch. + const definition = decorated.childForFieldName('definition'); + if (!definition || definition.type !== 'function_definition') return undefined; + + const name = definition.childForFieldName('name')?.text; + return name !== undefined && name.length > 0 ? name : undefined; +} diff --git a/gitnexus/src/core/ingestion/route-extractors/spring.ts b/gitnexus/src/core/ingestion/route-extractors/spring.ts index 5c0b0637b..36828a1f5 100644 --- a/gitnexus/src/core/ingestion/route-extractors/spring.ts +++ b/gitnexus/src/core/ingestion/route-extractors/spring.ts @@ -30,6 +30,7 @@ import { unquoteSpringLiteral, type SharedSpringType, } from './spring-shared.js'; +import { parseJavaConstOperands } from './java-const-resolver.js'; /** * Single predicate-free tree-sitter query that captures all route annotations @@ -53,6 +54,13 @@ import { * suppresses that class's method-level array routes rather than emit them with a * dropped prefix (a wrong route). Full class-array cross-product support is left * to a follow-up (#2280). + * + * The class-level `@value_expr` branches exist for the same reason: a + * CONSTANT-valued class prefix (`@RequestMapping(ApiPaths.BASE)`) cannot be + * folded here — the repo-wide constant map only exists in the parse phase — so + * they only DETECT it, and Phase 2 suppresses every method route under such a + * class. Without them the prefix was invisible and the method route was emitted + * unprefixed, i.e. at a path the application does not serve. */ const ROUTE_ANNOTATION_QUERY = new Parser.Query( Java, @@ -90,6 +98,42 @@ const ROUTE_ANNOTATION_QUERY = new Parser.Query( key: (identifier) @key value: [(string_literal) @value (element_value_array_initializer (string_literal) @value)]))))) @node + (class_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list + [(identifier) @value_expr + (field_access) @value_expr + (binary_expression) @value_expr])))) @node + (class_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key + value: [(identifier) @value_expr + (field_access) @value_expr + (binary_expression) @value_expr]))))) @node + (method_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list + [(identifier) @value_expr + (field_access) @value_expr + (binary_expression) @value_expr])))) @node + (method_declaration + (modifiers + (annotation + name: [(identifier) (scoped_identifier)] @ann + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key + value: [(identifier) @value_expr + (field_access) @value_expr + (binary_expression) @value_expr]))))) @node ] `, ); @@ -122,6 +166,11 @@ export function extractSpringRoutes( // class-array cross-product support is out of scope here. const prefixByClassId = new Map(); const classesWithArrayPrefix = new Set(); + // Classes whose `@RequestMapping` prefix is a constant reference or concat. + // Same treatment as the array form, for the same reason: no single prefix + // string is knowable at extraction time, so emitting the methods below would + // publish them at a WRONG (unprefixed) path rather than not at all. + const classesWithUnfoldablePrefix = new Set(); const classHttpMethodsById = new Map(); for (const match of TYPE_DECLARATION_QUERY.matches(tree.rootNode)) { const typeNode = match.captures.find((capture) => capture.name === 'type')?.node; @@ -139,11 +188,16 @@ export function extractSpringRoutes( const node = caps['node']; const valueNode = caps['value']; const keyNode = caps['key']; - if (!annNode || !node || !valueNode) continue; + const valueExprNode = caps['value_expr']; + if (!annNode || !node || (!valueNode && !valueExprNode)) continue; const capturedAnnotationName = annNode.text.split('.').pop() ?? annNode.text; if (node.type === 'class_declaration' && capturedAnnotationName === 'RequestMapping') { if (!isRouteMemberKey(keyNode)) continue; + if (!valueNode) { + classesWithUnfoldablePrefix.add(node.id); + continue; + } if (valueNode.parent?.type === 'element_value_array_initializer') { classesWithArrayPrefix.add(node.id); continue; @@ -166,7 +220,11 @@ export function extractSpringRoutes( const node = caps['node']; const valueNode = caps['value']; const keyNode = caps['key']; - if (!annNode || !node || !valueNode) continue; + // A constant-referencing value arrives as @value_expr, not @value — the + // match carries exactly one of the two. Require @value only when no + // @value_expr is present; the operand branch below folds the expression. + const valueExprCapture = match.captures.find((c) => c.name === 'value_expr')?.node ?? null; + if (!annNode || !node || (!valueNode && !valueExprCapture)) continue; if (node.type !== 'method_declaration') continue; @@ -181,8 +239,12 @@ export function extractSpringRoutes( if (methodMethods.length === 0) continue; if (!isRouteMemberKey(keyNode)) continue; - const routePath = unquoteSpringLiteral(valueNode.text); - if (routePath === null) continue; + // #2391-style non-literal path (constant ref or `+`-concat): emit with + // operands for cross-file folding in the parse phase. The match carries + // either @value (literal) or @value_expr (non-literal) — never both. + const valueExprNode = valueExprCapture; + const routePath = valueNode ? unquoteSpringLiteral(valueNode.text) : null; + if (routePath === null && !valueExprNode) continue; const enclosingType = findEnclosingType(node); // Interface-declared `@*Mapping`s are not concrete routes on their own — the @@ -206,10 +268,20 @@ export function extractSpringRoutes( // scan — safe under routeCoverage:'partial'. Full class-array cross-product // support is tracked in #2280. (Scalar method paths under an array class // prefix are left unchanged: that pre-existing divergence is out of scope.) - const isArrayElement = valueNode.parent?.type === 'element_value_array_initializer'; + const isArrayElement = valueNode?.parent?.type === 'element_value_array_initializer'; if (isArrayElement && enclosingClass && classesWithArrayPrefix.has(enclosingClass.id)) { continue; } + // Same rule for a CONSTANT-valued class prefix (`@RequestMapping(ApiPaths.BASE)`), + // and for every method route under it — not just array-form ones. The prefix + // needs the repo-wide constant map, which does not exist at extraction time, + // so the prefix would simply be dropped and the route emitted at a path the + // application never serves. On base such a route was not emitted at all; + // turning a missing fact into a wrong one is the failure this module's skip + // floor exists to prevent. Folding class prefixes cross-file is a follow-up. + if (enclosingClass && classesWithUnfoldablePrefix.has(enclosingClass.id)) { + continue; + } const classPrefix = enclosingClass ? (prefixByClassId.get(enclosingClass.id) ?? '') : ''; // `node` is the annotated `method_declaration`; its name field is the @@ -217,6 +289,25 @@ export function extractSpringRoutes( const handlerName = node.childForFieldName('name')?.text; for (const httpMethod of httpMethods) { + if (routePath === null && valueExprNode) { + // Non-literal annotation value: parse operands now; the parse phase + // folds them against the repo-wide Java constant map (KTD5 skip floor + // on failure — never a phantom `POST /`). + const operands = parseJavaConstOperands(valueExprNode); + if (operands === null) continue; + routes.push({ + filePath, + routePath: '', + routePathExpr: valueExprNode.text, + routePathOperands: operands, + httpMethod, + decoratorName: ann, + lineNumber: annNode.startPosition.row + lineOffset, + ...(classPrefix ? { prefix: classPrefix } : {}), + ...(handlerName ? { handlerName } : {}), + }); + continue; + } routes.push({ filePath, routePath, @@ -233,6 +324,13 @@ export function extractSpringRoutes( for (const match of TYPE_DECLARATION_QUERY.matches(tree.rootNode)) { const typeNode = match.captures.find((capture) => capture.name === 'type')?.node; if (typeNode?.type !== 'class_declaration') continue; + // A no-argument `@GetMapping` IS the class prefix, so a class prefix that + // cannot be folded here leaves nothing to emit — the route would ship with + // `routePath: ''` and no prefix, i.e. an empty-path Route. The Phase 2 loop + // above already suppresses these classes; this loop needs the same guard, or + // the suppression is one-sided and the group side (which routes both shapes + // through `methodRoutes`) disagrees with ingestion. + if (classesWithUnfoldablePrefix.has(typeNode.id)) continue; const classPrefix = prefixByClassId.get(typeNode.id) ?? ''; const classMethods = classHttpMethodsById.get(typeNode.id) ?? ['*']; for (const methodNode of directMethods(typeNode)) { diff --git a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts index b87fa3b19..17f639941 100644 --- a/gitnexus/src/core/ingestion/scope-extractor-bridge.ts +++ b/gitnexus/src/core/ingestion/scope-extractor-bridge.ts @@ -64,7 +64,17 @@ export function extractParsedFile( const message = `scope extraction failed for ${filePath}: ${ err instanceof Error ? err.message : String(err) }`; - if (onWarn !== undefined) onWarn(message); + if (onWarn !== undefined) { + try { + onWarn(message); + } catch (warnErr) { + logger.warn( + `scope extraction warning callback failed for ${filePath}: ${ + warnErr instanceof Error ? warnErr.message : String(warnErr) + }`, + ); + } + } logger.warn(message); return undefined; } diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index 39b8667ae..fde3c7b26 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -1775,6 +1775,7 @@ const KNOWN_SUB_TAGS: ReadonlySet = new Set([ '@scope.lexical-names', '@declaration.name', '@declaration.qualified_name', + '@declaration.is-synthetic', '@import.name', '@import.source', '@import.alias', diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts index c0289d496..55becdcb6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts @@ -22,6 +22,7 @@ import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexe import { generateId } from '../../../../lib/utils.js'; import { AMBIGUOUS_POSITION, + exactPositionKey, localNameKey, positionKey, qualifiedKey, @@ -35,6 +36,32 @@ import { import { templateConstraintsIdTag } from '../../utils/template-arguments.js'; import { parameterShapeIdTag } from '../../utils/method-props.js'; import { definitionIdPosition } from '../utils/definition-id.js'; + +const defGraphIdMemoByLookup = new WeakMap>(); + +const isResolveDefGraphIdMemoEnabled = (): boolean => { + const raw = process.env.GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO; + if (raw === undefined || raw.trim() === '') return true; + const value = raw.trim().toLowerCase(); + return value !== '0' && value !== 'false' && value !== 'off' && value !== 'no'; +}; + +const defGraphIdMemoKey = ( + filePath: string, + def: { + nodeId?: string; + qualifiedName?: string; + type?: NodeLabel; + parameterTypes?: readonly string[]; + parameterTypeClasses?: readonly ParameterTypeClass[]; + parameterCount?: number; + templateArguments?: readonly string[]; + templateConstraints?: unknown; + namespacePrefix?: string; + }, +): string => + `${filePath}\0${def.nodeId ?? ''}\0${def.type ?? ''}\0${def.qualifiedName ?? ''}\0${def.parameterCount ?? ''}\0${(def.parameterTypes ?? []).join(',')}\0${(def.parameterTypeClasses ?? []).join(',')}\0${def.namespacePrefix ?? ''}\0${(def.templateArguments ?? []).join(',')}\0${templateConstraintsIdTag(def.templateConstraints)}`; + /** * Labels that may legitimately ANCHOR a CALLS/ACCESSES edge as the * source ("caller"). A Variable / Property can be the TARGET of an @@ -168,14 +195,6 @@ function pickCallerCallableDef( * resolution working for languages that don't yet synthesize * qualifiers). */ -/** - * Extract the 1-based declaration line from a scope-resolution def id. - * Shape: `def:#::<...>`; `undefined` when it doesn't match. - */ -function defStartLine(nodeId: string | undefined, filePath: string): number | undefined { - return definitionIdPosition(nodeId, filePath)?.line; -} - /** * Trailing segment of a dotted qualified name (`Outer.inner` -> `inner`), * with any function-local `@line:col` identity suffix stripped @@ -237,6 +256,38 @@ export function resolveDefGraphId( namespacePrefix?: string; }, nodeLookup: GraphNodeLookup, +): string | undefined { + if (!isResolveDefGraphIdMemoEnabled()) { + return resolveDefGraphIdUncached(filePath, def, nodeLookup); + } + const qn = def.qualifiedName; + if (qn === undefined || qn.length === 0) return undefined; + let bucket = defGraphIdMemoByLookup.get(nodeLookup); + if (bucket === undefined) { + bucket = new Map(); + defGraphIdMemoByLookup.set(nodeLookup, bucket); + } + const key = defGraphIdMemoKey(filePath, def); + if (bucket.has(key)) return bucket.get(key); + const resolved = resolveDefGraphIdUncached(filePath, def, nodeLookup); + bucket.set(key, resolved); + return resolved; +} + +function resolveDefGraphIdUncached( + filePath: string, + def: { + nodeId?: string; + qualifiedName?: string; + type?: NodeLabel; + parameterTypes?: readonly string[]; + parameterTypeClasses?: readonly ParameterTypeClass[]; + parameterCount?: number; + templateArguments?: readonly string[]; + templateConstraints?: unknown; + namespacePrefix?: string; + }, + nodeLookup: GraphNodeLookup, ): string | undefined { const qn = def.qualifiedName; if (qn === undefined || qn.length === 0) return undefined; @@ -256,9 +307,34 @@ export function resolveDefGraphId( // AST nodes (outer wrapper vs inner callable), but the graph node's // `startLine` follows the initializer (#2735) so this join matches even // when the binding is split across lines. - const line = defStartLine(def.nodeId, filePath); + const definitionPosition = definitionIdPosition(def.nodeId, filePath); + const line = definitionPosition?.line; if (line !== undefined && isPositionQualifiedLocalLabel(def.type)) { const simple = simpleNameOf(qn); + if (definitionPosition !== undefined) { + const exactHit = nodeLookup.get( + exactPositionKey( + filePath, + def.type, + definitionPosition.line - 1, + definitionPosition.column, + ), + ); + if (exactHit !== undefined && exactHit !== AMBIGUOUS_POSITION) return exactHit; + if (exactHit === undefined && siblingLabel !== undefined) { + const siblingExactHit = nodeLookup.get( + exactPositionKey( + filePath, + siblingLabel, + definitionPosition.line - 1, + definitionPosition.column, + ), + ); + if (siblingExactHit !== undefined && siblingExactHit !== AMBIGUOUS_POSITION) { + return siblingExactHit; + } + } + } const posHit = nodeLookup.get(positionKey(filePath, def.type, line - 1, simple)); if (posHit !== undefined && posHit !== AMBIGUOUS_POSITION) return posHit; // Retry under the sibling callable label when the def's OWN label diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts index fb42f954b..df7f9b948 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts @@ -96,6 +96,16 @@ export function positionKey( return `

:${filePath}::${label}::${startLine}::${name}`; } +/** Exact source-position key used before the legacy line/name join. */ +export function exactPositionKey( + filePath: string, + label: NodeLabel, + startLine: number, + startColumn: number, +): string { + return `:${filePath}::${label}::${startLine}:${startColumn}`; +} + /** * Key recording that a FUNCTION-LOCAL callable with this simple name exists in the * file (#2699 follow-up). @@ -131,6 +141,7 @@ export function buildGraphNodeLookup(graph: KnowledgeGraph): GraphNodeLookup { name?: string; qualifiedName?: string; templateArguments?: readonly string[]; + startColumn?: number; }; if (props.filePath === undefined || props.name === undefined) continue; if (!isLinkableLabel(node.label)) continue; @@ -139,6 +150,10 @@ export function buildGraphNodeLookup(graph: KnowledgeGraph): GraphNodeLookup { // ambiguous rather than letting source order decide. const startLine = (props as { startLine?: number }).startLine; if (startLine !== undefined && isPositionQualifiedLocalLabel(node.label)) { + if (props.startColumn !== undefined) { + const exactK = exactPositionKey(props.filePath, node.label, startLine, props.startColumn); + lookup.set(exactK, lookup.has(exactK) ? AMBIGUOUS_POSITION : node.id); + } const posK = positionKey(props.filePath, node.label, startLine, props.name); lookup.set(posK, lookup.has(posK) ? AMBIGUOUS_POSITION : node.id); // A local-identity node carries `@:

` on its last name segment. Record diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts index 6a8e1ba83..4ce65c63a 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/callable-value-flow.ts @@ -82,6 +82,15 @@ export interface EmitCallableValueFlowInput { readonly isCallableValueTarget?: (def: SymbolDefinition) => boolean; readonly hasFileLocalCallableLinkage?: (def: SymbolDefinition) => boolean; readonly onWarn?: (warning: CallableValueFlowWarning) => void; + /** When set, skip a second `collectDeferredIndirectSites` walk. */ + readonly deferredIndirectSites?: ReadonlySet; + /** When set, skip a second `referenceSites` signature walk. */ + readonly callSignaturesBySite?: ReadonlyMap; +} + +export interface DeferredIndirectCollection { + readonly sites: ReadonlySet; + readonly callSignaturesBySite: ReadonlyMap; } /** Position key shared with the existing free/reference skip-set contract. */ @@ -100,7 +109,16 @@ export function collectDeferredIndirectSites( parsedFiles: readonly ParsedFile[], scopes?: ScopeResolutionIndexes, ): ReadonlySet { + return collectDeferredIndirectCollection(parsedFiles, scopes).sites; +} + +/** One `referenceSites` walk for deferred keys and call-signature evidence. */ +export function collectDeferredIndirectCollection( + parsedFiles: readonly ParsedFile[], + scopes?: ScopeResolutionIndexes, +): DeferredIndirectCollection { const out = new Set(); + const callSignaturesBySite = new Map(); const flowCells = new Set(); if (scopes !== undefined) { for (const parsed of parsedFiles) { @@ -113,11 +131,23 @@ export function collectDeferredIndirectSites( } } for (const parsed of parsedFiles) { - const canonical = new Set( - parsed.referenceSites - .filter((site) => site.kind === 'call') - .map((site) => callableFlowSiteKey(parsed.filePath, site.atRange)), - ); + const canonical = new Set(); + for (const site of parsed.referenceSites) { + if (site.kind !== 'call') continue; + const key = callableFlowSiteKey(parsed.filePath, site.atRange); + canonical.add(key); + const signature: CallableFlowExpectedSignature = { + ...(site.arity !== undefined ? { parameterCount: site.arity } : {}), + ...(site.argumentTypes !== undefined ? { parameterTypes: site.argumentTypes } : {}), + ...(site.argumentTypeClasses !== undefined + ? { parameterTypeClasses: site.argumentTypeClasses } + : {}), + }; + const previous = callSignaturesBySite.get(key); + if (previous === undefined || signatureEvidence(signature) > signatureEvidence(previous)) { + callSignaturesBySite.set(key, signature); + } + } for (const site of parsed.callableFlowSites ?? []) { if (site.kind !== 'invoke') continue; const key = callableFlowSiteKey(parsed.filePath, site.callSite); @@ -132,7 +162,7 @@ export function collectDeferredIndirectSites( } } } - return out; + return { sites: out, callSignaturesBySite }; } function flowCellOperand(site: CallableFlowSite): CallableFlowOperand | undefined { @@ -156,7 +186,8 @@ function flowCellOperand(site: CallableFlowSite): CallableFlowOperand | undefine export function emitCallableValueFlow(input: EmitCallableValueFlowInput): CallableValueFlowResult { const facts: FileFact[] = []; const invokes: FileInvoke[] = []; - const canonicalInvokeKeys = collectDeferredIndirectSites(input.parsedFiles, input.scopes); + const canonicalInvokeKeys = + input.deferredIndirectSites ?? collectDeferredIndirectSites(input.parsedFiles, input.scopes); let unmatchedInvokes = 0; for (const parsed of input.parsedFiles) { for (const site of parsed.callableFlowSites ?? []) { @@ -485,7 +516,7 @@ export function emitCallableValueFlow(input: EmitCallableValueFlowInput): Callab targetIndexes, aliasesByTargetId, ); - const callSignaturesBySite = indexCallSignatures(input.parsedFiles); + const callSignaturesBySite = input.callSignaturesBySite ?? indexCallSignatures(input.parsedFiles); const dynamicCallees = new Map>(); const dynamicOverflow = new Set(); const dynamicTargetHistory = new Map>(); diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 98484c1d7..04d6010f8 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -809,7 +809,7 @@ export function pickUniqueGlobalCallable( // because the list would then depend on the caller's scope, not just its file. const cacheKey = scopeDefsCache !== undefined && isCallerVisible === undefined - ? `${name}${callerFilePath}` + ? `${name}\0${callerFilePath}` : undefined; let scopeDefs: readonly SymbolDefinition[] | undefined = cacheKey !== undefined ? scopeDefsCache!.get(cacheKey) : undefined; diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index 0ca8b2bea..8218a9656 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -50,6 +50,7 @@ import { buildPropertyNameIndex } from '../passes/unique-name-properties.js'; import { PdgEmitSink, type PdgEmitManifest } from '../../../lbug/pdg-emit-sink.js'; import { resolveNativeSafeStorageDir } from '../../../lbug/lbug-config.js'; import type { ScopeResolver } from '../contract/scope-resolver.js'; +import { reconcileScopeExtractionFailures } from '../scope-extraction-failures.js'; import { logger } from '../../../logger.js'; export interface ScopeResolutionOutput { @@ -61,6 +62,8 @@ export interface ScopeResolutionOutput { readonly importsEmitted: number; /** Reference (CALLS / ACCESSES / INHERITS / USES) edges emitted. */ readonly referenceEdgesEmitted: number; + /** Files still missing scope captures after the main-thread fallback. */ + readonly scopeExtractionFailures: readonly string[]; /** Additive stream of resolver diagnostics; does not affect graph edges. */ readonly resolutionOutcomes: readonly ResolutionOutcome[]; /** @@ -125,6 +128,7 @@ const NOOP_OUTPUT: ScopeResolutionOutput = Object.freeze({ filesProcessed: 0, importsEmitted: 0, referenceEdgesEmitted: 0, + scopeExtractionFailures: [], resolutionOutcomes: [], // Deliberately absent, not `[]`: nothing ran, so nothing was decided either. perLanguage: new Map(), @@ -174,6 +178,7 @@ export const scopeResolutionPhase: PipelinePhase = { const { scannedFiles } = getPhaseOutput(deps, 'structure'); const parseOutput = getPhaseOutput(deps, 'parse'); const { model, parsedFiles: workerParsedFiles } = parseOutput; + const scopeExtractionFailures = new Set(parseOutput.scopeExtractionFailures); // SemanticModel populated during `parse`: scope-resolution consumes // TypeRegistry / MethodRegistry / SymbolTable lookups instead of // rebuilding parallel indexes. See ARCHITECTURE.md § "Semantic-model @@ -538,6 +543,14 @@ export const scopeResolutionPhase: PipelinePhase = { provider, ); + // Worker warnings are provisional: scope-resolution retries missing + // ParsedFiles on the main thread. Persist only final omissions. + reconcileScopeExtractionFailures( + scopeExtractionFailures, + files.map((file) => file.path), + stats.scopeExtractionFailedPaths, + ); + // Release file contents and pre-extracted entries after each language // to reduce memory pressure. For large codebases (16K+ PHP files), // holding all source code simultaneously with scope trees causes OOM. @@ -651,13 +664,20 @@ export const scopeResolutionPhase: PipelinePhase = { // Even when no language ran, surface a finalized manifest (its CSVs are on // disk) so loadGraphToLbug COPYs them rather than orphaning them — empty in // the no-files case, harmless. - if (!anyRan) return pdgEmitManifest ? { ...NOOP_OUTPUT, pdgEmitManifest } : NOOP_OUTPUT; + if (!anyRan) { + return { + ...NOOP_OUTPUT, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), + ...(pdgEmitManifest ? { pdgEmitManifest } : {}), + }; + } return { ran: true, filesProcessed: totalFiles, importsEmitted: totalImports, referenceEdgesEmitted: totalRefs, + scopeExtractionFailures: [...scopeExtractionFailures].sort(), resolutionOutcomes, undecidedSatisfaction, perLanguage, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index b7888cc8e..df823fecb 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -90,7 +90,7 @@ import { import { emitImportEdges } from '../graph-bridge/imports-to-edges.js'; import { callableFlowSiteKey, - collectDeferredIndirectSites, + collectDeferredIndirectCollection, emitCallableValueFlow, } from '../passes/callable-value-flow.js'; import type { ScopeResolver, UndecidedSatisfaction } from '../contract/scope-resolver.js'; @@ -464,6 +464,8 @@ interface RunScopeResolutionInput { interface RunScopeResolutionStats { readonly filesProcessed: number; readonly filesSkipped: number; + /** Files still missing a ParsedFile after the main-thread fallback. */ + readonly scopeExtractionFailedPaths: readonly string[]; readonly importsEmitted: number; readonly resolve: ResolveStats; readonly referenceEdgesEmitted: number; @@ -564,6 +566,7 @@ export function runScopeResolution( // ── Phase 1: extract each file → ParsedFile ──────────────────────────── const parsedFiles: ParsedFile[] = []; + const scopeExtractionFailedPaths: string[] = []; let filesSkipped = 0; const treeCache = input.treeCache; const preExtracted = input.preExtractedParsedFiles; @@ -587,15 +590,20 @@ export function runScopeResolution( } if (parsed === undefined) { const cachedTree = treeCache?.get(file.path); + let extractionWarned = false; parsed = extractParsedFile( provider.languageProvider, file.content, file.path, - onWarn, + (warning) => { + extractionWarned = true; + onWarn(warning); + }, cachedTree, ); if (parsed === undefined) { filesSkipped++; + if (extractionWarned) scopeExtractionFailedPaths.push(file.path); continue; } } @@ -643,6 +651,7 @@ export function runScopeResolution( return { filesProcessed: parsedFiles.length, filesSkipped, + scopeExtractionFailedPaths, importsEmitted: 0, resolve: { sitesProcessed: 0, referencesEmitted: 0, unresolved: 0 }, referenceEdgesEmitted: 0, @@ -680,6 +689,7 @@ export function runScopeResolution( return { filesProcessed: 0, filesSkipped, + scopeExtractionFailedPaths, importsEmitted: 0, resolve: { sitesProcessed: 0, referencesEmitted: 0, unresolved: 0 }, referenceEdgesEmitted: 0, @@ -978,7 +988,8 @@ export function runScopeResolution( // ── Phase 4: emit graph edges (LOAD-BEARING ORDER — see I1) ──────────── input.onProgress?.('linking symbols', files.length, files.length); const handledSites = new Set(preEmittedInheritanceSites); - const deferredIndirectSites = collectDeferredIndirectSites(emitParsedFiles, indexes); + const deferredIndirectCollection = collectDeferredIndirectCollection(emitParsedFiles, indexes); + const deferredIndirectSites = deferredIndirectCollection.sites; const callableArgumentSites = new Set(); if (input.pdg !== true && deferredIndirectSites.size > 0) { for (const parsed of emitParsedFiles) { @@ -1229,6 +1240,8 @@ export function runScopeResolution( collapseByCallerTarget: provider.collapseMemberCallsByCallerTarget === true, isCallableValueTarget: provider.isCallableValueTarget, hasFileLocalCallableLinkage: provider.hasFileLocalCallableLinkage, + deferredIndirectSites, + callSignaturesBySite: deferredIndirectCollection.callSignaturesBySite, onWarn: (warning) => logger.warn( warning, @@ -1646,6 +1659,7 @@ export function runScopeResolution( return { filesProcessed: parsedFiles.length, filesSkipped, + scopeExtractionFailedPaths, importsEmitted, resolve: resolveStats, referenceEdgesEmitted: diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts b/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts new file mode 100644 index 000000000..ce3f6de57 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/scope-extraction-failures.ts @@ -0,0 +1,49 @@ +export interface ScopeExtractionFailureSummary { + /** Exact number of unique files whose scope extraction failed. */ + readonly total: number; + /** Deterministic sample of repo-relative paths for diagnostics. */ + readonly paths: readonly string[]; + /** True when `paths` is a capped sample rather than the full set. */ + readonly truncated?: boolean; +} + +export const SCOPE_EXTRACTION_FAILURE_PATH_LIMIT = 25; + +/** Read a persisted summary without trusting its runtime JSON shape. */ +export function scopeExtractionFailureTotal(summary: unknown): number | undefined { + if (summary === undefined) return 0; + if (typeof summary !== 'object' || summary === null) return undefined; + const total = (summary as { total?: unknown }).total; + if (total === 0) return 0; + return typeof total === 'number' && Number.isInteger(total) && total > 0 ? total : undefined; +} + +/** Replace provisional worker failures with the final fallback outcome. */ +export function reconcileScopeExtractionFailures( + failures: Set, + attemptedPaths: readonly string[], + failedPaths: readonly string[], +): void { + const stillFailed = new Set(failedPaths); + for (const filePath of attemptedPaths) { + if (stillFailed.has(filePath)) failures.add(filePath); + else failures.delete(filePath); + } +} + +export function summarizeScopeExtractionFailures( + paths: readonly string[] = [], + limit: number = SCOPE_EXTRACTION_FAILURE_PATH_LIMIT, +): ScopeExtractionFailureSummary | undefined { + const unique = [ + ...new Set(paths.filter((path): path is string => typeof path === 'string' && path.length > 0)), + ].sort(); + if (unique.length === 0) return undefined; + const boundedLimit = + Number.isInteger(limit) && limit >= 0 ? limit : SCOPE_EXTRACTION_FAILURE_PATH_LIMIT; + return { + total: unique.length, + paths: unique.slice(0, boundedLimit), + ...(unique.length > boundedLimit ? { truncated: true } : {}), + }; +} diff --git a/gitnexus/src/core/ingestion/utils/ast-helpers.ts b/gitnexus/src/core/ingestion/utils/ast-helpers.ts index 75830ea0b..3af6d29f4 100644 --- a/gitnexus/src/core/ingestion/utils/ast-helpers.ts +++ b/gitnexus/src/core/ingestion/utils/ast-helpers.ts @@ -474,6 +474,25 @@ export function walkNamedTree(node: SyntaxNode, cb: (node: SyntaxNode) => void): } } +/** + * True when a node is, or contains, tree-sitter error recovery. + * + * After a syntax error the parser keeps going by guessing node boundaries, so + * the surviving tree stays WELL FORMED while describing text that was never + * written that way: an unterminated argument list can absorb the source of the + * next declaration into an `ERROR` child, and an assignment with no right-hand + * side gets a `MISSING` value node whose text is invented. A capture that reads + * such a subtree emits facts that look ordinary and are false, which is worse + * than emitting nothing — so callers that record source text verbatim should + * check this first and fail closed. + * + * `hasError` covers the subtree; `isMissing` is checked as well because a node + * inserted by recovery is the one case where the node itself carries the flag. + */ +export function hasRecoveredSyntax(node: SyntaxNode): boolean { + return node.hasError || node.isMissing; +} + /** Return the first matching ancestor unless a boundary ancestor is reached first. */ export function findAncestorBeforeBoundary( node: SyntaxNode, @@ -1230,6 +1249,34 @@ export interface ObjectLiteralBindingInfo { ownerName?: string; } +/** + * True when an object-literal member is contained by an array before reaching + * another callable or class boundary. + * + * An array does not provide a stable named owner for its elements, so members + * below one cannot use `.` identity or ownership. They still + * need distinct graph identities, however; callers use this predicate to opt + * into source-position qualification while keeping ownership suppressed. + */ +export const isArrayContainedObjectLiteralMember = (node: SyntaxNode): boolean => { + let current: SyntaxNode | null = node; + let sawObject = false; + + while (current) { + if (current.type === 'object') sawObject = true; + if (current.type === 'array' && sawObject) return true; + if ( + current !== node && + (FUNCTION_NODE_TYPES.has(current.type) || CLASS_CONTAINER_TYPES.has(current.type)) + ) { + return false; + } + current = current.parent; + } + + return false; +}; + /** * Block-statement AST types that disqualify an object-literal binding from * carrying a HAS_METHOD edge. A `const` declared inside one of these is block- @@ -1385,6 +1432,13 @@ export const findObjectLiteralBindingInfo = ( objectDepth += 1; } + if (current !== node && current.type === 'array') { + // `const handlers = [{ run() {} }]` has no `handlers.run` member. + // Crossing the array would mint a confident but false owner edge; keep + // the existing conservative under-approximation used for nested objects. + return null; + } + if (current.type === 'variable_declarator' && objectDepth >= 1) { if (objectDepth > 1) { // Method belongs to a nested object literal; safe under-approximation. diff --git a/gitnexus/src/core/ingestion/utils/symbol-labels.ts b/gitnexus/src/core/ingestion/utils/symbol-labels.ts index a21df10b6..0751324dc 100644 --- a/gitnexus/src/core/ingestion/utils/symbol-labels.ts +++ b/gitnexus/src/core/ingestion/utils/symbol-labels.ts @@ -13,8 +13,8 @@ import type { NodeLabel } from 'gitnexus-shared'; * Single source of truth so the set can't silently drift the way the inline copy * did in #2379. * - * NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY` - * carries a near-identical hand-list that is intentionally a SUBSET — it excludes + * NOTE: group extractor queries in `manifest-extractor.ts` and `graphql-extractor.ts` + * carry near-identical hand-lists that are intentionally SUBSETS — they exclude * `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution * behavior check (would widen which nodes resolve as contract symbols), so it is * deliberately left separate for now. diff --git a/gitnexus/src/core/ingestion/utils/test-file-path.ts b/gitnexus/src/core/ingestion/utils/test-file-path.ts new file mode 100644 index 000000000..ceee59c56 --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/test-file-path.ts @@ -0,0 +1,87 @@ +/** + * Test-file path classification — the single source of truth. + * + * WHY THIS MODULE EXISTS + * + * Two independent copies of this predicate existed and had drifted apart: + * + * - `core/ingestion/entry-point-scoring.ts` `isTestFile` — excludes test + * files from process entry-point detection. + * - `mcp/local/local-backend.ts` `isTestFilePath` — backs the + * `includeTests` flag on `impact` / `trace` / `context`. + * + * They answered "is this a test file?" differently, so the same path could be a + * test in one code path and not the other. The MCP copy recognized no C#, Java, + * or Swift test convention at all, meaning `includeTests: false` silently failed + * to filter them; the scoring copy missed `/conftest.` (it already matched + * `/test/`, so `/test/fixtures/` was never a scoring gap). + * + * The duplication was not gratuitous: `entry-point-scoring.ts` imports the + * language-provider registry, and #2802 deliberately cut that closure out of MCP + * server startup. Importing it back into `local-backend.ts` would reintroduce + * that cost. So the shared predicate lives here instead, with NO imports — pure + * string matching — and both callers delegate to it. + * + * Keep it dependency-free. Anything imported here lands in MCP startup. + */ + +/** + * Lowercase forward-slash path substrings. Directory needles include a leading + * slash so they match path components after the caller slash-prefixes relative + * paths. `/test/` already covers Maven `src/test` and `/test/fixtures/`; + * `/tests/` covers Laravel `tests/Feature` and `/tests/fixtures/`. + */ +const TEST_PATH_SUBSTRINGS: readonly string[] = [ + '.test.', + '.spec.', + '__tests__/', + '__mocks__/', + '/test/', + '/tests/', + '/testing/', + '/spec/', + '/test_', + '/conftest.', + '/uitests/', + '.tests/', + '.test/', + '.integrationtests/', + '.unittests/', + '/testproject/', +]; + +/** Case-insensitive suffixes that already include a delimiter (`_test.py`, not `test.py`). */ +const TEST_PATH_DELIMITED_SUFFIXES: readonly string[] = [ + '_test.py', + '_test.go', + '_spec.rb', + '_test.rb', +]; + +/** + * Case-sensitive `Test`/`Tests`/`Spec` suffixes. Lowercasing first would also + * match production names such as `Contest.swift` and `Latest.php`. + */ +const TEST_PATH_CASED_SUFFIXES: readonly string[] = [ + 'Tests.swift', + 'Test.swift', + 'Tests.cs', + 'Test.cs', + 'Test.php', + 'Spec.php', +]; + +/** Absent / empty paths are not test paths. */ +export function isTestFilePath(filePath: string | null | undefined): boolean { + if (!filePath) return false; + const slashed = filePath.replace(/\\/g, '/'); + const prefixed = slashed.startsWith('/') ? slashed : `/${slashed}`; + const lower = prefixed.toLowerCase(); + if (TEST_PATH_SUBSTRINGS.some((needle) => lower.includes(needle))) return true; + if (TEST_PATH_DELIMITED_SUFFIXES.some((suffix) => lower.endsWith(suffix))) return true; + // Xcode `{Product}UITests` targets. Slash-anchored `/uitests/` does not match + // `MyAppUITests`; an unanchored `uitests/` substring also matches `fruitests`. + if (prefixed.split('/').some((seg) => seg.endsWith('UITests'))) return true; + const basename = prefixed.slice(prefixed.lastIndexOf('/') + 1); + return TEST_PATH_CASED_SUFFIXES.some((suffix) => basename.endsWith(suffix)); +} diff --git a/gitnexus/src/core/ingestion/workers/callable-id.ts b/gitnexus/src/core/ingestion/workers/callable-id.ts index cb8cea942..7672dd8c8 100644 --- a/gitnexus/src/core/ingestion/workers/callable-id.ts +++ b/gitnexus/src/core/ingestion/workers/callable-id.ts @@ -43,12 +43,12 @@ function containsPosition(node: SyntaxNode, row: number, column: number): boolea } /** - * Zero-based start row that keys the graph-to-scope position join for a bound - * callable (#2735). + * Zero-based start position that keys the graph-to-scope join for a bound + * callable (#2735/#3041). * * Graph-node queries may anchor on an outer binding wrapper while the scope - * channel anchors on the inner callable. The join is line-only, so a multi-line - * binding needs the graph node's `startLine` to follow the semantic definition. + * channel anchors on the inner callable. A bound graph node therefore follows + * the semantic definition's line and column rather than the outer wrapper. * * `ParsedFile.localDefs` is the language-agnostic source of that position. * Matching uses only the canonical label, name, and source range; shared worker @@ -58,21 +58,26 @@ function containsPosition(node: SyntaxNode, row: number, column: number): boolea * Missing or ambiguous semantic matches retain the wrapper row, preserving the * existing fail-closed behavior. */ -export function boundCallableStartRow( +export function boundCallableStartPosition( definitionNode: SyntaxNode, nodeName: string, nodeLabel: NodeLabel, localDefs: readonly SymbolDefinition[] | undefined, nameNode?: SyntaxNode | null, -): number { - if (localDefs === undefined) return definitionNode.startPosition.row; +): { readonly row: number; readonly column: number } { + if (localDefs === undefined) return definitionNode.startPosition; const origin = nameNode?.startPosition ?? definitionNode.startPosition; - let best: { row: number; distance: number } | undefined; + let best: { row: number; column: number; distance: number } | undefined; let tied = false; for (const def of localDefs) { - if (def.type !== nodeLabel || simpleDefinitionName(def) !== nodeName) continue; + if ( + def.type !== nodeLabel || + (simpleDefinitionName(def) !== nodeName && def.qualifiedName !== nodeName) + ) { + continue; + } const position = definitionIdPosition(def.nodeId, def.filePath); if (position === undefined) continue; @@ -82,14 +87,29 @@ export function boundCallableStartRow( const distance = Math.abs(row - origin.row) * 1_000_000 + Math.abs(position.column - origin.column); if (best === undefined || distance < best.distance) { - best = { row, distance }; + best = { row, column: position.column, distance }; tied = false; - } else if (distance === best.distance && row !== best.row) { + } else if ( + distance === best.distance && + (row !== best.row || position.column !== best.column) + ) { tied = true; } } - return best !== undefined && !tied ? best.row : definitionNode.startPosition.row; + return best !== undefined && !tied + ? { row: best.row, column: best.column } + : definitionNode.startPosition; +} + +export function boundCallableStartRow( + definitionNode: SyntaxNode, + nodeName: string, + nodeLabel: NodeLabel, + localDefs: readonly SymbolDefinition[] | undefined, + nameNode?: SyntaxNode | null, +): number { + return boundCallableStartPosition(definitionNode, nodeName, nodeLabel, localDefs, nameNode).row; } /** * A function-local callable's own name segment: its name plus its declaration @@ -114,8 +134,13 @@ export function boundCallableStartRow( * bare/class-qualified ids, which is what keeps this off the symbols other * files, saved queries and stored references actually address. */ +export const positionQualifiedCallableName = ( + name: string, + position: { readonly row: number; readonly column: number }, +): string => `${name}@${position.row}:${position.column}`; + export const localIdentity = (node: SyntaxNode, name: string): string => - `${name}@${node.startPosition.row}:${node.startPosition.column}`; + positionQualifiedCallableName(name, node.startPosition); /** * The qualified name of a callable nested inside another callable — THE single diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 4af861f97..75240481b 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -1,9 +1,10 @@ import { parentPort, threadId, workerData } from 'node:worker_threads'; import { createRequire } from 'node:module'; import { - boundCallableStartRow, + boundCallableStartPosition, localIdentity, nestedCallableQualifiedName, + positionQualifiedCallableName, } from './callable-id.js'; import Parser from 'tree-sitter'; import JavaScript from 'tree-sitter-javascript'; @@ -99,6 +100,7 @@ import { getDefinitionNodeFromCaptures, findEnclosingClassInfo, findObjectLiteralBindingInfo, + isArrayContainedObjectLiteralMember, findReturnShapeOwnerInfo, isReturnShapeProperty, findMemberAssignmentOwnerInfo, @@ -149,6 +151,11 @@ import { templateConstraintsIdTag, } from '../utils/template-arguments.js'; import type { LanguageProvider } from '../language-provider.js'; +import { + mergeCanonicalDefinitionProperties, + runDefinitionPropertiesExtractor, + shouldHarvestModuleConstants, +} from '../language-provider.js'; import type { ParsedFile } from 'gitnexus-shared'; import { extractParsedFile, type ScopeCaptureSourceKind } from '../scope-extractor-bridge.js'; import { @@ -500,6 +507,12 @@ export interface ParseWorkerResult { * finalize-orchestrator. */ parsedFiles: ParsedFile[]; + /** + * Repo-relative paths whose scope-capture/extraction step threw. Optional for + * parse-cache compatibility; unlike transient worker telemetry this must be + * replayed on a cache hit so the persisted index cannot claim completeness. + */ + scopeExtractionFailures?: string[]; skippedLanguages: Record; /** * Files whose parse output carried a value the structured-clone algorithm @@ -865,6 +878,13 @@ const CALLABLE_PREFIX_BOUNDARY_TYPES: ReadonlySet = new Set([ 'anonymous_object_creation_expression', // C# ]); +/** + * Object-literal callables use the binding owner in their identity so spelling + * a member as a property or shorthand method cannot change its graph semantics. + */ +const shouldObjectOwnerQualifyCallable = (label: NodeLabel): boolean => + label === 'Function' || label === 'Method'; + const enclosingCallablePrefix = ( node: SyntaxNode, filePath: string, @@ -932,6 +952,11 @@ const callableOwnQualifiedName = ( // `ownName === null` branch below carries the position INSTEAD of a name, // never in addition to one, so the two spellings cannot stack. const ownName = efnResult?.funcName ?? genericFuncName(fnNode) ?? null; + let finalLabel = efnResult?.label ?? inferFunctionLabel(fnNode.type); + if (provider.labelOverride) { + const override = provider.labelOverride(fnNode, finalLabel); + if (override !== null) finalLabel = override; + } const prefix = enclosingCallablePrefix(fnNode, filePath, provider); const classInfo = @@ -945,7 +970,16 @@ const callableOwnQualifiedName = ( provider.resolveContainerTypeOwner, ) : null; - const owner = prefix ?? classInfo?.className; + const objectOwner = + prefix === undefined && classInfo === null && shouldObjectOwnerQualifyCallable(finalLabel) + ? findObjectLiteralBindingInfo(fnNode, filePath, { includeOwnerName: true })?.ownerName + : undefined; + const owner = prefix ?? classInfo?.className ?? objectOwner; + const needsArrayPosition = + owner === undefined && + ownName !== null && + shouldObjectOwnerQualifyCallable(finalLabel) && + isArrayContainedObjectLiteralMember(fnNode); const result = prefix !== undefined ? nestedCallableQualifiedName(prefix, fnNode, ownName ?? 'fn') @@ -953,7 +987,9 @@ const callableOwnQualifiedName = ( ? localIdentity(fnNode, 'fn') : owner ? `${owner}.${ownName}` - : ownName; + : needsArrayPosition + ? positionQualifiedCallableName(ownName, fnNode.startPosition) + : ownName; callableQualifiedNameCache.set(fnNode, result); return result; }; @@ -1009,8 +1045,21 @@ const findEnclosingFunctionId = ( // to the METHOD, not directly to the class, and a Go receiver method can // never itself be nested inside another callable. const nestedPrefix = enclosingCallablePrefix(current, filePath, provider); + const objectOwnerName = + nestedPrefix === undefined && + classInfo === null && + shouldObjectOwnerQualifyCallable(finalLabel) + ? findObjectLiteralBindingInfo(current, filePath, { includeOwnerName: true })?.ownerName + : undefined; const ownerName = - nestedPrefix ?? classInfo?.className ?? standaloneMethodInfo?.receiverType ?? undefined; + nestedPrefix ?? + classInfo?.className ?? + standaloneMethodInfo?.receiverType ?? + objectOwnerName; + const needsArrayPosition = + ownerName === undefined && + shouldObjectOwnerQualifyCallable(finalLabel) && + isArrayContainedObjectLiteralMember(current); // Lockstep with the other two id-building phases — see // `nestedCallableQualifiedName`, which is the shared rule. When a // nested prefix exists it IS `ownerName`, so this branch and the @@ -1020,7 +1069,9 @@ const findEnclosingFunctionId = ( ? nestedCallableQualifiedName(nestedPrefix, current, funcName) : ownerName ? `${ownerName}.${funcName}` - : funcName; + : needsArrayPosition + ? positionQualifiedCallableName(funcName, current.startPosition) + : funcName; // Include # suffix to match definition-phase Method/Constructor IDs. // Use the same MethodExtractor (getMethodInfo) as the definition phase. // When same-arity collisions exist, also append ~type1,type2. @@ -1472,11 +1523,11 @@ export function extractORMQueries( import { extractFastAPIRouterBindings } from '../route-extractors/fastapi-router-bindings.js'; import { - extractPythonModuleConstants, parseConstOperands, type ModuleConstants, type Operand, } from '../route-extractors/python-const-resolver.js'; +import { unfoldableDeclarationsOf } from '../route-extractors/constant-resolver.js'; /** * Report a non-fatal worker issue to the pool over IPC so a caught error is not @@ -1579,6 +1630,11 @@ const processFileGroup = ( } const provider = getProvider(language); + // Owner map for provider.synthesizeStructureMembers: type-declaration AST + // node id → graph node id for classes THIS file's capture loop materialized. + // Keyed by in-memory AST identity (never persisted); filled below. + const classOwnersByNodeId = new Map(); + // #2687: ONE pass over `matches` yields both suppression sets — the // definition-name claims by rank (callable > Property > value), so the dedup // below cannot depend on tree-sitter's match order, and the concrete-typedef @@ -1594,14 +1650,19 @@ const processFileGroup = ( // see parsedfile-store.ts). parse-impl flushes `result.parsedFiles` to disk // per chunk and does NOT retain them in main-thread heap, so this no longer // costs ~1× the semantic model in RAM during parse. + let scopeExtractionFailed = false; const parsedFile = extractParsedFile( provider, parseContent, file.path, - reportWarning, + (message) => { + scopeExtractionFailed = true; + reportWarning(message); + }, tree, scopeSourceKind, ); + if (scopeExtractionFailed) (result.scopeExtractionFailures ??= []).push(file.path); if (parsedFile !== undefined) { // Capture-time side-channel (#1983): `extractParsedFile` just ran the // provider's `emitScopeCaptures`, which (for C++ ADL/namespace marks, @@ -1786,12 +1847,14 @@ const processFileGroup = ( const httpMethod = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'].includes(method) ? method : 'GET'; + const handlerName = provider.decoratorRouteHandlerName?.(decoratorNode); const base = { filePath: file.path, httpMethod, decoratorName, lineNumber: decoratorNode.startPosition.row + lineOffset, ...(decoratorReceiver ? { decoratorReceiver } : {}), + ...(handlerName ? { handlerName } : {}), }; if (decoratorArgStr) { // String-literal path (the fast path, unchanged). Empty-string @@ -2334,23 +2397,24 @@ const processFileGroup = ( // wrapper while scope-resolution anchors on the INNER expression. The // position join is line-only, so `startLine` must follow the initializer // (ids still use `definitionNode` via `localIdentity`). - const startRow = + const startPosition = definitionNode && (nodeLabel === 'Function' || nodeLabel === 'Method' || nodeLabel === 'Constructor') - ? boundCallableStartRow( + ? boundCallableStartPosition( definitionNode, nodeName, nodeLabel, parsedFile?.localDefs, nameNode, ) - : definitionNode?.startPosition.row; + : definitionNode?.startPosition; const startLine = - startRow !== undefined - ? startRow + lineOffset + startPosition !== undefined + ? startPosition.row + lineOffset : nameNode ? nameNode.startPosition.row + lineOffset : lineOffset; + const startColumn = startPosition?.column ?? nameNode?.startPosition.column ?? 0; // Compute enclosing class BEFORE node ID — needed to qualify method IDs const needsOwner = @@ -2438,20 +2502,31 @@ const processFileGroup = ( // and COLLAPSE INTO ONE node — two distinct settings become one symbol, // and the merged name then looks workspace-unique to name inference, // which resolves reads of it to a node representing both. + const objectLiteralBindingInfo = + !enclosingClassId && + (nodeLabel === 'Function' || nodeLabel === 'Method' || nodeLabel === 'Property') && + definitionNode + ? findObjectLiteralBindingInfo(definitionNode, file.path, { + includeOwnerName: + shouldObjectOwnerQualifyCallable(nodeLabel) || nodeLabel === 'Property', + }) + : null; const objectLiteralOwnerInfo = - !enclosingClassId && (nodeLabel === 'Method' || nodeLabel === 'Property') && definitionNode + !enclosingClassId && + (nodeLabel === 'Function' || nodeLabel === 'Method' || nodeLabel === 'Property') && + definitionNode ? (findMemberAssignmentOwnerInfo(definitionNode, file.path) ?? - findObjectLiteralBindingInfo(definitionNode, file.path, { - // Only `Property` opts into the qualifier; `Method` ids must stay - // byte-identical or every object-literal method in every indexed - // repo changes id. - includeOwnerName: nodeLabel === 'Property', - }) ?? + objectLiteralBindingInfo ?? // R3-4: an anonymous literal in return position is owned by the // function whose shape it is. Last in the chain so a variable-bound // literal keeps its existing owner and its existing id. (nodeLabel === 'Property' ? findReturnShapeOwnerInfo(definitionNode, file.path) : null)) : null; + const isArrayContainedObjectCallable = + !enclosingClassId && + shouldObjectOwnerQualifyCallable(nodeLabel) && + definitionNode !== undefined && + isArrayContainedObjectLiteralMember(definitionNode); // Provenance for narrowing (R3-4). A return shape is a real definition but // the weaker one, and the unique-name pass ranks declared anchors above it // so indexing these cannot change an answer that already resolved. @@ -2549,7 +2624,9 @@ const processFileGroup = ( // define `bar` stay distinct nodes. objectLiteralOwnerInfo?.ownerName !== undefined ? `${objectLiteralOwnerInfo.ownerName}.${nodeName}` - : nodeName; + : isArrayContainedObjectCallable + ? positionQualifiedCallableName(nodeName, startPosition) + : nodeName; // #2742: qualify by the enclosing `mod` chain, so two same-named items at // different module depths in one file are DISTINCT nodes. Without this, @@ -2879,19 +2956,43 @@ const processFileGroup = ( } } + const isExported = + language === SupportedLanguages.Vue && isVueSetup + ? isVueSetupTopLevel(nameNode || definitionNode) + : cachedExportCheck(provider.exportChecker, nameNode || definitionNode, nodeName); + if (definitionNode && provider.definitionPropertiesExtractor) { + const definitionProperties = runDefinitionPropertiesExtractor( + provider.definitionPropertiesExtractor, + { + nodeLabel, + nodeName, + filePath: file.path, + definitionNode, + parsedImports: parsedFile?.parsedImports ?? [], + isExported, + }, + (error) => + reportWarning( + `Definition property extraction failed for ${file.path}:${nodeName}: ${error instanceof Error ? error.message : String(error)}`, + ), + ); + if (definitionProperties !== undefined) Object.assign(methodProps, definitionProperties); + } + result.nodes.push({ id: nodeId, label: nodeLabel, - properties: { + properties: mergeCanonicalDefinitionProperties(methodProps, { name: nodeName, filePath: file.path, startLine, + ...(shouldObjectOwnerQualifyCallable(nodeLabel) && + (objectLiteralBindingInfo?.ownerName || isArrayContainedObjectCallable) + ? { startColumn } + : {}), endLine: definitionNode ? definitionNode.endPosition.row + lineOffset : startLine, language: language, - isExported: - language === SupportedLanguages.Vue && isVueSetup - ? isVueSetupTopLevel(nameNode || definitionNode) - : cachedExportCheck(provider.exportChecker, nameNode || definitionNode, nodeName), + isExported, ...(qualifiedTypeName !== undefined ? { qualifiedName: qualifiedTypeName } : {}), ...(classTemplateArguments !== undefined && classTemplateArguments.length > 0 ? { templateArguments: classTemplateArguments } @@ -2906,10 +3007,9 @@ const processFileGroup = ( } : {}), ...(description !== undefined ? { description } : {}), - ...methodProps, ...(declaredType !== undefined ? { declaredType } : {}), ...(returnShapeProperty ? { fromReturnShape: true, isDetail: true } : {}), - }, + }), }); // enclosingClassId already computed above (before nodeId generation) @@ -2954,8 +3054,23 @@ const processFileGroup = ( : {}), }); - // Only emit File -> Symbol DEFINES for top-level symbols (issue #1944). - if (ownerId === undefined) { + // Class-like definitions register their AST node id → graph node id for + // provider.synthesizeStructureMembers. The definition node is the same + // type-declaration AST node that the provider-specific planner receives. + if ( + isClassLikeLabel && + definitionNode && + provider.classExtractor?.isTypeDeclaration(definitionNode) + ) { + classOwnersByNodeId.set(definitionNode.id, nodeId); + } + + // Object-literal callables remain file definitions as well as members of + // their exported binding. Class members still use HAS_METHOD alone. + const isTopLevelObjectCallable = + objectLiteralBindingInfo?.ownerName !== undefined && + shouldObjectOwnerQualifyCallable(nodeLabel); + if (ownerId === undefined || isTopLevelObjectCallable) { const fileId = generateId('File', file.path); const relId = generateId('DEFINES', `${fileId}->${nodeId}`); result.relationships.push({ @@ -2978,7 +3093,7 @@ const processFileGroup = ( type: memberEdgeType, confidence: 1.0, reason: objectLiteralOwnerInfo - ? 'object literal method belongs to exported object binding' + ? 'object literal member belongs to exported object binding' : '', }); } @@ -3021,12 +3136,29 @@ const processFileGroup = ( (result.routerModuleAliases ??= []), (result.routerConstructorPrefixes ??= []), ); - // #2391: harvest module-level string constants + from-imports so parse-impl - // can resolve non-literal decorator route paths cross-file. Only emit for - // files that carry something resolvable (a constant definition or an import - // binding) to keep the aggregate bounded on large repos. - const constants = extractPythonModuleConstants(tree); - if (constants.literals.size > 0 || constants.exprs.size > 0 || constants.imports.size > 0) { + } + + // #2391/#2980: harvest module-level string constants + import bindings via + // the provider hook so parse-impl can resolve non-literal decorator route + // paths cross-file. Cost-gated by the provider's syntax-driven heuristic; + // only files that carry something resolvable (a constant definition or an + // import binding) are emitted, keeping the aggregate bounded on large repos. + // A provider that declares no heuristic harvests unconditionally — see + // `shouldHarvestModuleConstants`, which owns that rule so it can be tested + // without booting a worker. + if (provider.extractModuleConstants && shouldHarvestModuleConstants(provider, parseContent)) { + const constants = provider.extractModuleConstants(tree); + const topLevelDeclarations = ( + constants as ModuleConstants & { readonly topLevelDeclarations?: unknown } + ).topLevelDeclarations; + if ( + constants.literals.size > 0 || + constants.exprs.size > 0 || + constants.imports.size > 0 || + (constants.wildcardImports?.length ?? 0) > 0 || + unfoldableDeclarationsOf(constants).size > 0 || + (topLevelDeclarations instanceof Set && topLevelDeclarations.size > 0) + ) { (result.moduleConstants ??= []).push({ filePath: file.path, constants }); } } @@ -3048,6 +3180,19 @@ const processFileGroup = ( if (springTypes.length > 0) (result.springTypes ??= []).push(...springTypes); } + if (provider.synthesizeStructureMembers) { + const synthetic = provider.synthesizeStructureMembers(tree, file.path, classOwnersByNodeId); + for (const node of synthetic.nodes) { + result.nodes.push(node as ParsedNode); + } + for (const sym of synthetic.symbols) { + result.symbols.push(sym as ParsedSymbol); + } + for (const rel of synthetic.relationships) { + result.relationships.push(rel as ParsedRelationship); + } + } + // Vue: emit CALLS edges for components used in