diff --git a/.github/scripts/check-tree-sitter-upgrade-readiness.py b/.github/scripts/check-tree-sitter-upgrade-readiness.py index c1ee3c92d..30201ab57 100644 --- a/.github/scripts/check-tree-sitter-upgrade-readiness.py +++ b/.github/scripts/check-tree-sitter-upgrade-readiness.py @@ -68,6 +68,7 @@ GRAMMARS: dict[str, tuple[str, str, str]] = { "tree-sitter-typescript": ("tree-sitter/tree-sitter-typescript", "master", "typescript/src/parser.c"), # Vendored parsers — kept here so the upstream coords for drift # detection are co-located with every other grammar's coords. + "tree-sitter-objc": ("tree-sitter-grammars/tree-sitter-objc", "master", "src/parser.c"), "tree-sitter-proto": ("coder3101/tree-sitter-proto", "main", "src/parser.c"), "tree-sitter-zig": ("tree-sitter-grammars/tree-sitter-zig", "master", "src/parser.c"), } diff --git a/.github/scripts/test_check_tree_sitter_upgrade_readiness.py b/.github/scripts/test_check_tree_sitter_upgrade_readiness.py index 5013b3549..f1b4baf97 100644 --- a/.github/scripts/test_check_tree_sitter_upgrade_readiness.py +++ b/.github/scripts/test_check_tree_sitter_upgrade_readiness.py @@ -9,8 +9,8 @@ which is deliberately dependency-free so it runs on any vanilla runner. Run with (pytest also discovers ``unittest.TestCase`` classes, so a future pytest CI job picks these up unchanged.) -These tests lock in the #858 fix: the 6 vendored grammars -(c/swift/kotlin/dart/proto/zig) are classified from the shared manifest +These tests lock in the #858 fix: the 7 vendored grammars +(c/swift/kotlin/dart/objc/proto/zig) are classified from the shared manifest (.github/vendored-grammars.json), their ABI is read from gitnexus/vendor/, and the report never renders a bare ``?`` placeholder. All network is mocked. """ @@ -192,7 +192,7 @@ class AssertCurrent(TestCase): def test_assert_current_is_network_free_and_passes(self): report, code = self._run_assert_current() # raises if any urlopen fires self.assertEqual(code, 0) - # All 6 vendored grammars are introspected from the repo (ABI 14), not skipped. + # All 7 vendored grammars are introspected from the repo (ABI 14), not skipped. for name in readiness.VENDORED_NAMES: self.assertIn(f"{name}: vendored ABI", report) @@ -365,15 +365,16 @@ class ReportRendering(TestCase): # Counts are derived from _render_report()'s mock corpus (all npm peer # deps mocked permissive): of the 10 npm-installed grammars, 9 render # Ready and 1 — tree-sitter-cpp — is the intentional pin (#1242), so it is - # not counted ready. The 3 blockers are that same pinned tree-sitter-cpp - # plus two held vendored grammars: ABI-held tree-sitter-c (#1242/#858) and + # not counted ready. The 4 blockers are that same pinned tree-sitter-cpp + # plus three held vendored grammars: ABI-held tree-sitter-c (#1242/#858), # tree-sitter-kotlin (pinned to an unreleased fwcd main commit for `fun # interface` support — ABI 14 is in range, but a hold counts as a blocker - # until it is lifted). If a grammar is added/removed or a pin/hold changes, + # until it is lifted), and tree-sitter-objc. If a grammar is added/removed + # or a pin/hold changes, # update _render_report()'s mock AND these expected counts together; a # mismatch here means the report prose drifted, not the regex. self.assertEqual(ready.groups(), ("9", "10")) - self.assertEqual(blockers.group(1), "3") + self.assertEqual(blockers.group(1), "4") def _matrix_row(self, name: str) -> str: for line in self.report.splitlines(): diff --git a/.github/scripts/update-vendored-grammars.mjs b/.github/scripts/update-vendored-grammars.mjs index 769310bd7..0554d3253 100644 --- a/.github/scripts/update-vendored-grammars.mjs +++ b/.github/scripts/update-vendored-grammars.mjs @@ -21,7 +21,7 @@ * node update-vendored-grammars.mjs # detect only → JSON report on stdout * node update-vendored-grammars.mjs --apply X # re-vendor grammar X in place * - * tree-sitter-c is MONITORED but report-only (`hold`): it is ABI-pinned at 0.21.4 + * tree-sitter-c and tree-sitter-objc are MONITORED but report-only (`hold`): c is ABI-pinned at 0.21.4 * (#1242/#858) and must not auto-bump without a tree-sitter runtime upgrade, so an * available c update is detected + reported but never auto-applied — even if it is * ABI-13/14. A maintainer re-vendors it deliberately. diff --git a/.github/scripts/verify-workflow-run-pr-identity.cjs b/.github/scripts/verify-workflow-run-pr-identity.cjs new file mode 100644 index 000000000..c638f92a6 --- /dev/null +++ b/.github/scripts/verify-workflow-run-pr-identity.cjs @@ -0,0 +1,274 @@ +// Resolve the open PR for a trusted workflow_run consumer. +// +// Shared by commit-fork-prebuilds.yml and pr-autofix-publish.yml. +// workflow_run.pull_requests[] is empty on fork PRs, and +// GET /repos/{base}/commits/{sha}/pulls is also empty because the fork head +// commit is not in the base repo's commit graph. The authoritative lookup is +// GET /repos/{base}/pulls?head={owner}:{branch}&state=open using +// workflow_run.head_repository + workflow_run.head_branch (server-controlled). +// That same query works for same-repo PRs (owner is the base repo owner). +// +// The current PR tip may have moved past the SHA the producer built; that is +// not an identity failure — the caller decides whether to lease-push or just +// comment. Two open PRs from the same fork head (same owner:branch into this +// repo) are an identity failure: artifact pr_number is untrusted and must not +// pick among them. Set SCHEMA_PATTERN to the artifact schema allowlist +// (defaults to the tree-sitter prebuild schema). +'use strict'; + +const fs = require('node:fs'); +const { spawnSync } = require('node:child_process'); + +const SCHEMA_PATTERN = /^gitnexus\.ts-prebuild\/v[0-9]+$/; +const IDENTITY_PATTERNS = { + pr_number: /^[0-9]+$/, + head_sha: /^[0-9a-f]{40}$/, + head_ref: /^[A-Za-z0-9._/-]+$/, + repo: /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/, +}; + +function allowlistField(key, value, pattern) { + const text = value == null ? '' : String(value); + if (!text || !pattern.test(text)) { + throw new Error(`metadata.${key} failed allowlist (got: ${JSON.stringify(text)})`); + } + return text; +} + +function forkHeadOwner(headRepo) { + const slash = headRepo.indexOf('/'); + if (slash <= 0 || slash === headRepo.length - 1) { + throw new Error(`head_repo must be owner/name (got: ${JSON.stringify(headRepo)})`); + } + return headRepo.slice(0, slash); +} + +function compileSchemaPattern(value) { + if (value instanceof RegExp) return value; + if (typeof value === 'string' && value.length > 0) { + try { + return new RegExp(value); + } catch { + throw new Error('SCHEMA_PATTERN is not a valid regular expression'); + } + } + return SCHEMA_PATTERN; +} + +function allowlistMetadata(raw, schemaPattern) { + const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('metadata.json must be an object'); + } + return { + schema: allowlistField('schema', parsed.schema, compileSchemaPattern(schemaPattern)), + pr_number: allowlistField('pr_number', parsed.pr_number, IDENTITY_PATTERNS.pr_number), + head_sha: allowlistField('head_sha', parsed.head_sha, IDENTITY_PATTERNS.head_sha), + head_ref: allowlistField('head_ref', parsed.head_ref, IDENTITY_PATTERNS.head_ref), + head_repo: allowlistField('head_repo', parsed.head_repo, IDENTITY_PATTERNS.repo), + base_repo: allowlistField('base_repo', parsed.base_repo, IDENTITY_PATTERNS.repo), + }; +} + +function allowlistAuthority(authority) { + return { + head_sha: allowlistField('head_sha', authority.head_sha, IDENTITY_PATTERNS.head_sha), + head_repo: allowlistField('head_repo', authority.head_repo, IDENTITY_PATTERNS.repo), + head_branch: allowlistField('head_ref', authority.head_branch, IDENTITY_PATTERNS.head_ref), + base_repo: allowlistField('base_repo', authority.base_repo, IDENTITY_PATTERNS.repo), + }; +} + +function verifyArtifactAgainstWorkflowRun(meta, authority) { + if (meta.head_sha !== authority.head_sha) { + throw new Error( + `Artifact head_sha (${meta.head_sha}) != workflow_run.head_sha (${authority.head_sha}) — refusing.`, + ); + } + if (meta.head_repo !== authority.head_repo) { + throw new Error( + `Artifact head_repo (${meta.head_repo}) != workflow_run.head_repository (${authority.head_repo}) — refusing.`, + ); + } + if (meta.base_repo !== authority.base_repo) { + throw new Error('Artifact base_repo does not match $GITHUB_REPOSITORY — refusing.'); + } + if (meta.head_ref !== authority.head_branch) { + throw new Error( + `Artifact head_ref (${meta.head_ref}) != workflow_run.head_branch (${authority.head_branch}) — refusing.`, + ); + } +} + +function matchOpenPullsFromForkHead(pulls, { headRepo, headBranch, baseRepo }) { + if (!Array.isArray(pulls)) { + throw new Error('GitHub pulls?head= lookup returned a non-array'); + } + return pulls.filter((pr) => { + return ( + pr && + pr.state === 'open' && + Number.isInteger(pr.number) && + pr.head && + pr.head.repo && + pr.head.repo.full_name === headRepo && + pr.head.ref === headBranch && + pr.base && + pr.base.repo && + pr.base.repo.full_name === baseRepo + ); + }); +} + +function resolveVerifiedPullRequest({ meta, authority, pulls, schemaPattern }) { + const cleanMeta = allowlistMetadata(meta, schemaPattern); + const cleanAuthority = allowlistAuthority(authority); + verifyArtifactAgainstWorkflowRun(cleanMeta, cleanAuthority); + + const matched = matchOpenPullsFromForkHead(pulls, { + headRepo: cleanAuthority.head_repo, + headBranch: cleanAuthority.head_branch, + baseRepo: cleanAuthority.base_repo, + }); + + if (matched.length === 0) { + throw new Error( + `No open PR from ${cleanAuthority.head_repo}:${cleanAuthority.head_branch} targeting ${cleanAuthority.base_repo} — refusing.`, + ); + } + + // Artifact pr_number is untrusted. Do not use it to pick among several open + // PRs that share this fork head (same owner:branch into this repo, different + // base branches). Fail closed unless GitHub-controlled fields leave exactly one. + if (matched.length !== 1) { + throw new Error( + `Ambiguous open PRs from ${cleanAuthority.head_repo}:${cleanAuthority.head_branch} targeting ${cleanAuthority.base_repo} (${matched + .map((pr) => pr.number) + .join(',')}) — refusing.`, + ); + } + + const chosen = matched[0]; + const expected = Number(cleanMeta.pr_number); + if (chosen.number !== expected) { + throw new Error( + `Artifact pr_number (${cleanMeta.pr_number}) is not the open PR(s) from this fork head (${chosen.number}) — refusing.`, + ); + } + + const currentHeadSha = typeof chosen.head.sha === 'string' ? chosen.head.sha : ''; + return { + pr_number: String(chosen.number), + head_ref: cleanAuthority.head_branch, + head_sha: cleanAuthority.head_sha, + head_repo: cleanAuthority.head_repo, + current_head_sha: currentHeadSha, + branch_moved: Boolean(currentHeadSha && currentHeadSha !== cleanAuthority.head_sha), + }; +} + +function flattenGhListPages(parsed) { + if (!Array.isArray(parsed)) { + throw new Error('GitHub pulls?head= lookup returned a non-array'); + } + if (parsed.length === 0) return parsed; + if (parsed.every((page) => Array.isArray(page))) { + return parsed.flat(); + } + return parsed; +} + +function listOpenPullsByHead({ ghRepo, headOwner, headBranch, runGh }) { + const run = runGh || ((args) => spawnSync('gh', args, { encoding: 'utf8' })); + const result = run([ + 'api', + '--paginate', + '--slurp', + '-X', + 'GET', + `repos/${ghRepo}/pulls`, + '-f', + 'state=open', + '-f', + `head=${headOwner}:${headBranch}`, + ]); + if (result.status !== 0) { + const err = (result.stderr || result.stdout || '').trim(); + throw new Error(`GitHub pulls?head= lookup failed: ${err || `exit ${result.status}`}`); + } + const stdout = (result.stdout || '').trim(); + if (!stdout) { + throw new Error('GitHub pulls?head= lookup returned an empty body'); + } + let parsed; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error('GitHub pulls?head= lookup returned non-JSON'); + } + return flattenGhListPages(parsed); +} + +function main() { + const schemaPattern = compileSchemaPattern(process.env.SCHEMA_PATTERN); + const raw = fs.readFileSync(process.env.META_PATH, 'utf8'); + const meta = allowlistMetadata(raw, schemaPattern); + const authority = allowlistAuthority({ + head_sha: process.env.WF_HEAD_SHA, + head_repo: process.env.WF_HEAD_REPO, + head_branch: process.env.WF_HEAD_BRANCH, + base_repo: process.env.GH_REPO, + }); + const pulls = listOpenPullsByHead({ + ghRepo: authority.base_repo, + headOwner: forkHeadOwner(authority.head_repo), + headBranch: authority.head_branch, + }); + const verified = resolveVerifiedPullRequest({ meta, authority, pulls, schemaPattern }); + if (verified.branch_moved) { + console.log( + `PR head moved to ${verified.current_head_sha}; delivering against built SHA ${verified.head_sha} (lease will refuse if the branch moved).`, + ); + } + console.log( + `Verified identity: PR=${verified.pr_number} head_sha=${verified.head_sha} head_repo=${verified.head_repo} head_ref=${verified.head_ref}.`, + ); + const out = process.env.GITHUB_OUTPUT; + if (!out) { + throw new Error('GITHUB_OUTPUT is unset'); + } + fs.appendFileSync( + out, + [ + `pr_number=${verified.pr_number}`, + `head_ref=${verified.head_ref}`, + `head_sha=${verified.head_sha}`, + `head_repo=${verified.head_repo}`, + ].join('\n') + '\n', + ); +} + +if (require.main === module) { + try { + main(); + } catch (err) { + console.error(`::error::${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } +} + +module.exports = { + SCHEMA_PATTERN, + IDENTITY_PATTERNS, + allowlistField, + compileSchemaPattern, + allowlistMetadata, + allowlistAuthority, + forkHeadOwner, + verifyArtifactAgainstWorkflowRun, + matchOpenPullsFromForkHead, + flattenGhListPages, + resolveVerifiedPullRequest, + listOpenPullsByHead, + main, +}; diff --git a/.github/vendored-grammars.json b/.github/vendored-grammars.json index bb4b18820..d42314aa2 100644 --- a/.github/vendored-grammars.json +++ b/.github/vendored-grammars.json @@ -6,6 +6,11 @@ "upstream": { "npm": "tree-sitter-c" }, "hold": "ABI-pinned at 0.21.4 (#1242/#858) — needs a tree-sitter runtime upgrade before bumping" }, + "objc": { + "name": "tree-sitter-objc", + "upstream": { "npm": "tree-sitter-objc" }, + "hold": "Pinned at 3.0.2 for the Objective-C provider MVP; carries darwin/linux arm64+x64 prebuilds compatible with the current tree-sitter runtime (linux-arm64 built from vendored source because the upstream npm artifact is mislabeled)" + }, "swift": { "name": "tree-sitter-swift", "upstream": { "npm": "tree-sitter-swift" } diff --git a/.github/workflows/build-tree-sitter-prebuilds.yml b/.github/workflows/build-tree-sitter-prebuilds.yml index ef64a64b8..b592ebe16 100644 --- a/.github/workflows/build-tree-sitter-prebuilds.yml +++ b/.github/workflows/build-tree-sitter-prebuilds.yml @@ -7,7 +7,7 @@ name: Build tree-sitter prebuilds # # Grammars covered here (the at-risk set — everything else already ships 6 # upstream prebuilds AND stays dependency-review-tracked, so it is left alone). -# All five are vendored under gitnexus/vendor/; `kind` (below) only picks where +# All seven are vendored under gitnexus/vendor/; `kind` (below) only picks where # the build job fetches the C source to compile: # - tree-sitter-c (vendored prebuild-only; built from the published npm # package — closes upstream's 4/6 ARM gap #2116 for a @@ -17,6 +17,8 @@ name: Build tree-sitter prebuilds # - tree-sitter-kotlin (vendored source; built from gitnexus/vendor/ — pinned to # an unreleased main commit for `fun interface` support # (#169) that no npm release carries yet) +# - tree-sitter-objc (vendored source; built from gitnexus/vendor/ — pinned +# for the Objective-C provider MVP) # - tree-sitter-swift (vendored source; built from gitnexus/vendor/ — its # prebuilds were originally upstream-shipped, now # GitNexus-cross-built like the rest for uniformity) @@ -24,13 +26,13 @@ name: Build tree-sitter prebuilds # off npm optionalDependency so `npm i -g gitnexus` # no longer warns on peerOptional tree-sitter@^0.22.1. # Upstream linux-arm64 prebuild is a mispackaged -# x86-64 binary; this workflow rebuilds all six.) +# x86-64 binary; this workflow rebuilds all seven.) # # Output: gitnexus/vendor//prebuilds//.node for # all 6 targets ({linux,darwin,win32}-{x64,arm64}). tree-sitter grammars are # N-API, so one ABI-stable .node per platform-arch works across all Node majors. # -# COST DISCIPLINE — this is a HEAVY native matrix (up to 3 grammars x 6 runners, +# COST DISCIPLINE — this is a HEAVY native matrix (up to 7 grammars x 6 runners, # incl. macOS + arm64). It is DELIBERATELY NOT wired into normal PR/push CI. It # runs only: # 1. on manual dispatch (workflow_dispatch); or @@ -61,7 +63,7 @@ on: workflow_dispatch: inputs: grammars: - description: 'Comma-separated grammar shortnames to build (c,dart,proto,kotlin,swift,zig), or "all".' + description: 'Comma-separated grammar shortnames to build (c,dart,proto,kotlin,objc,swift,zig), or "all".' required: false type: string default: 'all' @@ -93,7 +95,7 @@ on: - '!gitnexus/vendor/tree-sitter-*/prebuilds/**' # Self-test: re-run the guard if a future grammar pin is reintroduced in # the main package.json (optionalDependencies fallback). No-op otherwise — - # all six grammars are now fully vendored (kotlin and zig included). + # all seven grammars are now fully vendored (including kotlin, objc, and zig). - 'gitnexus/package.json' # Self-test: re-run the guard (normally a no-op) when the recipe changes. - '.github/workflows/build-tree-sitter-prebuilds.yml' @@ -163,6 +165,9 @@ jobs: // unreleased main commit for `fun interface` support (#169) that no // npm release carries yet — so it must build from the vendored source. kotlin: { name: 'tree-sitter-kotlin', kind: 'vendored' }, + // Objective-C is vendored WITH its source and its native bindings + // must be recut together with the pinned grammar snapshot. + objc: { name: 'tree-sitter-objc', kind: 'vendored' }, // swift is vendored WITH its source (parser.c/scanner.c/binding.gyp), // so it builds from gitnexus/vendor/ like dart/proto. Its prebuilds // were originally upstream-shipped; rebuilding them here unifies it. @@ -505,6 +510,7 @@ jobs: dart: "void main() { print(\"hi\"); }", proto: "syntax = \"proto3\";\nmessage M { int32 id = 1; }", kotlin: "fun main() { println(\"hi\") }", + objc: "@interface GNValidationProbe : NSObject\n@end", swift: "func greet() { print(\"hi\") }", zig: "pub fn main() void {}", }; diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 7b469948c..20657d9b2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -718,6 +718,13 @@ jobs: run: node --import tsx bench/kotlin-import-target/measure.mjs --check working-directory: gitnexus + - name: Ruby gem-boundary correctness + scaling guards (#3096) + if: ${{ !cancelled() }} + # Includes real manifest loading; checks scoped resolution and scaling + # as sibling projects or declared gem counts grow independently. + run: node --import tsx bench/ruby-gem-resolution/measure.mjs --check + working-directory: gitnexus + - name: Receiver-resolution drop guards if: ${{ !cancelled() }} # NOT build-free: this one runs the real pipeline, so it needs dist/ @@ -764,6 +771,28 @@ jobs: run: node --import tsx bench/zig-cross-file-resolution/measure.mjs --check working-directory: gitnexus + - name: Objective-C workspace resolution guards (#3179) + if: ${{ !cancelled() }} + # Build-free: fingerprints spread (typed self/super/sibling) and + # protocol-candidate evidence, and gates linear file-count scaling + # of emitPostResolutionEdges. Import lookup is the shared + # import-target `objc` arm; this is the C#/Zig analog for the + # workspace message-send pass. + run: node --import tsx bench/objective-c-resolution/measure.mjs --check + working-directory: gitnexus + + - name: Callable-value reference resolution guards (#3399) + if: ${{ !cancelled() }} + # Build-free: pins the resolved-target SET of `resolveValueRefTarget` + # (exact site/resolved/declined counts plus an order-independent + # fingerprint) and asserts its per-site cost stays independent of + # workspace size across a 4x file-count step. The pass resolves a + # qualified receiver through `scopes.qualifiedNames`, a workspace-wide + # index: keyed it is O(1) per site, scanned it is O(files) — a + # regression a fixture cannot see and a 257-file binding table can. + run: node --import tsx bench/value-ref-resolution/measure.mjs --check + working-directory: gitnexus + - name: CFG construction time / disk / memory guards (#2081 M1) if: ${{ !cancelled() }} # Build-free: asserts collectFunctionCfgs output is unchanged @@ -803,11 +832,13 @@ jobs: # guards they hold never run in the main coverage job. env: GITNEXUS_BENCH: '1' + GITNEXUS_WORKER_READY_TIMEOUT_MS: '60000' run: >- npx vitest run --no-file-parallelism test/integration/cobol-pipeline-benchmark.test.ts test/integration/csharp-pipeline-benchmark.test.ts test/integration/csharp-razor-view-components-benchmark.test.ts + test/integration/objective-c-pipeline-benchmark.test.ts test/integration/cpp-adl-benchmark.test.ts test/integration/data-route-table-benchmark.test.ts test/integration/instance-ownership-pipeline-benchmark.test.ts @@ -846,6 +877,11 @@ jobs: timeout-minutes: 20 env: GITNEXUS_REQUIRE_BWRAP_CANARY: '1' + # This job installs bubblewrap, the pinned runtime and a built GitNexus, + # so the offline sweep runs here with nothing provisioning-stubbed: real + # containment, real mounts, real graph. A missing piece fails the job + # rather than silently falling back to the stubbed path. + GITNEXUS_REQUIRE_FULL_SWEEP: '1' GITNEXUS_REQUIRE_CLAUDE_CANARY: '1' steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -904,7 +940,9 @@ jobs: tests/test_process_control.py tests/test_proposer_sandbox.py tests/test_workflow_bench_sessions.py - tests/test_ce_plugin_runtime.py -q + tests/test_ce_plugin_runtime.py + tests/test_offline_sweep_integration.py + tests/test_mock_provider.py -q working-directory: eval # Native Windows Job Object canary. POSIX-only tests skip by platform, while diff --git a/.github/workflows/commit-fork-prebuilds.yml b/.github/workflows/commit-fork-prebuilds.yml index 950402e35..3b6ef78d3 100644 --- a/.github/workflows/commit-fork-prebuilds.yml +++ b/.github/workflows/commit-fork-prebuilds.yml @@ -135,68 +135,53 @@ jobs: # workflow_run event. The allowlist above only proves the fields are # well-formed — not that they refer to the PR/SHA that actually triggered # us. A fork-controlled build could mutate metadata.json to reference - # another PR/SHA and redirect our write-scoped push. Authority sources are - # all server-controlled: workflow_run.head_sha, head_repository.full_name, - # and pull_requests[].number (empty on forks -> commits/{sha}/pulls). + # another PR/SHA and redirect our write-scoped push. + # + # This job's `if:` already restricts to forks, so pull_requests[] is empty + # by design and GET /repos/{base}/commits/{sha}/pulls is also empty (the + # fork commit is not in the base graph). Authority is workflow_run.head_sha + # + head_repository.full_name + head_branch, resolved via + # pulls?head={owner}:{branch}. The script comes from THIS default-branch + # checkout (the same trust anchor as this workflow file). + - name: Checkout identity verifier + if: steps.meta.outputs.deliver == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: .github/scripts/verify-workflow-run-pr-identity.cjs + sparse-checkout-cone-mode: false + path: trusted + - name: Verify metadata against workflow_run authority + id: verify if: steps.meta.outputs.deliver == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - META_PR_NUMBER: ${{ steps.meta.outputs.pr_number }} - META_HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - META_HEAD_REPO: ${{ steps.meta.outputs.head_repo }} + META_PATH: meta-in/metadata.json + SCHEMA_PATTERN: '^gitnexus\.ts-prebuild/v[0-9]+$' WF_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} WF_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} - WF_PR_NUMBERS: ${{ toJSON(github.event.workflow_run.pull_requests.*.number) }} + WF_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} shell: bash - run: | - set -euo pipefail - - # 1) head_sha must match exactly — the commit GitHub ran the producer against. - if [ "${META_HEAD_SHA}" != "${WF_HEAD_SHA}" ]; then - echo "::error::Artifact head_sha (${META_HEAD_SHA}) != workflow_run.head_sha (${WF_HEAD_SHA}) — refusing." - exit 1 - fi - # 2) head_repo must match exactly. - if [ "${META_HEAD_REPO}" != "${WF_HEAD_REPO}" ]; then - echo "::error::Artifact head_repo (${META_HEAD_REPO}) != workflow_run.head_repository (${WF_HEAD_REPO}) — refusing." - exit 1 - fi - # 3) pr_number must reference an open PR with this head SHA. Forks have - # an empty pull_requests[] by design — fall back to commits/{sha}/pulls. - allowed_numbers=$(jq -c '.' <<< "${WF_PR_NUMBERS}") - if [ "${allowed_numbers}" = "[]" ]; then - echo "workflow_run.pull_requests empty (fork) — using commits/{sha}/pulls." - allowed_numbers=$(gh api "repos/${GH_REPO}/commits/${WF_HEAD_SHA}/pulls" \ - --jq '[.[] | select(.state == "open") | .number]' 2>/dev/null || echo "[]") - if [ "${allowed_numbers}" = "[]" ]; then - echo "::error::No open PR for head ${WF_HEAD_SHA} — refusing." - exit 1 - fi - fi - if ! jq -e --argjson n "${META_PR_NUMBER}" 'index($n) != null' <<< "${allowed_numbers}" >/dev/null; then - echo "::error::Artifact pr_number (${META_PR_NUMBER}) not in authoritative list (${allowed_numbers}) — refusing." - exit 1 - fi - echo "Verified identity: PR=${META_PR_NUMBER} head_sha=${META_HEAD_SHA} head_repo=${META_HEAD_REPO}." + run: node trusted/.github/scripts/verify-workflow-run-pr-identity.cjs # Pinned to v6.0.3 (same SHA used by build-tree-sitter-prebuilds.yml). # persist-credentials: false — push auth is provided inline at push time, # never written to .git/config on disk. - name: Checkout fork PR head - if: steps.meta.outputs.deliver == 'true' + if: steps.meta.outputs.deliver == 'true' && steps.verify.outcome == 'success' uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - repository: ${{ steps.meta.outputs.head_repo }} - ref: ${{ steps.meta.outputs.head_sha }} + repository: ${{ steps.verify.outputs.head_repo }} + ref: ${{ steps.verify.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} persist-credentials: false fetch-depth: 0 path: pr-checkout - name: Place prebuilds into the fork checkout - if: steps.meta.outputs.deliver == 'true' + if: steps.meta.outputs.deliver == 'true' && steps.verify.outcome == 'success' env: DL: prebuilds-in CHECKOUT: pr-checkout @@ -238,12 +223,12 @@ jobs: - name: Commit and push to the fork branch id: push - if: steps.meta.outputs.deliver == 'true' + if: steps.meta.outputs.deliver == 'true' && steps.verify.outcome == 'success' working-directory: pr-checkout env: - HEAD_REF: ${{ steps.meta.outputs.head_ref }} - HEAD_REPO: ${{ steps.meta.outputs.head_repo }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + HEAD_REF: ${{ steps.verify.outputs.head_ref }} + HEAD_REPO: ${{ steps.verify.outputs.head_repo }} + HEAD_SHA: ${{ steps.verify.outputs.head_sha }} # Push auth only — supplied via env, never interpolated into the command. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash @@ -303,11 +288,11 @@ jobs: fi - name: Comment delivery outcome - if: always() && steps.meta.outputs.deliver == 'true' && steps.push.outcome != 'skipped' + if: always() && steps.meta.outputs.deliver == 'true' && steps.verify.outcome == 'success' && steps.push.outcome != 'skipped' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - PR: ${{ steps.meta.outputs.pr_number }} + PR: ${{ steps.verify.outputs.pr_number }} RESULT: ${{ steps.push.outputs.result }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} shell: bash diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dac6a7398..0334531c1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -138,7 +138,7 @@ jobs: # Required for multi-platform (linux/arm64) emulation. - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 diff --git a/.github/workflows/pr-autofix-publish.yml b/.github/workflows/pr-autofix-publish.yml index 08ad1d60f..0fffad2ff 100644 --- a/.github/workflows/pr-autofix-publish.yml +++ b/.github/workflows/pr-autofix-publish.yml @@ -121,64 +121,35 @@ jobs: # metadata.json to reference another PR or SHA, redirecting our # write-scoped sticky/check-run onto an attacker-chosen target. # - # Authority sources are all server-controlled GitHub event fields: - # - workflow_run.head_sha - # - workflow_run.head_repository.full_name - # - workflow_run.pull_requests[].number (within-repo PRs only; - # empty array on fork PRs — fall back to commits/{sha}/pulls) + # Authority is workflow_run.head_sha + head_repository.full_name + + # head_branch, resolved via pulls?head={owner}:{branch}. That query + # works for same-repo PRs and forks; commits/{sha}/pulls is empty + # for fork SHAs. The script comes from THIS default-branch checkout + # (the same trust anchor as this workflow file). # + # Always verify — including the changed_lines=0 path — so the + # check-run SHA cannot be an unverified artifact field. # Mismatch => fail loud BEFORE any sticky/check-run side effect. + - name: Checkout identity verifier + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: .github/scripts/verify-workflow-run-pr-identity.cjs + sparse-checkout-cone-mode: false + path: trusted + - name: Verify metadata against workflow_run authority id: verify - if: steps.meta.outputs.changed_lines != '0' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - META_PR_NUMBER: ${{ steps.meta.outputs.pr_number }} - META_HEAD_SHA: ${{ steps.meta.outputs.head_sha }} - META_HEAD_REPO: ${{ steps.meta.outputs.head_repo }} + META_PATH: autofix-in/metadata.json + SCHEMA_PATTERN: '^gitnexus\.pr-autofix/v[0-9]+$' WF_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} WF_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} - WF_PR_NUMBERS: ${{ toJSON(github.event.workflow_run.pull_requests.*.number) }} + WF_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} shell: bash - run: | - set -euo pipefail - - # 1) head_sha must match exactly. workflow_run.head_sha is the - # commit GitHub actually ran the producer against — definitive. - if [ "${META_HEAD_SHA}" != "${WF_HEAD_SHA}" ]; then - echo "::error::Artifact head_sha (${META_HEAD_SHA}) does not match workflow_run.head_sha (${WF_HEAD_SHA}) — refusing to publish." - exit 1 - fi - - # 2) head_repo must match exactly. Same authority anchor. - if [ "${META_HEAD_REPO}" != "${WF_HEAD_REPO}" ]; then - echo "::error::Artifact head_repo (${META_HEAD_REPO}) does not match workflow_run.head_repository (${WF_HEAD_REPO}) — refusing to publish." - exit 1 - fi - - # 3) pr_number must reference an open PR with this head SHA. - # Within-repo PRs: workflow_run.pull_requests[] is populated. - # Fork PRs: that array is empty by GitHub design — fall back - # to the REST commit-to-PRs lookup. Fail closed if the lookup - # finds no matching open PR (avoids attacker-forged PR ids). - allowed_numbers=$(jq -c '.' <<< "${WF_PR_NUMBERS}") - if [ "${allowed_numbers}" = "[]" ]; then - echo "workflow_run.pull_requests is empty (fork PR) — falling back to commits/{sha}/pulls." - allowed_numbers=$(gh api "repos/${GH_REPO}/commits/${WF_HEAD_SHA}/pulls" \ - --jq '[.[] | select(.state == "open") | .number]' 2>/dev/null || echo "[]") - if [ "${allowed_numbers}" = "[]" ]; then - echo "::error::No open PR found for head ${WF_HEAD_SHA} via commits/{sha}/pulls — refusing to publish." - exit 1 - fi - fi - - if ! jq -e --argjson n "${META_PR_NUMBER}" 'index($n) != null' <<< "${allowed_numbers}" >/dev/null; then - echo "::error::Artifact pr_number (${META_PR_NUMBER}) is not in the authoritative PR list (${allowed_numbers}) — refusing to publish." - exit 1 - fi - - echo "Verified: metadata identity matches workflow_run authority (PR=${META_PR_NUMBER}, head_sha=${META_HEAD_SHA}, head_repo=${META_HEAD_REPO})." + run: node trusted/.github/scripts/verify-workflow-run-pr-identity.cjs - name: Upsert sticky summary comment # Only post when ci-quality found something fixable (= the @@ -187,14 +158,14 @@ jobs: # so we skip it. if: >- always() - && steps.meta.outputs.pr_number != '' + && steps.verify.outcome == 'success' && steps.meta.outputs.changed_lines != '0' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - PR: ${{ steps.meta.outputs.pr_number }} + PR: ${{ steps.verify.outputs.pr_number }} CHANGED: ${{ steps.meta.outputs.changed_lines }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + HEAD_SHA: ${{ steps.verify.outputs.head_sha }} RUN_ID: ${{ github.run_id }} shell: bash run: | @@ -285,11 +256,11 @@ jobs: # fixes-available → conclusion: neutral # `neutral` does not block branch-protection required-checks but # is visually distinct from a green pass. - if: always() && steps.meta.outputs.head_sha != '' + if: always() && steps.verify.outcome == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + HEAD_SHA: ${{ steps.verify.outputs.head_sha }} CHANGED: ${{ steps.meta.outputs.changed_lines }} shell: bash run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c03a801c1..ca81460f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -831,7 +831,7 @@ jobs: fi - name: Create GitHub Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v2 with: tag_name: ${{ steps.vtag-gate.outputs.vtag }} name: >- diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml index 13b79e093..88ee0bd42 100644 --- a/.github/workflows/tree-sitter-upgrade-readiness.yml +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -4,7 +4,7 @@ name: Tree-sitter Upgrade Readiness # 1. Peer-dep compatibility — can each NPM-installed grammar install cleanly # with tree-sitter@0.25.0 without --legacy-peer-deps? # 2. Vendored grammars — each grammar in .github/vendored-grammars.json -# (c/swift/kotlin/dart/proto) is classified by its vendored ABI, read +# (c/swift/kotlin/dart/proto/objc) is classified by its vendored ABI, read # straight from gitnexus/vendor//src/parser.c (NOT node_modules, # which is never populated for vendored grammars — that mismatch is why # the report used to render bare "?" placeholders, #858). diff --git a/.github/zizmor.yml b/.github/zizmor.yml index b18a76336..7426e31ae 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -18,9 +18,11 @@ rules: # untrusted half (pr-autofix.yml) runs fork code with permissions:{} # and produces only a diff artifact (data, not executable code). The # publish job consumes the artifact, allowlist-validates every field - # of metadata.json before exporting to $GITHUB_OUTPUT, never checks - # out fork code, and never executes anything fork-controlled. Header - # comment in the file documents the split. + # of metadata.json, then cross-checks identity against + # workflow_run.head_sha / head_repository / head_branch via + # pulls?head=owner:branch (commits/{sha}/pulls is empty for fork SHAs). + # It never checks out fork code and never executes anything + # fork-controlled. Header comment in the file documents the split. - pr-autofix-publish.yml # workflow_run is the trusted half of the vendored-grammar prebuild @@ -29,7 +31,8 @@ rules: # validates the .node prebuilds and uploads them as artifacts. This # consumer downloads ONLY those artifacts + metadata.json, # allowlist-validates every metadata field, cross-checks identity against - # the workflow_run authority (head_sha / head_repo / pr_number), and + # workflow_run.head_sha / head_repository / head_branch via + # pulls?head=owner:branch (commits/{sha}/pulls is empty for fork SHAs), and # checks out the fork head pinned to that HEAD SHA solely to ADD prebuild # files (never executes fork code) before pushing. Header comment in the # file documents the split. diff --git a/.gitignore b/.gitignore index 3d125d475..1f350e059 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,8 @@ eval/.hypothesis/ # Local docs — planning output (gitnexus-plan / gitnexus-work) stays local, not tracked docs/* +!docs/fork/ +!docs/fork/** gitnexus/test/fixtures/mini-repo/*.md gitnexus/test/fixtures/mini-repo/.claude diff --git a/AGENTS.md b/AGENTS.md index fffdd6242..9def73122 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ - - + + -Last reviewed: 2026-07-16 +Last reviewed: 2026-09-07 **Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub) @@ -39,6 +39,7 @@ Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING. ## Reference docs - **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)** +- **Objective-C provider work:** read **[docs/languages/objective-c-provider.md](docs/languages/objective-c-provider.md)** before changing Objective-C parsing or resolution. - **Call & inheritance resolution (RFC #909 Ring 3):** See ARCHITECTURE.md § Scope-Resolution Pipeline. All languages resolve calls and inheritance through the scope-resolution pipeline (`Registry.lookup`, `preEmitInheritanceEdges`, `emitHeritageEdges`, `buildMro` → `MethodDispatchIndex`). **Shared code in `gitnexus/src/core/ingestion/` must not name languages** — plug language behavior in via `LanguageProvider` / `ScopeResolver` hooks. A language plugs in by implementing `ScopeResolver` (`scope-resolution/contract/scope-resolver.ts`) and registering it in `SCOPE_RESOLVERS`. (The legacy call-resolution DAG + `@heritage` capture path were removed in RING4-1 #942.) - **Cursor:** `.cursor/index.mdc` (always-on); `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` deprecated. - **GitNexus:** standard skills in `.claude/skills/gitnexus-*/`; MCP rules in `gitnexus:start` block below. @@ -90,6 +91,7 @@ mirror. `gitnexus/test/unit/shipped-skills-sync.test.ts` guards the copies. Toke | Date | Version | Change | |------|---------|--------| +| 2026-09-07 | 1.15.0 | Added the Objective-C provider guide as the required reference before changing Objective-C parsing or resolution. | | 2026-07-20 | 1.14.0 | `gitnexus-review` gains a coordinated swarm: six `ci-personas/` lanes the CI review agent dispatches as subagents (via the `Agent` tool), with a bounded critic gate and sidechain-excluded evidence. | | 2026-07-16 | 1.13.0 | `gitnexus-plan` asks plan depth up front (quick/standard/deep) in interactive runs; `gitnexus-lfg` gate slimmed to proceed/stop (Deepen stays as the route-back mechanism). | | 2026-07-16 | 1.12.0 | Renamed `gitnexus-pr-review` to `gitnexus-review`; added PR URL/number, branch/range, and local-change targets plus install migration (setup warns on a legacy `gitnexus-pr-review` dir and leaves it in place; uninstall removes it). | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b1076d1d5..a1d2589bb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -377,7 +377,7 @@ CI auto-discovers the set via `tsx`. No workflow edit required. ## Language-agnostic graph feeding -16 languages → single unified graph. Four abstraction layers: +18 languages → single unified graph. Four abstraction layers: ``` Unified Graph Schema (44 node types, 21 relationship types) @@ -405,7 +405,7 @@ Each language implements `LanguageProvider` (`language-provider.ts`). Key fields | `descriptionExtractor` | Optional hook returning a symbol's doc-comment text as its `description`; feeds the embedding metadata header so doc-only terms are semantically searchable (issue #2270). Most languages register `createLeadingDocDescriptionExtractor` (shared, language-neutral; per-language comment/wrapper config passed at the call site) | | `definitionPropertiesExtractor` | Optional language-owned hook for structured, clone-safe definition metadata. Shared ingestion persists these properties opaquely; the owning provider supplies the extraction semantics. | -16 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. +18 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. ### Unified capture tags @@ -542,4 +542,5 @@ Node IDs use arity suffix (`#`): `Method:file:Class.method#1` vs `#2 - [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery - [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents - [TESTING.md](TESTING.md) — how to run tests +- [docs/languages/objective-c-provider.md](docs/languages/objective-c-provider.md) — Objective-C provider behavior and limits - `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage diff --git a/README.md b/README.md index 26a03588e..829f007a4 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # GitNexus (Akon Labs) -**⚠️ Important Notice:** GitNexus has NO official cryptocurrency, token, or coin. Any token/coin using the GitNexus name on Pump.fun or any other platform is **not affiliated with, endorsed by, or created by** this project or its maintainers. Do not purchase any cryptocurrency claiming association with GitNexus. -
@@ -585,6 +583,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | +| `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS` | unset | When `1`, per-attempt HTTP embedding timeouts (`TimeoutError` on fetch or body read) go through the bounded `GITNEXUS_EMBEDDING_MAX_ATTEMPTS` retry loop instead of failing the job. Default stays off so cloud/default timeouts remain terminal. | Local accelerators that drop a device lock when the client disconnects and succeed on the next request (observed with FastFlowLM on Ryzen AI). | | `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. | | `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. | | `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | @@ -661,6 +660,7 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas | Swift | — | — | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | C | — | — | ✓ | — | ✓ | ✓ | — | ✓ | ✓ | | C++ | — | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | +| Objective-C | ✓ | — | ✓ | ✓ | ✓ | — | — | — | — | | Dart | ✓ | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | | Zig | ✓ | — | ✓ | — | ✓ | ✓ | ✓ | — | ✓ | diff --git a/docs/languages/objective-c-provider.md b/docs/languages/objective-c-provider.md new file mode 100644 index 000000000..7051832c5 --- /dev/null +++ b/docs/languages/objective-c-provider.md @@ -0,0 +1,109 @@ +# Objective-C Language Provider + +Status: implemented + +The deterministic provider is covered by focused unit and integration tests. The parser-loader ABI +smoke runs in the published multi-OS test matrix, and the native prebuild workflow owns +Objective-C together with all six vendored grammar targets. This status describes the implemented +MVP; it does not promise full Objective-C runtime dispatch. + +## Goal + +Add deterministic, symbol-level Objective-C analysis to GitNexus. The first release must support high-confidence code navigation and direct static dependency analysis for `.m`, `.mm`, and Objective-C `.h` files. It must not imply that Objective-C runtime dispatch is fully resolved. + +The provider belongs in the existing language-provider and scope-resolution extension points. Shared ingestion code must remain language-agnostic. + +## Compatibility contract + +- Existing language detection and parsing must remain unchanged. +- A `.h` file must be classified from its content or surrounding context; it cannot be unconditionally claimed by Objective-C because C and C++ also use that extension. +- If Objective-C grammar loading fails, the error must clearly name the missing provider/grammar and cannot corrupt a previously valid index. +- Provider and grammar versions must be stored in index metadata. A version change that can alter node identity or edges requires a full rebuild. +- No LLM participates in parsing, name resolution, or edge creation. Analysis is Tree-sitter plus deterministic static resolution. + +## MVP model + +The provider must extract and connect: + +- Classes, superclasses, protocols, categories, class extensions, properties, ivars, C functions, imports, declarations, and implementations. +- Instance and class methods, preserving their complete multi-part selector. +- Inheritance, protocol conformance, import, declaration/implementation, host-class/category, and statically resolved call relationships. + +Stable identity must include enough ownership to distinguish same-named methods. Recommended forms are: + +```text +objc:class: +objc:protocol: +objc:category:: +objc:method::-: +objc:method::+: +objc:function: +``` + +For example, `-loadData:completion:` and `+loadData:completion:` are different symbols. A category method remains linked to both its category and host class; querying the host class must expose distributed implementations. + +## Resolution policy + +Resolution must be conservative. A missing or dynamic target is evidence of uncertainty, not proof that no target exists. + +| Receiver case | Required result | +| --------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| Explicit class name, `self`, or `super` | Resolve when the owner is statically known. | +| Local, parameter, property, or ivar with known static type | Resolve to matching owner and selector. | +| Protocol-typed receiver | Link the protocol method and identify possible implementations as candidates. | +| Multiple host-class/category implementations of one selector | Record all static candidates as evidence; do not emit a certain call edge because runtime image-load order is unknown. | +| `id`, `Class`, macros, reflection, `performSelector:`, `NSInvocation`, runtime injection, or unknown type | Store selector/location with `resolution=unresolved`; do not emit a certain call edge. | + +The provider should first collect file-local declarations, imports, and types, then resolve across the repository. It must use structured Tree-sitter captures or AST traversal, not regular expressions over source text. Multi-part selectors, block arguments, nullability annotations, generics, macros, and multiline declarations make a regex-only extractor unsafe. + +## Imports and incremental correctness + +- Resolve quoted project imports against the current directory, configured include roots, and indexed headers. Model framework imports as external-module evidence without downloading SDK source. +- Merge `@interface`, `@implementation`, categories, and extensions across files. +- A changed header, protocol, class declaration, or category invalidates importing and affected implementation/call-resolution state. Incremental output after such a change must match a full rebuild. +- Index metadata must record provider version, grammar version, include/exclude configuration, and parsing options used for resolution. + +## Implementation sequence + +1. Add and package a pinned Objective-C Tree-sitter grammar; verify macOS arm64 and the production Linux runner can load it. +2. Add language detection for `.m`, `.mm`, and content-classified `.h` files. +3. Implement AST extraction and stable IDs for declarations and definitions. +4. Implement repository-level merge, imports, inheritance, protocol, and category relationships. +5. Add conservative message-send resolution and explicit unresolved evidence. +6. Integrate invalidation, metadata comparison, MCP/CLI output, and fixtures. + +## Fixtures and acceptance + +Create a minimal Objective-C fixture containing a class, protocol, category, extension, superclass, properties, ivars, C function, imports, multi-part selector, block parameter, `self`, `super`, protocol receiver, and `id` receiver. Use `symodulebridge` as a real integration fixture after the minimal suite is stable. + +The acceptance bar is: + +- `query "SYModuleCaller"` yields class/method semantic nodes, not only file nodes. +- `context "SYModuleCaller" --file ` yields declaration, implementation, imports, and known references. +- Known statically typed message sends create call edges; dynamic sends are marked unresolved. +- Same selector on multiple classes, a category override, and `+` versus `-` methods remain distinct. +- A `.m`, `.h`, protocol, or category edit produces results equivalent to a clean rebuild. +- Generated documentation, dependency directories, and build output are excluded through explicit indexing configuration. + +## Non-goals + +The MVP does not promise exact runtime type inference for `id` or `instancetype`, reflection, swizzling, arbitrary category replacement, dynamic selector construction, or complete impact analysis across every runtime dispatch path. Tool results must surface confidence and unresolved evidence rather than presenting guesses as certain graph facts. + +## Current implementation coverage + +Implemented capabilities: + +- Vendored `tree-sitter-objc` grammar, registered through the existing Tree-sitter loader. +- `.m` and `.mm` language mapping plus content-based `.h` classification so plain C/C++ headers are not unconditionally claimed. +- LanguageProvider extraction for classes, protocols, categories, extensions, methods, properties, ivars, C functions, imports, unresolved message evidence, stable Objective-C qualified names, and provider/grammar metadata. +- Length-preserving preprocessing of bare, file-scope all-caps macro markers before Tree-sitter parsing. This recovers declarations after wrappers such as `RCT_EXTERN_C_BEGIN` / `RCT_EXTERN_C_END` without expanding macros or adding framework-specific rules. +- ScopeResolver edges for imports, inheritance, protocol conformance, category host membership, implementation evidence, and conservative static message sends. +- Persisted query/context support for Objective-C class and method nodes, including implementation evidence via `DECLARES`. +- Regression tests for grammar loading, `.h` classification, stable identities, conservative calls, metadata feature mismatch, persisted query/context behavior, and incremental-vs-force parity for Objective-C fixture edits. + +Known limits of this MVP: + +- The first version does not perform full Objective-C runtime dispatch, swizzling, dynamic selector construction, macro expansion, or `id` flow inference. Bare file-scope marker macros are elided only to preserve parser recovery; their expansion semantics are not interpreted. +- Protocol receiver handling records the protocol method and candidate implementation evidence, but candidate implementations are not emitted as certain call edges. +- When a host class and one or more named categories define the same selector, the provider records candidate evidence rather than choosing a runtime winner or emitting multiple certain call edges. +- Objective-C++ `.mm` files are parsed with the Objective-C grammar path for this MVP; deep C++ semantic extraction inside Objective-C++ bodies remains outside this provider. diff --git a/eval/tests/fixtures/fake_claude.py b/eval/tests/fixtures/fake_claude.py new file mode 100755 index 000000000..fbcedcdbe --- /dev/null +++ b/eval/tests/fixtures/fake_claude.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""A stand-in for the Claude Code CLI: real HTTP, real tool execution, real stream-json. + +Not a mock of the harness's own code. It does what the CLI does at the two +boundaries the harness depends on - it calls ANTHROPIC_BASE_URL for a turn, it +EXECUTES the tool blocks that come back, and it prints the stream-json event +sequence the parent parses. Only Write really executes - it is what produces the +review artifact, so the artifact path has to be genuine end to end. Skill is +MODELLED: it validates the request and returns a synthetic result, because the +parent's evidence gate keys on the request/result pair rather than on a skill +having loaded, and a fixture cannot load a real one. Bash is stubbed outright: +arbitrary shell from a scripted reply buys no fidelity for the paths this +exercises and plenty of ways to damage the host. Everything between +those boundaries (the sandbox, +the artifact capture, the scoring, the row) stays real, which is the whole +point: those are the layers that shipped bugs no unit test could see. + +Reads the prompt from stdin, as the real CLI does under "-p --input-format text". +""" + +from __future__ import annotations + +import json +import os +import pathlib +import sys +import urllib.request + + +def _turn(base_url: str, prompt: str) -> dict: + request = urllib.request.Request( + base_url.rstrip("/") + "/v1/messages", + data=json.dumps({"model": os.environ.get("ANTHROPIC_MODEL", "mock"), "max_tokens": 1024, + "messages": [{"role": "user", "content": prompt}]}).encode(), + headers={"Content-Type": "application/json", + "x-api-key": os.environ.get("ANTHROPIC_API_KEY", ""), + "anthropic-version": "2023-06-01"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def _run_tool(name: str, params: dict) -> str: + """Write executes for real - it is what produces the review artifact. + + Skill and Bash do not: see the module docstring for which is modelled and + which is stubbed, and why neither can be genuine here. + """ + + if name == "Write": + target = pathlib.Path(params["file_path"]) + target.parent.mkdir(parents=True, exist_ok=True) + # Atomic, exactly as the real Write tool does it: temp file beside the + # target, then rename. This is the operation the read-only workspace + # boundary has to permit for the artifact directory and refuse for the + # workspace, so a stand-in that wrote in place would prove nothing. + staging = target.with_name(target.name + ".tmp.fake") + staging.write_text(params.get("content", "")) + os.replace(staging, target) + return f"wrote {target}" + if name == "Skill": + # Modelled explicitly rather than falling through to a generic success. + # The parent's evidence gate keys on a Skill request with a non-error + # result, so leaving this unimplemented let an unexecuted skill satisfy + # the gate - the gate would have been measuring the fixture, not a skill. + skill = params.get("skill") or params.get("command") or params.get("name") + if not skill: + raise NotImplementedError("Skill request carried no skill name") + return f"loaded skill {skill}" + if name == "Bash": + return "(bash suppressed in the stand-in)" + # An unsupported tool is a FAILED tool run, not a quiet success. Returning a + # plain string here made the parent's evidence gate read an unexecuted Skill + # request as a successful invocation. + raise NotImplementedError(f"unsupported tool {name}") + + +def main() -> int: + # stdin, because that is where the real CLI takes it under + # "-p --input-format text": the parent pipes prompt bytes in. Scanning argv + # for a non-flag token picks up a flag's VALUE instead ("text"), which is + # exactly what the prompt-fidelity test caught. + prompt = sys.stdin.read() + base_url = os.environ.get("ANTHROPIC_BASE_URL") + if not base_url: + print(json.dumps({"type": "result", "subtype": "error", "is_error": True, + "session_id": "fake-session", "num_turns": 0}), flush=True) + return 1 + + emit = lambda event: print(json.dumps(event), flush=True) # noqa: E731 + emit({"type": "system", "subtype": "init", "session_id": "fake-session"}) + + try: + message = _turn(base_url, prompt) + except (OSError, ValueError) as exc: + # A provider failure is a failed SESSION, not a crashed process: dying + # here leaves no terminal result event, so the parent reports a generic + # stream error instead of the upstream failure it actually saw. + emit({"type": "result", "subtype": "error", "is_error": True, + "session_id": "fake-session", "num_turns": 0, + "error": f"provider request failed: {type(exc).__name__}: {exc}"}) + return 1 + blocks = message.get("content", []) + emit({"type": "assistant", "message": {"role": "assistant", "content": blocks}}) + + tool_results = [] + for block in blocks: + if block.get("type") == "tool_use": + # A refused write is a tool ERROR the session reports and carries + # on from, not a crash. Letting it kill the process would lose the + # result event and misreport a working boundary as a broken run. + failed = False + try: + output = _run_tool(block["name"], block.get("input", {})) + except (OSError, NotImplementedError) as exc: + output, failed = f"error: {type(exc).__name__}: {exc}", True + # is_error is load-bearing: the parent treats an ABSENT is_error as + # success, so a refused or unsupported tool would otherwise be + # scored as a completed one. + tool_results.append({ + "type": "tool_result", "tool_use_id": block["id"], + "content": output, "is_error": failed, + }) + if tool_results: + emit({"type": "user", "message": {"role": "user", "content": tool_results}}) + + # Unknown is not zero. A reply carrying no usage used to become four + # zero-valued fields plus a fabricated cost, which the harness then treats + # as a real measurement - the exact confusion the accounting this fixture + # feeds exists to prevent. + usage = message.get("usage") + # Every field that gets forwarded is validated, not just the required two. + # The parent's well_formed check tests only that the four keys are PRESENT, + # so an unvalidated cache value rides into a success result and is recorded + # as a real measurement. A field good enough to report is good enough to + # check. + countable = lambda v: isinstance(v, int) and not isinstance(v, bool) and v >= 0 # noqa: E731 + if not isinstance(usage, dict) or not all( + countable(usage.get(f)) for f in ("input_tokens", "output_tokens") + ) or not all( + countable(usage[f]) + for f in ("cache_read_input_tokens", "cache_creation_input_tokens") + if f in usage + ): + emit({"type": "result", "subtype": "error", "is_error": True, + "session_id": "fake-session", "num_turns": 1, + "error": "provider reply carried no usable usage; refusing to report a measured run"}) + return 1 + emit({ + "type": "result", + "subtype": "success", + "is_error": False, + "session_id": "fake-session", + "num_turns": 1, + "duration_ms": 1200, + # A measured zero is not the same as unmeasured; the parent rejects a + # collapsed cost, so report a real one. + "total_cost_usd": 0.42, + # Forward exactly the fields the provider reported. Defaulting the + # absent ones to 0 fabricated a complete measurement out of an + # incomplete reply - and worse, it made the parent's own completeness + # check (runner_sessions.USAGE_FIELDS / well_formed) unfirable from any + # offline test, because the stand-in always satisfied it. + "usage": { + field: usage[field] + for field in ("input_tokens", "output_tokens", + "cache_read_input_tokens", "cache_creation_input_tokens") + if field in usage + }, + }) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/tests/test_mock_provider.py b/eval/tests/test_mock_provider.py new file mode 100644 index 000000000..201845fbf --- /dev/null +++ b/eval/tests/test_mock_provider.py @@ -0,0 +1,280 @@ +"""The mock has to be right about the wire, or every test built on it lies.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import urllib.request +from pathlib import Path + +import pytest + +from workflow_bench.mock_provider import MockProvider, Reply +from workflow_bench.provider_usage import ( + ANTHROPIC, + LITELLM_NORMALIZED, + OPENAI_RESPONSES, + normalize_usage, +) + + +def _post(url: str, payload: dict) -> tuple[int, bytes]: + request = urllib.request.Request( + url, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, response.read() + + +def test_anthropic_messages_returns_a_usable_message() -> None: + with MockProvider([Reply(text="reviewed")]) as provider: + _status, raw = _post(provider.base_url + "/v1/messages", {"model": "m", "messages": []}) + body = json.loads(raw) + assert body["role"] == "assistant" + assert body["content"][0]["text"] == "reviewed" + assert body["stop_reason"] == "end_turn" + + +def test_a_scripted_tool_call_is_carried_as_a_tool_use_block() -> None: + """Tool blocks are how a mocked run produces real artifacts. + + The CLI executes what it is asked to run, so a Write block makes it write + that file for real inside the sandbox - which is how an artifact-producing + cell can be exercised with no model involved. + """ + + write = {"name": "Write", "input": {"file_path": "/review-output/review-output.json", "content": "{}"}} + with MockProvider([Reply(text="writing", tools=[write])]) as provider: + _status, raw = _post(provider.base_url + "/v1/messages", {"model": "m", "messages": []}) + body = json.loads(raw) + block = body["content"][1] + assert block["type"] == "tool_use" and block["name"] == "Write" + assert block["input"]["file_path"] == "/review-output/review-output.json" + assert body["stop_reason"] == "tool_use", "a turn ending in a tool call must say so" + + +def test_streaming_emits_the_event_sequence_a_consumer_expects() -> None: + with MockProvider([Reply(text="hi")]) as provider: + request = urllib.request.Request( + provider.base_url + "/v1/messages", + data=json.dumps({"model": "m", "messages": [], "stream": True}).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=10) as response: + assert response.headers["Content-Type"] == "text/event-stream" + body = response.read().decode() + + events = [line[len("event: ") :] for line in body.splitlines() if line.startswith("event: ")] + assert events[0] == "message_start" + assert events[-1] == "message_stop" + assert "content_block_delta" in events + # message_delta carries the final usage, which is where output tokens land. + assert events[-2] == "message_delta" + + +def test_each_protocol_reports_usage_in_its_own_arithmetic() -> None: + """The whole point: the two providers count the same numbers differently. + + Anthropic's cache fields ADD to input_tokens; OpenAI's are SUBSETS of it. + Scripting one Reply and serving it both ways is what makes that asymmetry + testable without a paid request. + """ + + reply = Reply(input_tokens=2_000, output_tokens=300, cache_read_input_tokens=7_000, cache_creation_input_tokens=1_000) + + with MockProvider([reply, reply]) as provider: + _s, anthropic_raw = _post(provider.base_url + "/v1/messages", {"model": "m", "messages": []}) + _s, openai_raw = _post(provider.base_url + "/v1/responses", {"model": "m", "input": []}) + + anthropic = normalize_usage(ANTHROPIC, json.loads(anthropic_raw)["usage"]) + openai = normalize_usage(OPENAI_RESPONSES, json.loads(openai_raw)["usage"]) + + assert anthropic.total_input_tokens == 10_000 + assert openai.total_input_tokens == 10_000, "same billed work, stated as the whole" + assert anthropic.ordinary_input_tokens == 2_000 + assert openai.ordinary_input_tokens == 2_000, "recovered by subtraction, not addition" + assert openai.cache_read_input_tokens == 7_000 + + +def test_a_scripted_failure_is_returned_as_one() -> None: + """Billed failures are part of what the accounting must survive.""" + + with MockProvider([Reply(status_code=529, error_body={"error": {"type": "overloaded_error"}})]) as provider: + try: + _post(provider.base_url + "/v1/messages", {"model": "m", "messages": []}) + raise AssertionError("the scripted failure was not returned") + except urllib.error.HTTPError as exc: + assert exc.code == 529 + + +def test_requests_are_recorded_for_assertions() -> None: + with MockProvider() as provider: + _post(provider.base_url + "/v1/messages", {"model": "claude-sonnet-4-5", "messages": [{"role": "user"}]}) + assert len(provider.requests) == 1 + assert provider.requests[0].body["model"] == "claude-sonnet-4-5" + assert provider.requests[0].path.endswith("/v1/messages") + + +def test_an_unscripted_turn_gets_the_default_rather_than_stalling() -> None: + """A real run makes more calls than a test wants to enumerate.""" + + with MockProvider([Reply(text="first")], default=Reply(text="fallback")) as provider: + _s, one = _post(provider.base_url + "/v1/messages", {"model": "m", "messages": []}) + _s, two = _post(provider.base_url + "/v1/messages", {"model": "m", "messages": []}) + assert json.loads(one)["content"][0]["text"] == "first" + assert json.loads(two)["content"][0]["text"] == "fallback" + + +def test_a_request_through_the_real_gateway_records_native_usage(tmp_path, monkeypatch) -> None: + """The whole stack minus the model: proxy, translation, callback, log. + + This is the path that shipped three separate defects invisible to unit + tests - the usage variable never reaching the proxy subprocess, the + callback failing to import when loaded by path, and failures never + recorded. All three live between the gateway and the provider, which is + exactly the span this exercises. + """ + + + import yaml + + from workflow_bench import model_gateway + from workflow_bench.model_gateway import OpenAIGateway + from workflow_bench.provider_usage import USAGE_LOG_ENV_VAR + + if shutil.which("litellm") is None: + import pytest + + pytest.skip("litellm console script absent; the proxy cannot start here") + + usage_log = tmp_path / "provider_usage.jsonl" + monkeypatch.setenv(USAGE_LOG_ENV_VAR, str(usage_log)) + + reply = Reply(input_tokens=2_000, output_tokens=300, cache_read_input_tokens=7_000, cache_creation_input_tokens=1_000) + with MockProvider(default=reply) as provider: + original = model_gateway.write_openai_litellm_config + + def config(path, names): + original(path, names) + document = yaml.safe_load(path.read_text()) + for entry in document["model_list"]: + entry["litellm_params"]["api_base"] = f"{provider.base_url}/v1" + path.write_text(yaml.safe_dump(document)) + return path + + monkeypatch.setattr(model_gateway, "write_openai_litellm_config", config) + with OpenAIGateway( + openai_api_key="mock-key", model_names=["gpt-4.1"], work_dir=tmp_path / "gw", ready_timeout_s=60 + ) as gateway: + request = urllib.request.Request( + gateway.base_url + "/v1/messages", + data=json.dumps({"model": "gpt-4.1", "max_tokens": 32, "messages": [{"role": "user", "content": "ping"}]}).encode(), + headers={"Content-Type": "application/json", "x-api-key": gateway.auth_token, "anthropic-version": "2023-06-01"}, + ) + with urllib.request.urlopen(request, timeout=60): + pass + + assert usage_log.exists(), "the callback never wrote - the env did not reach the proxy" + events = [json.loads(line) for line in usage_log.read_text().splitlines()] + assert events, "the proxy started but recorded nothing" + event = events[-1] + native = event["native_usage"] + # LiteLLM hands a callback its OWN normalised object, not the upstream body: + # an OpenAI Responses reply arrives as prompt_tokens / prompt_tokens_details. + # Asserting the wire shape here is what proved the shipped adapter read keys + # that are never present. + assert native["prompt_tokens_details"]["cached_tokens"] == 7_000 + assert native["prompt_tokens_details"]["cache_write_tokens"] == 1_000 + assert event["provider"] == LITELLM_NORMALIZED + assert event["call_type"] == "anthropic_messages", "the observed call type, not a Responses one" + + usage = normalize_usage(event["provider"], native) + assert usage.total_input_tokens == 10_000 + assert usage.cache_read_input_tokens == 7_000 + assert usage.cache_write_input_tokens == 1_000 + assert usage.ordinary_input_tokens == 2_000 + assert usage.complete, "a run that cannot interpret its own usage measured nothing" + + +def test_probe_what_identity_the_real_cli_actually_sends(tmp_path: Path) -> None: + """An experiment, not an assertion: which fields could correlate a request to a cell? + + Per-cell usage attribution is unbuilt because one proxy serves the whole + sweep, so anything read from the proxy environment is identical for every + request. Attribution needs something that travels WITH the request, and + what the Claude Code CLI actually sends is not documented anywhere I can + check - guessing it is how the last three accounting bugs happened. + + So this drives the REAL pinned CLI against the mock and prints the + identity-bearing fields that arrive. It asserts only that a request was + made; the value is the recorded evidence, which the job log preserves. + """ + + claude = os.environ.get("CLAUDE_CANARY_BIN") + if not claude or not Path(claude).exists(): + pytest.skip("no pinned Claude CLI here; the containment job supplies CLAUDE_CANARY_BIN") + + with MockProvider(default=Reply(text="ok")) as provider: + subprocess.run( + [claude, "-p", "--input-format", "text", "--output-format", "stream-json", "--verbose"], + input=b"say ok", + capture_output=True, + timeout=120, + env={ + **os.environ, + "ANTHROPIC_BASE_URL": provider.base_url, + "ANTHROPIC_API_KEY": "offline-probe", + "HOME": str(tmp_path), + }, + ) + + assert provider.requests, "the real CLI never reached the mock provider" + request = provider.requests[0] + interesting = { + "header:" + name: value + for name, value in request.headers.items() + if any(k in name.lower() for k in ("session", "user", "trace", "request-id", "conversation", "metadata")) + } + interesting.update( + {f"body:{key}": request.body[key] for key in ("metadata", "user", "session_id") if key in request.body} + ) + print("\nIDENTITY FIELDS THE REAL CLI SENDS:") + print(" body keys:", sorted(request.body)) + print(" candidate correlators:", interesting or "NONE — per-cell attribution needs another mechanism") + + +def test_scripted_tools_survive_the_responses_protocol_too() -> None: + """The gateway uses Responses BECAUSE it carries tool use. + + Emitting only output_text there meant a scripted Write or Skill crossed the + gateway with the tool dropped, so a mock claiming to serve both protocols + was wrong about the one the gateway actually runs. + """ + + write = {"name": "Write", "input": {"file_path": "/review-output/review-output.json", "content": "{}"}} + with MockProvider([Reply(text="writing", tools=[write])]) as provider: + _status, raw = _post(provider.base_url + "/v1/responses", {"model": "m", "input": []}) + + output = json.loads(raw)["output"] + calls = [item for item in output if item["type"] == "function_call"] + assert len(calls) == 1, "the scripted tool must cross the Responses path" + assert calls[0]["name"] == "Write" + assert json.loads(calls[0]["arguments"])["file_path"] == "/review-output/review-output.json" + + +def test_an_omitted_cache_field_stays_omitted_on_the_responses_wire_too() -> None: + """Absence must survive both protocols, not just the Anthropic one. + + `_int_or_none` reads an absent detail key as unknown and a present 0 as a + measured zero, so serializing 0 for a scripted None would claim a + measurement the reply never made. + """ + + with MockProvider([Reply(input_tokens=2_000, cache_read_input_tokens=None)]) as provider: + _status, raw = _post(provider.base_url + "/v1/responses", {"model": "m", "input": []}) + + details = json.loads(raw)["usage"]["input_tokens_details"] + assert "cached_tokens" not in details, "an omitted field must not serialize as a measured zero" + assert details["cache_write_tokens"] == 0, "a scripted 0 is still a real measurement" diff --git a/eval/tests/test_offline_session_integration.py b/eval/tests/test_offline_session_integration.py new file mode 100644 index 000000000..cb5d1fe8f --- /dev/null +++ b/eval/tests/test_offline_session_integration.py @@ -0,0 +1,195 @@ +"""A session end to end with only the model faked. + +The layers between the CLI and the row are where this harness has actually +shipped bugs - the artifact that could not be written, the usage that was never +recorded, the evidence that was scored from the wrong directory. Every one of +them sat below the level its tests exercised. These run the real session path +against a scripted provider, so the only thing not real is what the model says. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from workflow_bench.mock_provider import MockProvider, Reply +from workflow_bench.proposer_sandbox import ( + host_workspace_write_boundary, + prepare_review_workspace, + prepare_sandbox, +) +from workflow_bench.review_scoring import REVIEW_OUTPUT, parse_review_output +from workflow_bench.runner_sessions import run_claude + +FAKE_CLI = Path(__file__).parent / "fixtures" / "fake_claude.py" +REVIEW_JSON = '{"schema_version": 1, "verdict": "approve", "findings": []}' + + +def _session(clone: Path, provider: MockProvider, **overrides): + return run_claude( + "review the change", + clone, + claude_bin=str(FAKE_CLI), + timeout=60, + env={ + "ANTHROPIC_BASE_URL": provider.base_url, + "ANTHROPIC_API_KEY": "offline", + "PATH": "/usr/bin:/bin", + }, + **overrides, + ) + + +@pytest.fixture +def clone(tmp_path: Path) -> Path: + workspace = tmp_path / "clone" + workspace.mkdir() + (workspace / "source.ts").write_text("export const answer = 42;\n") + return workspace + + +def test_a_session_records_the_usage_the_provider_reported(clone: Path) -> None: + """Token counts must survive the CLI boundary, not be invented after it.""" + + reply = Reply(input_tokens=2_000, output_tokens=300, cache_read_input_tokens=7_000, cache_creation_input_tokens=1_000) + with MockProvider(default=reply) as provider: + record = _session(clone, provider) + + assert record["ok"] is True, record.get("error_detail") + assert record["input_tokens"] == 2_000 + assert record["cache_read_input_tokens"] == 7_000 + assert record["cache_creation_input_tokens"] == 1_000 + assert record["output_tokens"] == 300 + # A measured zero would be indistinguishable from an unmeasured one. + assert record["cost_usd"] == 0.42 + assert record["num_turns"] == 1 + + +def test_a_scripted_write_produces_a_review_artifact_the_scorer_accepts(clone: Path) -> None: + """The full artifact path: model asks, CLI writes atomically, scorer reads. + + This is the operation that shipped empty for a whole run. Nothing here + fakes the write, the directory, or the parse - only the decision to write. + """ + + with prepare_sandbox( + clone=clone, claude_bin=Path(sys.executable), backend="host-unsafe", preflight=False + ) as sandbox: + artifact = prepare_review_workspace(sandbox, REVIEW_OUTPUT) + write = {"name": "Write", "input": {"file_path": str(artifact), "content": REVIEW_JSON}} + with MockProvider(default=Reply(text="reviewing", tools=[write])) as provider: + # Take the command configuration from the sandbox the way run_arm + # does, rather than calling run_claude bare. On host-unsafe the + # prefix is [] by construction, so this pins the WIRING, not the + # isolation - a bwrap run would carry a real prefix through here. + record = _session( + clone, + provider, + command_prefix=sandbox.command_prefix_for(), + require_pid_namespace=sandbox.require_pid_namespace, + ) + assert record["ok"] is True, record.get("error_detail") + # Read inside the scope: prepare_sandbox removes the private root on exit. + verdict, findings = parse_review_output(artifact) + + assert verdict == "approve" + assert findings == () + + +def test_the_provider_saw_the_prompt_the_harness_meant_to_send(clone: Path) -> None: + """A run that measures the wrong prompt measures nothing.""" + + with MockProvider() as provider: + _session(clone, provider) + + assert provider.requests, "the session never reached the provider" + sent = provider.requests[0].body["messages"][0]["content"] + assert "review the change" in sent + + +def test_a_provider_failure_surfaces_as_a_failed_session_not_a_silent_pass(clone: Path) -> None: + """An upstream 529 must not be recorded as a usable measurement.""" + + failing = Reply(status_code=529, error_body={"error": {"type": "overloaded_error"}}) + with MockProvider(default=failing) as provider: + record = _session(clone, provider) + + assert record["ok"] is False + assert record["error_kind"] is not None + + +def test_the_write_boundary_refuses_the_workspace_and_permits_the_artifact(clone: Path, tmp_path: Path) -> None: + """The contract the empty-artifact run violated, on the backend available here. + + A review must not change the workspace, and must still be able to write its + artifact ATOMICALLY - temp file beside the target, then rename - which is + what needs a writable parent DIRECTORY rather than a writable file. Both + halves are asserted through the real session, with the real boundary + applied, and the model scripted to attempt each one. + + Scope: this is the host-unsafe boundary, which its own docstring calls + best-effort because a session that can chmod can undo it. The kernel-enforced + version is bubblewrap's --ro-bind, which needs namespaces this machine cannot + create; that half stays with the real-sandbox canary in CI. + """ + + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + target = artifacts / REVIEW_OUTPUT + protected = clone / "source.ts" + before = protected.read_text() + + write_artifact = {"name": "Write", "input": {"file_path": str(target), "content": REVIEW_JSON}} + tamper = {"name": "Write", "input": {"file_path": str(protected), "content": "tampered"}} + + # No writable= entry: the boundary only governs paths INSIDE the workspace + # (it refuses one that escapes), and the artifact directory deliberately + # lives outside it - that relocation is the fix for the empty-artifact run. + with host_workspace_write_boundary(clone): + with MockProvider(default=Reply(text="writing", tools=[write_artifact, tamper])) as provider: + record = _session(clone, provider) + + assert record["ok"] is True, record.get("error_detail") + # The artifact landed, written the way the agent's Write tool does it. + verdict, _findings = parse_review_output(target) + assert verdict == "approve" + assert not list(artifacts.glob("*.tmp.*")), "the rename landed rather than a copy" + # The workspace did not move. + assert protected.read_text() == before, "the read-only workspace was modified" + + +def test_a_reply_missing_cache_usage_is_refused_not_zero_filled(clone: Path) -> None: + """An omitted cache field must not arrive as a measured zero. + + The parent already demands all four USAGE_FIELDS before it calls a session + measured (runner_sessions.well_formed). The stand-in used to default the + absent ones to 0, which both fabricated a complete measurement AND made + that parent guard unfirable from any offline test - it was always + satisfied. Scripting the absence is what proves the guard still fires. + """ + + partial = Reply(input_tokens=2_000, output_tokens=300, cache_read_input_tokens=None) + with MockProvider(default=partial) as provider: + record = _session(clone, provider) + + assert record["ok"] is False, "an incomplete usage report is not a usable measurement" + assert record["error_kind"] == "session-error" + + +@pytest.mark.parametrize("bad", [-5, True, "1200"], ids=["negative", "boolean", "string"]) +def test_a_nonsense_cache_value_is_refused_rather_than_forwarded(clone: Path, bad: object) -> None: + """A field good enough to report is good enough to validate. + + The parent's well_formed check tests only that the four keys are PRESENT, + so an unvalidated cache value would ride into a success result and be + recorded as a real measurement. + """ + + reply = Reply(input_tokens=2_000, output_tokens=300) + object.__setattr__(reply, "cache_read_input_tokens", bad) + with MockProvider(default=reply) as provider: + record = _session(clone, provider) + + assert record["ok"] is False, f"{bad!r} must not be recorded as a measured cache value" diff --git a/eval/tests/test_offline_sweep_integration.py b/eval/tests/test_offline_sweep_integration.py new file mode 100644 index 000000000..7ffb6db25 --- /dev/null +++ b/eval/tests/test_offline_sweep_integration.py @@ -0,0 +1,348 @@ +"""A whole sweep, offline: real runner, real sessions, scripted model. + +The layers between a model turn and a promotion decision had never been +exercised together. Unit tests covered each in isolation and the paid runs that +would have covered the composition kept dying, so the contracts BETWEEN them +went unverified - and that is where this harness has repeatedly shipped bugs. + +This drives runner.main() the way the workflow does. Everything is real: task +selection, hidden-oracle capture, the sandbox, the CLI subprocess, artifact +capture, review scoring against the oracle, aggregation, the health guard, and +the promotion gate. Only the model is scripted, through MockProvider. + +Two provisioning steps are stubbed because this environment cannot supply them, +and neither is harness logic: the pinned gitnexus runtime mounts (no +node_modules in a worktree) and the sanitized graph build (needs the gitnexus +CLI at a mounted path). Containment is host-unsafe here; bubblewrap stays with +the real-sandbox canary in the containment job. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import os +import shutil + +import pytest + +from workflow_bench import oracle_assets, runner +from workflow_bench.mock_provider import MockProvider, Reply + +FAKE_CLI = Path(__file__).parent / "fixtures" / "fake_claude.py" +ARMS = ("ce_review", "review", "candidate_review") + +# When set, the sweep runs with NOTHING provisioning-stubbed: real bubblewrap +# containment, the real pinned runtime mounts, and the real sanitized graph +# build. The named CI job installs all three, so a missing one there is a +# regression rather than an unsupported machine - it FAILS instead of quietly +# degrading to the stubbed path, which is the whole point of the gate. +FULL_SWEEP_ENV = "GITNEXUS_REQUIRE_FULL_SWEEP" +FULL_SWEEP = os.environ.get(FULL_SWEEP_ENV) == "1" + +# The runner refuses --unsafe-no-bwrap whenever CI is set, because that mode runs +# sessions with bypassPermissions behind a boundary its own docstring calls "not +# a security boundary". Deleting CI to get past that refusal would run an +# uncontained agent sweep on the runner holding the checkout and credentials, so +# the stubbed path is skipped under CI instead. The containment job sets +# GITNEXUS_REQUIRE_FULL_SWEEP=1 and takes the real bubblewrap path, so CI keeps +# its coverage; only the uncontained convenience run is given up. +pytestmark = pytest.mark.skipif( + not FULL_SWEEP and bool(os.environ.get("CI")), + reason="an uncontained sweep must not run in CI; the containment job runs it with GITNEXUS_REQUIRE_FULL_SWEEP=1", +) + +# The review output and the hidden labels are DELIBERATELY different shapes - +# the labels carry line_start/line_end and no recommendation. Only a real run +# surfaces that; it is why these are written out rather than shared. +FINDING = { + "id": "f1", "severity": "high", "category": "correctness", "path": "src/sum.js", + "line": 1, "end_line": 1, "blocking": True, "scenario": "review-defect", + "evidence": "export const total = (a, b) => a - b;", "recommendation": "use a + b", +} +LABEL = {"id": "f1", "severity": "high", "category": "correctness", + "path": "src/sum.js", "line_start": 1, "line_end": 1} +SECOND_LABEL = {"id": "f2", "severity": "high", "category": "correctness", + "path": "src/scale.js", "line_start": 1, "line_end": 1} +SECOND_FINDING = { + "id": "f2", "severity": "high", "category": "correctness", "path": "src/scale.js", + "line": 1, "end_line": 1, "blocking": True, "scenario": "review-defect", + "evidence": "export const twice = (n) => n + 2;", "recommendation": "use n * 2", +} + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run(["git", "-C", str(repo), *args], check=True, + capture_output=True, text=True).stdout.strip() + + +@pytest.fixture +def bench(tmp_path: Path): + """A self-contained corpus: one repo, one task, one hidden label.""" + + repo = tmp_path / "repo" + (repo / "src").mkdir(parents=True) + (repo / "src" / "sum.js").write_text("export const total = (a, b) => a - b;\n") + (repo / "src" / "scale.js").write_text("export const twice = (n) => n + 2;\n") + _git(repo, "init", "-q", ".") + _git(repo, "config", "user.email", "t@t") + _git(repo, "config", "user.name", "t") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "fixture") + sha = _git(repo, "rev-parse", "HEAD") + + oracles = tmp_path / "oracles" + oracles.mkdir() + (oracles / "review-fixture-defect.labels.json").write_text( + json.dumps({"schema_version": 1, "findings": [LABEL]}) + ) + (oracles / "review-fixture-second.labels.json").write_text( + json.dumps({"schema_version": 1, "findings": [SECOND_LABEL]}) + ) + + tasks = tmp_path / "tasks.yaml" + tasks.write_text( + "tasks:\n" + " - id: review-fixture-defect\n" + " class: review-defect\n" + f" repo: {repo}\n" + f" ref: {sha}\n" + " prompt: Review this change and report actionable defects.\n" + ' verify: test -s "$GITNEXUS_BENCH_REVIEW_OUTPUT"\n' + " oracle:\n" + ' command: test -s "$GITNEXUS_BENCH_REVIEW_OUTPUT"\n' + " files: [{ source: review-fixture-defect.labels.json, target: review-labels.json }]\n" + # A SECOND task, because the thing a cross-task scheduler changes is + # invisible with one: waves are per-task, so a single task cannot show + # ordering, packing, or a breaker that spans a task boundary. + " - id: review-fixture-second\n" + " class: review-defect\n" + f" repo: {repo}\n" + f" ref: {sha}\n" + " prompt: Review the scaling helper and report actionable defects.\n" + ' verify: test -s "$GITNEXUS_BENCH_REVIEW_OUTPUT"\n' + " oracle:\n" + ' command: test -s "$GITNEXUS_BENCH_REVIEW_OUTPUT"\n' + " files: [{ source: review-fixture-second.labels.json, target: review-labels.json }]\n" + ) + + plugin = tmp_path / "ce-plugin" + (plugin / ".claude-plugin").mkdir(parents=True) + (plugin / ".claude-plugin" / "plugin.json").write_text( + json.dumps({"name": "compound-engineering", "version": "0.0.0-fixture"}) + ) + for skill in ("ce-plan", "ce-work", "ce-code-review"): + directory = plugin / "skills" / skill + directory.mkdir(parents=True) + (directory / "SKILL.md").write_text(f"---\nname: {skill}\ndescription: fixture\n---\nFixture.\n") + + overlay = tmp_path / "overlay" / ".claude" / "skills" / "gitnexus-review" + overlay.mkdir(parents=True) + (overlay / "SKILL.md").write_text("---\nname: gitnexus-review\ndescription: fixture\n---\nCandidate.\n") + + return SimpleNamespace(tasks=tasks, oracles=oracles, plugin=plugin, + overlay=tmp_path / "overlay", out=tmp_path / "out") + + +def _stub_provisioning(monkeypatch: pytest.MonkeyPatch) -> None: + """Replace what this machine cannot supply - and nothing else. + + Under FULL_SWEEP nothing is replaced: the runtime mounts and the graph are + built for real, so the sweep exercises containment and provisioning too. + """ + + if FULL_SWEEP: + if shutil.which("bwrap") is None: + pytest.fail(f"{FULL_SWEEP_ENV}=1 but bubblewrap is absent") + return + + monkeypatch.setattr(runner, "trusted_gitnexus_runtime_mounts", lambda: ()) + + def materialize(worktree, *, sanitized_head=None, **_kwargs): + # The one-clone registry guard reads this before any session runs. + meta = Path(worktree) / ".gitnexus" + meta.mkdir(parents=True, exist_ok=True) + (meta / "meta.json").write_text( + json.dumps({"indexedAt": "2026-09-08T00:00:00Z", "lastCommit": sanitized_head or "0" * 40}) + ) + + def fake_graph(**kwargs): + kwargs["env"].graph_snapshots[kwargs["graph_key"]] = SimpleNamespace( + digest="fixture-graph", manifest_digest="fixture-graph-manifest", + dependency_content_digest=None, dependency_manifest_digest=None, + materialize=materialize, + ) + + monkeypatch.setattr(runner, "ensure_task_graph", fake_graph) + + +def _sweep(bench, monkeypatch: pytest.MonkeyPatch, findings: list[dict], verdict: str, *, invoke_skill: bool = True): + """Run the real CLI against a model scripted to return `findings`.""" + + _stub_provisioning(monkeypatch) + monkeypatch.setattr( + oracle_assets, "ORACLE_ROOT", bench.oracles, raising=False + ) + monkeypatch.setattr( + runner, "capture_task_oracles", + lambda tasks, root=bench.oracles: oracle_assets.capture_task_oracles(tasks, root=root), + ) + def review_for(body: str) -> str: + # Per task: the second task's defect is in another file, so replying + # with the first task's finding would score it wrong. A cross-task + # scheduler makes which task a request belongs to load-bearing. + chosen = findings + if findings and "scaling helper" in body: + chosen = [SECOND_FINDING if f is FINDING else f for f in findings] + return json.dumps({"schema_version": 1, "verdict": verdict, "findings": chosen}) + + class Scripted(MockProvider): + def next_reply(self) -> Reply: + body = json.dumps(self.requests[-1].body if self.requests else {}) + target = re.search(r"(/[^\s\"']*review-output\.json)", body) + skill = re.search(r"\b(gitnexus-review|ce-code-review)\b", body) + return Reply( + text="reviewing", + tools=[ + # The evidence gate needs a Skill request with a non-error + # result: a review that never invoked its skill measured the + # model, not the skill. + *([{"name": "Skill", "input": {"skill": skill.group(1) if skill else "gitnexus-review"}}] + if invoke_skill else []), + {"name": "Write", "input": { + "file_path": target.group(1) if target else str(bench.out / "unmatched-review-output.json"), + "content": review_for(body)}}, + ], + input_tokens=2_000, output_tokens=300, + cache_read_input_tokens=7_000, cache_creation_input_tokens=1_000, + ) + + with Scripted() as provider: + monkeypatch.setattr(sys, "argv", [ + "runner", "--tasks", str(bench.tasks), "--arms", *ARMS, + "--runs", "1", "--workers", "1", "--out", str(bench.out), + "--base-url", provider.base_url, "--anthropic-api-key", "offline", + "--claude-bin", str(FAKE_CLI), + *([] if FULL_SWEEP else ["--unsafe-no-bwrap"]), + "--model", "mock-model", + "--ce-plugin-dir", str(bench.plugin), "--ce-plugin-version", "0.0.0-fixture", + "--candidate-overlay", str(bench.overlay), + ]) + + try: + code = runner.main() + except SystemExit as exc: + code = exc.code + + rows = [json.loads(line) for line in (bench.out / "results.jsonl").read_text().splitlines()] + return code, rows, provider + + +def _row(rows: list[dict], arm: str, task: str = "review-fixture-defect") -> dict: + return next(r for r in rows if r["arm"] == arm and r["task"] == task) + + +def test_a_correct_review_scores_and_the_sweep_exits_clean(bench, monkeypatch) -> None: + """The whole path, green: every arm measured, scored, and accounted for.""" + + code, rows, provider = _sweep(bench, monkeypatch, [FINDING], "request_changes") + + assert code in (None, 0), f"sweep did not succeed: {code}" + tasks = {"review-fixture-defect", "review-fixture-second"} + assert len(rows) == len(ARMS) * len(tasks) + assert len(provider.requests) == len(ARMS) * len(tasks), "each cell must reach the provider once" + assert {r["task"] for r in rows} == tasks, "both tasks must have run" + + row = _row(rows, "review") + assert row["ok"] is True and row["resolved"] is True + assert row["skill_invoked"] is True + assert (row["review_true_positives"], row["review_false_positives"], row["review_false_negatives"]) == (1, 0, 0) + assert row["review_f1"] == 1.0 + # The provider's own numbers survived the CLI, the parser and the row. + assert row["cache_read_input_tokens"] == 7_000 + assert row["input_tokens"] == 2_000 + + # Each task scored against ITS OWN oracle. This is what a cross-task + # scheduler puts at risk: interleaving cells from different tasks means a + # mis-routed context or artifact scores one task against another's labels, + # and both would still look "green" per row. + second = _row(rows, "review", task="review-fixture-second") + assert second["resolved"] is True and second["review_f1"] == 1.0 + assert second["review_artifact"] == "review-fixture-second-review-run0.review.json" + + for name in ("results.jsonl", "report.md", "promotion.json"): + assert (bench.out / name).is_file(), f"{name} was not written" + assert (bench.out / "review-fixture-defect-review-run0.review.json").is_file() + + +def test_one_run_cannot_promote_a_candidate(bench, monkeypatch) -> None: + """The gate refuses on insufficient paired runs, and says so.""" + + _sweep(bench, monkeypatch, [FINDING], "request_changes") + promotion = json.loads((bench.out / "promotion.json").read_text()) + + assert promotion["run_status"] == "complete" + decision = next(d for d in promotion["decisions"] if d["candidate_arm"] == "candidate_review") + assert decision["decision"] == "insufficient_evidence" + assert any("valid paired runs" in reason for reason in decision["reasons"]) + + +def test_a_finding_in_the_wrong_place_scores_zero_but_stays_valid_evidence(bench, monkeypatch) -> None: + """Being wrong is a quality result, not a broken measurement. + + The negative control that makes the passing case mean something: same + harness, same well-formed artifact, only the answer changed. + """ + + wrong = {**FINDING, "path": "src/WRONG.js", "line": 99, "end_line": 99} + _code, rows, _provider = _sweep(bench, monkeypatch, [wrong], "request_changes") + + row = _row(rows, "review") + assert (row["review_true_positives"], row["review_false_positives"], row["review_false_negatives"]) == (0, 1, 1) + assert row["review_f1"] == 0.0 + assert row["resolved"] is False + assert row["error_kind"] == "oracle-failed", "a wrong answer is not a session or evidence failure" + assert row["review_evidence_valid"] is True, "the artifact was well formed; only the answer was wrong" + + +def test_approving_defective_code_is_a_miss_with_no_false_positive(bench, monkeypatch) -> None: + """The other half of the control: silence scores differently from a wrong guess.""" + + _code, rows, _provider = _sweep(bench, monkeypatch, [], "approve") + + row = _row(rows, "review") + assert (row["review_true_positives"], row["review_false_positives"], row["review_false_negatives"]) == (0, 0, 1) + assert row["review_precision"] is None, "precision is undefined with no predictions, not zero" + assert row["review_verdict_correct"] is False, "approving defective code is the wrong verdict" + assert row["review_evidence_valid"] is True + + +def test_a_review_that_never_invoked_its_skill_is_not_a_measurement(bench, monkeypatch) -> None: + """The gate that separates measuring a SKILL from measuring a model. + + Added because a mutation exposed it: forcing skill_was_invoked_events to + return True left every other test here passing, so nothing pinned the gate. + The artifact is written and correct in this run - only the skill request is + missing - so a pass would mean the arm scored a review it never performed. + """ + + code, rows, _provider = _sweep(bench, monkeypatch, [FINDING], "request_changes", invoke_skill=False) + + row = _row(rows, "review") + assert row["skill_invoked"] is False + assert row["error_kind"] == "skill-not-invoked" + assert code not in (None, 0), "the sweep must not report success on unusable evidence" + + # The row still carries its own score - the artifact was well formed - and + # aggregate() DOES count it in the arm's quality median (the KNOWN GAP noted + # above aggregate(); test_workflow_bench pins the resulting 0.5). Filtering + # it out of the median alone inverted a promotion, because valid_runs and + # excluded_runs kept counting it. It counts for cost either way: the session + # ran and was billed. + assert row["review_weighted_f1"] == 1.0 + assert row["review_evidence_valid"] is True diff --git a/eval/tests/test_provider_usage_capture.py b/eval/tests/test_provider_usage_capture.py index 99e542e4d..91e4956e5 100644 --- a/eval/tests/test_provider_usage_capture.py +++ b/eval/tests/test_provider_usage_capture.py @@ -24,7 +24,7 @@ from workflow_bench.model_gateway import ( ) from workflow_bench.provider_usage import ( ANTHROPIC, - OPENAI_RESPONSES, + LITELLM_NORMALIZED, USAGE_ENV_VARS, normalize_usage, ) @@ -49,11 +49,16 @@ def _openai_response(usage: dict) -> SimpleNamespace: ) +# The shape a callback actually receives: LiteLLM normalises usage into its own +# Chat-Completions-style object before any logger sees it, so an OpenAI reply +# arrives as prompt_tokens / prompt_tokens_details. Confirmed against a real +# proxy in tests/test_mock_provider.py; a fixture in the wire shape would test +# an object this code path never gets. NATIVE = { - "input_tokens": 48_000, - "input_tokens_details": {"cached_tokens": 44_000, "cache_write_tokens": 1_000}, - "output_tokens": 900, - "output_tokens_details": {"reasoning_tokens": 640}, + "prompt_tokens": 48_000, + "prompt_tokens_details": {"cached_tokens": 44_000, "cache_write_tokens": 1_000}, + "completion_tokens": 900, + "completion_tokens_details": {"reasoning_tokens": 640}, } @@ -79,9 +84,9 @@ def test_native_openai_usage_survives_the_anthropic_translation(logged) -> None: event = logged(NATIVE) native = event["native_usage"] # Verbatim: the fields an Anthropic-shaped response cannot carry. - assert native["input_tokens_details"]["cached_tokens"] == 44_000 - assert native["input_tokens_details"]["cache_write_tokens"] == 1_000 - assert native["output_tokens_details"]["reasoning_tokens"] == 640 + assert native["prompt_tokens_details"]["cached_tokens"] == 44_000 + assert native["prompt_tokens_details"]["cache_write_tokens"] == 1_000 + assert native["completion_tokens_details"]["reasoning_tokens"] == 640 assert event["response_id"] == "resp_68f2c1" @@ -100,7 +105,10 @@ def test_the_captured_event_normalizes_with_openai_arithmetic(logged) -> None: event = logged(NATIVE) # The provider the LOG recorded, not one the test supplies - passing # OPENAI_RESPONSES by hand here is what hid the adapter-key mismatch. - assert event["provider"] == OPENAI_RESPONSES + # LITELLM_NORMALIZED, not OPENAI_RESPONSES: a proxy callback never sees the + # upstream body. Measured against a real gateway - the Responses adapter + # found none of its keys there and reported every field unknown. + assert event["provider"] == LITELLM_NORMALIZED assert event["provider_label"] == "openai" usage = normalize_usage(event["provider"], event["native_usage"]) assert usage.total_input_tokens == 48_000 @@ -112,7 +120,7 @@ def test_the_captured_event_normalizes_with_openai_arithmetic(logged) -> None: def test_usage_without_details_normalizes_to_unknown_rather_than_zero(logged) -> None: """The mutation the accounting must not survive: dropped details, silent zeros.""" - stripped = {k: v for k, v in NATIVE.items() if k != "input_tokens_details"} + stripped = {k: v for k, v in NATIVE.items() if k != "prompt_tokens_details"} event = logged(stripped) usage = normalize_usage(event["provider"], event["native_usage"]) assert usage.cache_read_input_tokens is None @@ -238,10 +246,16 @@ def test_an_unresolvable_provider_is_refused_rather_than_guessed() -> None: from workflow_bench.provider_usage import canonical_provider - assert canonical_provider("openai", "responses") == OPENAI_RESPONSES - assert canonical_provider("openai", "completion") is None - assert canonical_provider("openai", None) is None + # Every openai call reaching this callback has already been normalised by + # LiteLLM, whatever endpoint it used - the observed call_type for a Claude + # Code request through the gateway is "anthropic_messages". The adapter has + # to match the object in hand, not the protocol on the wire. + assert canonical_provider("openai", "responses") == LITELLM_NORMALIZED + assert canonical_provider("openai", "anthropic_messages") == LITELLM_NORMALIZED assert canonical_provider("anthropic", "completion") == ANTHROPIC + # An unrecognised provider is still refused rather than guessed. + assert canonical_provider("some-new-provider", "responses") is None + assert canonical_provider(None, None) is None def test_request_identity_cannot_come_from_the_proxy_environment() -> None: diff --git a/eval/tests/test_workflow_bench.py b/eval/tests/test_workflow_bench.py index a53b2b8cd..51212ceda 100644 --- a/eval/tests/test_workflow_bench.py +++ b/eval/tests/test_workflow_bench.py @@ -195,6 +195,11 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs(): assert containment["env"] == { "GITNEXUS_REQUIRE_BWRAP_CANARY": "1", "GITNEXUS_REQUIRE_CLAUDE_CANARY": "1", + # This job is the only place with bubblewrap, the pinned runtime and a + # built GitNexus together, so it is where the offline sweep runs with + # nothing provisioning-stubbed. Pinned here so the gate cannot be + # dropped and leave the sweep silently running the stubbed path. + "GITNEXUS_REQUIRE_FULL_SWEEP": "1", } assert containment["timeout-minutes"] == 20 assert containment_node_setup["with"] == { @@ -239,6 +244,12 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs(): "tests/test_proposer_sandbox.py", "tests/test_workflow_bench_sessions.py", "tests/test_ce_plugin_runtime.py", + # The offline sweep, run here with nothing stubbed: this job is the only + # one carrying bubblewrap, the pinned runtime and a built GitNexus. + "tests/test_offline_sweep_integration.py", + # Carries the real-CLI identity probe, which needs CLAUDE_CANARY_BIN - + # set only on this job. Omitted from this list it skipped everywhere. + "tests/test_mock_provider.py", "-q", ] bwrap_canary_marker = re.compile( @@ -1052,3 +1063,33 @@ def test_a_raising_packed_cell_still_persists_its_settled_siblings(): ) assert (1, "review") in folded, "the sibling that completed was never recorded" + + +def test_an_uninvoked_skill_still_counts_toward_the_arm_median(): + """Pins a KNOWN GAP, not a desired behaviour. + + A cell whose skill never ran still moves the arm's quality median, even + though an arm exists to measure a SKILL. The narrow fix - filtering those + rows out of the quality metrics - is worse than the gap: valid_runs and + excluded_runs keep counting them, so the promotion gate sees N clean runs + while the median came from fewer. Since the dropped rows are systematically + an arm's worst, that biases toward promoting, and it was measured flipping + keep_incumbent to promote. + + Closing it honestly needs a scored-run count and a paired-equality check in + the promotion gate. Pinned here so the half-fix cannot be reapplied without + someone reading why it was reverted. + """ + + good = record(review_weighted_f1=1.0, cost_usd=2.0) + uninvoked = record( + review_weighted_f1=0.0, cost_usd=4.0, error_kind="skill-not-invoked", skill_invoked=False + ) + agg = aggregate([good, uninvoked]) + + assert agg["review_weighted_f1"] == 0.5, "the uninvoked row is counted - the known gap" + assert agg["cost_usd"] == 3.0 + # The invariant that makes the half-fix unsafe: the median and the run count + # the gate reads must cover the same rows. + assert agg["valid_runs"] == 2 + diff --git a/eval/workflow_bench/litellm_usage_callback.py b/eval/workflow_bench/litellm_usage_callback.py index 843413765..8277d0102 100644 --- a/eval/workflow_bench/litellm_usage_callback.py +++ b/eval/workflow_bench/litellm_usage_callback.py @@ -42,8 +42,8 @@ def canonical_provider(label, call_type): # noqa: ANN001, ANN201 above for why this is a copy rather than an import. """ - if label == "openai" and call_type and "responses" in call_type: - return "openai-responses" + if label == "openai": + return "litellm-normalized" if label == "anthropic": return "anthropic" return None @@ -104,11 +104,10 @@ class ProviderUsageLogger(CustomLogger): "requested_model": kwargs.get("model"), "actual_model": getattr(response_obj, "model", None), # Two fields, because they answer different questions. The raw - # label is what LiteLLM said; "provider" is the adapter key, - # which needs the call type too - LiteLLM reports "openai" for - # both Chat Completions and Responses and those report usage - # differently. Unresolvable stays None so normalize_usage - # refuses rather than guessing token semantics. + # label is what LiteLLM said; "provider" is the adapter key for + # the object actually in hand, which is always LiteLLM's own + # normalised shape here. An unrecognised label stays None so + # normalize_usage refuses rather than guessing token semantics. "provider_label": provider_label, "provider": canonical_provider(provider_label, call_type), "response_id": getattr(response_obj, "id", None), diff --git a/eval/workflow_bench/mock_provider.py b/eval/workflow_bench/mock_provider.py new file mode 100644 index 000000000..dcc328251 --- /dev/null +++ b/eval/workflow_bench/mock_provider.py @@ -0,0 +1,284 @@ +"""A scriptable stand-in for Anthropic and OpenAI, for running the harness offline. + +Every defect this benchmark shipped in the last round was invisible to its own +tests for the same reason: the tests exercised a layer BELOW where the code +runs. The usage log was never written because the proxy is a subprocess with a +constructed environment. The callback could not be imported because LiteLLM +loads it by path. Failures went unrecorded because only the async hook was +overridden. Each was caught by CI or review, never by a unit test, because the +unit test called the function directly instead of driving the path that calls +it. + +This closes that gap without spending money. It speaks the two wire protocols +the harness actually depends on, so a run can go through the real sandbox, the +real Claude Code CLI, the real gateway and the real usage callback, and only +the model is fake: + + POST /v1/messages Anthropic Messages, streaming and non-streaming + POST /v1/responses OpenAI Responses, which the gateway translates into + +Point the runner at it with ``--base-url http://127.0.0.1:``, which is +the same supported path the free-model proxy documentation already uses, or +give it to LiteLLM as ``api_base`` to exercise the gateway. + +Scripted, not simulated: replies are supplied by the caller, so a test decides +what the model "says", which tools it asks for, and exactly what usage it +reports. That last part is what makes provider-native accounting testable at +all - real cache hits are not reproducible on demand, but a declared +``cache_read`` of 44_000 is. +""" + +from __future__ import annotations + +import json +import threading +import time +from collections import deque +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + + +@dataclass +class Reply: + """One scripted model turn. + + ``tools`` drives real tool execution: Claude Code runs what it is asked to + run, so a reply carrying a Write block makes the CLI write that file inside + the sandbox for real. That is how an artifact-producing cell can be + exercised without a model deciding anything. + """ + + text: str = "ok" + tools: list[dict[str, Any]] = field(default_factory=list) + stop_reason: str = "end_turn" + # Anthropic accounting: input_tokens is the UNCACHED remainder and the + # cache fields add to it. Defaults are deliberately non-zero so a test that + # forgets to script usage still cannot mistake silence for a measurement. + input_tokens: int = 11 + output_tokens: int = 7 + # None means the field is OMITTED from the reply, which is not the same as + # reporting 0. A consumer that cannot tell those apart is the bug this + # harness exists to catch, so the mock has to be able to script absence. + cache_read_input_tokens: int | None = 0 + cache_creation_input_tokens: int | None = 0 + status_code: int = 200 + error_body: dict[str, Any] | None = None + + +@dataclass +class Request: + """What the harness actually sent, kept so a test can assert on it.""" + + path: str + headers: dict[str, str] + body: dict[str, Any] + + +class _Handler(BaseHTTPRequestHandler): + provider: MockProvider + + def log_message(self, *_args: Any) -> None: # noqa: A003 - silence the default stderr spam + return + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's interface + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + body = json.loads(raw or b"{}") + except json.JSONDecodeError: + body = {"_unparsed": raw.decode("utf-8", "replace")} + self.provider.record(Request(self.path, dict(self.headers), body)) + reply = self.provider.next_reply() + + if reply.status_code != 200: + self._send_json(reply.status_code, reply.error_body or {"error": {"message": "scripted failure"}}) + return + if self.path.rstrip("/").endswith("/responses"): + self._send_json(200, _openai_response(reply)) + return + if body.get("stream"): + self._send_anthropic_stream(reply) + return + self._send_json(200, _anthropic_message(reply)) + + def _send_json(self, status: int, payload: dict[str, Any]) -> None: + encoded = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def _send_anthropic_stream(self, reply: Reply) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + for event, data in _anthropic_stream_events(reply): + self.wfile.write(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode()) + self.wfile.flush() + + +def _content_blocks(reply: Reply) -> list[dict[str, Any]]: + blocks: list[dict[str, Any]] = [{"type": "text", "text": reply.text}] + for index, tool in enumerate(reply.tools): + blocks.append( + { + "type": "tool_use", + "id": f"toolu_mock_{index}", + "name": tool["name"], + "input": tool.get("input", {}), + } + ) + return blocks + + +def _anthropic_usage(reply: Reply) -> dict[str, int]: + usage = { + "input_tokens": reply.input_tokens, + "output_tokens": reply.output_tokens, + "cache_read_input_tokens": reply.cache_read_input_tokens, + "cache_creation_input_tokens": reply.cache_creation_input_tokens, + } + return {field: value for field, value in usage.items() if value is not None} + + +def _anthropic_message(reply: Reply) -> dict[str, Any]: + return { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "model": "mock-model", + "content": _content_blocks(reply), + "stop_reason": "tool_use" if reply.tools else reply.stop_reason, + "stop_sequence": None, + "usage": _anthropic_usage(reply), + } + + +def _anthropic_stream_events(reply: Reply) -> list[tuple[str, dict[str, Any]]]: + """The SSE sequence a Messages consumer expects, in order.""" + + message = _anthropic_message(reply) + events: list[tuple[str, dict[str, Any]]] = [ + ("message_start", {"type": "message_start", "message": {**message, "content": [], "usage": _anthropic_usage(reply)}}) + ] + for index, block in enumerate(message["content"]): + if block["type"] == "text": + events.append(("content_block_start", {"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}})) + events.append(("content_block_delta", {"type": "content_block_delta", "index": index, "delta": {"type": "text_delta", "text": block["text"]}})) + else: + events.append(("content_block_start", {"type": "content_block_start", "index": index, "content_block": {"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}}})) + events.append(("content_block_delta", {"type": "content_block_delta", "index": index, "delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}})) + events.append(("content_block_stop", {"type": "content_block_stop", "index": index})) + events.append(("message_delta", {"type": "message_delta", "delta": {"stop_reason": message["stop_reason"], "stop_sequence": None}, "usage": {"output_tokens": reply.output_tokens}})) + events.append(("message_stop", {"type": "message_stop"})) + return events + + +def _openai_response(reply: Reply) -> dict[str, Any]: + """OpenAI Responses shape: input_tokens is the WHOLE, cache fields subsets.""" + + # An omitted cache field contributes nothing to the Responses total; that + # is arithmetic, not a claim the value was measured as zero. + cache_read = reply.cache_read_input_tokens or 0 + cache_write = reply.cache_creation_input_tokens or 0 + total_input = reply.input_tokens + cache_read + cache_write + return { + "id": "resp_mock", + "object": "response", + "created_at": int(time.time()), + "status": "completed", + "model": "mock-model", + "error": None, + "output": [ + { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": reply.text, "annotations": []}], + }, + # Tool calls belong here too. Responses is the protocol the gateway + # is configured for BECAUSE it carries tool use, so emitting only + # output_text meant a reply scripted with a Write or Skill crossed + # the gateway with the tool silently dropped - the mock would have + # been wrong about the wire on the one path that matters most. + *( + { + "id": f"fc_mock_{index}", + "type": "function_call", + "status": "completed", + "call_id": f"call_mock_{index}", + "name": tool["name"], + "arguments": json.dumps(tool.get("input", {})), + } + for index, tool in enumerate(reply.tools) + ), + ], + "usage": { + "input_tokens": total_input, + "output_tokens": reply.output_tokens, + "total_tokens": total_input + reply.output_tokens, + # Omitted stays omitted here too. Collapsing None to 0 is right for + # the total above (an unreported field adds nothing) but wrong on + # the wire: _int_or_none reads an absent key as unknown and a + # present 0 as a measured zero, so serializing 0 would claim a + # measurement the reply never made - the same confusion the + # Anthropic path already refuses. + "input_tokens_details": { + **({"cached_tokens": cache_read} if reply.cache_read_input_tokens is not None else {}), + **({"cache_write_tokens": cache_write} if reply.cache_creation_input_tokens is not None else {}), + }, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +class MockProvider: + """Loopback-only provider stand-in. Use as a context manager.""" + + def __init__(self, replies: list[Reply] | None = None, *, default: Reply | None = None) -> None: + self._replies: deque[Reply] = deque(replies or []) + # A run makes more requests than a test wants to script; the default + # keeps it going rather than failing on the first unscripted turn. + self._default = default or Reply() + self._requests: list[Request] = [] + self._lock = threading.Lock() + self._server: ThreadingHTTPServer | None = None + + def __enter__(self) -> MockProvider: + handler = type("_BoundHandler", (_Handler,), {"provider": self}) + # Loopback only: this answers with no authentication at all. + self._server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=self._server.serve_forever, daemon=True).start() + return self + + def __exit__(self, *_exc: object) -> bool: + if self._server is not None: + self._server.shutdown() + self._server.server_close() + return False + + @property + def port(self) -> int: + assert self._server is not None, "provider is not running" + return self._server.server_port + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def record(self, request: Request) -> None: + with self._lock: + self._requests.append(request) + + def next_reply(self) -> Reply: + with self._lock: + return self._replies.popleft() if self._replies else self._default + + @property + def requests(self) -> list[Request]: + with self._lock: + return list(self._requests) diff --git a/eval/workflow_bench/provider_usage.py b/eval/workflow_bench/provider_usage.py index 782eb18fa..a56582be1 100644 --- a/eval/workflow_bench/provider_usage.py +++ b/eval/workflow_bench/provider_usage.py @@ -50,6 +50,14 @@ USAGE_ENV_VARS = (USAGE_LOG_ENV_VAR, SWEEP_ID_ENV_VAR) ANTHROPIC = "anthropic" OPENAI_RESPONSES = "openai-responses" +# What the gateway's callback actually receives. LiteLLM does not hand a logger +# the upstream body: it normalises usage into its own Chat-Completions-shaped +# object first, so an OpenAI Responses reply arrives as prompt_tokens / +# prompt_tokens_details even though the wire carried input_tokens / +# input_tokens_details. Measured against a real proxy, not assumed - the +# Responses adapter below found none of its keys and reported every field +# unknown. The arithmetic is still OpenAI's (the whole, with subsets). +LITELLM_NORMALIZED = "litellm-normalized" class UsageSemanticsError(ValueError): @@ -131,6 +139,35 @@ def _normalize_openai_responses(usage: Mapping[str, Any]) -> NormalizedUsage: ) +def _normalize_litellm(usage: Mapping[str, Any]) -> NormalizedUsage: + """prompt_tokens is the WHOLE; the details are subsets of it.""" + + total = _int_or_none(usage, "prompt_tokens") + details = usage.get("prompt_tokens_details") + cache_read = _int_or_none(details, "cached_tokens") + cache_write = _int_or_none(details, "cache_write_tokens") + if cache_write is None: + cache_write = _int_or_none(details, "cache_creation_tokens") + output_details = usage.get("completion_tokens_details") + + ordinary: int | None = None + if total is not None and cache_read is not None and cache_write is not None: + ordinary = total - cache_read - cache_write + if ordinary < 0: + raise UsageSemanticsError( + f"LiteLLM cached ({cache_read}) + cache_write ({cache_write}) " + f"exceed prompt_tokens ({total})" + ) + return NormalizedUsage( + ordinary_input_tokens=ordinary, + cache_read_input_tokens=cache_read, + cache_write_input_tokens=cache_write, + total_input_tokens=total, + output_tokens=_int_or_none(usage, "completion_tokens"), + reasoning_output_tokens=_int_or_none(output_details, "reasoning_tokens"), + ) + + def _normalize_anthropic(usage: Mapping[str, Any]) -> NormalizedUsage: """input_tokens is the uncached REMAINDER; the cache fields add to it.""" @@ -154,16 +191,24 @@ def _normalize_anthropic(usage: Mapping[str, Any]) -> NormalizedUsage: def canonical_provider(label: str | None, call_type: str | None) -> str | None: """Map LiteLLM's provider label onto an adapter key, or None if unsure. - LiteLLM reports ``custom_llm_provider`` as "openai" for both Chat - Completions and Responses, and those two report usage differently, so the - label alone cannot pick an adapter. The call type is what distinguishes - them. Returning None when it does not is deliberate: normalize_usage - refuses an unknown provider rather than guessing token semantics, which is - the whole point of keeping the native object authoritative. + Every "openai" label maps to LITELLM_NORMALIZED regardless of call type, + because anything reaching a proxy callback has already been normalised by + LiteLLM into its own object - measured against a real gateway, where the + observed call type is "anthropic_messages" and the upstream Responses shape + never arrives. OPENAI_RESPONSES stays in the adapter table for a RAW + upstream body, which only direct callers and the wire-shape tests pass. + + An unrecognised label still returns None, so normalize_usage refuses rather + than guessing token semantics. """ - if label == "openai" and call_type and "responses" in call_type: - return OPENAI_RESPONSES + if label == "openai": + # Anything reaching a proxy callback has already been normalised by + # LiteLLM, whichever endpoint the caller used - the observed call_type + # for a Claude Code request through this gateway is "anthropic_messages", + # not a Responses one. The adapter has to match the object in hand, not + # the protocol on the wire. + return LITELLM_NORMALIZED if label in _ADAPTERS: return label return None @@ -172,6 +217,7 @@ def canonical_provider(label: str | None, call_type: str | None) -> str | None: _ADAPTERS = { ANTHROPIC: _normalize_anthropic, OPENAI_RESPONSES: _normalize_openai_responses, + LITELLM_NORMALIZED: _normalize_litellm, } diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index a2223a20a..42cbfc9cb 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -1591,6 +1591,21 @@ def aggregate(records: list[dict[str, Any]]) -> dict[str, Any]: "review_category_accuracy", "review_grounded_evidence", ) + # NOTE: a skill-not-invoked row still contributes to these medians. That is + # a real measurement gap - an arm exists to measure a SKILL, and a cell + # where the skill never ran did not measure it - but the narrow fix is + # WORSE than the gap, so it is deliberately not applied here. + # + # Filtering those rows out of the quality metrics alone leaves valid_runs + # and excluded_runs counting them, so the promotion gate sees N clean runs + # while the median was taken over fewer. Because the dropped rows are + # systematically an arm's worst, that biases toward PROMOTING: measured on + # one real run at 0.9 plus two uninvoked rows at 0.0, the gate flipped from + # keep_incumbent to promote. The three verdict fields below compound it - + # they are all() reducers, so one uninvoked cell flips a whole arm. + # Closing this honestly needs a scored-run count and a paired-equality + # check in the gate itself: a promotion-semantics change, not an + # aggregation fix. if any("review_weighted_f1" in record for record in valid): for metric in review_metrics: values = [record[metric] for record in valid if record.get(metric) is not None] diff --git a/gitnexus-shared/src/graph/types.ts b/gitnexus-shared/src/graph/types.ts index 58d96e42f..8b2b13596 100644 --- a/gitnexus-shared/src/graph/types.ts +++ b/gitnexus-shared/src/graph/types.ts @@ -14,6 +14,8 @@ export type NodeLabel = | 'Folder' | 'File' | 'Class' + | 'Protocol' + | 'Category' | 'Function' | 'Method' | 'Variable' diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index 9857a60cc..7958cfa9d 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -137,6 +137,7 @@ export type { FinalizeOutput, FinalizedScc, FinalizeStats, + AmbiguousWildcardExport, } from './scope-resolution/finalize-algorithm.js'; // Scope-aware registries + 7-step lookup (RFC §4; Ring 2 SHARED #917) diff --git a/gitnexus-shared/src/language-detection.ts b/gitnexus-shared/src/language-detection.ts index e805d4e4c..3fe27f7b7 100644 --- a/gitnexus-shared/src/language-detection.ts +++ b/gitnexus-shared/src/language-detection.ts @@ -32,6 +32,7 @@ const EXTENSION_MAP: Record = { [SupportedLanguages.Python]: ['.py'], [SupportedLanguages.Java]: ['.java'], [SupportedLanguages.C]: ['.c'], + [SupportedLanguages.ObjectiveC]: ['.m', '.mm'], [SupportedLanguages.CPlusPlus]: [ '.cpp', '.cc', @@ -111,6 +112,7 @@ const SYNTAX_MAP: Record = { [SupportedLanguages.Python]: 'python', [SupportedLanguages.Java]: 'java', [SupportedLanguages.C]: 'c', + [SupportedLanguages.ObjectiveC]: 'objectivec', [SupportedLanguages.CPlusPlus]: 'cpp', [SupportedLanguages.CSharp]: 'csharp', [SupportedLanguages.Go]: 'go', diff --git a/gitnexus-shared/src/languages.ts b/gitnexus-shared/src/languages.ts index d16adbd1a..eec45e612 100644 --- a/gitnexus-shared/src/languages.ts +++ b/gitnexus-shared/src/languages.ts @@ -11,6 +11,7 @@ export enum SupportedLanguages { Java = 'java', C = 'c', CPlusPlus = 'cpp', + ObjectiveC = 'objective-c', CSharp = 'csharp', Go = 'go', Ruby = 'ruby', diff --git a/gitnexus-shared/src/lbug/schema-constants.ts b/gitnexus-shared/src/lbug/schema-constants.ts index 217c382a6..aea908ef9 100644 --- a/gitnexus-shared/src/lbug/schema-constants.ts +++ b/gitnexus-shared/src/lbug/schema-constants.ts @@ -13,6 +13,8 @@ export const NODE_TABLES = [ 'Folder', 'Function', 'Class', + 'Protocol', + 'Category', 'Interface', 'Method', 'CodeElement', diff --git a/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts b/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts index f6038b3f0..2dc14b398 100644 --- a/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts +++ b/gitnexus-shared/src/scope-resolution/finalize-algorithm.ts @@ -5,13 +5,14 @@ * Pure logic that takes per-file parse output (`ParsedImport[]` + * `SymbolDefinition[]`) and returns: * - * - Linked `ImportEdge[]` per module scope, with `targetModuleScope` and - * `targetDefId` filled where resolvable; edges that could not be - * resolved within the hard fixpoint cap are marked + * - Linked `ImportEdge[]` keyed by binding scope (`fromScope`; module + * scope unless `importsBindAtLexicalScope` is on), with + * `targetModuleScope` and `targetDefId` filled where resolvable; edges + * that could not be resolved within the hard fixpoint cap are marked * `linkStatus: 'unresolved'`. - * - Materialized `bindings` per module scope — local defs merged with - * imported / wildcard-expanded / re-exported names via the provider's - * `mergeBindings` precedence. + * - Materialized `bindings` keyed by the same scopes — local defs merged + * with imported / wildcard-expanded / re-exported names via the + * provider's `mergeBindings` precedence. * - The SCC condensation of the import graph, exposed so disjoint SCCs * can be processed in parallel by callers that want that. * @@ -39,7 +40,7 @@ import type { BindingRef, ImportEdge, ParsedImport, ScopeId, WorkspaceIndex } fr /** Per-file input for the finalize pass. */ export interface FinalizeFile { readonly filePath: string; - /** The module scope id for this file; owns the finalized imports + bindings. */ + /** Default binding scope for imports without lexical provenance or opt-in. */ readonly moduleScope: ScopeId; readonly parsedImports: readonly ParsedImport[]; /** @@ -84,6 +85,10 @@ export interface FinalizeInput { * expects pure answers. */ export interface FinalizeHooks { + /** Bind imports at their extracted lexical scope. Missing provenance retains + * the legacy module-scope behavior. Opt-in: lexical position and language + * import-binding semantics are distinct facts. */ + readonly importsBindAtLexicalScope?: boolean; /** * Resolve a raw import target to the concrete file path that owns it. * Return `null` when no target file is resolvable (e.g., `np.foo` when @@ -114,6 +119,34 @@ export interface FinalizeHooks { */ expandsWildcardTo(targetModuleScope: ScopeId, workspaceIndex: WorkspaceIndex): readonly string[]; + /** + * Does this language make two `wildcard` re-exports that both DECLARE the + * same name AMBIGUOUS (no winner), rather than overloads or redeclarations + * of one entity? + * + * True for ECMAScript modules: `export * from './a'; export * from './b'` + * with `collide` declared in both excludes the name from the module's + * exports, so binding either source is a guess. False (the default) for + * languages whose wildcard import is `#include`, `require`, or a package + * fan-out, where the same name declared in two files is an overload set + * (C++ `write_audit(int)` / `write_audit(int, int)` across two headers), a + * redeclaration of one function, or a per-file `init` — legal, and resolved + * downstream by arity or by definition. Only a language that opts in has + * its collisions refused and reported via `ambiguousWildcardExports`. + */ + readonly wildcardCollisionIsAmbiguous?: boolean; + + /** + * A named import / named re-export binds only to MODULE-LEVEL declarations + * of the target file. Opt-in for languages whose `import { x }` can never + * reach a class member: without it, a class method sharing a name with a + * top-level value — or standing alone — wins the callable preference in + * `findExportByName` and the import binds to a symbol the module cannot + * export (a confident wrong edge). Languages that bind module-level members + * by bare name (static members, module functions) leave it off. + */ + readonly namedImportsBindTopLevelOnly?: boolean; + /** * Merge `incoming` bindings into `existing` for a given name. Called * once per name at each scope. Typical rules: @@ -164,12 +197,32 @@ export interface FinalizeStats { readonly unresolvedEdges: number; readonly sccCount: number; readonly largestSccSize: number; + /** + * Names a file re-exported through two or more `export *` sources that each + * DECLARE the name, so the language names no winner. The finalize pass + * refuses to bind them (they are absent from the file's re-export closure + * AND from its wildcard-expanded module-scope bindings) instead of taking the + * first-listed source and publishing the guess as `import-resolved`. + * Reported so the caller can record the refusal — an importer of that name + * stays unresolved, and the reason must be auditable rather than silent. + */ + readonly ambiguousWildcardExports: readonly AmbiguousWildcardExport[]; +} + +/** One refused `export *` collision — see `FinalizeStats.ambiguousWildcardExports`. */ +export interface AmbiguousWildcardExport { + readonly filePath: string; + readonly name: string; + /** `nodeId`s of the colliding declarations, in `export *` declaration order. */ + readonly candidateDefIds: readonly string[]; } export interface FinalizeOutput { - /** Linked `ImportEdge[]` per module scope, in original input order. */ + /** Linked `ImportEdge[]` keyed by binding scope (`fromScope`). Module scope + * for languages that bind file-wide; nested lexical scopes when + * `importsBindAtLexicalScope` is on. */ readonly imports: ReadonlyMap; - /** Materialized bindings per module scope. */ + /** Materialized bindings keyed by the same scopes as `imports`. */ readonly bindings: ReadonlyMap>; /** SCCs in reverse-topological order (leaves first). */ readonly sccs: readonly FinalizedScc[]; @@ -223,7 +276,33 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu // SCC-condensed). Eliminates the recursive crawl that the per-edge // `tryFinalize` call site used to do; lookups are O(1) afterwards. // See `buildReexportClosures` for the algorithm. - const reexportClosures = buildReexportClosures(input.files, byFilePath, edgeIndex); + // A local import is a dependency of its file, not a re-export of that file. + const moduleEdgeIndex = new Map(); + for (const file of input.files) { + moduleEdgeIndex.set( + file.filePath, + (edgeIndex.get(file.filePath) ?? []).filter((d) => d.fromScope === file.moduleScope), + ); + } + const ambiguityByFile = collectAmbiguityByFile( + input.files, + byFilePath, + moduleEdgeIndex, + hooks.wildcardCollisionIsAmbiguous === true, + hooks.namedImportsBindTopLevelOnly === true, + ); + const ambiguousByFile = new Map>(); + for (const [filePath, byName] of ambiguityByFile) { + ambiguousByFile.set(filePath, new Set(byName.keys())); + } + const topLevelOnly = hooks.namedImportsBindTopLevelOnly === true; + const reexportClosures = buildReexportClosures( + input.files, + byFilePath, + moduleEdgeIndex, + ambiguousByFile, + topLevelOnly, + ); // ── Phase 3: process SCCs in reverse-topological order (leaves first). // Within each SCC, run a bounded fixpoint that resolves intra-SCC edges. @@ -231,6 +310,19 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu // already finalized); edges inside the SCC may need multiple passes. const linkedByScope = new Map(); let linkedEdges = 0; + // Every refused wildcard name, reported from the ambiguity map rather than from + // the edges phase 4 happens to drop: a language whose `expandsWildcardTo` + // returns nothing (TypeScript — `export *` never binds names locally) drops + // no expanded edge, yet its importers were refused through the closure just + // the same, and that refusal must still be visible. Named-vs-named conflicts + // remain refused above but are not export-star collisions in this audit. + const ambiguousWildcardExports: AmbiguousWildcardExport[] = []; + for (const [filePath, byName] of ambiguityByFile) { + for (const [name, ambiguity] of byName) { + if (ambiguity.kind !== 'wildcard') continue; + ambiguousWildcardExports.push({ filePath, name, candidateDefIds: ambiguity.candidateDefIds }); + } + } for (const scc of sccs) { const sccFiles = new Set(scc.files); @@ -249,7 +341,7 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu if (drafts === undefined) continue; for (const draft of drafts) { if (draft.finalized !== null) continue; - const finalized = tryFinalize(draft, byFilePath, reexportClosures); + const finalized = tryFinalize(draft, byFilePath, reexportClosures, topLevelOnly); if (finalized !== null) { draft.finalized = finalized; progressed = true; @@ -272,13 +364,25 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu } } - // ── Phase 4: collect finalized `ImportEdge[]` per module scope, preserving + // ── Phase 4: collect finalized `ImportEdge[]` per binding scope, preserving // input order within each file, and wildcard-expand where applicable. for (const file of input.files) { const drafts = edgeIndex.get(file.filePath); if (drafts === undefined) continue; - const finalized: ImportEdge[] = []; + const finalizedByScope = new Map([[file.moduleScope, []]]); + // Names this file's `export *` sources collide on (see + // `collectAmbiguousWildcards`). Their expanded edges are dropped here, so + // the file's own module scope does not bind an arbitrary winner either — + // suppressing them only in the closure would leave this binding standing, + // and it was this binding, not the closure, that produced the published + // `import-resolved` guess. + const ambiguousHere = ambiguousByFile.get(file.filePath) ?? EMPTY_NAME_SET; for (const d of drafts) { + let finalized = finalizedByScope.get(d.fromScope); + if (finalized === undefined) { + finalized = []; + finalizedByScope.set(d.fromScope, finalized); + } const edge = d.finalized; if (edge === null) { throw new Error(`Invariant violated: import edge was not finalized for ${file.filePath}`); @@ -286,16 +390,25 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu if (d.source.kind === 'wildcard' && edge.linkStatus !== 'unresolved') { // Produce one `wildcard-expanded` ImportEdge per exported name. const expanded = expandWildcard(edge, byFilePath, hooks, input.workspaceIndex); - for (const e of expanded) finalized.push(e); + for (const e of expanded) { + if ( + d.fromScope === file.moduleScope && + e.kind === 'wildcard-expanded' && + ambiguousHere.has(e.localName) + ) + continue; + finalized.push(e); + } } else { finalized.push(edge); } if (edge.linkStatus !== 'unresolved') linkedEdges++; } - linkedByScope.set(file.moduleScope, Object.freeze(finalized)); + for (const [scopeId, edges] of finalizedByScope) + linkedByScope.set(scopeId, Object.freeze(edges)); } - // ── Phase 5: materialize module-scope bindings (local + imports + wildcards), + // ── Phase 5: materialize bindings (local + scoped imports + wildcards), // delegating precedence to `provider.mergeBindings`. const bindingsByScope = materializeBindings(input.files, linkedByScope, hooks); @@ -312,6 +425,7 @@ export function finalize(input: FinalizeInput, hooks: FinalizeHooks): FinalizeOu unresolvedEdges: totalEdges - linkedEdges, sccCount, largestSccSize, + ambiguousWildcardExports: Object.freeze(ambiguousWildcardExports), }; return Object.freeze({ @@ -339,6 +453,10 @@ function makeEdgeDrafts( hooks: FinalizeHooks, workspace: WorkspaceIndex, ): ImportEdgeDraft[] { + const fromScope = + hooks.importsBindAtLexicalScope === true + ? (parsed.declaredAtScope ?? file.moduleScope) + : file.moduleScope; // Dynamic-unresolved passes through — no `BindingRef`, no target file. if (parsed.kind === 'dynamic-unresolved') { const base: ImportEdge = { @@ -351,7 +469,7 @@ function makeEdgeDrafts( { source: parsed, fromFile: file.filePath, - fromScope: file.moduleScope, + fromScope, targetFile: null, base, finalized: base, // already fully finalized @@ -381,7 +499,7 @@ function makeEdgeDrafts( { source: parsed, fromFile: file.filePath, - fromScope: file.moduleScope, + fromScope, targetFile: null, base, finalized: base, @@ -417,7 +535,7 @@ function makeEdgeDrafts( return { source: parsed, fromFile: file.filePath, - fromScope: file.moduleScope, + fromScope, targetFile: tf, base, finalized: isFileLevelTerminal ? base : null, @@ -529,6 +647,7 @@ function tryFinalize( draft: ImportEdgeDraft, byFilePath: Map, reexportClosures: ReadonlyMap, + topLevelOnly: boolean, ): ImportEdge | null { const targetFile = draft.targetFile; if (targetFile === null) return draft.base; // already terminal @@ -552,7 +671,11 @@ function tryFinalize( // so consumers can reach the module as a symbol — but its absence is not // a failure. if (draft.base.kind === 'namespace') { - const moduleDef = findExportByName(targetModule.localDefs, extractExportedName(draft.source)); + const moduleDef = findExportByName( + targetModule.localDefs, + extractExportedName(draft.source), + topLevelOnly, + ); return { ...draft.base, targetModuleScope: targetModule.moduleScope, @@ -564,7 +687,7 @@ function tryFinalize( // local defs. Multi-hop re-export chains settle iteratively — each hop // resolves once its prior hop is finalized. const importedName = extractExportedName(draft.source); - const exported = findExportByName(targetModule.localDefs, importedName); + const exported = findExportByName(targetModule.localDefs, importedName, topLevelOnly); if (exported !== undefined) { const transitiveVia = @@ -691,15 +814,18 @@ function buildReexportClosures( files: readonly FinalizeFile[], byFilePath: ReadonlyMap, edgeIndex: ReadonlyMap, + ambiguous: ReadonlyMap>, + topLevelOnly: boolean, ): ReadonlyMap { const closures = new Map>(); for (const file of files) closures.set(file.filePath, new Map()); // ── Step 1: build the re-export sub-graph (only resolvable wildcard / - // reexport / flagged-named targets contribute edges), and collect the - // per-file ambiguous names in the same walk. + // reexport / flagged-named targets contribute edges). The per-file + // ambiguous-name sets arrive precomputed (`collectAmbiguityByFile`) because + // phase 4 consults the same sets when it expands wildcards into module + // scope — one source of truth for "this name has no winner". const subGraph = new Map>(); - const ambiguous = new Map>(); for (const file of files) { const targets = new Set(); const drafts = edgeIndex.get(file.filePath); @@ -710,7 +836,6 @@ function buildReexportClosures( if (!byFilePath.has(d.targetFile)) continue; targets.add(d.targetFile); } - ambiguous.set(file.filePath, collectAmbiguousReexports(drafts, byFilePath)); } subGraph.set(file.filePath, targets); } @@ -726,7 +851,7 @@ function buildReexportClosures( if (!scc.isCycle) { const filePath = scc.files[0]; if (filePath !== undefined) { - populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous); + populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous, topLevelOnly); } continue; } @@ -740,7 +865,9 @@ function buildReexportClosures( progressed = false; iter++; for (const filePath of scc.files) { - if (populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous)) { + if ( + populateFileClosure(filePath, byFilePath, edgeIndex, closures, ambiguous, topLevelOnly) + ) { progressed = true; } } @@ -820,6 +947,220 @@ function isNamedReexport(draft: ImportEdgeDraft): draft is ImportEdgeDraft & { * are still filling in, so detecting them needs a set that grows during the * fixpoint — the thing this pre-pass exists to avoid. */ +/** + * Per-file set of re-exported names that have NO decidable winner, from both + * detectors: `collectAmbiguousReexports` (flagged-named vs flagged-named) and + * `collectAmbiguousWildcards` (`export *` vs `export *`, direct declarations). + * Fixed for the whole run; consulted by the closure fixpoint AND by phase 4's + * wildcard expansion, so a refused name is absent from BOTH the exports an + * importer can reach and the module-scope bindings the file itself sees. + */ +interface ReexportAmbiguity { + readonly kind: 'named' | 'wildcard'; + readonly candidateDefIds: readonly string[]; +} + +function collectAmbiguityByFile( + files: readonly FinalizeFile[], + byFilePath: ReadonlyMap, + edgeIndex: ReadonlyMap, + wildcardCollisionIsAmbiguous: boolean, + topLevelOnly: boolean, +): ReadonlyMap> { + const out = new Map>(); + for (const file of files) { + const drafts = edgeIndex.get(file.filePath); + if (drafts === undefined) continue; + const byName = new Map(); + for (const name of collectAmbiguousReexports(drafts, byFilePath)) { + byName.set(name, { + kind: 'named', + candidateDefIds: namedReexportCandidates(drafts, byFilePath, name, topLevelOnly), + }); + } + // Wildcard-vs-wildcard is a language rule (`FinalizeHooks. + // wildcardCollisionIsAmbiguous`): ECMAScript excludes the name, C++ + // overloads it. Without the opt-in this half stays first-wins. + if (wildcardCollisionIsAmbiguous) { + for (const [name, ids] of collectAmbiguousWildcards(file, drafts, byFilePath)) { + if (!byName.has(name)) byName.set(name, { kind: 'wildcard', candidateDefIds: ids }); + } + } + if (byName.size === 0) continue; + out.set(file.filePath, byName); + } + return out; +} + +/** The declarations a flagged-named collision on `localName` points at. */ +function namedReexportCandidates( + drafts: readonly ImportEdgeDraft[], + byFilePath: ReadonlyMap, + localName: string, + topLevelOnly: boolean, +): readonly string[] { + const ids: string[] = []; + for (const draft of drafts) { + if (!isNamedReexport(draft) || draft.source.localName !== localName) continue; + const targetFile = draft.targetFile; + if (targetFile === null) continue; + const target = byFilePath.get(targetFile); + if (target === undefined) continue; + const def = findExportByName(target.localDefs, draft.source.importedName, topLevelOnly); + if (def !== undefined && !ids.includes(def.nodeId)) ids.push(def.nodeId); + } + return Object.freeze(ids); +} + +/** + * `export * from './a'; export * from './b'` where BOTH `a` and `b` declare + * `collide`: the language names no winner (ECMAScript excludes the name from + * the module's exports entirely; a direct `import { collide }` of it is a + * SyntaxError-class ambiguity). First-wins here published the `a` binding as + * `import-resolved` at full confidence — a definite target for a call that has + * none, which is the incorrect-context-over-missing-context failure in its + * purest form. The name is refused instead and reported. + * + * Decidable in this pre-pass because it reads only the targets' own + * `localDefs` — nothing that fills in during the closure fixpoint. Collisions + * that arrive TRANSITIVELY (two wildcards whose targets each re-export the + * name from somewhere else) are still first-wins; detecting them needs a set + * that grows mid-fixpoint, the thing this pre-pass exists to avoid. + * + * A name the file DECLARES itself, or re-exports by NAME, is excluded: an + * explicit export shadows every `export *`, so those collisions are legal and + * resolved by precedence, not ambiguous. + * + * Only MODULE-LEVEL, EXPORT-SHAPED declarations can collide. `localDefs` also + * carries class members, properties and parameters (a `Property:value` on two + * unrelated classes, an interface field named `move`), which no `export *` + * publishes. Counting those produced thousands of phantom collisions on a real + * monorepo (2,640 on grafana) and — the dangerous half — would have refused a + * genuinely exported `move()` because some class elsewhere had a `move` + * property. The wildcard closure loop tolerates the wider set because nobody + * imports a property by name; a refusal cannot afford the same tolerance. + * + * Export evidence, when the language supplies it (`SymbolDefinition.isExported`, + * tri-state), settles the rest: a def marked `false` is module-private and is + * neither a provider here nor published by the closure + * (`indexTopLevelExportsByName`), so a private `function foo` beside an exported + * one no longer refuses the export — and, the half that matters more, cannot be + * the first-listed winner the closure binds either. A def marked `true` counts + * whatever its label, `Variable` included: the closure publishes a `Variable`, + * so two sources each exporting `const alpha` are a real collision and must be + * refused rather than first-wins. + * + * Without evidence (`isExported` undefined — most languages) `Variable` is + * excluded from the COLLISION set only: the typical top-level `const` in a + * barrel's sources is module-private (`const category = ['Axis']` in fourteen + * option-builder files), so counting it would refuse a real exported constant + * of the same name for nothing. Residual risk, accepted, for that evidence-free + * case: a non-exported `function`/`class` sharing its name with an exported one + * behind the same barrel is counted as a collision and the export is refused — + * a missing edge, never a wrong one. `ownerId` is only set for class members, so + * a callable nested in an object literal (`showIf: (cfg) => …` across fourteen + * option-builder files) still counts as a provider when unmarked. Measured + * before the export marker existed: grafana@871af0720 refuses 52 names (from + * 2,640 before the member exclusion), discourse@3f71fa15c 5. + */ + +/** Labels that are never a module export, whatever their owner. */ +const NON_EXPORTABLE_MEMBER_LABELS: readonly string[] = [ + 'Property', + 'Method', + 'Constructor', + 'Parameter', + 'Field', +]; +/** Labels excluded from the collision set when no export evidence is present. */ +const UNMARKED_NON_COLLIDING_LABELS: ReadonlySet = new Set([ + ...NON_EXPORTABLE_MEMBER_LABELS, + 'Variable', +]); +/** + * Labels a module can never export by name: class/interface members and + * parameters. Filtered by LABEL, not `ownerId` — `ownerId` is populated in a + * later pass and is not reliable while the closure is built. + */ +const MEMBER_LABELS: ReadonlySet = new Set(NON_EXPORTABLE_MEMBER_LABELS); + +/** + * A declaration `export *` could publish, for COLLISION purposes: top-level, of + * an exportable kind, and not marked module-private. With export evidence the + * label rule yields to the marker (an exported `Variable` collides; a private + * `function` does not); without it `Variable` is left out — see the header. + */ +function isWildcardPublishable(def: SymbolDefinition): boolean { + // Explicit evidence wins over the label: a CommonJS `module.exports = { + // alpha() {} }` member is labeled Method and IS the module's export. + if (def.isExported === true) return true; + if (def.isExported === false) return false; + if (def.ownerId !== undefined) return false; + return !UNMARKED_NON_COLLIDING_LABELS.has(def.type); +} + +/** + * Can a declaration of the barrel's OWN shadow a name its `export *` sources + * collide on? Only a module-level binding can — ECMAScript's explicit-export + * precedence is about the module's own exports. A class MEMBER named `clash` + * (`export class Unrelated { clash() {} }`) is not such a binding and must not + * switch the collision check off; it did, and a confident edge to one source's + * `clash` was emitted where the import should have been refused. + */ +function canShadowWildcard(def: SymbolDefinition): boolean { + if (def.isExported === true) return true; + if (def.isExported === false) return false; + if (def.ownerId !== undefined) return false; + return !MEMBER_LABELS.has(def.type); +} +function collectAmbiguousWildcards( + file: FinalizeFile, + drafts: readonly ImportEdgeDraft[], + byFilePath: ReadonlyMap, +): ReadonlyMap { + const shadowed = new Set(); + for (const def of file.localDefs) { + if (!canShadowWildcard(def)) continue; + const name = deriveSimpleName(def); + if (name !== null) shadowed.add(name); + } + for (const draft of drafts) { + if (isNamedReexport(draft)) shadowed.add(draft.source.localName); + } + + // name → (target file → declaring def ids), in declaration order. + const providers = new Map>(); + for (const draft of drafts) { + if (draft.source.kind !== 'wildcard') continue; + const targetFile = draft.targetFile; + if (targetFile === null) continue; + const target = byFilePath.get(targetFile); + if (target === undefined) continue; + for (const def of target.localDefs) { + if (!isWildcardPublishable(def)) continue; + const name = deriveSimpleName(def); + if (name === null || shadowed.has(name)) continue; + let byTarget = providers.get(name); + if (byTarget === undefined) { + byTarget = new Map(); + providers.set(name, byTarget); + } + const ids = byTarget.get(targetFile); + if (ids === undefined) byTarget.set(targetFile, [def.nodeId]); + else ids.push(def.nodeId); + } + } + const conflicting = new Map(); + for (const [name, byTarget] of providers) { + // Two DIFFERENT source files declaring the name. The same file declaring + // it twice (overloads, a declaration merged with its namespace) is one + // provider and not a collision. + if (byTarget.size < 2) continue; + conflicting.set(name, Object.freeze([...byTarget.values()].flat())); + } + return conflicting; +} + function collectAmbiguousReexports( drafts: readonly ImportEdgeDraft[], byFilePath: ReadonlyMap, @@ -859,6 +1200,7 @@ function populateFileClosure( edgeIndex: ReadonlyMap, closures: Map>, ambiguousByFile: ReadonlyMap>, + topLevelOnly: boolean, ): boolean { const myClosure = closures.get(filePath); if (myClosure === undefined) return false; @@ -883,7 +1225,7 @@ function populateFileClosure( if (ambiguous.has(localName) || myClosure.has(localName)) continue; const importedName = draft.source.importedName; - const direct = findExportByName(targetModule.localDefs, importedName); + const direct = findExportByName(targetModule.localDefs, importedName, topLevelOnly); if (direct !== undefined) { myClosure.set(localName, { def: direct, via: Object.freeze([targetFile]) }); continue; @@ -909,9 +1251,27 @@ function populateFileClosure( const targetModule = byFilePath.get(targetFile); if (targetModule === undefined) continue; - for (const def of targetModule.localDefs) { - const name = deriveSimpleName(def); - if (name === null || ambiguous.has(name) || myClosure.has(name)) continue; + // Fan out the WINNER per name, not every def. `export const alpha = () => + // {}` emits both a `Variable` (the lexical declaration) and a `Function` + // (the arrow) under the same simple name; iterating `localDefs` raw let + // whichever came first — the `Variable` — claim the closure slot, and a + // call bound to a value shadow emits no CALLS edge. Named re-exports + // already go through `findExportByName`'s callable-preferred index; the + // wildcard hop is the same lookup and must apply the same preference. + // Measured: grafana `Button`/`clearButtonStyles` (arrow consts behind + // `export *`) resolved 8 of 475 ledger entries before this. + // Over TOP-LEVEL declarations only. `localDefs` also carries class members; + // `Foo.render` (label `Method`, callable) outranked the file's real + // `const render` in the callable-preferred index and `import { render }` + // bound to a symbol `export *` can never publish — a confident wrong edge + // where the value shadow used to yield none. Gated by the same hook as the + // named-import path: only a language that opted in (ECMAScript, where + // `export *` cannot publish a class member) narrows; every other language's + // wildcard keeps the wide index, whose members are legitimately reachable. + for (const [name, def] of (topLevelOnly ? indexTopLevelExportsByName : indexExportsByName)( + targetModule.localDefs, + )) { + if (ambiguous.has(name) || myClosure.has(name)) continue; myClosure.set(name, { def, via: Object.freeze([targetFile]) }); } const targetClosure = closures.get(targetFile); @@ -993,6 +1353,13 @@ function deriveSimpleName(def: SymbolDefinition): string | null { function findExportByName( defs: readonly SymbolDefinition[], name: string, + /** + * `true` (a `namedImportsBindTopLevelOnly` language): consult only + * module-level declarations, so a class member can neither outrank a + * top-level value nor bind on its own. Phase-4 wildcard expansion keeps + * the wide index — that is the path languages use to bind members. + */ + topLevelOnly: boolean = false, ): SymbolDefinition | undefined { // GENERIC RULE (applies to every language using this finalize // algorithm): when MULTIPLE `SymbolDefinition`s share the same simple @@ -1018,7 +1385,7 @@ function findExportByName( // // See `gitnexus/test/integration/resolvers/typescript-hof-callbacks.test.ts` // for the cross-file regression this rule prevents. - return indexExportsByName(defs).get(name); + return (topLevelOnly ? indexTopLevelExportsByName(defs) : indexExportsByName(defs)).get(name); } /** @@ -1062,6 +1429,47 @@ function indexExportsByName( return index; } +/** + * `indexExportsByName` restricted to declarations a module publishes by name: + * members (by LABEL — `ownerId` is stamped by a later reconcile pass and is not + * reliable while the closure is built) are skipped unless the language marked + * them exported (a CommonJS `module.exports = { alpha() {} }` member), and so + * is any def the language marked module-private (`isExported === false`) — a + * function nested inside another function carries the Function label and used + * to displace the real exported value of the same name here; a barrel cannot + * republish what its source never exported, and binding it would put a private + * `function foo` in front of the exported one another source provides. + * `Variable` stays, since a barrel legitimately republishes a `const`. Same + * memoization contract. + */ +const TOP_LEVEL_EXPORTS_BY_NAME = new WeakMap< + readonly SymbolDefinition[], + ReadonlyMap +>(); + +function indexTopLevelExportsByName( + defs: readonly SymbolDefinition[], +): ReadonlyMap { + const cached = TOP_LEVEL_EXPORTS_BY_NAME.get(defs); + if (cached !== undefined) return cached; + const index = new Map(); + for (const d of defs) { + // Evidence over label, both ways: a marked-private def (a function nested + // in another function carries the Function label too) is skipped, and a + // marked-exported member (`module.exports = { alpha() {} }`) is admitted. + if (d.isExported === false) continue; + if (d.isExported !== true && MEMBER_LABELS.has(d.type)) continue; + const name = deriveSimpleName(d); + if (name === null) continue; + const existing = index.get(name); + if (existing === undefined) index.set(name, d); + else if (!isCallableOrTypeLike(existing.type) && isCallableOrTypeLike(d.type)) + index.set(name, d); + } + TOP_LEVEL_EXPORTS_BY_NAME.set(defs, index); + return index; +} + const EMPTY_NAME_SET: ReadonlySet = new Set(); const CALLABLE_OR_TYPE_LIKE: ReadonlySet = new Set([ @@ -1179,7 +1587,7 @@ function materializeBindings( linkedByScope: ReadonlyMap, hooks: FinalizeHooks, ): ReadonlyMap> { - const out = new Map>(); + const buckets = new Map>(); // Build a `nodeId → SymbolDefinition` index once across all files // (O(N_files × D_defs)) so the per-edge lookup below is O(1) instead @@ -1204,8 +1612,17 @@ function materializeBindings( scopeBindings.set(name, hooks.mergeBindings(existing, incoming, file.moduleScope)); } - // Layer in finalized imports. - const imports = linkedByScope.get(file.moduleScope) ?? []; + buckets.set(file.moduleScope, scopeBindings); + } + + // Layer imports into their binding scope; lexical locals already live in + // the scope tree and must not be copied into unrelated scope buckets. + for (const [scopeId, imports] of linkedByScope) { + let scopeBindings = buckets.get(scopeId); + if (scopeBindings === undefined) { + scopeBindings = new Map(); + buckets.set(scopeId, scopeBindings); + } for (const edge of imports) { if (edge.targetDefId === undefined || edge.linkStatus === 'unresolved') continue; const def = defById.get(edge.targetDefId); @@ -1224,15 +1641,18 @@ function materializeBindings( if (name === null) continue; const incoming: BindingRef[] = [{ def, origin, via: edge }]; const existing = scopeBindings.get(name) ?? []; - scopeBindings.set(name, hooks.mergeBindings(existing, incoming, file.moduleScope)); + scopeBindings.set(name, hooks.mergeBindings(existing, incoming, scopeId)); } + } + const out = new Map>(); + for (const [scopeId, scopeBindings] of buckets) { // Freeze nested buckets for immutability. const frozen = new Map(); for (const [name, refs] of scopeBindings) { frozen.set(name, Object.freeze(refs.slice())); } - out.set(file.moduleScope, frozen); + out.set(scopeId, frozen); } return out; diff --git a/gitnexus-shared/src/scope-resolution/language-classification.ts b/gitnexus-shared/src/scope-resolution/language-classification.ts index 20059c3e9..2e635c8e2 100644 --- a/gitnexus-shared/src/scope-resolution/language-classification.ts +++ b/gitnexus-shared/src/scope-resolution/language-classification.ts @@ -9,7 +9,8 @@ * Initial classification (locked in Ring 1 #910): * - production: javascript, typescript, python, java, c, cpp, csharp, go, * ruby, rust, php, kotlin, swift, dart - * - experimental: vue (embedded-language / SFC complexity), + * - experimental: objective-c (fork provider MVP), + * vue (embedded-language / SFC complexity), * cobol (regex-provider path) * - quarantined: (none) * @@ -34,6 +35,7 @@ export const LanguageClassifications: Readonly = new Set = new Set([ 'Class', + 'Protocol', + 'Category', 'Interface', 'Enum', 'Struct', diff --git a/gitnexus-shared/src/scope-resolution/symbol-definition.ts b/gitnexus-shared/src/scope-resolution/symbol-definition.ts index e90b0be85..4334d1534 100644 --- a/gitnexus-shared/src/scope-resolution/symbol-definition.ts +++ b/gitnexus-shared/src/scope-resolution/symbol-definition.ts @@ -111,6 +111,18 @@ export interface SymbolDefinition { * source (for example an anonymous class). Consumers may use this only as a * conservative priority hint; it does not change graph-node identity. */ isSynthetic?: boolean; + /** + * Whether the producing language saw EXPORT EVIDENCE on this declaration — + * an `export` modifier, a later `export { name }` specifier, an `export + * default name`. TRI-STATE, and the absence is load-bearing: `undefined` + * means the language emitted no verdict (most languages, and any ECMAScript + * file whose export surface is a CommonJS assignment the marker cannot read), + * which readers MUST treat as "unknown" and fall back to their prior + * behavior. Only `false` says "this module does not publish the name": a + * `false` keeps a module-private `function foo` from being counted as a + * wildcard provider or bound through a barrel's `export *` closure. + */ + isExported?: boolean; /** Links Method/Constructor/Property to owning Class/Struct/Trait nodeId */ ownerId?: string; /** #1982/#1993: bridge-held enclosing-namespace path (e.g. `NS1`, `Outer.Inner`) diff --git a/gitnexus-shared/src/scope-resolution/types.ts b/gitnexus-shared/src/scope-resolution/types.ts index 94a085daa..50a968146 100644 --- a/gitnexus-shared/src/scope-resolution/types.ts +++ b/gitnexus-shared/src/scope-resolution/types.ts @@ -107,7 +107,14 @@ export type CaptureMatch = Readonly>; * produced when `expandsWildcardTo` materializes a wildcard against target * exports — a provider must never emit it at parse time. */ -export type ParsedImport = +export type ParsedImport = ParsedImportSyntax & { + /** Lexical location retained by extraction, independently of execution timing. + * Absent for legacy or synthesized imports. Binding semantics remain opt-in + * through FinalizeHooks.importsBindAtLexicalScope. */ + readonly declaredAtScope?: ScopeId; +}; + +type ParsedImportSyntax = /** * Per-name import without rename. * @@ -183,14 +190,15 @@ export type ParsedImport = * field exists.** The natural place to decide it looks like the graph * bridge, by walking the scope the finalized edges hang off; that is * exactly what `graph-bridge/imports-to-edges.ts` once attempted, and it - * is dead code by construction. `finalize-algorithm.ts:295` publishes - * every file's finalized edges as - * `linkedByScope.set(file.moduleScope, …)`, so the map the bridge - * receives is keyed by the file's **Module** scope and by nothing else: - * the walk starts at a `Module` every time and answers `false` for every - * import in the tree. Finalize cannot recover the position either — - * `FinalizeFile.parsedImports` is a flat per-file `ParsedImport[]` with - * no scope attached. The extractor is the last stage that still knows + * is dead code by construction. Default finalize still publishes each + * file's edges under `file.moduleScope`, so a bridge walk that starts + * there answers `false` for every import. Resolvers that opt into + * `importsBindAtLexicalScope` instead key the edge by + * `parsed.declaredAtScope` when extraction placed the statement (Rust + * `use` in a function body is the example). Even then the *execution* + * question — does this import run at module init? — is not the same as + * the lexical key, and `FinalizeFile.parsedImports` is still a flat + * per-file list. The extractor is the last stage that still knows * where the statement sat (`scope-extractor.ts`, Pass 3), so it marks the * fact here and it rides the edge from there — see * {@link ImportEdge.runsOnlyWhenCalled}. diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 282a9027b..90271c2bd 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -11,7 +11,7 @@ "@langchain/anthropic": "^1.5.8", "@langchain/core": "^1.2.8", "@langchain/google-genai": "^2.2.0", - "@langchain/langgraph": "^1.4.9", + "@langchain/langgraph": "^1.4.14", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.3", "@sigma/edge-curve": "^3.1.0", @@ -36,7 +36,7 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.12", + "react-i18next": "^17.0.13", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -51,14 +51,14 @@ "@playwright/test": "^1.62.1", "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.6", + "@testing-library/user-event": "^14.6.7", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", - "@types/react": "^19.2.14", + "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", "@vercel/node": "^5.10.2", - "@vitejs/plugin-react": "^6.0.5", + "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", @@ -1165,14 +1165,14 @@ } }, "node_modules/@langchain/langgraph": { - "version": "1.4.9", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.9.tgz", - "integrity": "sha512-EvD9rS66Cya09y6rbMgD3Ir8miAkJQFo7FyJOPRPO736Kz3y5TeyeBDOS8ctff/jRc788bPijHx2NVFM79Qqig==", + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.14.tgz", + "integrity": "sha512-uWAdRYTllfKCnTrlyovExPJCHJwcf3Wl2LzUlnaqsT7Rmoo3aCeYtq/7MV/Pw4q11motG8pR8bjr6T6V8Pe1gQ==", "license": "MIT", "dependencies": { - "@langchain/langgraph-checkpoint": "^1.1.3", - "@langchain/langgraph-sdk": "~1.9.28", - "@langchain/protocol": "^0.0.18", + "@langchain/langgraph-checkpoint": "^1.1.5", + "@langchain/langgraph-sdk": "~1.10.2", + "@langchain/protocol": "^0.0.19", "@standard-schema/spec": "1.1.0" }, "engines": { @@ -1184,9 +1184,9 @@ } }, "node_modules/@langchain/langgraph-checkpoint": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.3.tgz", - "integrity": "sha512-wgzdQNeEsdw1e+4lvlj0tdq/RYR/k1vPin10g0ymGoehZDDgd9nvIllGXSXN4TFgF9sf5qQP/KTkOcLfeseIhA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", + "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", "license": "MIT", "engines": { "node": ">=18" @@ -1196,12 +1196,12 @@ } }, "node_modules/@langchain/langgraph-sdk": { - "version": "1.9.28", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.28.tgz", - "integrity": "sha512-4j3XuM0PvtmAbL8mPfBS99ez3+ytRfgbOpAR/nOeaejTRF3Q9dNw2QnaGLGng8wLPtGLoSj+SYgUOVxy9Bv9vg==", + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.10.2.tgz", + "integrity": "sha512-86qsfdBZWu1ZgywLN8AThU/jXi9rjPDZPWcTJp4SA1A/L62ypTNoSXbvtiwZt1odokXccYTxK1XWS8tmVdvEmw==", "license": "MIT", "dependencies": { - "@langchain/protocol": "^0.0.18", + "@langchain/protocol": "^0.0.19", "@types/json-schema": "^7.0.15", "p-queue": "^9.0.1", "p-retry": "^7.1.1" @@ -1209,9 +1209,7 @@ "peerDependencies": { "@langchain/core": "^1.1.48", "react": "^18 || ^19", - "react-dom": "^18 || ^19", - "svelte": "^4.0.0 || ^5.0.0", - "vue": "^3.0.0" + "react-dom": "^18 || ^19" }, "peerDependenciesMeta": { "react": { @@ -1219,12 +1217,6 @@ }, "react-dom": { "optional": true - }, - "svelte": { - "optional": true - }, - "vue": { - "optional": true } } }, @@ -1295,9 +1287,9 @@ } }, "node_modules/@langchain/protocol": { - "version": "0.0.18", - "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.18.tgz", - "integrity": "sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==", + "version": "0.0.19", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.19.tgz", + "integrity": "sha512-9hKcRrH7cBX6gfutdfXPoft1OCchHe4FEpALoDJMl5Qu+n/YG5ynZmyu8+8cxORlPwHBoKTxggvXz+76M1yX1Q==", "license": "MIT" }, "node_modules/@mapbox/node-pre-gyp": { @@ -2101,9 +2093,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.6", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", - "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", + "version": "14.6.7", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.7.tgz", + "integrity": "sha512-MPCpX8bxe8zS+JmmTwLp8jd0dy1rAm60Te/SL8JrQM3qvQJcBOs1d7IefJMyZzqM3EWBrDn/LWDt1BCGu4ASfg==", "dev": true, "license": "MIT", "engines": { @@ -2535,9 +2527,9 @@ "license": "MIT" }, "node_modules/@types/react": { - "version": "19.2.14", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", - "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -2716,9 +2708,9 @@ } }, "node_modules/@vitejs/plugin-react": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.5.tgz", - "integrity": "sha512-BOVzne/NL162sMdResB25mUv+vWMF5NoAjNf09TeGlE7ZpszZWSD3winycicLJw72yeVsoCn/2kOhEuCvEShMA==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", "dev": true, "license": "MIT", "dependencies": { @@ -2730,6 +2722,7 @@ "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", "vite": "^8.0.0" }, "peerDependenciesMeta": { @@ -2738,6 +2731,9 @@ }, "babel-plugin-react-compiler": { "optional": true + }, + "oxc-transform-react": { + "optional": true } } }, @@ -5077,9 +5073,9 @@ } }, "node_modules/joi": { - "version": "18.2.3", - "resolved": "https://registry.npmjs.org/joi/-/joi-18.2.3.tgz", - "integrity": "sha512-N5A3KTWQpPWT4ExxxPlUx7WmykGXRzhNidWhV41d6Abu9YfI2NyWCJuxdPnslJCPWtbRpSVOWSnSS6GakLM/Rg==", + "version": "18.2.8", + "resolved": "https://registry.npmjs.org/joi/-/joi-18.2.8.tgz", + "integrity": "sha512-G2TX62h58ZHuwqetJgP2F4ualakqAmZtBYe3jWen7gxQRw5xApX6crnFtuB91WC0c3ESBnva+kGSnb3+6pIQDQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -7304,9 +7300,9 @@ } }, "node_modules/react-i18next": { - "version": "17.0.12", - "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.12.tgz", - "integrity": "sha512-lFWPEGkxQ6RhusdUkysFBD58VHfSSzvHBzqMgN0SvfVpdQGfwtNkStTqdy08/sJd7s807qqutgx93fRpD0DJ3Q==", + "version": "17.0.13", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.0.13.tgz", + "integrity": "sha512-Cc1PscmblIHA1kljTqDwrcVMI21ydgmUzw0UAeQBe7pAOgfuRLfzXze4EUBQoeDiICzFIXXhHFoZxuetNg5D0Q==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.29.7", diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index eef658136..fee794c47 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -21,7 +21,7 @@ "@langchain/anthropic": "^1.5.8", "@langchain/core": "^1.2.8", "@langchain/google-genai": "^2.2.0", - "@langchain/langgraph": "^1.4.9", + "@langchain/langgraph": "^1.4.14", "@langchain/ollama": "^1.3.0", "@langchain/openai": "^1.5.3", "@sigma/edge-curve": "^3.1.0", @@ -46,7 +46,7 @@ "pandemonium": "^2.4.0", "react": "^19.2.5", "react-dom": "^19.2.8", - "react-i18next": "^17.0.12", + "react-i18next": "^17.0.13", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", @@ -61,14 +61,14 @@ "@playwright/test": "^1.62.1", "@testing-library/jest-dom": "^7.0.0", "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.6", + "@testing-library/user-event": "^14.6.7", "@types/dompurify": "^3.2.0", "@types/node": "^26.0.1", - "@types/react": "^19.2.14", + "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", "@types/react-syntax-highlighter": "^15.5.13", "@vercel/node": "^5.10.2", - "@vitejs/plugin-react": "^6.0.5", + "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", diff --git a/gitnexus-web/src/components/FileTreePanel.tsx b/gitnexus-web/src/components/FileTreePanel.tsx index a3c7487a9..3004e9353 100644 --- a/gitnexus-web/src/components/FileTreePanel.tsx +++ b/gitnexus-web/src/components/FileTreePanel.tsx @@ -186,6 +186,10 @@ const getNodeTypeIcon = (label: NodeLabel) => { return FileCode; case 'Class': return Box; + case 'Protocol': + return Hash; + case 'Category': + return Box; case 'Function': return Braces; case 'Method': diff --git a/gitnexus-web/src/lib/constants.ts b/gitnexus-web/src/lib/constants.ts index 5a85754cf..8e5259741 100644 --- a/gitnexus-web/src/lib/constants.ts +++ b/gitnexus-web/src/lib/constants.ts @@ -8,6 +8,8 @@ export const NODE_COLORS: Record = { Folder: '#6366f1', // Indigo File: '#3b82f6', // Blue Class: '#f59e0b', // Amber - stands out + Protocol: '#ec4899', // Pink - like Interface + Category: '#14b8a6', // Teal - like Method Function: '#10b981', // Emerald Method: '#14b8a6', // Teal Variable: '#64748b', // Slate - muted (less important) @@ -51,6 +53,8 @@ export const NODE_SIZES: Record = { Folder: 10, // Structural - clearly bigger than files File: 6, // Common element - smaller than folders Class: 8, // Important code structure + Protocol: 7, // Like Interface + Category: 3, // Like Method Function: 4, // Common code element - small Method: 3, // Smaller than function Variable: 2, // Tiny - leaf node @@ -115,6 +119,8 @@ export const DEFAULT_VISIBLE_LABELS: NodeLabel[] = [ 'Folder', 'File', 'Class', + 'Protocol', + 'Category', 'Function', 'Method', 'Property', // Kotlin/Java fields (HAS_PROPERTY + DEFINES File→Property) @@ -129,6 +135,8 @@ export const FILTERABLE_LABELS: NodeLabel[] = [ 'Folder', 'File', 'Class', + 'Protocol', + 'Category', 'Interface', 'Enum', 'Type', diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index 50b9f2683..1cf1abde2 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -23,6 +23,21 @@ export interface BackendRepo { repoPath?: string; // git HEAD returns "repoPath"; older versions return "path" indexedAt: string; lastCommit?: string; + /** + * Branch this index was built from. Absent on legacy entries and non-git + * repos. Since #3199 a branch-pinned analyze registers its own entry, so this + * is what tells two entries for the same repository apart — the name is + * derived from the clone directory and is not a contract. + */ + branch?: string; + /** Non-primary branch indexes recorded for the same path. */ + branches?: Array<{ branch: string; indexedAt?: string; lastCommit?: string }>; + /** + * Present only when the index is behind the repo's checked-out HEAD; absent + * means either up to date or not answerable (see the server's + * `repo-projection.ts`). Same shape MCP `list_repos` returns. + */ + staleness?: { commitsBehind: number; hint?: string }; stats?: { files?: number; nodes?: number; @@ -663,7 +678,14 @@ export type BackendProbeStatus = 'ok' | 'unauthorized' | 'unreachable'; */ export const probeBackendStatus = async (): Promise => { try { - const response = await fetchWithTimeout(`${_backendUrl}/api/repos`, {}, PROBE_TIMEOUT_MS); + // `/api/health` rather than `/api/repos`: this is a liveness question on a + // 2s budget, and `/api/repos` now spawns a `git rev-list` per registered + // repo to answer freshness. Probing it made the cost of "is the server up?" + // scale with the number of indexed repos, and a failed probe re-polls, + // stacking more children on the way (#3232 review). `/api/health` is a + // constant, and still sits behind the same `/api/*` edge gate, so the 401 + // branch below keeps distinguishing "gated" from "not there". + const response = await fetchWithTimeout(`${_backendUrl}/api/health`, {}, PROBE_TIMEOUT_MS); if (response.status === 200) return 'ok'; return response.status === 401 ? 'unauthorized' : 'unreachable'; } catch { diff --git a/gitnexus-web/test/unit/constants.test.ts b/gitnexus-web/test/unit/constants.test.ts index 10fe238b9..216a3d52f 100644 --- a/gitnexus-web/test/unit/constants.test.ts +++ b/gitnexus-web/test/unit/constants.test.ts @@ -65,6 +65,8 @@ describe('FILTERABLE_LABELS', () => { expect(FILTERABLE_LABELS).toContain('Type'); expect(FILTERABLE_LABELS).toContain('Decorator'); expect(FILTERABLE_LABELS).toContain('Variable'); + expect(FILTERABLE_LABELS).toContain('Protocol'); + expect(FILTERABLE_LABELS).toContain('Category'); }); it('every filterable label has a defined color in NODE_COLORS', () => { diff --git a/gitnexus-web/test/unit/filter-panel.test.ts b/gitnexus-web/test/unit/filter-panel.test.ts index 67ab3397c..9bc9c5132 100644 --- a/gitnexus-web/test/unit/filter-panel.test.ts +++ b/gitnexus-web/test/unit/filter-panel.test.ts @@ -7,6 +7,8 @@ const LEGEND_LABELS: NodeLabel[] = [ 'Folder', 'File', 'Class', + 'Protocol', + 'Category', 'Interface', 'Enum', 'Type', @@ -20,6 +22,8 @@ const ICON_MAP: Record = { Folder: 'Folder', File: 'FileCode', Class: 'Box', + Protocol: 'Hash', + Category: 'Box', Function: 'Braces', Method: 'Braces', Interface: 'Hash', @@ -62,6 +66,8 @@ describe('color legend', () => { expect(LEGEND_LABELS).toContain('Type'); expect(LEGEND_LABELS).toContain('Decorator'); expect(LEGEND_LABELS).toContain('Variable'); + expect(LEGEND_LABELS).toContain('Protocol'); + expect(LEGEND_LABELS).toContain('Category'); }); it('every legend label has a color defined', () => { @@ -76,6 +82,8 @@ describe('color legend', () => { 'Folder', 'File', 'Class', + 'Protocol', + 'Category', 'Interface', 'Enum', 'Type', diff --git a/gitnexus/bench/emit-persistence/baselines.json b/gitnexus/bench/emit-persistence/baselines.json index a6f47f7e8..9c04fb248 100644 --- a/gitnexus/bench/emit-persistence/baselines.json +++ b/gitnexus/bench/emit-persistence/baselines.json @@ -1,5 +1,6 @@ { - "fingerprint": "011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2", + "fingerprint": "d3c3a53ead47663816344369dc0e501f31d60c462039de190ab6f5361037bd79", + "_rebaselined_objective_c_node_tables": "Objective-C support adds Protocol and Category node tables to csv-generator.ts's MULTI_LANG_TYPES, so this deterministic 2,400-entity synthetic emit now creates 38 CSV files rather than 36. The two additions, category.csv and protocol.csv, are both expected 55-byte header-only files because the synthetic graph contains no Objective-C nodes. Re-emitting the graph and recomputing the fingerprint after excluding exactly those two files yields the exact prior baseline 011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2, proving all 36 pre-existing files retain their bytes, routing, and within-file order. Prior 011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2 -> d3c3a53ead47663816344369dc0e501f31d60c462039de190ab6f5361037bd79. The timing guard remains within budget: local scaling_ratio 0.965 against 1.8 and elapsed_ms_large 69.24ms against 1000ms.", "_rebaselined_3161_static_gated_column": "Every relationship row gained a trailing `staticGated` BOOLEAN column (RELATION_SCHEMA in src/core/lbug/schema.ts, REL_CSV_HEADER + buildRelRow in csv-generator.ts, the fallback CREATE in lbug-adapter.ts), written as `0` for every edge that does not carry the flag and `1` for a Zig call site inside a comptime-false branch (PR #3161). Unlike the earlier header-only rebaselines this one touches ROWS, so the evidence is the inverse operation rather than a file-set diff: re-emitting this bench's 2,400-entity graph on the branch produces the same 36 CSV files; exactly 3 of them differ from a copy with the new column stripped (`rel_File_Class.csv` 156228 -> 151416 bytes, `rel_File_Function.csv` 334008 -> 324396, `rel_Function_Function.csv` 185028 -> 180216: one header field plus `,0` per row, 4,800 / 9,600 / 4,800 rows, 2 bytes each), the other 33 files are byte-identical, and the per-file fingerprint over the stripped copies is EXACTLY the prior baseline 72096279092d4f118de7e179333705c19c9aff2664f77d71a7da48cd9f73fb5a. So no row moved between pair files and no row reordered; the only bytes that changed are the appended column. Prior 72096279092d4f118de7e179333705c19c9aff2664f77d71a7da48cd9f73fb5a -> 011485e5180c005be9dfec7ac8dc4e3bb0fcfc4a680a16dcae2cf2d8af5ef7e2. Timing gates passed while the guard was red: scaling_ratio 0.82 against the 1.8 budget, elapsed_ms_large 185.54ms against the 1000ms backstop, so no throughput claim is being rebaselined away.", "scaling_budget": 1.8, "max_ms_large": 1000, diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index a02f23e0a..37fa78535 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -2,6 +2,8 @@ "_what": "Baselines for bench/import-target/measure.mjs cover every import-target resolver registered in SCOPE_RESOLVERS on a shared corpus, plus a configured C# arm for the branch the default call cannot reach. measure.mjs derives its arm inventory from LANG_REGISTRY and --check reconciles the registered languages in both directions. The single PHP arm supplies its production PSR-4 Composer mapping; csharp_csproj supplies csproj configuration. C and C++ also receive their production resolutionConfig header corpus. The timing, shape, fingerprint, context, and retained-heap gates therefore cover each production resolver path without splitting PHP into configured and unconfigured identities.", "_rebaselined_2960_kotlin_declared_packages": "Kotlin now resolves only from parsed package facts and local module bindings. This deliberately changes its five fingerprints, removes path-depth sensitivity, adds the context probe, and reduces the 32000-file retained index from 40.82 MiB to 4.31 MiB. External same-name path decoys now remain unresolved.", "_zig_arm_1432": "zig was registered in SCOPE_RESOLVERS by PR #1432 with no arm here, which is the exact hole the inventory arm exists for, and it reported it. The arm passes NO build config: resolveZigImportInternal walks an importer-relative @import path component by component and probes allFiles.has() twice (as spelled, then + '.zig'), so its fingerprints pin that walk alone and do not move when build.zig / build.zig.zon parsing (the bare-name legs, gated by test/unit/zig-import-resolver.test.ts) changes. Same corpus proportions as rust — 979 of 3200 resolve at 400 files — and rust's collide design, a deep tree whose spellings carry thirteen components against the unique arm's three, because component count is the only axis the cost has. Measured on one box: small 1.036 ms, collide 1.700 ms, scaling 1.050, collide scaling 1.011, depth 1.573; budgets take the file's usual ~1.5x on ratios (2.4 depth) and ~4x on absolute ms (4 / 7). Heap reads 16 B at both 8000 and 32000 files — it builds nothing, exactly rust's reading — so it sits in heap_bound_bytes at rust's absolute 1048576 B.", + "_objc_arm_3179": "Objective-C resolves local header and source imports through a cached suffix map. The map preserves the previous first-path tie-break, which is covered directly in objective-c-provider.test.ts. This arm covers the registered resolver's five dispatch paths, including the depth-sensitive index construction and a collision layout. Measured on this branch: small 1.844 ms, collide 1.861 ms, scaling 1.111, collide scaling 1.067, depth 2.779, and 44101944 retained bytes. The 3.6 depth budget allows normal timing variation while remaining below a full per-import file scan; collision scaling stays at the shared linear 1.8 budget because suffix lookup is keyed.", + "_cobol_copybook_dir_preference_2967": "COBOL now prefers well-known copybook directories (copybooks/, cpy/, copy/, plus importer dir) over vendor paths when resolving COPY statements, so COPY EXTERNAL no longer hits same-named decoys outside the project structure. This intentional behavior change moves the resolver from depth-free (prior measured ~0.885) to depth-sensitive (measured 1.751 on CI), because the new preferredCopybookDirs check walks path components. The 2.4 depth budget is ~1.37x the measured ratio, consistent with the ~1.5x convention for depth arms. The unique-arm fingerprints and resolved counts change to reflect the new target set: small/deep drop from 1153 to 442 resolved because files outside preferred directories are now correctly filtered. The collide arm (svc${d}/copybooks layout) keeps 1153 resolved because ALL its files are in preferred copybook directories, so the filter does not change its resolution outcomes — the collide arm now measures collision behavior within the preferred class rather than across mixed layouts. Fixes #2967.", "_php_composer_gate_2962": "The canonical PHP arm supplies an authoritative App PSR-4 mapping. A deterministic Vendor0 suffix decoy makes deletion of the external gate change every timing fingerprint, while the heap arm separately pins a mapped miss, the rendered mapping, and the external null result. Three serial samples measured depth ratios 1.058, 1.114, and 1.158; the 1.8 budget is 1.55x the observed maximum. The mapped-miss heap reading peaked at 39607216 retained bytes at 32000 files.", "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the languages these PRs changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files \u2014 that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) for Kotlin's current declared-package behavior in test/unit/kotlin-module-resolution.test.ts, and for the four resolvers added there in test/unit/scope-resolution/{php,java,cobol}-import-target-parity.test.ts and test/unit/import-resolvers/csharp-csproj-parity.test.ts, and for JavaScript in test/unit/scope-resolution/javascript-import-target-parity.test.ts (a differential over 211200 old-vs-new pairs, PR #2911). The eight languages added last have no per-language parity harness against a pre-change implementation and do NOT need one: nothing about their resolution changed, so there is no before to diff against. Their fingerprints are pure forward guards, minted from the current implementations, and their adapter-boundary index reuse is covered for every registered language at once by test/unit/scope-resolution/import-target-index-reuse.contract.test.ts. NOTE for csharp_csproj: on this corpus the #2902 indexed leg (step 3 of resolveCSharpImportInternal) is reached by 2221 of the 3200 small-arm imports but answers null for every one of them \u2014 the 979 that resolve do so at step 2 \u2014 so this fingerprint pins that legs cost and its null answers, while its positive tie-breaks (unanchored substring, iteration order) are pinned by csharp-csproj-parity.test.ts. NOTE for kotlin, go, csharp and java: twenty fingerprints across these four languages were re-baselined in #2881, the one deliberate behaviour change any language in this file has had. It landed in two steps and the second is the reason the first is not a special case: Kotlin first, then the shared package-dir-index (go, java, csharp) and the csproj namespace index once the same rule was found live there. `getKotlinFileIndex` no longer requires a file's package directory to be the FIRST occurrence of that name in its own path, so the unique arm's `d % 7` nested slice (`mod{d}/src/main/kotlin/com/example/pkg{d}/inner/pkg{d}`) now belongs to package `pkg{d}` and its wildcard imports resolve: resolved 1100 -> 1153 small and deep, 4456 -> 4681 large. The collide arm needed a CORPUS edit alongside it, not just a new number \u2014 its `d % 7` slice deliberately imported `com.example.vendor{d}`, a package that exists nowhere, purely to mirror the unique arm's nested-slice MISS, so leaving it would have left collide at 1100 against small's 1153 and broken the same-workload invariant the arm is built on (that assertion is what caught it). It now uses the same `com.example.models.*` spelling as the rest of the arm, which is why its distinct_outcomes fell (2775 -> 2744, 11087 -> 10961): one shared target instead of one per d. The record-level evidence for the resolver change \u2014 235 of 19968 records moved, 54 null -> resolved, 0 buckets losing a member \u2014 is in bench/kotlin-import-target/baselines.json `_provenance`. The kotlin heap_reading_bytes and heap_ceiling_bytes moved with it, together as `_heap_reading_note` requires: 48073096 -> 48200224 bytes_large (+127128, +0.264%), ceiling still exactly 1.5x. Small, and it is worth saying WHY it is small rather than reading the number as evidence that the change is cheap. `dirChildren` grows by one entry per component-suffix the old rule used to skip, and this arm can only see part of that: the heap corpus is built with HEAP_PAD 8, which prefixes every path with `d0/\u2026/d7/`, so no path can begin with a suffix of its own directory and the leading-segment half of the old rule is structurally invisible here. What moves the reading is the `d % 7` nested slice alone. Read +0.264% as this arm's ceiling on the effect, not as the effect. GO NEEDED A CORPUS EDIT TO BE GATED AT ALL. Its nested slice was `src/pkg{d}/internal/pkg{d}`, repeating only the LAST segment, while a Go query addresses the whole package path `src/pkg{d}` \u2014 so the directory never even ended with the query and the first-occurrence rule was never reached. Every go arm sat unchanged through the resolver fix. `uniqueDir`/`collideDir` now repeat the shape at the granularity Go actually queries (`src/pkg{d}/internal/src/pkg{d}`, `svc{d}/internal/sub/svc{d}/internal`), which is what moved go from 979 to 1153 resolved and bumped `languages.go.heap.path_segments` 13 -> 14. The general lesson: a corpus that carries a shape the QUERY cannot express does not gate that shape. CSHARP AND JAVA HIT THE SAME COLLIDE-ARM TRAP AS KOTLIN. Both collide arms sent their `d % 7` slice to a namespace that exists nowhere (`App.Src{d}.Vendor`, `com.svc{d}.vendor`) purely to MIRROR the unique arm's nested-slice miss; once that miss became a hit, collide sat at 979/1100 against small's 1153 and the same-workload assertion failed. Both now use the same spelling as the rest of their arm. HEAP: no reading here moved for the resolver change. An earlier revision of this branch re-recorded `csharp_csproj` 73703384 -> 73116520 as a -0.79% effect of the step-2 filter; review measured base and branch three times each and got the same 73.10e6 on BOTH sides \u2014 the recorded 73703384 was simply not reproducible on this box, and re-recording it would have dropped that language's derived floor by 0.8% for no reason belonging to this change. Reverted. Everything else sat within +/-0.03%. Note that `_heap_reading_note`'s claim that these readings 'reproduce to the byte across processes on one box' did NOT hold on the box this was measured on: go, dart, ruby, python, php and cpp all wandered by a few hundred to a few thousand bytes between processes with no code change touching them. Treat sub-0.05% movement as jitter, not signal. HEAP, kotlin, second movement: 48200224 -> 42802456 (-11.20%), re-recorded with its ceiling. `getKotlinFileIndex` now compacts each `dirChildren` bucket as it freezes it. `addChild` mints a bucket as `[raw]` and pushes the rest, and V8 grows a backing store by `old + old/2 + 16`, so the second child takes a 1-slot store to 17: 61144 buckets, 52.9% of their slots empty, 88 B each. Same fix and same accounting as the python `byBasename` sentence above. Note what this means for the gate: a memory WIN of this size passes every arm \u2014 it is under the ceiling and over the 0.5x floor \u2014 so it is recorded because the convention says a reading and its ceiling move together, not because anything went red. kotlin now reads 40.82 MiB. The prose in measure.mjs calling it '45.85 MiB, the second-largest reading in this file' is corrected with it \u2014 and was already wrong on the ranking before this change, since csharp_csproj (69.73) and php (47.28) both read higher; kotlin was third. A measurement written into prose is not re-taken, which is the finding `_heap_bound_note` records about this very file. One further corpus edit, made in review and MEASURED rather than assumed: kotlin's collide layout repeated only the `models` leaf (`\u2026/com/example/models/inner/models`) while a Kotlin query addresses the whole dotted path, so a full revert of the Kotlin guards left both collide fingerprints UNMOVED \u2014 the arm was blind to the rule it was re-baselined for. Deepening it to `\u2026/models/inner/com/example/models` makes the revert move both, and those two fingerprints are the only ones that changed for it. The same deepening was applied to the java and kotlin UNIQUE arms and REVERTED: it moved ten more fingerprints, grew java's heap reading 43%, and bought nothing \u2014 progressive stripping lands those queries on the same file with or without the rule, so the control still failed only on go.", "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 \u2014 which deletes the entire depth arm \u2014 moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir. THE HEAP ARM IS ASSERTED THE SAME WAY, by the same loop, and was not before: files_small, files_large, path_segments and probe decide WHAT it measures, and every one of them was reported and compared to nothing. Swapping HEAP_PROBE_TARGET.csharp_csproj for a target matching no CSPROJ_CONFIGS rootNamespace skips the whole config loop, so the getFilesInDir and getInsensitive legs never run and the arm the header calls the witness that the read pattern IS the footprint quietly becomes a two-map arm \u2014 73703384 -> 59921216 B, ratio 1.017 -> 1.011, ceiling and floor both still passing and --check still exiting 0. Setting HEAP_SMALL equal to HEAP_LARGE is the same hole from the other side: ratio goes to ~1.0 by construction and bytes_large never moves. bytes_small and bytes_large are deliberately NOT asserted for equality \u2014 heap_ceiling_bytes and the heap_reading_bytes floor bound them with ~50% either way, because heapUsed accounting moves across platforms and Node majors and an exact byte assertion would be a re-baseline per runner. THE CONTEXT ARM IS ASSERTED THE SAME WAY, by the same loop, and more strictly than either: target, with_context and without_context are exact strings with no tolerance at all, because the arm resolves one import over a three-file corpus and has no measurement noise to tolerate. A separate check requires the last two to DIFFER, for the same reason deep.fingerprint must differ from small.fingerprint \u2014 a probe on which both call shapes agree asserts one number twice. Both halves run through resolveOne, so what the arm gates is this bench threading run.ts's fifth argument, not the resolvers' behaviour.", @@ -28,6 +30,7 @@ "vue": 1.8, "c": 3.8, "cpp": 4, + "objc": 1.8, "zig": 1.8 }, "depth_budget": { @@ -39,7 +42,7 @@ "kotlin": 1.8, "php": 1.8, "java": 1.8, - "cobol": 1.6, + "cobol": 2.4, "swift": 2.3, "rust": 2.1, "python": 2.2, @@ -48,6 +51,7 @@ "vue": 2.3, "c": 3, "cpp": 3, + "objc": 3.6, "zig": 2.4 }, "small_ms_ceiling": { @@ -68,6 +72,7 @@ "vue": 81, "c": 7, "cpp": 7, + "objc": 8, "zig": 4 }, "collide_ms_ceiling": { @@ -88,6 +93,7 @@ "vue": 93, "c": 11, "cpp": 12, + "objc": 8, "zig": 7 }, "heap_ceiling_bytes": { @@ -95,6 +101,7 @@ "go": 4497696, "dart": 11751300, "cpp": 15035016, + "objc": 66200000, "csharp": 44900000, "csharp_csproj": 110600000, "ruby": 61600000, @@ -110,6 +117,7 @@ "go": 2998464, "dart": 7834200, "cpp": 10023344, + "objc": 44101944, "csharp": 29869080, "csharp_csproj": 73703384, "ruby": 41020808, @@ -118,9 +126,9 @@ "python": 6360936, "c": 10018816 }, - "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 3500000 (1.508x), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 2.2x below the smallest real index measured here (cobol's 2.32 MB at the same file count), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", + "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 6200000 (1.508x of 4112464 B measured after copybook-dir preference; previously 3500000 based on 2320456 B before the behavior change), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 3.9x below the smallest real index measured here (cobol's 3.92 MB at the same file count after copybook-dir preference), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", "heap_bound_bytes": { - "cobol": 3500000, + "cobol": 6200000, "swift": 5200000, "rust": 1048576, "javascript": 1048576, @@ -561,23 +569,23 @@ "small": { "files": 400, "imports": 3200, - "resolved": 1153, + "resolved": 442, "distinct_outcomes": 2941, - "fingerprint": "e5bf9c2a74cad64df6ac18299b56fc9139943baec6118036b2e765ac3d4252f2" + "fingerprint": "f204d1f7c98374ad1dee5d32e6b2d885f9c25939ed8eb6e463c64473e65a67b7" }, "large": { "files": 1600, "imports": 12800, - "resolved": 4681, + "resolved": 1643, "distinct_outcomes": 11791, - "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e" + "fingerprint": "715153083398570bb301307f153c19f084d4d0066f9a8fc5d2ac7d359a4e29c2" }, "deep": { "files": 400, "imports": 3200, - "resolved": 1153, + "resolved": 442, "distinct_outcomes": 2941, - "fingerprint": "c690c6abc5c7aab31f27a97e5ef25d32daa483d9c08ceb48bc0b85ac406e6e37" + "fingerprint": "8dd5ef1c3813bae02793f9bbb7f5c2ce702f2750d855a287aeff686085c48e8d" }, "collide": { "files": 400, @@ -593,7 +601,7 @@ "distinct_outcomes": 11393, "fingerprint": "8bc1d506b54e800d060eb4c92fca01c5b06c5a7130248dc1a79521bdea53982a" }, - "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e", + "fingerprint": "715153083398570bb301307f153c19f084d4d0066f9a8fc5d2ac7d359a4e29c2", "heap": { "files_small": 8000, "files_large": 32000, @@ -603,7 +611,7 @@ "_measured": { "collide_ms": 0.197, "collide_scaling_ratio": 1.046, - "depth_ratio": 0.885, + "depth_ratio": 1.751, "scaling_ratio": 0.936, "small_ms": 0.286 } @@ -1021,6 +1029,57 @@ "small_ms": 1.626 } }, + "objc": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 3154, + "fingerprint": "1fc1ece7761a8227b6f78fd801bcef0717b45492ff3236d1800ef84f982a5782" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 12614, + "fingerprint": "41c51f110131c6615c1008934a9f83d30f6a106d435cfb59a3282a469077456d" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 3154, + "fingerprint": "5bf07f919d00696a3bccdd802afc34feb1b35758d281a44707214043e0abde05" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 3154, + "fingerprint": "5d9fb65101efaa9244b46288556d6ddfd8b77e849c682c1c4f71faa25bf1018f" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 12614, + "fingerprint": "37f1fd29df9c5b9c17dfc46c38b124b1a8b064856c1b9047e8846fa59263e1dd" + }, + "fingerprint": "41c51f110131c6615c1008934a9f83d30f6a106d435cfb59a3282a469077456d", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/missing.m" + }, + "_measured": { + "collide_ms": 1.861, + "collide_scaling_ratio": 1.067, + "depth_ratio": 2.779, + "scaling_ratio": 1.111, + "small_ms": 1.844 + } + }, "zig": { "small": { "files": 400, diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index 68193154d..35c8361a1 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -490,6 +490,7 @@ import { makeVueResolveImportTarget } from '../../src/core/ingestion/languages/v import { typescriptScopeResolver } from '../../src/core/ingestion/languages/typescript/scope-resolver.ts'; import { cScopeResolver } from '../../src/core/ingestion/languages/c/scope-resolver.ts'; import { cppScopeResolver } from '../../src/core/ingestion/languages/cpp/scope-resolver.ts'; +import { objectiveCScopeResolver } from '../../src/core/ingestion/languages/objective-c/scope-resolver.ts'; // `SCOPE_RESOLVERS` is NOT imported here — see the inventory arm at the bottom, // which loads it dynamically. Statically it costs 6-10 s of module load // depending on the box (measured both ways there), because reaching the @@ -598,6 +599,7 @@ const HEAP_BUDGETED = [ 'dart', 'go', 'cpp', + 'objc', ]; // javascript, typescript and vue were budgeted here until #2953 and are now // BOUNDED, which is a demotion in gate strength and a promotion in what the @@ -756,6 +758,7 @@ const EXTENSION = { vue: '.vue', c: '.c', cpp: '.cpp', + objc: '.m', zig: '.zig', }; /** C and C++ resolve `#include` against HEADERS, which reach the resolver @@ -879,6 +882,7 @@ function uniqueDir(lang, d, i) { // C and C++ split headers from sources — the shape that makes // `resolutionConfig` load-bearing. Odd `i` is the header. if (lang === 'c' || lang === 'cpp') return i % 2 === 1 ? `include/comp${d}` : `src/comp${d}`; + if (lang === 'objc') return `src/comp${d}`; if (lang === 'ruby') return `lib/mod${d}`; // One flat `src/mod{d}/` per index and NO nested slice, on purpose: a Zig // import is spelled RELATIVE TO THE IMPORTER, and `uniqueTarget` does not @@ -974,6 +978,7 @@ function collideDir(lang, d, i) { if (lang === 'javascript' || lang === 'typescript') return `pkg${d}/src`; if (lang === 'vue') return `src/pkg${d}/components`; if (lang === 'c' || lang === 'cpp') return i % 2 === 1 ? `svc${d}/include` : `svc${d}/src`; + if (lang === 'objc') return `svc${d}/src`; if (lang === 'ruby') return `svc${d}/lib/models`; // Rust's reasoning, verbatim: the resolver walks path components and probes // `.has()`, never searches, so file count is not an axis its cost has and a @@ -1400,6 +1405,9 @@ function uniqueTarget(lang, { local, r, d, j, dirs }) { ? ['stdio.h', 'stdlib.h', 'string.h'][(r >>> 4) % 3] : `vendor${(r >>> 4) % 97}/missing${h}`; } + if (lang === 'objc') { + return local ? `comp${j % dirs}/file${j}.m` : `vendor${(r >>> 4) % 97}/missing.m`; + } if (lang === 'ruby') { return local ? `mod${d}/file${j}` @@ -1630,6 +1638,9 @@ function collideTarget(lang, { local, r, d, j, dirs }) { ? ['stdio.h', 'stdlib.h', 'string.h'][(r >>> 4) % 3] : `vendor${(r >>> 4) % 97}/mod0${h}`; } + if (lang === 'objc') { + return local ? `src/file${j}.m` : `vendor${(r >>> 4) % 97}/missing.m`; + } if (lang === 'ruby') { // `models/mod{n}.rb` in every package. Ruby answers `require` from a keyed // suffix map, so the repeated basename cannot grow a bucket: this arm @@ -1912,6 +1923,9 @@ function resolveOne(lang, from, target, pass) { if (lang === 'cpp') { return cppScopeResolver.resolveImportTarget(target, from, allFilePaths, pass.config); } + if (lang === 'objc') { + return objectiveCScopeResolver.resolveImportTarget(target, from, allFilePaths, pass.config); + } if (lang === 'csharp' || lang === 'csharp_csproj') { return resolveCsharpImportTarget( { kind: 'namespace', localName: '_', importedName: '_', targetRaw: target }, @@ -2132,6 +2146,7 @@ const HEAP_PROBE_TARGET = { javascript: 'vendor0/lib/missing', python: 'vendor0.deep.missing', c: 'vendor0/missing.h', + objc: 'vendor0/missing.m', // The entries below cover the BOUNDED tier — see `HEAP_BOUNDED`, which // derives to cobol, swift and rust; the rest were promoted. Same rule as the // budgeted ones above: a spelling `uniqueTarget` already mints for that language, and @@ -2394,6 +2409,7 @@ const LANG_REGISTRY = { vue: SupportedLanguages.Vue, c: SupportedLanguages.C, cpp: SupportedLanguages.CPlusPlus, + objc: SupportedLanguages.ObjectiveC, zig: SupportedLanguages.Zig, }; const LANGS = Object.keys(LANG_REGISTRY); @@ -2701,6 +2717,9 @@ for (const lang of LANGS) { } for (const arm of ['deep', 'collide']) { if (got[arm].resolved !== got.small.resolved) { + // COBOL #2967 exception: the collide arm (all files in copybook dirs) legitimately + // resolves MORE than the unique arm (mixed layouts) after preferred-dir filtering. + if (lang === 'cobol' && arm === 'collide') continue; failures.push( `${lang}: ${arm} arm resolved ${got[arm].resolved} vs small ${got.small.resolved} — the ` + `${arm} arm was supposed to change ${arm === 'deep' ? 'path depth' : 'directory and file NAMING'} ` + diff --git a/gitnexus/bench/objective-c-resolution/baseline.json b/gitnexus/bench/objective-c-resolution/baseline.json new file mode 100644 index 000000000..14108eb6d --- /dev/null +++ b/gitnexus/bench/objective-c-resolution/baseline.json @@ -0,0 +1,41 @@ +{ + "_comment": "Correctness counts are exact. Both arms are linear file-count gates. protocol implementer USES live on one shared node per (protocol, selector): N objc-protocol-candidate edges, not N². Measured after the shared-set hoist: spread 0.62→2.37 ms (linear_factor 0.956); protocol 0.69→2.14 ms (linear_factor 0.775) with candidate uses 40→160.", + "spread": { + "small": { + "classes": 40, + "calls": 119, + "extends": 39, + "implements": 0, + "protocol_candidate_uses": 0, + "ms_budget": 80 + }, + "large": { + "classes": 160, + "calls": 479, + "extends": 159, + "implements": 0, + "protocol_candidate_uses": 0, + "ms_budget": 320 + }, + "linear_scaling_slack": 1.375 + }, + "protocol": { + "small": { + "classes": 40, + "calls": 40, + "extends": 0, + "implements": 40, + "protocol_candidate_uses": 40, + "ms_budget": 80 + }, + "large": { + "classes": 160, + "calls": 160, + "extends": 0, + "implements": 160, + "protocol_candidate_uses": 160, + "ms_budget": 80 + }, + "linear_scaling_slack": 1.375 + } +} diff --git a/gitnexus/bench/objective-c-resolution/measure.mjs b/gitnexus/bench/objective-c-resolution/measure.mjs new file mode 100644 index 000000000..577abca52 --- /dev/null +++ b/gitnexus/bench/objective-c-resolution/measure.mjs @@ -0,0 +1,386 @@ +#!/usr/bin/env node +/** + * Build-free scaling and correctness guard for Objective-C workspace + * resolution (#3179). + * + * Import lookup already lives in bench/import-target (the `objc` arm). This + * bench isolates `emitPostResolutionEdges`: heritage, category membership, + * and static message-send dispatch. Two shapes: + * + * spread — typed self / super / sibling receivers. Cost must stay + * linear in class count (the C# "namespaces spread" analog). + * protocol — every class conforms to one protocol and sends one + * protocol-typed message. Implementer USES live on one + * shared node per (protocol, selector); messages only add + * the source→shared hop. Both arms must stay linear. + * + * Usage: + * node --import tsx bench/objective-c-resolution/measure.mjs + * node --import tsx bench/objective-c-resolution/measure.mjs --check + */ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.ts'; +import { + OBJECTIVE_C_GRAMMAR_PACKAGE, + OBJECTIVE_C_GRAMMAR_VERSION, + OBJECTIVE_C_PROVIDER_VERSION, + objcClassQualifiedName, + objcMethodQualifiedName, + objcProtocolQualifiedName, +} from '../../src/core/ingestion/languages/objective-c/facts.ts'; +import { objectiveCScopeResolver } from '../../src/core/ingestion/languages/objective-c/scope-resolver.ts'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SMALL_CLASSES = 40; +const LARGE_CLASSES = 160; +const REPS = 7; + +const graphNodeId = (label, qualifiedName) => `${label}:${qualifiedName}`; + +function emptyFacts(filePath) { + return { + providerVersion: OBJECTIVE_C_PROVIDER_VERSION, + grammarPackage: OBJECTIVE_C_GRAMMAR_PACKAGE, + grammarVersion: OBJECTIVE_C_GRAMMAR_VERSION, + filePath, + containers: [], + methods: [], + members: [], + functions: [], + imports: [], + messages: [], + unresolvedMessages: [], + }; +} + +function classContainer(filePath, name, extras = {}) { + const qualifiedName = objcClassQualifiedName(name); + return { + kind: 'class', + declarationRole: 'implementation', + name, + qualifiedName, + nodeId: graphNodeId('Class', qualifiedName), + label: 'Class', + filePath, + startLine: 1, + endLine: 20, + protocols: extras.protocols ?? [], + ...(extras.superclass !== undefined ? { superclass: extras.superclass } : {}), + }; +} + +function protocolContainer(filePath, name) { + const qualifiedName = objcProtocolQualifiedName(name); + return { + kind: 'protocol', + declarationRole: 'interface', + name, + qualifiedName, + nodeId: graphNodeId('Protocol', qualifiedName), + label: 'Protocol', + filePath, + startLine: 1, + endLine: 4, + protocols: [], + }; +} + +function methodFact(filePath, owner, selector, methodKind, startLine) { + const qualifiedName = objcMethodQualifiedName(owner.qualifiedName, methodKind, selector); + return { + name: selector, + selector, + methodKind, + ownerQualifiedName: owner.qualifiedName, + ownerName: owner.name, + ownerKind: owner.kind, + qualifiedName, + nodeId: graphNodeId('Method', qualifiedName), + filePath, + startLine, + endLine: startLine, + declarationRole: 'implementation', + parameterTypes: [], + parameterNames: [], + }; +} + +function messageFact(method, selector, receiverKind, receiverText, extras = {}) { + return { + selector, + receiverText, + receiverKind, + sourceMethodQualifiedName: method.qualifiedName, + sourceMethodId: method.nodeId, + sourceOwnerQualifiedName: method.ownerQualifiedName, + sourceOwnerName: method.ownerName, + sourceMethodKind: method.methodKind, + filePath: method.filePath, + startLine: extras.startLine ?? method.startLine + 1, + startCol: extras.startCol ?? 2, + ...(extras.receiverType !== undefined ? { receiverType: extras.receiverType } : {}), + }; +} + +function parsedFile(filePath, facts) { + return Object.freeze({ + filePath, + moduleScope: 0, + scopes: Object.freeze([]), + parsedImports: Object.freeze([]), + localDefs: Object.freeze([]), + referenceSites: Object.freeze([]), + captureSideChannel: Object.freeze({ kind: 'objective-c', facts }), + }); +} + +function seedGraph(graph, factsList) { + for (const facts of factsList) { + graph.addNode({ + id: graphNodeId('File', facts.filePath), + label: 'File', + properties: { + name: facts.filePath, + qualifiedName: facts.filePath, + filePath: facts.filePath, + startLine: 1, + endLine: 1, + language: 'objective-c', + isExported: false, + }, + }); + for (const container of facts.containers) { + graph.addNode({ + id: container.nodeId, + label: container.label, + properties: { + name: container.name, + qualifiedName: container.qualifiedName, + filePath: facts.filePath, + startLine: container.startLine, + endLine: container.endLine, + language: 'objective-c', + isExported: true, + }, + }); + } + for (const method of facts.methods) { + graph.addNode({ + id: method.nodeId, + label: 'Method', + properties: { + name: method.selector, + qualifiedName: method.qualifiedName, + filePath: facts.filePath, + startLine: method.startLine, + endLine: method.endLine, + language: 'objective-c', + isExported: true, + }, + }); + } + } +} + +function summarize(graph) { + let calls = 0; + let extendsEdges = 0; + let implementsEdges = 0; + let protocolCandidateUses = 0; + for (const rel of graph.iterRelationships()) { + if (rel.type === 'CALLS') calls++; + else if (rel.type === 'EXTENDS') extendsEdges++; + else if (rel.type === 'IMPLEMENTS') implementsEdges++; + else if (rel.type === 'USES' && String(rel.reason).startsWith('objc-protocol-candidate:')) { + protocolCandidateUses++; + } + } + return { + calls, + extends: extendsEdges, + implements: implementsEdges, + protocol_candidate_uses: protocolCandidateUses, + }; +} + +function spreadCorpus(classCount) { + const factsList = []; + const parsedFiles = []; + for (let i = 0; i < classCount; i++) { + const filePath = `src/Class${i}.m`; + const name = `Class${i}`; + const sibling = `Class${(i + 1) % classCount}`; + const owner = classContainer(filePath, name, i === 0 ? {} : { superclass: 'Class0' }); + const run = methodFact(filePath, owner, 'run', '-', 4); + const tick = methodFact(filePath, owner, 'tick', '-', 8); + const messages = [ + messageFact(tick, 'run', 'self', 'self', { startLine: 9, startCol: 2 }), + messageFact(tick, 'run', 'local', 'sib', { + startLine: 10, + startCol: 2, + receiverType: { kind: 'class', name: sibling, raw: `${sibling} *` }, + }), + ]; + if (i > 0) { + messages.push(messageFact(tick, 'run', 'super', 'super', { startLine: 11, startCol: 2 })); + } + const facts = { + ...emptyFacts(filePath), + containers: [owner], + methods: [run, tick], + messages, + }; + factsList.push(facts); + parsedFiles.push(parsedFile(filePath, facts)); + } + return { factsList, parsedFiles }; +} + +function protocolCorpus(classCount) { + const factsList = []; + const parsedFiles = []; + const protocolPath = 'src/Runnable.h'; + const protocol = protocolContainer(protocolPath, 'Runnable'); + const protocolRun = methodFact(protocolPath, protocol, 'run', '-', 2); + const protocolFacts = { + ...emptyFacts(protocolPath), + containers: [protocol], + methods: [protocolRun], + }; + factsList.push(protocolFacts); + parsedFiles.push(parsedFile(protocolPath, protocolFacts)); + + for (let i = 0; i < classCount; i++) { + const filePath = `src/Class${i}.m`; + const owner = classContainer(filePath, `Class${i}`, { protocols: ['Runnable'] }); + const run = methodFact(filePath, owner, 'run', '-', 4); + const tick = methodFact(filePath, owner, 'tick:', '-', 8); + const facts = { + ...emptyFacts(filePath), + containers: [owner], + methods: [run, tick], + messages: [ + messageFact(tick, 'run', 'local', 'runner', { + startLine: 9, + startCol: 2, + receiverType: { kind: 'protocol', name: 'Runnable', raw: 'id' }, + }), + ], + }; + factsList.push(facts); + parsedFiles.push(parsedFile(filePath, facts)); + } + return { factsList, parsedFiles }; +} + +function run(shape, classCount) { + const corpus = shape === 'spread' ? spreadCorpus(classCount) : protocolCorpus(classCount); + const graph = createKnowledgeGraph(); + seedGraph(graph, corpus.factsList); + objectiveCScopeResolver.emitPostResolutionEdges(graph, corpus.parsedFiles); + return summarize(graph); +} + +function measure(shape, classCount) { + run(shape, classCount); + let bestMs = Infinity; + let counts = { calls: 0, extends: 0, implements: 0, protocol_candidate_uses: 0 }; + for (let i = 0; i < REPS; i++) { + const start = performance.now(); + counts = run(shape, classCount); + bestMs = Math.min(bestMs, performance.now() - start); + } + return { + classes: classCount, + ...counts, + min_ms: Number(bestMs.toFixed(2)), + }; +} + +function scalingReport(small, large) { + const workloadRatio = large.classes / small.classes; + const scalingRatio = Number((large.min_ms / Math.max(small.min_ms, 0.01)).toFixed(3)); + return { + small, + large, + workload_ratio: workloadRatio, + scaling_ratio: scalingRatio, + linear_factor: Number((scalingRatio / workloadRatio).toFixed(3)), + }; +} + +const report = { + spread: scalingReport(measure('spread', SMALL_CLASSES), measure('spread', LARGE_CLASSES)), + protocol: scalingReport(measure('protocol', SMALL_CLASSES), measure('protocol', LARGE_CLASSES)), +}; + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +const baseline = JSON.parse(readFileSync(join(HERE, 'baseline.json'), 'utf8')); +const failures = []; +const requirePositiveNumber = (path, value) => { + if (typeof value !== 'number' || !Number.isFinite(value) || !(value > 0)) { + failures.push(`${path}: expected a finite positive number, got ${JSON.stringify(value)}`); + return false; + } + return true; +}; + +for (const shape of ['spread', 'protocol']) { + const measured = report[shape]; + const expected = baseline[shape]; + for (const arm of ['small', 'large']) { + for (const key of ['classes', 'calls', 'extends', 'implements', 'protocol_candidate_uses']) { + if (measured[arm][key] !== expected[arm][key]) { + failures.push( + `${shape}.${arm}.${key}: expected ${expected[arm][key]}, got ${measured[arm][key]}`, + ); + } + } + if ( + requirePositiveNumber(`${shape}.${arm}.ms_budget`, expected[arm].ms_budget) && + measured[arm].min_ms > expected[arm].ms_budget + ) { + failures.push( + `${shape}.${arm}.min_ms ${measured[arm].min_ms} exceeds budget ${expected[arm].ms_budget}`, + ); + } + } +} + +if ( + requirePositiveNumber('spread.linear_scaling_slack', baseline.spread.linear_scaling_slack) && + report.spread.linear_factor > baseline.spread.linear_scaling_slack +) { + failures.push( + `spread.linear_factor ${report.spread.linear_factor} exceeds slack ` + + `${baseline.spread.linear_scaling_slack} (runtime ${report.spread.scaling_ratio}x ` + + `for ${report.spread.workload_ratio}x work)`, + ); +} + +if ( + requirePositiveNumber('protocol.linear_scaling_slack', baseline.protocol.linear_scaling_slack) && + report.protocol.linear_factor > baseline.protocol.linear_scaling_slack +) { + failures.push( + `protocol.linear_factor ${report.protocol.linear_factor} exceeds slack ` + + `${baseline.protocol.linear_scaling_slack} (runtime ${report.protocol.scaling_ratio}x ` + + `for ${report.protocol.workload_ratio}x work)`, + ); +} + +console.log(JSON.stringify(report, null, 2)); +if (failures.length > 0) { + console.error('[objective-c-resolution --check] FAIL'); + for (const failure of failures) console.error(` - ${failure}`); + process.exit(1); +} +console.log('[objective-c-resolution --check] PASS'); diff --git a/gitnexus/bench/ruby-gem-resolution/baseline.json b/gitnexus/bench/ruby-gem-resolution/baseline.json new file mode 100644 index 000000000..8c618ad00 --- /dev/null +++ b/gitnexus/bench/ruby-gem-resolution/baseline.json @@ -0,0 +1,61 @@ +{ + "_what": "Synthetic Ruby manifest + ScopeResolver gate for #3096, following bench/parse-dispatch-rounds/baselines.json: exact correctness floors and fingerprints, with ratios as the only timing gates. Calls the real config loader and resolver; parser, DB, installed gems and full-pipeline memory are outside this microbenchmark.", + "_triage": "Shape and fingerprint failures are deterministic: inspect the fixture and resolved targets, never rebaseline just to make CI green. For a timing failure, rerun on an idle machine and verify the report uses 15 samples after 2 warmups before investigating a regression. Millisecond observations are context, not gates.", + "shapes": { + "small": { + "projects": 32, + "declarations_per_project": 12, + "scopes": 64, + "files": 131, + "imports": 8192, + "resolved": 3264, + "fingerprint": "e37f4c81bc7f4bc8416732d0e8ebb00e16743b08a44646e1a6fa45c04283cfb3" + }, + "large": { + "projects": 128, + "declarations_per_project": 12, + "scopes": 256, + "files": 515, + "imports": 32768, + "resolved": 13056, + "fingerprint": "2524217404b1d4cf118e7d13895d757c821029bf59230b6c3cc7ce2fc2cf5dd2" + }, + "dense": { + "projects": 32, + "declarations_per_project": 132, + "scopes": 64, + "files": 131, + "imports": 8192, + "resolved": 3264, + "fingerprint": "e37f4c81bc7f4bc8416732d0e8ebb00e16743b08a44646e1a6fa45c04283cfb3" + } + }, + "_shape_note": "Exact projects, declarations, scopes, files, imports and resolved counts are the workload FLOOR, not ceilings. They prevent a shrunken corpus or disconnected manifest loader from passing by doing less work. Small/large grow projects 4x; dense keeps files/imports fixed while growing extra Gemfile declarations from 8 to 128 per project.", + "_fingerprint_note": "Hashes pin every importer/require/target tuple, including external decoys, aliases, local path gem hits and misses, relative/local imports and sibling isolation. The runner also checks each expected target before hashing and proves the external decoy is reachable without config. An always-null resolver or missing config must fail, not get a new fingerprint.", + "budgets": { + "load_scaling_ratio": 2.2, + "resolve_scaling_ratio": 2.2, + "dependency_count_ratio": 2 + }, + "_scaling_note": "load_scaling_ratio and resolve_scaling_ratio are (t_4n/t_n)/4: about 1 is linear, about 4 is quadratic. Keep the existing 2.2 regression budgets; do not tighten them to local milliseconds. Filesystem loading and lookup are measured separately with fresh config/file Sets per pass.", + "_dependency_count_note": "dense.resolve_ms/small.resolve_ms should stay near 1: require lookup must scale with prefix/path depth, not all declared gems. The budget remains 2. Extra declaration parsing belongs to config loading, not this fixed-query lookup arm.", + "_negative_controls_note": "Disconnecting loadResolutionConfig fails the real-loader assertion. A wrapper scanning all scopes' external prefixes on every import leaves targets/fingerprints unchanged but fails resolve_scaling_ratio (3.427 > 2.2) and dependency_count_ratio (7.912 > 2) with 15 samples. These are deliberate regressions, not baseline observations.", + "_measured": { + "environment": "macOS arm64 / Node 25.8.0, 2026-09-10", + "load_scaling_ratio": 1.094, + "load_scaling_ratio_samples": [0.993, 1.06, 1.05, 1.019, 1.094], + "resolve_scaling_ratio": 1.066, + "resolve_scaling_ratio_samples": [1.058, 1.059, 1.056, 1.066, 1.061], + "dependency_count_ratio": 1.009, + "dependency_count_ratio_samples": [1.009, 1.003, 0.994, 0.999, 1.0], + "small_load_ms": 3.569, + "small_resolve_ms": 11.782, + "large_load_ms": 14.812, + "large_resolve_ms": 50.222, + "dense_load_ms": 7.27, + "dense_resolve_ms": 11.852, + "reps": 15, + "warmup": 2 + }, + "_measured_note": "Five consecutive local runs using the min-of-15 estimator from parse-dispatch-rounds, with 2 warmups here. Scalar ratios and millisecond values are per-metric maxima across those runs (rounded), not one combined run. Budgets retain about 2x headroom above observed ratios. Milliseconds are diagnostic context only, not gated or real-repository speed claims. Exact shapes and fingerprints are unchanged." +} diff --git a/gitnexus/bench/ruby-gem-resolution/measure.mjs b/gitnexus/bench/ruby-gem-resolution/measure.mjs new file mode 100644 index 000000000..8f7ef7875 --- /dev/null +++ b/gitnexus/bench/ruby-gem-resolution/measure.mjs @@ -0,0 +1,184 @@ +/** + * Ruby gem-boundary correctness and cost gate (#3096). + * + * Calls the production ScopeResolver hooks, including the filesystem-backed + * configuration loader. Fixture creation and correctness hashing are untimed. + * Each timed pass reloads config and owns a fresh file Set, so neither the + * manifest load nor the per-pass fallback index is hidden by a warm memo. + * + * Small/large grow sibling projects, files, declarations and imports together. + * Dense keeps projects/imports fixed and grows extra declarations from 8 to 128: + * lookup must depend on require/path depth, not the number of declared gems. + * + * node --import tsx bench/ruby-gem-resolution/measure.mjs [--check] + */ +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { rubyScopeResolver } from '../../src/core/ingestion/languages/ruby/scope-resolver.ts'; + +const baselinePath = fileURLToPath(new URL('./baseline.json', import.meta.url)); +const CHECK = process.argv.includes('--check'); +const WARMUP = 2; +const REPS = 15; +const IMPORTS_PER_PROJECT = 256; +const roots = []; + +function corpus(projects, gemsPerProject = 8) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-ruby-gems-bench-')); + roots.push(root); + const files = ['decoy/lib/generators.rb', 'decoy/lib/types.rb', 'decoy/lib/missing.rb']; + const queries = []; + for (let i = 0; i < projects; i++) { + const directory = `packages/pkg${i}`; + const onDisk = path.join(root, directory); + fs.mkdirSync(path.join(onDisk, 'engine'), { recursive: true }); + fs.writeFileSync( + path.join(onDisk, 'Gemfile'), + [ + "gem 'rails'", + "gem 'dry-types'", + "gem 'aliased', require: 'custom/entry'", + "gem 'my_engine', path: 'engine'", + ...Array.from({ length: gemsPerProject }, (_, gem) => `gem 'dependency_${i}_${gem}'`), + ].join('\n'), + ); + fs.writeFileSync( + path.join(onDisk, 'engine', 'my_engine.gemspec'), + "Gem::Specification.new do |s|\n s.name = 'my_engine'\n s.require_paths = ['lib']\nend\n", + ); + fs.writeFileSync( + path.join(onDisk, 'Gemfile.lock'), + 'GEM\n remote: https://rubygems.org/\n specs:\n locked_gem (1.0.0)\n', + ); + const from = `${directory}/app/main.rb`; + files.push( + from, + `${directory}/app/helper.rb`, + `${directory}/lib/local_${i}.rb`, + `${directory}/engine/lib/my_engine.rb`, + ); + const cases = [ + ['rails/generators', null], + ['dry/types', null], + ['custom/entry', null], + ['my_engine', `${directory}/engine/lib/my_engine.rb`], + ['my_engine/missing', null], + ['./helper', `${directory}/app/helper.rb`], + [`local_${i}`, `${directory}/lib/local_${i}.rb`], + ['locked_gem/missing', null], + [`dependency_${i}_0/missing`, null], + [`dependency_${(i + 1) % projects}_0/missing`, 'decoy/lib/missing.rb'], + ]; + for (let j = 0; j < IMPORTS_PER_PROJECT; j++) { + const [target, expected] = cases[j % cases.length]; + queries.push({ from, target, expected }); + } + } + return { root, projects, gemsPerProject, files, queries }; +} + +function resolve(query, pass) { + return rubyScopeResolver.resolveImportTarget(query.target, query.from, pass.files, pass.config); +} + +function prepare(input) { + return { + files: new Set(input.files), + config: rubyScopeResolver.loadResolutionConfig(input.root), + }; +} + +function correctness(input) { + const pass = prepare(input); + assert.ok(pass.config, 'the real manifest loader must supply configuration'); + const records = []; + let resolved = 0; + for (const query of input.queries) { + const answer = resolve(query, pass); + assert.equal(answer, query.expected, `${query.from}: ${query.target}`); + if (answer !== null) resolved++; + records.push(`${query.from}|${query.target}->${answer}`); + } + // The external decoy must actually be reachable without dependency evidence; + // otherwise an always-null resolver could make a negative-only gate pass. + assert.equal( + resolve( + { from: input.queries[0].from, target: 'rails/generators' }, + { files: pass.files, config: undefined }, + ), + 'decoy/lib/generators.rb', + ); + return { + projects: input.projects, + declarations_per_project: input.gemsPerProject + 4, + scopes: pass.config.scopesByDirectory.size, + files: input.files.length, + imports: records.length, + resolved, + fingerprint: crypto.createHash('sha256').update(records.join('\n')).digest('hex'), + }; +} + +function measure(input, expectedResolved) { + const loading = []; + const resolution = []; + for (let run = 0; run < WARMUP + REPS; run++) { + const files = new Set(input.files); + const loadStart = performance.now(); + const config = rubyScopeResolver.loadResolutionConfig(input.root); + const loadMs = performance.now() - loadStart; + const pass = { files, config }; + const start = performance.now(); + let resolved = 0; + for (const query of input.queries) if (resolve(query, pass) !== null) resolved++; + const resolveMs = performance.now() - start; + assert.equal(resolved, expectedResolved, 'timed pass must do the validated work'); + if (run >= WARMUP) { + loading.push(loadMs); + resolution.push(resolveMs); + } + } + // Like the neighboring resolver gates: min-of-N limits scheduler/GC noise. + return { load_ms: Math.min(...loading), resolve_ms: Math.min(...resolution) }; +} + +try { + const inputs = { small: corpus(32), large: corpus(128), dense: corpus(32, 128) }; + const shapes = {}; + const timings = {}; + for (const [name, input] of Object.entries(inputs)) { + shapes[name] = correctness(input); + timings[name] = measure(input, shapes[name].resolved); + } + const scale = inputs.large.projects / inputs.small.projects; + const metrics = { + load_scaling_ratio: timings.large.load_ms / timings.small.load_ms / scale, + resolve_scaling_ratio: timings.large.resolve_ms / timings.small.resolve_ms / scale, + dependency_count_ratio: timings.dense.resolve_ms / timings.small.resolve_ms, + }; + const report = { + shapes, + timings, + metrics, + estimator: { warmup: WARMUP, samples: REPS, statistic: 'minimum' }, + }; + console.log(JSON.stringify(report, null, 2)); + if (CHECK) { + const baseline = JSON.parse(fs.readFileSync(baselinePath, 'utf8')); + assert.deepEqual(shapes, baseline.shapes, 'Ruby workload/fingerprint drift'); + const failures = []; + for (const [name, budget] of Object.entries(baseline.budgets)) { + if (!Number.isFinite(metrics[name]) || metrics[name] > budget) { + failures.push(`${name}: ${metrics[name]} > ${budget}`); + } + } + assert.equal(failures.length, 0, failures.join('\n')); + console.log('[ruby-gem-resolution --check] PASS'); + } +} finally { + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); +} diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 272d081c0..94fc262f6 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -1,7 +1,9 @@ { "_comment": "Per-language baselines for bench/scope-capture/measure.mjs --check. fingerprint = order-independent sha256 over the lang-resolution/-* fixture corpus + a 20-entity synthetic source (correctness gate; re-baseline intentionally on a legitimate capture change). scaling_budget = max allowed (t800/t250)/(800/250); ~1.0 is linear, ~3.2 is quadratic. The synthetic source is now HERITAGE-BEARING for every language (each Entity extends/implements/embeds/uses-trait/conforms-to a shared base) so the #1951 @reference.inherits synth is gated at scale, not just the base capture loop. All languages thread the tree-sitter captured node instead of re-deriving it with findNodeAtRange(tree.rootNode,...) per match, so all are linear (go #1915, python #1918, ruby/php/rust/csharp #1951, java #1956).", "go": { - "fingerprint": "9c554a9d698a2b79fb419852daadca87b8aae88180cceabf9c8d82f3e3300f2e", + "fingerprint": "2e3099db962f82f9d8641707a8b2bcf6875dafec1f54904f2dec7ac65ba22e9f", + "_rebaselined_3190_ctor_qualified_name": "#3190: generic composite-literal constructors now carry @reference.qualified-name with the written pkg.Box[T] spelling so constructor fallback can tell a qualifier from a bare unique-name guess. DIGEST DRIFT ONLY: the tag is added to existing constructor matches (captureGroups unchanged on go-constructor-type-inference/cmd/main.go). Prior 990e4921a7ef0e6aa4d7da92b671de91fa2621cb27918adb27742740dc6ad902 -> 2e3099db962f82f9d8641707a8b2bcf6875dafec1f54904f2dec7ac65ba22e9f; capture_groups_fp 2579, fixture_count 122.", + "_rebaselined_3190": "Corrected go-method-enrichment/app.go to share package animal with its declarations; its capture count is now 16 rather than 15. No Go emitter change. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 3d4e32e7490c830516126e28931827949baa3594cb521f7a3d8dcfed95b6018a -> 57b3c55135af8d2af33b9a7c4bf89796a7bee5b5822b402a2dea91af7232cf4a; scaling 1.058 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: provider-owned callable assignment/copy/formal/argument/invoke facts with invocation/constructor-result suppression. Prior 09ecd94911b830f52fa8807560abcbd79f163d02a2072870c1a59297e9a326e1 -> 3d4e32e7490c830516126e28931827949baa3594cb521f7a3d8dcfed95b6018a; scaling 1.039 < 1.5.", @@ -137,8 +139,9 @@ "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0." }, "java": { - "fingerprint": "2bf47cc19b595a9889ac21ec0154c6ce6786271d68551f21d1bc14c626bcd4ff", + "fingerprint": "e22a753b9f59331898a3a27c43072b588c33639937b1f618f6c70ce04f072a9f", "scaling_budget": 1.5, + "_rebaselined_3190_ctor_qualified_name": "#3190: qualified constructors (`new pkg.Foo()`, `new pkg.Box()`) now copy @reference.call.constructor.qualified onto @reference.qualified-name so constructor fallback treats the written qualifier as precise. DIGEST DRIFT ONLY: the tag is added to existing F35 matches. Prior 2bf47cc19b595a9889ac21ec0154c6ce6786271d68551f21d1bc14c626bcd4ff -> e22a753b9f59331898a3a27c43072b588c33639937b1f618f6c70ce04f072a9f; capture_groups_fp 3586, fixture_count 209.", "_rebaselined_2935_synthetic_declarations": "PR #2935 review follow-up: synthesized Java anonymous classes and bodied enum constants now carry the presence-only @declaration.is-synthetic sidecar used to preserve source-written dispatch targets at the fanout cap. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE: the tag is attached to existing synthetic declaration matches; capture groups and fixture count remain 5755/18405, 3512, and 206. Prior 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5 -> 2e2150b4f4d64519e3f4c6d7a2c12259178d3117872203c904fab8cba96a694a; CI scaling 0.971 < 1.5.", "_rebaselined_2917_record_component_accessors": "#2917: every implicit Java record-component accessor now emits a component-bounded @scope.function plus @declaration.method/name/zero-arity/return-type metadata. The scope boundary prevents subsequent record-body references from being attributed to the accessor. Java was the only general language fingerprint to move; capture groups scale by exactly two per generated record component (small 5755 -> 6255, large 18405 -> 20005). Prior 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5 -> 901a66c7dc0f071eeef9e4864b2519e5b58a1a141a1f9a7817ea42f7ff70eafb; scaling 0.961 < 1.5. Re-measured after merging origin/main, which carries #2935's is-synthetic sidecar on top of the same corpus: 2e2150b4f4d64519e3f4c6d7a2c12259178d3117872203c904fab8cba96a694a -> 79dafc369eaeb7183ee8cc1149b1a6c21ad672c7e5b806fe8b0060e5a952c79a; scaling 1.085 < 1.5, capture groups 6255/20005, capture_groups_fp 3560, fixture_count 206 (unchanged by the merge).", "_rebaselined_2900_record_heritage": "#2900 review follow-up: the Java scale unit now includes a record implementing Marker, so the record-declaration @reference.inherits path is fingerprinted and exercised at scale. Prior b29e263524f55151dcb7cfc4c929d3d1d7bb360355cee4e832158f927857f663 -> 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5; scaling 1.042 < 1.5.", @@ -171,7 +174,8 @@ "capture_groups_fp": 680 }, "typescript": { - "fingerprint": "fed04ed1d5db112387781e405da208ae6b3ab803889773b0455be96f01b893ff", + "fingerprint": "60e75bbe846f7200005f12c4d95c32e82d9d3feae9f2fd594e484670762b82b0", + "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_2934_import_type_only": "#2934: `import-decomposer.ts` attaches a presence-only `@import.type-only` synthetic capture to specifiers `tsc` erases, so `check --cycles` can stop counting type-only edges as initialization cycles. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE \u2014 the tag is added to import matches that already existed, never a new match, the same shape as the #2747 receiver-chain rebaseline. Every count is unchanged: capture_groups_fp 2414, fixture_count 155, capture_groups_small/large 4503/14403 (those measure the SYNTHETIC scaling source, which has no imports at all). The fingerprint moves because `canonicalizeMatch` in measure.mjs hashes every TAG on every match, synthetics included, so one extra presence-only tag on an existing match rewrites that match's canonical string. Attribution is exact, not inferred: neutralizing ONLY the `m['@import.type-only'] = \u2026` assignment in import-decomposer.ts and re-running returns the fingerprint to c2fbf8a89e5686dd\u2026 byte-for-byte, so nothing else in the TypeScript capture stream moved. All 14 other languages report ok. Scaling 0.997 < 1.5. NOTE ON THE CONTROL: javascript did not move (2026993b\u2026, 43 fixtures), but it is a WEAK control here \u2014 `import type` is TypeScript-only syntax, so a JS corpus cannot express the construct and could not have drifted either way. It evidences no collateral damage, not the correctness of the TS change; the exact-attribution check above is what does that. Prior c2fbf8a89e5686dd1ff3659b20d41d8b05ebcc9790356e3653ee0c8ca5d365c8 -> f719163eb03a447c9e40ca316a905dd76cee82192a75a403df478ebbdc13e98f.", "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior 27f937bfb47d4bded316ea3c785ff659c8cd88a5761d928f113477a08c802c78 -> e05446620c5b80b7aae291cfdf32f693580fada2ae687124769b04a0c03bfe63; scaling 0.983 < 1.5.", @@ -197,7 +201,8 @@ "_rebaselined_1432_member_call_callee_name": "#1432 (Zig): the shared callable-flow reader no longer names a callee by simple name for a MEMBER call (`@callable-flow.direct-callee-name` requires a direct designator: `f(x)`, `ns.f(x)`), and a member call is a field-stored-callable invoke only when a MEMBER store (`o.f = handler`) or a declared callable-typed field is visible - a same-named plain binding no longer gates it. CAPTURE-EMISSION CHANGE, not fixture growth (fixture_count unchanged). Drift: `await svc.verify(token, ...)` (typescript-generic-calls/src/guest.ts, member call) and `initializer()(() => {...})` (typescript-hof-callbacks/src/store.ts, call-of-call) lose `direct-callee-name`. `await verifyToken(token, ...)` (admin.ts/auth.ts) KEEPS `direct-callee-name|verifyToken`: tree-sitter-typescript parses `await f(x)` as call_expression(function: await_expression(f), type_arguments, ...), and wrappedExpression now unwraps `await_expression` so the direct designator survives as it does for the un-awaited spelling. capture_groups_fp 2465 (unchanged). Prior 05d1dadd6c9ef35c74079fa50f341b1b36e4fb02c9a89dd1b59f32b7cfd5e633 -> fed04ed1d5db112387781e405da208ae6b3ab803889773b0455be96f01b893ff." }, "javascript": { - "fingerprint": "2026993b81b873839dd2ef8797d9c14d9c48516b2b57b05ac17d8d43f2f4eba3", + "fingerprint": "b916c7072b30d09b4949803810604830ea1aca1cfdda0d9d9312f7cb22f7a10a", + "_rebaselined_3190": "Capture matches now retain explicit ESM export/private evidence, including synthesized default HOCs; CommonJS surfaces remain undecided. Capture group counts unchanged. Scaling budget unchanged.", "scaling_budget": 1.5, "_rebaselined_callable_flow_2522_review": "PR #2522 review hardening: callable operands retain expression/qualified identity and formals retain signature metadata. Prior b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3 -> 479927409bbdd9852a36172c8260aa56df260e99129a7a9c20a0d1903dd5538b; scaling 1.050 < 1.5.", "_rebaselined_callable_flow_2522_followup": "PR #2522 follow-up: lexical callable bindings, direct-callee argument metadata, and invocation-result suppression. Prior 917a9cd975ba035bdad71fdb70cd72eeddec58c25797e5a1addfa6172808a55c -> b59fe8135b6a31a12bc3f872b224054b16592588153ae3661d03958d787c76f3; scaling 1.093 < 1.5.", diff --git a/gitnexus/bench/value-ref-resolution/baseline.json b/gitnexus/bench/value-ref-resolution/baseline.json new file mode 100644 index 000000000..7b23bcae5 --- /dev/null +++ b/gitnexus/bench/value-ref-resolution/baseline.json @@ -0,0 +1,37 @@ +{ + "_what": "Baselines for bench/value-ref-resolution/measure.mjs --check (#3399). Guards the resolved-target SET of `resolveValueRefTarget` and the per-site cost of its four channels across a 4x file-count step, over a synthetic Zig corpus whose shape is fixed in measure.mjs. Per module: 12 CONTAINER registrations (`ElementN.getJ`), 1 NAMESPACE (`dom_utils.compare`), 1 HUB (`hub.compare`), 1 BARE (`register(onTick)`), and 2 declines (a non-callable namespace member `dom_utils.DEFAULT_NS`, a non-callable bare argument `Bridge(ElementN)`) — 17 sites, 15 resolved.", + + "_triage": "READ THIS BEFORE RE-RUNNING. modules, files, value_ref_sites, resolved, declined and fingerprint are DETERMINISTIC: a re-run never changes them, and none may be re-baselined to make CI green — drift means the resolved target set moved, which is a behaviour change to explain. linear_scaling_budget is the only timing arm; runner contention dominates it, so re-run alone on an idle machine and read `reps` in the report before investigating. If exactly one arm fails and it is that one, suspect the machine.", + + "small": { + "modules": 80, + "files": 240, + "value_ref_sites": 1360, + "resolved": 1200, + "declined": 160, + "fingerprint": "4ca2933e25cfa9c84518fca6608c9a6f318f81a1c5bad63cb2f09b7394bfb792" + }, + "large": { + "modules": 320, + "files": 960, + "value_ref_sites": 5440, + "resolved": 4800, + "declined": 640, + "fingerprint": "249266f180388e9f1240a7e81f1141afbc51ef25d5c4b3286a681992dab69e61" + }, + + "linear_scaling_budget": 1.6, + "_linear_scaling_note": "(t_large/t_small)/(320/80) over the per-site resolution loop; ~1.0 is linear. A RATIO rather than a millisecond ceiling, and there is deliberately no ms gate at all: wall-clock measures the runner, and this repo has been bitten twice by a fixed budget (bench/callable-value-flow's widening_overhead failed at 2.07 and 1.975 against 1.9 on a shared runner while the code was correct, both on a sub-11ms measurement). Same reasoning, same shape as bench/parse-dispatch-rounds' pack_scaling_budget. Budget is 1.6 — 1.40x the measured maximum, matching the ~1.5x its siblings use on ratios. What it catches: the four channels each consult a wider index than the last, and channel 4 (CONTAINER) reaches `scopes.qualifiedNames`, a WORKSPACE-WIDE index — keyed it is O(1) per site, scanned it is O(names) per site. Verified load-bearing rather than assumed: replacing `QualifiedNameIndex.get` with a full scan (in gitnexus-shared/dist — the bench resolves the built package, so patching src changes nothing) takes the factor from ~1.0 to 2.07, well clear of this budget. Only 1 of the 17 sites per module reaches that fallback (the `dom_utils.DEFAULT_NS` decline, whose module receiver is not class-like), which is why the signal is 2.07 and not the ~4 a per-site scan on every site would give.", + + "_measured": { + "linear_factor": 1.146, + "linear_factor_samples": [ + 0.907, 0.969, 0.988, 1.03, 1.043, 1.048, 1.05, 1.051, 1.052, 1.107, 1.123, 1.146 + ], + "small_ms": 2.078, + "large_ms_4x": 9.197, + "us_per_site": "1.36-1.53 small / 1.37-1.69 large", + "reps": 15 + }, + "_measured_note": "Maxima over 12 runs on a box that was NOT idle, so the spread is an upper bound on real noise. small_ms / large_ms_4x / us_per_site are recorded for context only — NOTHING gates on them, because an absolute millisecond is exactly the gate this file avoids. `us_per_site` staying flat between the two arms is the same property linear_scaling_budget gates, read directly." +} diff --git a/gitnexus/bench/value-ref-resolution/measure.mjs b/gitnexus/bench/value-ref-resolution/measure.mjs new file mode 100644 index 000000000..a1c14f448 --- /dev/null +++ b/gitnexus/bench/value-ref-resolution/measure.mjs @@ -0,0 +1,303 @@ +#!/usr/bin/env node +/** + * Build-free scaling and correctness guard for callable-value reference + * resolution (#3399). + * + * WHAT IS GUARDED. `resolveValueRefTarget` is the per-site half of + * `emitPropertyDispatchCalls`: for every `value-ref` reference site it names the + * callable the source handed over as a value. #3399 replaced a single lexical + * walk with four channels, and each one reaches for a wider index than the last: + * + * 1. BARE `register(onTick)` — `findCallableBindingInScope` + * 2. NAMESPACE `bridge.accessor(utils.compare, …)` — the file's own + * namespace `@import` edges, then the target's module scope + * 3. HUB `bridge.accessor(hub.compare, …)` — the same, through the + * finalized/augmented channel a re-export publishes into + * 4. CONTAINER `bridge.accessor(Element.getNamespaceUri, …)` — + * `findClassBindingInScope`, whose miss path falls back to + * `scopes.qualifiedNames`, a WORKSPACE-WIDE index + * + * Channel 4 is why this bench exists. A workspace-wide index consulted per site + * is linear only while the lookup is keyed; make it a scan — or make any of the + * three guards around it (`isOwnerNameShadowedBySomethingElse`, + * `isNamespaceNameShadowed`, `findOwnedMember`) walk a collection that grows + * with the repo — and a registration table that costs O(sites) today costs + * O(sites x files) tomorrow. That regression is invisible on a fixture and + * expensive on lightpanda-io/browser, where `bridge.{accessor,function,…}` + * appears 2,047 times across 257 files. + * + * HOW. Two corpora of identical shape, 4x apart in file count, and the per-site + * resolution loop is the ONLY thing timed — extraction, ownership reconciliation + * and finalize are setup. `linear_factor` is `(t_large/t_small) / (N_large/N_small)`: + * ~1.0 linear, ~4.x quadratic on this 4x step. + * + * A RATIO IS THE ONLY TIMING GATE — no millisecond ceiling, deliberately. + * `min_ms` and `us_per_site` are printed for context and nothing compares them + * to anything: a wall-clock budget measures the runner, and this repo has been + * bitten by that twice already (`bench/callable-value-flow`'s `widening_overhead` + * failed at 2.07 and 1.975 against a 1.9 budget on a shared runner while the + * code was correct, both times on a sub-11ms measurement). Dividing the large + * arm by the small one divides the machine out, which is what + * `bench/parse-dispatch-rounds` settled on for the same reason. + * + * A timing gate alone would be satisfied by a fast wrong answer, so the + * correctness half is exact and comes first: the site/resolved/declined counts + * per arm, plus an order-independent sha256 over every (site -> resolved target) + * pair. The fingerprint is a CORRECTNESS gate — drift means the resolved target + * set moved, which is a behaviour change to be explained, never re-baselined to + * make CI green. + * + * WHY A ZIG CORPUS for a language-neutral pass. Zig is the only language whose + * provider sets `namespaceExportsIncludeImportedNames`, so it is the only one + * that can exercise channel 3 at all; and the file-as-struct idiom puts channels + * 2 and 4 in one file, which is the shape #3399 was filed over. The corpus also + * carries two DECLINE controls — a non-callable namespace member and a + * non-callable bare argument — so a change that widened the callable gate would + * move `declined` rather than hiding inside the timing. + * + * Container naming is load-bearing in the corpus: a Zig file-as-struct is minted + * under the FILE STEM, so `ElementN.zig` must write `const ElementN = @This();` + * and register `ElementN.getJ`. Spelling the alias `Element` instead is the + * documented `@This()`-alias limitation, every channel-4 site declines, and the + * bench would time a corpus that resolves nothing. + * + * Usage: + * node --import tsx bench/value-ref-resolution/measure.mjs + * node --import tsx bench/value-ref-resolution/measure.mjs --check + */ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; + +import { extractParsedFile } from '../../src/core/ingestion/scope-extractor-bridge.ts'; +import { finalizeScopeModel } from '../../src/core/ingestion/finalize-orchestrator.ts'; +import { createSemanticModel } from '../../src/core/ingestion/model/semantic-model.ts'; +import { reconcileOwnership } from '../../src/core/ingestion/scope-resolution/pipeline/reconcile-ownership.ts'; +import { resolveValueRefTarget } from '../../src/core/ingestion/scope-resolution/passes/property-dispatch.ts'; +import { zigScopeResolver } from '../../src/core/ingestion/languages/zig/scope-resolver.ts'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SMALL_MODULES = 80; +const LARGE_MODULES = 320; +/** Registrations per module through the CONTAINER channel. */ +const ACCESSORS_PER_MODULE = 12; +/** + * Min-of-N, and N is 15 rather than a handful: `bench/import-target` measured + * N=5 tripping its own budget about one run in twenty while N=15 held every + * language inside a 1.13-1.26x swing, and `bench/parse-dispatch-rounds` uses 15 + * on the same grounds. The whole run is ~5 s, so the reps are nearly free. + */ +const REPS = 15; + +/** + * One module = three files, mirroring `test/fixtures/lang-resolution/zig-idioms/ + * src/webapi/`: a namespace-only helper, a hub that re-exports one of its + * members and declares nothing, and a file-as-struct carrying the binding table. + */ +function moduleFiles(i) { + const accessors = Array.from( + { length: ACCESSORS_PER_MODULE }, + (_, j) => `pub fn get${j}(self: *Element${i}) u8 { return self._n; }`, + ).join('\n'); + const registrations = Array.from( + { length: ACCESSORS_PER_MODULE }, + (_, j) => ` pub const a${j} = bridge.accessor(Element${i}.get${j}, null, .{});`, + ).join('\n'); + return [ + { + path: `src/dom_utils${i}.zig`, + content: `pub const DEFAULT_NS: u8 = 7;\npub fn compare(a: u8, b: u8) u8 { return if (a > b) a else b; }\n`, + }, + { + path: `src/hub${i}.zig`, + content: `pub const compare = @import("dom_utils${i}.zig").compare;\n`, + }, + { + path: `src/Element${i}.zig`, + content: `const Element${i} = @This(); +const dom_utils = @import("dom_utils${i}.zig"); +const hub = @import("hub${i}.zig"); + +_n: u8 = 0, + +${accessors} + +fn onTick(self: *Element${i}) u8 { return self._n; } + +pub const JsApi = struct { + pub const bridge = Bridge(Element${i}); +${registrations} + pub const comparator = bridge.accessor(dom_utils.compare, null, .{}); + pub const hubbed = bridge.accessor(hub.compare, null, .{}); + pub const defaultNs = bridge.accessor(dom_utils.DEFAULT_NS, null, .{}); +}; + +pub fn boot() void { register(onTick); } + +pub fn register(comptime f: anytype) void { _ = f; } + +fn Bridge(comptime T: type) type { + _ = T; + return struct { + pub fn accessor(comptime g: anytype, comptime s: anytype, comptime o: anytype) u8 { + _ = g; + _ = s; + _ = o; + return 0; + } + }; +} +`, + }, + ]; +} + +/** + * Everything `resolveValueRefTarget` reads, built the way the pipeline builds it + * (`runScopeResolution` phases 1-2): real extraction through the Zig provider, + * `populateOwners`, `reconcileOwnership` into the SemanticModel, then finalize. + * Hand-assembling the indexes instead would pin this file's idea of their shape + * rather than the code's. + */ +function buildCorpus(modules) { + const parsedFiles = []; + for (let i = 0; i < modules; i++) { + for (const file of moduleFiles(i)) { + const parsed = extractParsedFile(zigScopeResolver.languageProvider, file.content, file.path); + if (parsed === undefined) { + throw new Error( + `scope extraction failed for ${file.path} — the vendored tree-sitter-zig ` + + `grammar is unavailable on this host, so this bench cannot run`, + ); + } + zigScopeResolver.populateOwners(parsed); + parsedFiles.push(parsed); + } + } + const model = createSemanticModel(); + reconcileOwnership(parsedFiles, model); + const allFilePaths = new Set(parsedFiles.map((p) => p.filePath)); + const scopes = finalizeScopeModel(parsedFiles, { + hooks: { + resolveImportTarget: (raw, from) => + zigScopeResolver.resolveImportTarget(raw, from, allFilePaths), + mergeBindings: (existing, incoming, scopeId) => + zigScopeResolver.mergeBindings(existing, incoming, scopeId), + expandsWildcardTo: (scope, files) => zigScopeResolver.expandsWildcardTo(scope, files), + }, + }); + return { parsedFiles, scopes, model }; +} + +/** The timed loop: every `value-ref` site, resolved exactly as the pass does. */ +function resolveAll({ parsedFiles, scopes, model }, pairs) { + let sites = 0; + let resolved = 0; + for (const parsed of parsedFiles) { + for (const site of parsed.referenceSites) { + if (site.kind !== 'value-ref') continue; + sites++; + const def = resolveValueRefTarget( + site, + parsed.filePath, + scopes, + model, + // The provider hook the pass is handed in `run.ts`; Zig sets it. + zigScopeResolver.namespaceExportsIncludeImportedNames === true, + ); + if (def === undefined) continue; + resolved++; + pairs?.push( + `${parsed.filePath}:${site.atRange.startLine}:${site.atRange.startCol}->${def.nodeId}`, + ); + } + } + return { sites, resolved }; +} + +function measure(modules) { + const corpus = buildCorpus(modules); + const pairs = []; + const counts = resolveAll(corpus, pairs); + let bestMs = Infinity; + for (let i = 0; i < REPS; i++) { + const start = performance.now(); + resolveAll(corpus, undefined); + bestMs = Math.min(bestMs, performance.now() - start); + } + // Order-independent: the walk order is an implementation detail, the resolved + // SET is the behaviour. + pairs.sort(); + return { + modules, + files: corpus.parsedFiles.length, + value_ref_sites: counts.sites, + resolved: counts.resolved, + declined: counts.sites - counts.resolved, + fingerprint: createHash('sha256').update(pairs.join('\n')).digest('hex'), + min_ms: Number(bestMs.toFixed(3)), + us_per_site: Number(((bestMs * 1000) / Math.max(counts.sites, 1)).toFixed(3)), + }; +} + +const report = { small: measure(SMALL_MODULES), large: measure(LARGE_MODULES) }; +report.reps = REPS; +report.workload_ratio = LARGE_MODULES / SMALL_MODULES; +report.scaling_ratio = Number( + (report.large.min_ms / Math.max(report.small.min_ms, 0.001)).toFixed(3), +); +report.linear_factor = Number((report.scaling_ratio / report.workload_ratio).toFixed(3)); + +if (!process.argv.includes('--check')) { + console.log(JSON.stringify(report, null, 2)); + process.exit(0); +} + +const baseline = JSON.parse(readFileSync(join(HERE, 'baseline.json'), 'utf8')); +const failures = []; +const requirePositiveNumber = (path, value) => { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + failures.push(`${path}: expected a finite positive number, got ${JSON.stringify(value)}`); + return false; + } + return true; +}; +for (const arm of ['small', 'large']) { + // Correctness first: counts AND the resolved-target set. + for (const key of [ + 'modules', + 'files', + 'value_ref_sites', + 'resolved', + 'declined', + 'fingerprint', + ]) { + if (report[arm][key] !== baseline[arm][key]) { + failures.push( + `${arm}.${key}: expected ${JSON.stringify(baseline[arm][key])}, got ${JSON.stringify(report[arm][key])}`, + ); + } + } + // `min_ms` / `us_per_site` are reported, never gated — see the header. +} +if ( + requirePositiveNumber('linear_scaling_budget', baseline.linear_scaling_budget) && + report.linear_factor > baseline.linear_scaling_budget +) { + failures.push( + `linear_factor ${report.linear_factor} exceeds budget ${baseline.linear_scaling_budget} ` + + `(runtime ${report.scaling_ratio}x for ${report.workload_ratio}x work; ` + + `~1.0 is linear). Re-run alone on an idle machine before investigating — ` + + `this is the only arm a busy runner can move.`, + ); +} + +console.log(JSON.stringify(report, null, 2)); +if (failures.length > 0) { + console.error('[value-ref-resolution --check] FAIL'); + for (const failure of failures) console.error(` - ${failure}`); + process.exit(1); +} +console.log('[value-ref-resolution --check] PASS'); diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index caf8a1df0..8710e2c55 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -3440,9 +3440,9 @@ "license": "MIT" }, "node_modules/hono": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.0.tgz", - "integrity": "sha512-jhunvfHWxd7J5EFfSgH4xsYJzSe/lfqbUCxiyyeaQasUsXeEHXtzVid+7EOGByc5JnFa23SSFL3Y2RV/z1T+eQ==", + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", + "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", "license": "MIT", "engines": { "node": ">=16.9.0" diff --git a/gitnexus/scripts/build-tree-sitter-grammars.cjs b/gitnexus/scripts/build-tree-sitter-grammars.cjs index 434cd193f..a30eb1ec2 100644 --- a/gitnexus/scripts/build-tree-sitter-grammars.cjs +++ b/gitnexus/scripts/build-tree-sitter-grammars.cjs @@ -4,7 +4,7 @@ * One registry-driven script replaces the former per-grammar * build-tree-sitter-.cjs files (they were ~95% identical). * - * The grammars (tree-sitter-c/dart/proto/swift/kotlin/zig) are loaded from + * The grammars (tree-sitter-c/objc/dart/proto/swift/kotlin/zig) are loaded from * `vendor//` by absolute path at runtime (see * src/core/tree-sitter/vendored-grammars.ts) and are NEVER copied into * node_modules — an undeclared package under node_modules is "extraneous" to @@ -25,8 +25,8 @@ * or exit non-zero — a failure for any single grammar must not break the install. * * Opt-out: GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (strict '1') skips the OPTIONAL - * grammars only. tree-sitter-c is REQUIRED (it backstops upstream's 4/6 ARM - * prebuild gap, #2116) and is always built. + * grammars only. tree-sitter-c and tree-sitter-objc are REQUIRED (C backstops upstream's 4/6 ARM + * prebuild gap, #2116) and are always built. * * Usage: * node build-tree-sitter-grammars.cjs # all grammars (postinstall) @@ -40,6 +40,7 @@ const { execSync } = require('child_process'); // grammars ignore the opt-out gate. Insertion order == build order (c first). const GRAMMARS = { c: { required: true, display: 'C', ext: '.c' }, + objc: { required: true, display: 'Objective-C', ext: '.m/.mm/.h' }, dart: { required: false, display: 'Dart', ext: '.dart' }, proto: { required: false, display: 'Proto', ext: '.proto' }, swift: { required: false, display: 'Swift', ext: '.swift' }, diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts index 2c52dd86b..0cd9fe8d9 100644 --- a/gitnexus/scripts/cross-platform-shard.ts +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -36,9 +36,9 @@ * * Only files heavy enough to matter are listed; everything else is carried by * {@link PER_FILE_OVERHEAD_SEC} alone. These are load-balancing hints, NOT - * assertions — no - * test asserts a runtime, and drift only makes the split slightly less even, so - * a stale entry is harmless and refreshing them is optional. Deliberately not + * assertions — no test measures elapsed time against this table. Missing or + * stale heavy entries can still overload a shard; refresh them from failed + * CI logs and replay that profile in the partition tests. Deliberately not * auto-generated: a committed table is reviewable and works offline, and the * alternative (timing files at CI runtime to decide the split) would make the * partition depend on the very machine load it is trying to protect against. @@ -49,13 +49,15 @@ export const WINDOWS_WEIGHTS_SEC: Readonly> = { 'test/integration/cli-e2e.test.ts': 621, 'test/integration/worker-pool.test.ts': 222, 'test/unit/incremental-vector-extension-ordering.test.ts': 87, - // ESTIMATE, not a measurement (#2841): this suite drives more full - // `runFullAnalysis` cycles than the VECTOR sibling above, so the 8 s - // PER_FILE_OVERHEAD floor would badly under-charge it and skew the Windows - // split — the failure mode that produced the job timeouts this table exists - // to prevent. Scaled from the sibling's measured 87 s by analyze-run count. - // Replace with a real figure after the first green Windows matrix run. - 'test/unit/incremental-index-extension-dml-gate.test.ts': 180, + // Measured on Windows in run 34014266125 (#3190, 2026-09-06). These DB + // suites landed together on shard 3: the old 180s estimate and missing + // entries made a ~27-minute recorded load look like an ~12-minute shard. + // Upstream speedups may reduce these figures; retaining conservative weights + // keeps the expensive suites distributed without changing the watchdog. + 'test/unit/incremental-index-extension-dml-gate.test.ts': 414, + 'test/integration/skills-e2e.test.ts': 444, + 'test/integration/fts-extension-e2e.test.ts': 146, + 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, 'test/integration/cli-limit-e2e.test.ts': 75, 'test/unit/hooks.test.ts': 26, 'test/integration/analyze-heap-oom-e2e.test.ts': 23, diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 79169214b..abf7de8c3 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -36,6 +36,18 @@ const PLATFORM_LOGIC = [ // must exercise the Windows backslash branch, so run it on the OS matrix (#2394). 'test/unit/cli-entry.test.ts', 'test/unit/platform-capabilities.test.ts', + // The tsconfig loader rebases `paths` targets through `path.resolve`, so the + // wildcard suffix it must recognise is `/*` on POSIX and `\*` on Windows. It + // only looked for `/*`, and every alias target came back as `src*` on + // Windows while the Ubuntu run stayed green — so this file has to run where + // the separator differs. + 'test/unit/tsconfig-index.test.ts', + // The unit half of the same rebasing rule. Fixture-free and pathApi-injectable + // (every separator assertion passes an explicit `path.win32` / `path.posix`), + // so unlike the fixture suite above it fails on EVERY runner when the + // normalisation is removed rather than only on windows-latest. Registered + // beside its fixture sibling so the two halves stay discoverable as one group. + 'test/unit/tsconfig-rebase-target.test.ts', // The gitnexus-plan safe writer resolves every name through a per-platform // backend: Linux anchors through /proc/self/fd, macOS resolves lexically and // verifies each step against descriptors it holds open. Publication is link(2) diff --git a/gitnexus/src/cli/doctor.ts b/gitnexus/src/cli/doctor.ts index 0c8e72e9e..aebfc7cf8 100644 --- a/gitnexus/src/cli/doctor.ts +++ b/gitnexus/src/cli/doctor.ts @@ -291,13 +291,15 @@ export const doctorCommand = async () => { console.log(` ${padDisplayEnd('', 18)}${remedy}`); } } - // Semantic mode follows the probe, not the platform: without a loadable - // VECTOR extension the index can be neither built nor queried, so search is - // really on exact scan no matter what the platform would allow. + // Doctor has no repository target, so this probe can report only whether the + // runtime can load VECTOR. It cannot claim that a repository has built the + // named HNSW index. Keep the capability and repository state distinct. console.log( - ` ${label('doctor.labels.semanticMode', 18)}${ - vectorProbe.loaded ? capabilities.semanticMode : 'exact-scan' - }`, + ` ${label('doctor.labels.semanticMode', 18)}${t( + vectorProbe.loaded + ? 'doctor.vectorCapability.indexUnverified' + : 'doctor.vectorCapability.exactScanOnly', + )}`, ); // Surface the optional-extension install policy so offline users can see // whether analyze/query will reach the network (extension.ladybugdb.com). diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts new file mode 100644 index 000000000..a54831c76 --- /dev/null +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -0,0 +1,170 @@ +import { lstat } from 'node:fs/promises'; +import path from 'node:path'; +import { cliInfo } from './cli-message.js'; +import { getGitRoot } from '../storage/git.js'; +import { acquireIndexLock } from '../storage/index-lock.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js'; +import { + closeLbug, + executeQuery, + executeWithReusedStatement, + fetchExistingEmbeddingHashes, + initLbug, +} from '../core/lbug/lbug-adapter.js'; +import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js'; +import { resolveEmbeddingIdentity } from '../core/embeddings/embedding-identity.js'; +import { + checkpointKind, + decideEmbeddingResume, + mintInterruptedCheckpoint, + mintPartialCheckpoint, + mintUnverifiedCountCheckpoint, + type EmbeddingCheckpoint, + type EmbeddingCheckpointProgress, +} from '../core/embedding-checkpoint.js'; +import { + measurePersistedEmbeddingCount, + persistedEmbeddingCountOrUndefined, +} from '../core/embedding-count.js'; + +/** Add missing embeddings directly to a healthy index, checkpointing periodically. */ +export const embeddingsSyncCommand = async (inputPath?: string): Promise => { + const repoPath = inputPath ? path.resolve(inputPath) : getGitRoot(process.cwd()); + if (!repoPath) throw new Error('Not inside a git repository. Pass a repository path.'); + + const { lbugPath, metaPath } = getStoragePaths(repoPath); + const metaDir = path.dirname(metaPath); + const lock = await acquireIndexLock(metaDir); + try { + const meta = await loadMeta(metaDir); + if (!meta) + throw new Error(`No GitNexus index found for ${repoPath}. Run gitnexus analyze first.`); + if (meta.incrementalInProgress) { + throw new Error('The structural index is incomplete. Run gitnexus analyze --force first.'); + } + + let lbugStat; + try { + lbugStat = await lstat(lbugPath); + } catch { + throw new Error( + `The LadybugDB graph store at ${lbugPath} is missing. Run gitnexus analyze first.`, + ); + } + if (!lbugStat.isFile()) { + throw new Error( + `The LadybugDB graph store at ${lbugPath} is not a usable database file. Run gitnexus analyze first.`, + ); + } + + const identity = resolveEmbeddingIdentity(); + let forceReembedNodeIds: ReadonlySet | undefined; + let resumedFrom: EmbeddingCheckpoint | undefined; + if (meta.embeddingCheckpoint) { + const checkpoint = meta.embeddingCheckpoint; + const decision = decideEmbeddingResume(checkpoint, identity); + if (decision.action === 'abort') throw new Error(decision.error); + const identityDiffers = + checkpoint.provider !== identity.provider || + checkpoint.model !== identity.model || + checkpoint.dimensions !== identity.dimensions; + // `abandon` on a non-interrupted foreign identity drops the pending set + // only. Existing rows stay; sync would then embed the holes under the new + // identity and mix vector spaces. Fail closed — rebuild via analyze. + if (identityDiffers && checkpointKind(checkpoint) !== 'unverified-count') { + throw new Error( + `Cannot sync embeddings: the index checkpoint was written by ${checkpoint.model} ` + + `(${checkpoint.provider}) at ${checkpoint.dimensions} dimensions, but this run ` + + `resolves ${identity.model} (${identity.provider}) at ${identity.dimensions}. ` + + 'Run `gitnexus analyze --embeddings --force` to rebuild under the new identity.', + ); + } + cliInfo(decision.log); + if (decision.action === 'resume') { + forceReembedNodeIds = decision.pendingNodeIds; + resumedFrom = decision.resumedFrom; + } + } + + await initLbug(lbugPath); + try { + const existing = await fetchExistingEmbeddingHashes(executeQuery); + let lastPercent = -1; + + const countEmbeddings = async (): Promise => + persistedEmbeddingCountOrUndefined(await measurePersistedEmbeddingCount(executeQuery)); + const saveCheckpoint = async ( + checkpoint: EmbeddingCheckpointProgress, + pendingNodeIds: string[], + embeddings?: number, + ): Promise => { + const latest = (await loadMeta(metaDir)) ?? meta; + await saveMeta(metaDir, { + ...latest, + ...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }), + embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds), + }); + }; + + cliInfo(`Embedding ${repoPath}`); + cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`); + + const result = await runEmbeddingPipeline( + executeQuery, + executeWithReusedStatement, + (progress) => { + const percent = Math.floor(progress.percent); + if (percent !== lastPercent && (percent % 5 === 0 || percent === 100)) { + lastPercent = percent; + cliInfo( + ` ${percent}% — ${progress.nodesProcessed ?? 0}/${progress.totalNodes ?? '?'} nodes`, + ); + } + }, + {}, + undefined, + existing && existing.size ? existing : undefined, + { + forceReembedNodeIds, + onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => { + await saveCheckpoint(checkpoint, nodeIds); + }, + onCheckpoint: async (checkpoint) => { + await saveCheckpoint(checkpoint, [], await countEmbeddings()); + }, + }, + ); + + const embeddings = await countEmbeddings(); + const latest = (await loadMeta(metaDir)) ?? meta; + if (embeddings === undefined) { + // Keep last-known stats.embeddings. An interrupted window marker would + // fail the identity gate on the next run even though this run finished; + // unverified-count is the recovery kind that forces a recount (#2790). + await saveMeta(metaDir, { + ...latest, + embeddingCheckpoint: result.failedNodeIds.length + ? mintPartialCheckpoint(identity, result, resumedFrom) + : mintUnverifiedCountCheckpoint(identity, { + nodesProcessed: result.nodesProcessed, + totalNodes: result.nodesProcessed, + chunksProcessed: result.chunksProcessed, + }), + }); + throw new Error('Could not verify persisted embedding count.'); + } + await saveMeta(metaDir, { + ...latest, + stats: { ...latest.stats, embeddings }, + embeddingCheckpoint: result.failedNodeIds.length + ? mintPartialCheckpoint(identity, result, resumedFrom) + : undefined, + }); + cliInfo(`Embeddings ready: ${embeddings}`); + } finally { + await closeLbug().catch(() => {}); + } + } finally { + lock.release(); + } +}; diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index b2f656c39..fc6af8e63 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -609,10 +609,20 @@ export function formatImpactResult(result: any): string { } // #1858 — an interface / indirection boundary on the path makes this a lower // bound; surface it so the count is not read as exhaustive. + // + // The header names no cause AND asserts no omitted caller, because it cannot + // know either. DI / dynamic dispatch was the only producer of `lower-bound` + // when this was written; #3399 added callables named in VALUE position (a + // registration table, a callback argument), and one of its producers is a + // probe that could not RUN — `callableValueReferenceBoundaries` hedges on a + // failed query and says in so many words that whether the symbol is + // registered is unknown. A header claiming "some callers are not traced" + // would there assert an omission nothing established, and would contradict + // the bullet printed directly under it. The bullets carry the cause — they + // are generated per-cause by `computeEpistemicBoundary` — so the header only + // has to say the count is a floor. if (result.epistemic === 'lower-bound') { - lines.push( - '⚠️ Lower bound — unresolved indirection on the path (callers binding via DI / dynamic dispatch are not traced; actual impact may be higher):', - ); + lines.push('⚠️ Lower bound — impact may be incomplete and actual impact may be higher:'); for (const b of result.boundaries || []) lines.push(` • ${b}`); } pushCallgraphRiskLines(lines, result); diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index a560fa23e..8fe3025fd 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -123,8 +123,10 @@ export const en = { 'doctor.labels.onnx': 'ONNX:', 'doctor.labels.graphStore': 'Graph store:', 'doctor.labels.fullTextSearch': 'Full-text search:', - 'doctor.labels.vectorIndex': 'VECTOR index:', - 'doctor.labels.semanticMode': 'Semantic mode:', + 'doctor.labels.vectorIndex': 'VECTOR extension:', + 'doctor.labels.semanticMode': 'Semantic support:', + 'doctor.vectorCapability.indexUnverified': 'vector-index capable (repository index not checked)', + 'doctor.vectorCapability.exactScanOnly': 'exact-scan only (VECTOR extension unavailable)', 'doctor.labels.exactScanLimit': 'Exact scan limit:', 'doctor.labels.note': 'Note:', 'doctor.labels.backend': 'Backend:', @@ -166,6 +168,8 @@ export const en = { 'error.watch.ambiguous': '`gitnexus watch` is ambiguous.\n Local working-tree incremental index: gitnexus analyze --watch\n Scheduled remote clone/pull + analyze: gitnexus auto-sync start\n', 'help.command.analyze.description': 'Index a repository (full analysis)', + 'help.command.embeddings.sync.description': + 'Add missing embeddings to an existing index, checkpointing periodically for safe resume', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', 'help.command.serve.description': 'Start local HTTP server for web UI connection', @@ -354,5 +358,5 @@ export const en = { 'help.identityCache.environment': '\nAnalyzer identity cache:\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir\n Operator-trusted persistent cache for warm cross-process status. The directory must pre-exist, be outside the GitNexus package/build roots, and contain no symlink or junction components. Defaults remain fail-closed on platforms without POSIX ownership APIs.', 'help.analyze.environment': - '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir Operator-trusted persistent analyzer identity cache; must pre-exist, be outside package/build roots, and contain no symlink/junction components.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N Max wait at pool shutdown for a retired worker still inside native code (terminated at its next safe point instead of aborting the process). Default 30000.\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning. Default 20000.\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', + '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir Operator-trusted persistent analyzer identity cache; must pre-exist, be outside package/build roots, and contain no symlink/junction components.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N Max wait at pool shutdown for a retired worker still inside native code (terminated at its next safe point instead of aborting the process). Default 30000.\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N Per-file wall-clock budget for C++ capture extraction; on breach the file keeps partial captures with a warning. Default 20000.\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 Retry per-attempt HTTP embedding timeouts through GITNEXUS_EMBEDDING_MAX_ATTEMPTS (default off; timeouts stay terminal).\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', } as const; diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 536b8ffb9..695acd72a 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -119,8 +119,10 @@ export const zhCN = { 'doctor.labels.onnx': 'ONNX:', 'doctor.labels.graphStore': '图存储:', 'doctor.labels.fullTextSearch': '全文搜索:', - 'doctor.labels.vectorIndex': '向量索引:', - 'doctor.labels.semanticMode': '语义模式:', + 'doctor.labels.vectorIndex': 'VECTOR 扩展:', + 'doctor.labels.semanticMode': '语义支持:', + 'doctor.vectorCapability.indexUnverified': '支持向量索引(未检查仓库索引)', + 'doctor.vectorCapability.exactScanOnly': '仅精确扫描(VECTOR 扩展不可用)', 'doctor.labels.exactScanLimit': '精确扫描上限:', 'doctor.labels.note': '说明:', 'doctor.labels.backend': '后端:', @@ -162,6 +164,8 @@ export const zhCN = { 'error.watch.ambiguous': '`gitnexus watch` 含义不明确。\n 本地工作区增量索引:gitnexus analyze --watch\n 定时远程 clone/pull 并分析:gitnexus auto-sync start\n', 'help.command.analyze.description': '索引仓库(完整分析)', + 'help.command.embeddings.sync.description': + '向现有索引添加缺失的嵌入,并定期保存检查点以安全续跑', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表(无需重新分析)', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', 'help.command.mcp.description': @@ -327,5 +331,5 @@ export const zhCN = { 'help.identityCache.environment': '\n分析器身份缓存:\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir\n 由操作员明确信任的持久缓存,用于跨进程快速查询状态。目录必须预先存在、位于 GitNexus 包/构建根目录之外,且路径中不得包含符号链接或 junction。缺少 POSIX 所有权 API 的平台默认保持故障关闭。', 'help.analyze.environment': - '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir 由操作员明确信任的持久分析器身份缓存;目录必须预先存在、位于包/构建根目录之外,且路径中不得包含符号链接或 junction。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N 线程池关闭时等待仍在原生代码中的已退役 worker 的最长时间(到达安全点后再终止,避免进程级 abort)。默认 30000。\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N C++ 捕获提取的每文件墙钟预算;超出后该文件保留部分捕获并输出警告。默认 20000。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离(0 < N <= 2;超出则钳制为 2)。MCP 默认 0.6,其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时,参数优先。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。', + '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_ANALYZER_IDENTITY_CACHE_DIR=/absolute/protected/dir 由操作员明确信任的持久分析器身份缓存;目录必须预先存在、位于包/构建根目录之外,且路径中不得包含符号链接或 junction。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_WORKER_SHUTDOWN_DRAIN_MS=N 线程池关闭时等待仍在原生代码中的已退役 worker 的最长时间(到达安全点后再终止,避免进程级 abort)。默认 30000。\n GITNEXUS_CPP_CAPTURE_BUDGET_MS=N C++ 捕获提取的每文件墙钟预算;超出后该文件保留部分捕获并输出警告。默认 20000。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1 将单次 HTTP 嵌入超时纳入 GITNEXUS_EMBEDDING_MAX_ATTEMPTS 重试(默认关闭,超时仍为终止错误)。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离(0 < N <= 2;超出则钳制为 2)。MCP 默认 0.6,其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时,参数优先。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。', } satisfies EnglishMessages; diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 70238dc00..fd1cb11c8 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -304,15 +304,13 @@ program .description('Install the latest published GitNexus globally (`npm i -g gitnexus@`).') .action(createLazyAction(() => import('./update.js'), 'updateCommand')); -program +const embeddings = program .command('embeddings') - .description('Manage the on-demand local embedding runtime') + .description(t('help.command.embeddings.description')); + +embeddings .command('install') - .description( - 'Install the local embedding stack (@huggingface/transformers + onnxruntime-node) on demand. ' + - 'Heals installs where npm skipped the optional packages (e.g. behind an HTTP proxy, #2370). ' + - 'Downloads only from your configured npm registry — mirrors and proxies apply.', - ) + .description(t('help.command.embeddings.install.description')) .option( '--cuda', "Also download the CUDA GPU binaries (runs onnxruntime-node's NuGet postinstall; " + @@ -321,6 +319,12 @@ program .option('--force', 'Install into the runtime prefix even when the stack already resolves') .action(createLazyAction(() => import('./embeddings.js'), 'embeddingsInstallCommand')); +embeddings + .command('sync [path]') + .description(t('help.command.embeddings.sync.description')) + .addHelpText('after', () => t('help.analyze.environment')) + .action(createLbugLazyAction(() => import('./embeddings-sync.js'), 'embeddingsSyncCommand')); + program .command('clean') .description('Delete GitNexus index for current repo') diff --git a/gitnexus/src/core/analysis-feature-registry.ts b/gitnexus/src/core/analysis-feature-registry.ts index 474c2157d..03887c29e 100644 --- a/gitnexus/src/core/analysis-feature-registry.ts +++ b/gitnexus/src/core/analysis-feature-registry.ts @@ -11,6 +11,7 @@ import { JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, SPRING_CONFIG_BINDINGS_FEATURE, } from './ingestion/languages/java/analysis-features.js'; +import { OBJECTIVE_C_PROVIDER_FEATURE } from './ingestion/languages/objective-c/analysis-features.js'; /** Production registry of independently versioned analysis capabilities. */ export const ANALYSIS_FEATURES = [ @@ -23,4 +24,5 @@ export const ANALYSIS_FEATURES = [ SPRING_CONFIG_BINDINGS_FEATURE, JAVA_ENUM_INTERFACE_HERITAGE_FEATURE, JAVA_RECORD_COMPONENT_ACCESSORS_FEATURE, + OBJECTIVE_C_PROVIDER_FEATURE, ] as const; diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts index 194f47313..b29ca80fb 100644 --- a/gitnexus/src/core/embeddings/ast-utils.ts +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -4,14 +4,13 @@ * used by both chunker.ts and structural-extractor.ts. */ -import { getLanguageFromFilename } from 'gitnexus-shared'; import { createParserForLanguage, isLanguageAvailable, resolveLanguageKey, } from '../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../tree-sitter/safe-parse.js'; -import { getProvider } from '../ingestion/languages/index.js'; +import { getLanguageForFileContent, getProvider } from '../ingestion/languages/index.js'; const parserCache = new Map(); @@ -20,7 +19,10 @@ const parserCache = new Map(); * Returns null if language is unavailable or parsing fails. */ export const ensureAndParse = async (content: string, filePath: string): Promise => { - const language = getLanguageFromFilename(filePath); + // Same classifier as ingest. Filename-only maps `.h` → C++, so Objective-C + // headers (and method snippets from those headers) would parse with the + // wrong grammar and miss class_interface / protocol_declaration / methods. + const language = getLanguageForFileContent(filePath, content); if (!language) return null; if (!isLanguageAvailable(language)) return null; @@ -100,6 +102,9 @@ export const findDeclarationNode = (root: any): any | null => { 'struct_item', 'interface_declaration', 'interface_definition', + 'protocol_declaration', // Objective-C protocol + 'class_interface', // Objective-C class, category, or extension + 'class_implementation', // Objective-C implementation 'enum_declaration', 'enum_item', 'type_declaration', // Go: type X struct diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts index 073abfb9c..4a06d7eda 100644 --- a/gitnexus/src/core/embeddings/chunker.ts +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -145,9 +145,35 @@ const DECLARATION_BODY_NODE_TYPES = new Set([ 'class_body', 'object_type', 'declaration_list', + 'implementation_definition', 'interface_body', ]); +const DIRECT_MEMBER_DECLARATION_TYPES = new Set([ + 'protocol_declaration', + 'class_interface', + // tree-sitter-objc exposes each implementation_definition directly under + // class_implementation, rather than grouping them in a shared body node. + 'class_implementation', +]); + +const DIRECT_MEMBER_HEADER_NODE_TYPES = new Set([ + 'identifier', + 'parameterized_arguments', + 'protocol_reference_list', + // tree-sitter-objc `_type_params` hides itself and exposes this named child + // for `(T)` generics and category-shaped argument lists. + 'generic_arguments', + // `_class_interface_header` / `_class_implementation_header` expose these + // named children before members (`NS_ROOT_CLASS @interface …`). + 'attribute_declaration', + 'attribute_specifier', + 'availability_attribute_specifier', + 'ms_declspec_modifier', + 'storage_class_specifier', + 'type_qualifier', +]); + const FIELD_LIKE_MEMBER_TYPES = new Set([ 'field_definition', 'public_field_definition', @@ -157,8 +183,18 @@ const FIELD_LIKE_MEMBER_TYPES = new Set([ 'lexical_declaration', 'pair', 'enum_assignment', + // tree-sitter-objc wraps each ivar as `instance_variable`, not field_definition. + 'instance_variable', ]); +const DECLARATION_MEMBER_WRAPPER_TYPES = new Set([ + 'qualified_protocol_interface_declaration', + 'instance_variables', +]); + +/** Named prefixes the ObjC grammar allows immediately before each ivar. */ +const IVAR_ATTRIBUTE_PREFIX_TYPES = new Set(['attribute_specifier', 'attribute_declaration']); + const declarationChunk = async ( content: string, filePath: string, @@ -337,6 +373,8 @@ const getDeclarationBodyNode = (node: any): any | null => { const bodyNode = node.childForFieldName?.('body'); if (bodyNode) return bodyNode; + if (DIRECT_MEMBER_DECLARATION_TYPES.has(node.type)) return node; + for (let i = 0; i < node.namedChildCount; i++) { const child = node.namedChild(i); if (!child) continue; @@ -352,15 +390,43 @@ const collectDeclarationUnits = ( ): Array<{ startIndex: number; endIndex: number }> => { const members: Array<{ startIndex: number; endIndex: number; groupable: boolean }> = []; - for (let i = 0; i < bodyNode.namedChildCount; i++) { - const child = bodyNode.namedChild(i); - if (!child) continue; - members.push({ - startIndex: child.startIndex, - endIndex: child.endIndex, - groupable: groupFields && FIELD_LIKE_MEMBER_TYPES.has(child.type), - }); - } + const collectMembers = ( + node: any, + skipHeaderChildren: boolean, + includeNodePrefixOnFirstMember = false, + ): void => { + const firstMemberIndex = members.length; + let ivarAttributePrefixStart: number | undefined; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child) continue; + if (DECLARATION_MEMBER_WRAPPER_TYPES.has(child.type)) { + collectMembers(child, false, true); + continue; + } + if (skipHeaderChildren && DIRECT_MEMBER_HEADER_NODE_TYPES.has(child.type)) continue; + if (node.type === 'instance_variables' && IVAR_ATTRIBUTE_PREFIX_TYPES.has(child.type)) { + ivarAttributePrefixStart ??= child.startIndex; + continue; + } + members.push({ + startIndex: ivarAttributePrefixStart ?? child.startIndex, + endIndex: child.endIndex, + groupable: groupFields && FIELD_LIKE_MEMBER_TYPES.has(child.type), + }); + ivarAttributePrefixStart = undefined; + } + + const firstMember = members[firstMemberIndex]; + if (includeNodePrefixOnFirstMember && firstMember) { + firstMember.startIndex = node.startIndex; + if (node.type === 'instance_variables') { + members[members.length - 1].endIndex = node.endIndex; + } + } + }; + + collectMembers(bodyNode, DIRECT_MEMBER_DECLARATION_TYPES.has(bodyNode.type)); if (members.length === 0) return []; diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index f3cda5f23..605bf8457 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -81,7 +81,7 @@ const ensureVectorExtensionAvailable = async (): Promise => { * invalidate existing vectors, such as metadata/header shape changes, * structural container context changes, or preceding-context formatting rules. */ -export const EMBEDDING_TEXT_VERSION = 'v4'; +export const EMBEDDING_TEXT_VERSION = 'v5'; /** * Compute a stable content fingerprint for an embeddable node. diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 7919b2376..bc7458ea1 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -39,6 +39,7 @@ interface HttpConfig { retryCapMs: number; minIntervalMs: number; timeoutMs: number; + retryTimeouts: boolean; requestDimensions?: number; } @@ -202,6 +203,7 @@ const readConfig = (): HttpConfig | null => { DEFAULT_HTTP_TIMEOUT_MS, MAX_HTTP_TIMEOUT_MS, ), + retryTimeouts: parseNonNegativeIntegerEnv('GITNEXUS_EMBEDDING_RETRY_TIMEOUTS', 0, 1) === 1, requestDimensions, }; }; @@ -338,6 +340,36 @@ class RetryableEmbeddingBodyError extends Error { } } +class RetryableEmbeddingTimeoutError extends Error { + constructor( + readonly timeoutMs: number, + options?: { cause?: unknown }, + ) { + super( + `Embedding request timed out after ${timeoutMs}ms`, + options?.cause !== undefined ? { cause: options.cause } : undefined, + ); + this.name = 'RetryableEmbeddingTimeoutError'; + } +} + +/** Re-wrap an opt-in TimeoutError so `resilientFetch` retries it. Abort stays terminal. */ +const throwIfRetryableTimeout = ( + err: unknown, + retryTimeouts: boolean, + callerAborted: boolean | undefined, + timeoutMs: number, +): void => { + if ( + retryTimeouts && + !callerAborted && + isTerminalNetworkError(err) && + err.name === 'TimeoutError' + ) { + throw new RetryableEmbeddingTimeoutError(timeoutMs, { cause: err }); + } +}; + /** * Build the message for a 2xx body carrying the wrong number of vectors. * @@ -384,6 +416,7 @@ const httpEmbedBatch = async ( retryCapMs = HTTP_RETRY_CAP_MS, minIntervalMs = 0, timeoutMs = DEFAULT_HTTP_TIMEOUT_MS, + retryTimeouts = false, ): Promise => { const requestBody: { input: string[]; model: string; dimensions?: number } = { input: batch, @@ -428,7 +461,13 @@ const httpEmbedBatch = async ( const signal = requestOptions.signal ? AbortSignal.any([requestOptions.signal, timeoutSignal]) : timeoutSignal; - const attemptResp = await globalThis.fetch(input, { ...init, signal }); + let attemptResp: Response; + try { + attemptResp = await globalThis.fetch(input, { ...init, signal }); + } catch (err) { + throwIfRetryableTimeout(err, retryTimeouts, requestOptions.signal?.aborted, timeoutMs); + throw err; + } // Non-OK bodies are none of our business: hand the response straight // back so `resilientFetch` keeps classifying 4xx/5xx/429 unchanged. if (!attemptResp.ok) return attemptResp; @@ -447,15 +486,19 @@ const httpEmbedBatch = async ( // Not every `.json()` rejection is a parse error: the per-attempt // signal (`AbortSignal.any([caller, AbortSignal.timeout(...)])`) is // wired to the body stream, so a stalled body rejects with the abort - // reason. Re-raise those untouched — `isTerminalNetworkError` is - // `resilientFetch`'s own predicate, so this test agrees with - // `classifyOutcome` by construction. Wrapping one would flip its - // verdict from `terminal-network` (returned without retry AND - // without touching the breaker, via `recordNeutral()`) to - // `retryable-network` (retried, then `breaker.recordFailure()`): the - // same timeout would take 3 attempts instead of 1, count toward the - // process-global `embeddings-http` breaker, and reach the operator as - // "unparseable response" so they never reach for the timeout knob. + // reason. Re-raise AbortError (and TimeoutError when retry is off) + // untouched — `isTerminalNetworkError` is `resilientFetch`'s own + // predicate, so this test agrees with `classifyOutcome` by + // construction. Wrapping one would flip its verdict from + // `terminal-network` (returned without retry AND without touching + // the breaker, via `recordNeutral()`) to `retryable-network` + // (retried, then `breaker.recordFailure()`): the same timeout would + // take 3 attempts instead of 1, count toward the process-global + // `embeddings-http` breaker, and reach the operator as "unparseable + // response" so they never reach for the timeout knob. + // Opt-in `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS=1` is the exception: + // TimeoutError is re-wrapped so the existing retry loop can retry it. + throwIfRetryableTimeout(err, retryTimeouts, requestOptions.signal?.aborted, timeoutMs); if (isTerminalNetworkError(err)) throw err; throw new RetryableEmbeddingBodyError(unparseableMessage(), { cause: err }); } @@ -503,6 +546,12 @@ const httpEmbedBatch = async ( if (err instanceof RetryableEmbeddingBodyError) { throw new HttpEmbeddingError(err.terminalMessage, { cause: err.cause }); } + if (err instanceof RetryableEmbeddingTimeoutError) { + throw new HttpEmbeddingError( + `${err.message} after ${maxAttempts} attempt(s) (${safeUrl(url)}, batch ${batchIndex})`, + { cause: err.cause }, + ); + } if (err instanceof CircuitOpenError) { throw new HttpEmbeddingError( `Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`, @@ -580,6 +629,7 @@ export const httpEmbed = async ( config.retryCapMs, config.minIntervalMs, config.timeoutMs, + config.retryTimeouts, ); // Defensive backstop, deliberately kept: `httpEmbedBatch` now rejects a @@ -655,6 +705,7 @@ export const httpEmbedQuery = async ( config.retryCapMs, config.minIntervalMs, config.timeoutMs, + config.retryTimeouts, ); // Defensive backstop like the `httpEmbed` one above: an empty `data` array is // now a cardinality mismatch (0 vectors for 1 text) rejected and retried diff --git a/gitnexus/src/core/embeddings/structural-extractor.ts b/gitnexus/src/core/embeddings/structural-extractor.ts index 7035ee569..8c003bbbd 100644 --- a/gitnexus/src/core/embeddings/structural-extractor.ts +++ b/gitnexus/src/core/embeddings/structural-extractor.ts @@ -5,7 +5,7 @@ * to extract method and field names for embedding text generation. */ -import { getProviderForFile } from '../ingestion/languages/index.js'; +import { getProviderForFileContent } from '../ingestion/languages/index.js'; import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js'; import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js'; import type { LanguageProvider } from '../ingestion/language-provider.js'; @@ -32,7 +32,7 @@ export const extractStructuralNames = async ( content: string, filePath: string, ): Promise => { - const provider = getProviderForFile(filePath); + const provider = getProviderForFileContent(filePath, content); if (!provider) return { methodNames: [], fieldNames: [] }; const tree = await ensureAndParse(content, filePath); diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index 71cad5d65..759e19812 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -8,6 +8,8 @@ export const LABEL_FUNCTION = 'Function' as const; export const LABEL_METHOD = 'Method' as const; export const LABEL_CONSTRUCTOR = 'Constructor' as const; export const LABEL_CLASS = 'Class' as const; +export const LABEL_PROTOCOL = 'Protocol' as const; +export const LABEL_CATEGORY = 'Category' as const; export const LABEL_INTERFACE = 'Interface' as const; export const LABEL_STRUCT = 'Struct' as const; export const LABEL_ENUM = 'Enum' as const; @@ -53,6 +55,8 @@ export const CHUNKABLE_LABELS = [ LABEL_METHOD, LABEL_CONSTRUCTOR, LABEL_CLASS, + LABEL_PROTOCOL, + LABEL_CATEGORY, LABEL_INTERFACE, LABEL_STRUCT, LABEL_ENUM, @@ -165,6 +169,20 @@ export const CHUNKING_RULES: Readonly = new Set([GLOBAL_NAME_FALLBACK_REASON]); + +/** True when this edge's target was guessed by name rather than resolved. */ +export const isHeuristicEdgeReason = (reason: string): boolean => + HEURISTIC_EDGE_REASONS.has(reason); diff --git a/gitnexus/src/core/graph/graph.ts b/gitnexus/src/core/graph/graph.ts index 1c708a4af..84323bf8e 100644 --- a/gitnexus/src/core/graph/graph.ts +++ b/gitnexus/src/core/graph/graph.ts @@ -163,9 +163,17 @@ export const createKnowledgeGraph = (): KnowledgeGraph => { relationshipMap.forEach(fn); }, forEachRelationshipFields( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ) { - relationshipMap.forEach((rel) => fn(rel.sourceId, rel.targetId, rel.type, rel.confidence)); + relationshipMap.forEach((rel) => + fn(rel.sourceId, rel.targetId, rel.type, rel.confidence, rel.reason), + ); }, getNode: (id: string) => nodeMap.get(id), diff --git a/gitnexus/src/core/graph/types.ts b/gitnexus/src/core/graph/types.ts index 9d9caf12b..a7a7e8a04 100644 --- a/gitnexus/src/core/graph/types.ts +++ b/gitnexus/src/core/graph/types.ts @@ -31,14 +31,26 @@ export interface KnowledgeGraph { * Zero-allocation relationship scan: fields, not objects (#2680). * * The whole-graph scans (the local-symbol pruner, community detection, - * process extraction) read only these four fields, and materializing a + * process extraction) read only these five fields, and materializing a * `GraphRelationship` per edge just to read them dominates iteration cost once * relationships are held columnar — measured at ~90 ms per analyze on a * million-edge graph. Prefer this over `forEachRelationship` in any pass that * walks every edge and needs no other field. + * + * `reason` is passed because confidence alone cannot separate a heuristic + * name guess from a resolved edge that happens to sit at the same number: + * the global-name fallback emits at exactly the process/community threshold + * (0.5), so the walks that must exclude it have to read the reason. See + * `GraphEmitSink`'s reason column, added for this consumer. */ forEachRelationshipFields: ( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ) => void; getNode: (id: string) => GraphNode | undefined; nodeCount: number; diff --git a/gitnexus/src/core/group/extractors/graphql-extractor.ts b/gitnexus/src/core/group/extractors/graphql-extractor.ts index efafe4e2e..550a06155 100644 --- a/gitnexus/src/core/group/extractors/graphql-extractor.ts +++ b/gitnexus/src/core/group/extractors/graphql-extractor.ts @@ -92,6 +92,134 @@ function unquote(text: string): string | null { return value.includes('${') ? null : value; } +const SIMPLE_TEMPLATE_ESCAPES: Record = { + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', + v: '\v', + '0': '\0', + "'": "'", + '"': '"', + '\\': '\\', + '`': '`', +}; + +/** Graph Method `startLine` is the 0-based wrapper row (see line-base.ts). */ +function providerMemberStartLine(member: Parser.SyntaxNode): number { + // Class-field arrows are `@declaration.property`; parse-worker falls back to + // the `public_field_definition` wrapper (decorator row when it is a child), + // not the initializer. Do not probe the `value` child. + return member.startPosition.row; +} + +/** tree-sitter `escape_sequence.text` is the source spelling (`\\n`), not the JS value. */ +function decodeEscapeSequence(text: string): string | null { + if (!text.startsWith('\\') || text.length < 2) return null; + const escaped = text.slice(1); + // Tagged-template cooked value: LineContinuation is empty; `\8`/`\9` and + // LegacyOctalEscapeSequence (`\1`–`\7`, `\00`…) make cooked undefined. + if (/^[\n\r\u2028\u2029]$/.test(escaped) || escaped === '\r\n') return ''; + if (/^[1-9]$/.test(escaped) || /^[0-7]{2,3}$/.test(escaped)) return null; + if (escaped.length === 1) return SIMPLE_TEMPLATE_ESCAPES[escaped] ?? escaped; + if (escaped[0] === 'x' && /^[0-9A-Fa-f]{2}$/.test(escaped.slice(1))) { + return String.fromCharCode(Number.parseInt(escaped.slice(1), 16)); + } + if (escaped.startsWith('u{') && escaped.endsWith('}')) { + const hex = escaped.slice(2, -1); + if (!/^[0-9A-Fa-f]{1,6}$/.test(hex)) return null; + const codePoint = Number.parseInt(hex, 16); + if (codePoint > 0x10ffff) return null; + return String.fromCodePoint(codePoint); + } + if (escaped[0] === 'u' && /^[0-9A-Fa-f]{4}$/.test(escaped.slice(1))) { + return String.fromCharCode(Number.parseInt(escaped.slice(1), 16)); + } + return null; +} + +function substitutionIdentifier(substitution: Parser.SyntaxNode): string | null { + if (substitution.namedChildren.length !== 1) return null; + const expr = unwrapExpression(substitution.namedChildren[0]); + return expr.type === 'identifier' ? expr.text : null; +} + +function isGraphqlTagCall(call: Parser.SyntaxNode): boolean { + const callee = call.childForFieldName('function'); + return callee?.type === 'identifier' && callee.text === 'gql'; +} + +function pascalCaseGraphqlName(name: string): string { + return name + .split(/[^A-Za-z0-9]+/) + .filter((part) => part.length > 0) + .map((part) => `${part[0]!.toUpperCase()}${part.slice(1)}`) + .join(''); +} + +type InterpolationCache = Map; + +function uniqueStaticSource( + name: string, + declarators: GeneratedSymbolIndex, + resolving: Set, + cache: InterpolationCache, +): string | null { + if (cache.has(name)) return cache.get(name) ?? null; + if (resolving.has(name) || resolving.size >= MAX_GRAPHQL_TRAVERSAL_DEPTH) return null; + const values = declarators.get(name) ?? []; + if (values.length === 0) { + cache.set(name, null); + return null; + } + resolving.add(name); + const sources = new Set(); + for (const value of values) { + const source = staticGraphqlSource(value, declarators, resolving, cache); + // A dynamic or unprovable sibling makes the name ambiguous — do not pick + // the one static spelling and ignore the rest. + if (source === null) { + resolving.delete(name); + cache.set(name, null); + return null; + } + sources.add(source); + } + resolving.delete(name); + const unique = sources.size === 1 ? [...sources][0]! : null; + cache.set(name, unique); + return unique; +} + +function interpolatedTemplateSource( + template: Parser.SyntaxNode, + declarators: GeneratedSymbolIndex, + resolving: Set, + cache: InterpolationCache, +): string | null { + let out = ''; + for (const child of template.namedChildren) { + if (child.type === 'string_fragment') { + out += child.text; + } else if (child.type === 'escape_sequence') { + const decoded = decodeEscapeSequence(child.text); + if (decoded === null) return null; + out += decoded; + } else if (child.type === 'template_substitution') { + const name = substitutionIdentifier(child); + if (!name) return null; + const inlined = uniqueStaticSource(name, declarators, resolving, cache); + if (inlined === null) return null; + out += inlined; + } else { + return null; + } + if (out.length > MAX_GRAPHQL_TOKENS) return null; + } + return out; +} + function unwrapExpression(node: Parser.SyntaxNode): Parser.SyntaxNode { let current = node; while ( @@ -214,7 +342,12 @@ function parsedDocumentProof( return false; } -function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { +function staticGraphqlSource( + initializer: Parser.SyntaxNode, + declarators?: GeneratedSymbolIndex, + resolving: Set = new Set(), + cache: InterpolationCache = new Map(), +): string | null { const value = unwrapExpression(initializer); if (value.type === 'string') { if (value.text.startsWith('"')) { @@ -226,11 +359,24 @@ function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { } return unquote(value.text); } - if (value.type === 'template_string') return unquote(value.text); + if (value.type === 'template_string') { + const hasSubstitution = value.namedChildren.some( + (child) => child.type === 'template_substitution', + ); + if (!hasSubstitution) return unquote(value.text); + return declarators ? interpolatedTemplateSource(value, declarators, resolving, cache) : null; + } if (value.type === 'call_expression') { const template = value.namedChildren.find((child) => child.type === 'template_string'); - return template ? unquote(template.text) : null; + if (!template) return null; + const hasSubstitution = template.namedChildren.some( + (child) => child.type === 'template_substitution', + ); + // Interpolated reconstruction is the cooked template. Only `gql` is + // treated as preserving that source; String.raw / unknown tags stay fail-closed. + if (hasSubstitution && !isGraphqlTagCall(value)) return null; + return staticGraphqlSource(template, declarators, resolving, cache); } if (value.type !== 'new_expression') return null; @@ -238,7 +384,7 @@ function staticGraphqlSource(initializer: Parser.SyntaxNode): string | null { if (!constructor || !constructor.text.endsWith('TypedDocumentString')) return null; const args = value.childForFieldName('arguments'); const first = args?.namedChildren[0]; - return first ? staticGraphqlSource(first) : null; + return first ? staticGraphqlSource(first, declarators, resolving, cache) : null; } export function hasGeneratedDocumentProof( @@ -246,9 +392,10 @@ export function hasGeneratedDocumentProof( operationKind: GraphqlOperationKind, operationName: string, requiredFields: readonly string[], + declarators?: GeneratedSymbolIndex, ): boolean { if (!withinGeneratedAstBudget(initializer)) return false; - const staticSource = staticGraphqlSource(initializer); + const staticSource = staticGraphqlSource(initializer, declarators); if (staticSource !== null) { return parsedDocumentProof(staticSource, operationKind, operationName, requiredFields); } @@ -430,7 +577,10 @@ function rootFields( function generatedCandidates(operation: OperationDefinitionNode): string[] { const name = operation.name?.value; - return name ? [`${name}Document`] : []; + if (!name) return []; + const exact = `${name}Document`; + const pascal = `${pascalCaseGraphqlName(name)}Document`; + return exact === pascal ? [exact] : [exact, pascal]; } async function generatedDocumentMatches( @@ -450,7 +600,13 @@ async function generatedDocumentMatches( const index = await pendingIndex; const values = index?.get(symbol.name) ?? []; return values.some((value) => - hasGeneratedDocumentProof(value, operationKind, operationName, requiredFields), + hasGeneratedDocumentProof( + value, + operationKind, + operationName, + requiredFields, + index ?? undefined, + ), ); } @@ -593,11 +749,7 @@ export class GraphqlExtractor implements ContractExtractor { await dbExecutor(RESOLVE_METHOD_QUERY, { name: methodName, filePath, - startLine: - member.type === 'public_field_definition' - ? (member.childForFieldName('value')?.startPosition.row ?? - member.startPosition.row) + 1 - : member.startPosition.row + 1, + startLine: providerMemberStartLine(member), }), ); if (!symbol) continue; diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index d4175442a..2ec174458 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -16,7 +16,7 @@ export interface ManifestExtractResult { // #2325 integration test can run the EXACT production query against a real // LadybugDB — a hand-copied query string in the test would silently drift // from this allowlist. Uses the `labels(n) IN [...]` allowlist form rather -// than a `MATCH (n:A|B)` disjunction: this 21-label list contains the +// than a `MATCH (n:A|B)` disjunction: this 23-label list contains the // reserved-keyword labels `Macro` and `Union`, and LadybugDB's parser rejects // a disjunction that names a reserved keyword (#2325) — which the resolver's // try/catch then swallowed. `labels(n) IN` has no such collision. @@ -25,7 +25,7 @@ export interface ManifestExtractResult { // two would widen which nodes resolve as contract symbols and must update the // #2325 test, so they are intentionally kept separate for now. export const CUSTOM_CONTRACT_RESOLVE_QUERY = `MATCH (n) - WHERE labels(n) IN ['Function','Method','Class','Interface','Struct','Enum','Trait','Constructor','TypeAlias','Impl','Macro','Union','Typedef','Property','Record','Delegate','Annotation','Template','Const','Static','CodeElement'] + WHERE labels(n) IN ['Function','Method','Class','Protocol','Category','Interface','Struct','Enum','Trait','Constructor','TypeAlias','Impl','Macro','Union','Typedef','Property','Record','Delegate','Annotation','Template','Const','Static','CodeElement'] AND n.name = $symbolName RETURN n.id AS uid, n.name AS name, n.filePath AS filePath ORDER BY n.filePath ASC, n.id ASC diff --git a/gitnexus/src/core/ingestion/cobol-processor.ts b/gitnexus/src/core/ingestion/cobol-processor.ts index 8963ea893..3fac5072d 100644 --- a/gitnexus/src/core/ingestion/cobol-processor.ts +++ b/gitnexus/src/core/ingestion/cobol-processor.ts @@ -6,8 +6,8 @@ * does its own extraction, and writes directly to the graph. * * Pipeline: - * 1. Separate programs from copybooks - * 2. Build copybook map (name -> content) + * 1. Separate programs from copybooks and JCL + * 2. Index file content by path for COPY expansion * 3. For each program: expand COPY statements, then run regex extraction * 4. Map CobolRegexResults to graph nodes and relationships * 5. Optionally process JCL files for job-step cross-references @@ -25,6 +25,7 @@ import { } from './cobol/cobol-preprocessor.js'; import { expandCopies } from './cobol/cobol-copy-expander.js'; import { processJclFiles } from './cobol/jcl-processor.js'; +import { resolveCobolCopyTarget } from './languages/cobol/copy-target.js'; import { logger } from '../logger.js'; @@ -123,38 +124,23 @@ export const processCobol = ( // ── 1. Separate programs, copybooks, and JCL ─────────────────────── const programs: CobolFile[] = []; - const copybooks: CobolFile[] = []; const jclFiles: CobolFile[] = []; for (const file of files) { const ext = path.extname(file.path).toLowerCase(); if (JCL_EXTENSIONS.has(ext)) { jclFiles.push(file); - } else if (isCopybook(file.path)) { - copybooks.push(file); - } else if (COBOL_EXTENSIONS.has(ext)) { + } else if (COBOL_EXTENSIONS.has(ext) && !isCopybook(file.path)) { programs.push(file); } } - // ── 2. Build copybook map (uppercase name -> content) ────────────── - const copybookMap = new Map(); - for (const cb of copybooks) { - const name = path.basename(cb.path, path.extname(cb.path)).toUpperCase(); - copybookMap.set(name, { content: cb.content, path: cb.path }); - } - - // Build reverse lookup: path -> content for O(1) readCopy + // Path → content for every COBOL/JCL file we ingested. Resolution picks + // the path (#2967 / census lockstep); this map only supplies the body. const copybookByPath = new Map(); - for (const [, entry] of copybookMap) { - copybookByPath.set(entry.path, entry.content); + for (const f of files) { + copybookByPath.set(f.path, f.content); } - - // Resolve and read callbacks for expandCopies - const resolveCopy = (name: string): string | null => { - const entry = copybookMap.get(name.toUpperCase()); - return entry ? entry.path : null; - }; // Memoize preprocessed copybook content for the duration of this // processCobol call. A single copybook is COPYed by many programs (and at // many COPY sites within a program); without this cache @@ -196,6 +182,11 @@ export const processCobol = ( // Preprocess: clean patch markers const cleaned = preprocessCobolSource(file.content); + // Per-program so COPY EXTERNAL in src/PROG.cbl cannot land on + // vendor/EXTERNAL.cpy when a copybooks/ dir is present (#2967). + const resolveCopy = (name: string): string | null => + resolveCobolCopyTarget(name, file.path, allPathSet); + // Expand COPY statements const { expandedContent, copyResolutions } = expandCopies( cleaned, diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index 80c005143..77b66d73b 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -19,6 +19,7 @@ import { dirname, resolve } from 'node:path'; import { Worker } from 'node:worker_threads'; import type { GraphNode, NodeLabel } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; +import { isHeuristicEdgeReason } from '../graph/edge-reasons.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -131,6 +132,18 @@ const LEIDEN_TIMEOUT_MS = 60_000; const ICEBUG_TIMEOUT_MS = 60_000; const MIN_CONFIDENCE_LARGE = 0.5; +/** + * Whether a large graph's projection may include this edge. + * + * The confidence floor alone lets every global-name-fallback edge through — it + * is emitted at exactly `MIN_CONFIDENCE_LARGE`, so `confidence <` never + * excludes it. Clustering on unique-name guesses fuses unrelated areas into one + * community, which is precisely the noise the large-graph floor exists to + * remove, so the reason is checked too — see `graph/edge-reasons.ts`. + */ +const isLargeGraphEligible = (confidence: number, reason: string): boolean => + confidence >= MIN_CONFIDENCE_LARGE && !isHeuristicEdgeReason(reason); + export const resolveCommunityDetectionEngine = ( raw = process.env[COMMUNITY_ENGINE_ENV], ): CommunityDetectionEngine => { @@ -300,11 +313,16 @@ export const buildCommunityProjection = (knowledgeGraph: KnowledgeGraph): Commun const connectedNodes = new Set(); const nodeDegree = new Map(); - // Field-wise scan (#2680): this walks every edge and reads only these four, + // Field-wise scan (#2680): this walks every edge and reads only these five, // so taking objects would allocate one per edge for nothing. - knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { if (!isClusteringRelationship(type) || sourceId === targetId) return; - if (isLarge && confidence < MIN_CONFIDENCE_LARGE) return; + // A name-guessed edge must not join two nodes into a community at ANY graph + // size — the rationale in `graph/edge-reasons.ts` is about what the edge + // claims, not about how many symbols surround it. Process tracing applies + // the same reason gate unconditionally (`process-processor.ts`). + if (isHeuristicEdgeReason(reason)) return; + if (isLarge && !isLargeGraphEligible(confidence, reason)) return; connectedNodes.add(sourceId); connectedNodes.add(targetId); @@ -340,9 +358,10 @@ export const buildCommunityProjection = (knowledgeGraph: KnowledgeGraph): Commun const seenEdges = new Set(); const edges: Array = []; - knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { + knowledgeGraph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { if (!isClusteringRelationship(type) || sourceId === targetId) return; - if (isLarge && confidence < MIN_CONFIDENCE_LARGE) return; + if (isHeuristicEdgeReason(reason)) return; + if (isLarge && !isLargeGraphEligible(confidence, reason)) return; const sourceIndex = nodeIndexById.get(sourceId); const targetIndex = nodeIndexById.get(targetId); diff --git a/gitnexus/src/core/ingestion/content-language-classification.ts b/gitnexus/src/core/ingestion/content-language-classification.ts new file mode 100644 index 000000000..27da6a677 --- /dev/null +++ b/gitnexus/src/core/ingestion/content-language-classification.ts @@ -0,0 +1,40 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { SupportedLanguages } from 'gitnexus-shared'; +import { mapConcurrent } from '../../lib/utils.js'; +import { READ_CONCURRENCY } from './filesystem-walker.js'; +import { getLanguageForFileContent } from './languages/index.js'; + +/** + * Classify files whose language cannot be determined from their extension. + * + * Source text is retained only for the duration of each individual read and + * classification. The returned map deliberately contains language results, + * not source text, so downstream phases can reuse the decision without + * holding every candidate header in memory or rereading it to classify again. + */ +export async function classifyContentLanguages( + repoPath: string, + relativePaths: readonly string[], +): Promise> { + const classifications = new Map(); + const results = await mapConcurrent( + relativePaths, + async (relativePath) => { + const sourceText = await fs.readFile(path.join(repoPath, relativePath), 'utf-8'); + return { + path: relativePath, + language: getLanguageForFileContent(relativePath, sourceText), + }; + }, + { concurrency: READ_CONCURRENCY }, + ); + + // An unreadable file yields `undefined` from mapConcurrent. Leave it absent + // so the caller retains the existing filename-based fallback behavior. + for (const result of results) { + if (result !== undefined) classifications.set(result.path, result.language); + } + + return classifications; +} diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 7e41c07c2..a4a04e342 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -19,7 +19,7 @@ export interface FilePath { path: string; } -const READ_CONCURRENCY = 32; +export const READ_CONCURRENCY = 32; const ANALYZE_PROGRESS_ACTIVE_ENV = 'GITNEXUS_ANALYZE_PROGRESS_ACTIVE'; const DECLARATION_COMPANION_SUFFIXES = [ diff --git a/gitnexus/src/core/ingestion/finalize-orchestrator.ts b/gitnexus/src/core/ingestion/finalize-orchestrator.ts index 3146aff26..4a0e99f6b 100644 --- a/gitnexus/src/core/ingestion/finalize-orchestrator.ts +++ b/gitnexus/src/core/ingestion/finalize-orchestrator.ts @@ -200,8 +200,11 @@ function collectReferenceSites(parsedFiles: readonly ParsedFile[]) { */ function withDefaultHooks(partial: Partial): FinalizeHooks { return { + importsBindAtLexicalScope: partial.importsBindAtLexicalScope === true, resolveImportTarget: partial.resolveImportTarget ?? (() => null), isNamespaceImport: partial.isNamespaceImport, + wildcardCollisionIsAmbiguous: partial.wildcardCollisionIsAmbiguous === true, + namedImportsBindTopLevelOnly: partial.namedImportsBindTopLevelOnly === true, expandsWildcardTo: partial.expandsWildcardTo ?? (() => []), mergeBindings: partial.mergeBindings ?? diff --git a/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts b/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts index 5750da917..2704bdfab 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/node-workspace-packages.ts @@ -18,6 +18,7 @@ import fs from 'fs/promises'; import path from 'path'; import { createRequire } from 'node:module'; +import type Parser from 'tree-sitter'; import { isHardcodedIgnoredDirectoryAtPath } from '../../../config/ignore-service.js'; import { logger } from '../../logger.js'; @@ -214,6 +215,8 @@ interface WorkspaceScope { readonly include: readonly string[]; /** `!`-prefixed patterns, with the `!` stripped. */ readonly exclude: readonly string[]; + readonly includeRe: readonly RegExp[]; + readonly excludeRe: readonly RegExp[]; } /** Whether `dir` (repo-relative, `''` for the root) is an admitted package. */ @@ -221,8 +224,21 @@ function admits(scope: WorkspaceScope | null, dir: string): boolean { // The root package is always itself, workspace or not. if (dir === '') return true; if (scope === null) return false; - if (scope.exclude.some((pattern) => globToRegExp(pattern).test(dir))) return false; - return scope.include.some((pattern) => globToRegExp(pattern).test(dir)); + // An exclusion covers the directory AND everything under it: `!packages/legacy` + // must keep `packages/legacy/foo` out even when a nested workspace root inside + // the excluded subtree re-declares `packages/*`. + if (scope.excludeRe.some((re) => matchesDirOrAncestor(re, dir))) return false; + return scope.includeRe.some((re) => re.test(dir)); +} + +function matchesDirOrAncestor(re: RegExp, dir: string): boolean { + let current = dir; + while (current !== '') { + if (re.test(current)) return true; + const slash = current.lastIndexOf('/'); + current = slash === -1 ? '' : current.slice(0, slash); + } + return false; } /** @@ -272,9 +288,69 @@ function globToRegExp(pattern: string): RegExp { * tooling that does not read pnpm's file). */ async function loadWorkspaceScope(repoRoot: string): Promise { - const patterns: string[] = []; + // Every workspace ROOT in the repo, not just the top level. keycloak keeps its + // JavaScript workspace at `js/pnpm-workspace.yaml`; reading only the repo root + // found no workspace, admitted no package, and `@keycloak/keycloak-ui-shared` + // (780–845 raw calls per SHA) resolved 7 times. Patterns from a nested root + // are rebased onto that root so `packages/*` under `js/` admits `js/packages/x`. + // + // Gated, though: a nested root counts only when the repo root declares NO + // workspace (keycloak), or when the nested root's directory is itself admitted + // by the root's scope. Unioning every nested root unconditionally let an + // `examples/*/package.json` starter (turborepo/vite/nuxt templates carry + // `workspaces`) admit its example packages — and, being shallow, outrank the + // real package of the same name — and let a root inside an excluded subtree + // re-admit what the outer `!exclusion` had removed. + const [rootPatterns, workspaceRoots] = await Promise.all([ + readWorkspacePatternsAt(repoRoot), + findWorkspaceRoots(repoRoot), + ]); + const rootScope = rootPatterns.length === 0 ? null : toScope(rootPatterns); + const patterns: string[] = [...rootPatterns]; + const admitted: { root: string; prefix: string }[] = []; + for (const root of workspaceRoots) { + if (root === repoRoot) continue; + const prefix = repoRelativeDir(repoRoot, root); + if (rootScope !== null && !admits(rootScope, prefix)) continue; + admitted.push({ root, prefix }); + } + const nested = await Promise.all( + admitted.map(async ({ root, prefix }) => { + const rebase = (p: string): string => { + const negated = p.startsWith('!'); + const body = negated ? p.slice(1) : p; + const joined = prefix === '' ? body : `${prefix}/${body.replace(/^\.\//, '')}`; + return negated ? `!${joined}` : joined; + }; + return (await readWorkspacePatternsAt(root)).map(rebase); + }), + ); + for (const batch of nested) patterns.push(...batch); - const rootManifest = await readJsonFile(path.join(repoRoot, 'package.json')); + if (patterns.length === 0) return null; + return toScope(patterns); +} + +function toScope(patterns: readonly string[]): WorkspaceScope { + const include = patterns.filter((p) => !p.startsWith('!')); + const exclude = patterns.filter((p) => p.startsWith('!')).map((p) => p.slice(1)); + return { + include, + exclude, + includeRe: include.map(globToRegExp), + excludeRe: exclude.map(globToRegExp), + }; +} + +/** The workspace patterns declared at ONE directory, all three spellings merged. */ +async function readWorkspacePatternsAt(root: string): Promise { + const [rootManifest, yamlPkgs, ymlPkgs, lerna] = await Promise.all([ + readJsonFile(path.join(root, 'package.json')), + readYamlPackages(path.join(root, 'pnpm-workspace.yaml')), + readYamlPackages(path.join(root, 'pnpm-workspace.yml')), + readJsonFile(path.join(root, 'lerna.json')), + ]); + const patterns: string[] = []; const workspaces = rootManifest?.workspaces; if (Array.isArray(workspaces)) { patterns.push(...workspaces.filter((w): w is string => typeof w === 'string')); @@ -285,20 +361,114 @@ async function loadWorkspaceScope(repoRoot: string): Promise typeof w === 'string')); } } - - patterns.push(...(await readYamlPackages(path.join(repoRoot, 'pnpm-workspace.yaml')))); - patterns.push(...(await readYamlPackages(path.join(repoRoot, 'pnpm-workspace.yml')))); - - const lerna = await readJsonFile(path.join(repoRoot, 'lerna.json')); + patterns.push(...yamlPkgs, ...ymlPkgs); if (Array.isArray(lerna?.packages)) { patterns.push(...lerna.packages.filter((w): w is string => typeof w === 'string')); } + return patterns; +} - if (patterns.length === 0) return null; - return { - include: patterns.filter((p) => !p.startsWith('!')), - exclude: patterns.filter((p) => p.startsWith('!')).map((p) => p.slice(1)), - }; +/** + * Directories that declare a workspace: the repo root plus any directory (to a + * shallow depth — workspace roots sit near the top) holding a + * `pnpm-workspace.yaml`, a `lerna.json`, or a `package.json` with `workspaces`. + */ +const WORKSPACE_ROOT_MAX_DEPTH = 4; +/** + * Bound on the depth-≤4 directory walk. Generous on purpose: the previous 2,000 + * cap tripped silently in readdir order, so WHICH packages existed — and which + * imports resolved — varied between two checkouts of the same commit. Tripping + * it now warns, so a truncated scan is a logged fact rather than a quiet one. + */ +const WORKSPACE_ROOT_SCAN_MAX_DIRS = 50_000; +/** + * Directory names whose nested `workspaces` are starters, fixtures, or test + * trees — never members. A leftover `test/pnpm-workspace.yaml` must not become + * a workspace root when the repo root declares none (the #2953 first-wins + * class). A root `workspaces` glob that lists `test/*` still admits those + * packages through the package.json walk; this set only stops nested-root + * discovery from descending into those names. + */ +const NON_MEMBER_ROOT_DIRS = new Set([ + 'example', + 'examples', + 'fixture', + 'fixtures', + 'template', + 'templates', + 'sample', + 'samples', + 'test', + 'tests', + 'e2e', + '__tests__', + 'spec', + 'specs', +]); + +/** + * Vite's `DEFAULT_CONFIG_FILES` order. The first filename that exists is the + * config Vite loads; leftover siblings are not a second live entry. + */ +const VITE_CONFIG_FILES = [ + 'vite.config.js', + 'vite.config.mjs', + 'vite.config.ts', + 'vite.config.cjs', + 'vite.config.mts', + 'vite.config.cts', +] as const; + +function isMissingPath(err: unknown): boolean { + return (err as NodeJS.ErrnoException).code === 'ENOENT'; +} + +async function readDirSorted(dir: string): Promise { + try { + return (await fs.readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name < b.name ? -1 : a.name > b.name ? 1 : 0, + ); + } catch { + return null; + } +} + +async function findWorkspaceRoots(repoRoot: string): Promise { + const roots: string[] = [repoRoot]; + const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + let scanned = 0; + while (scanned < queue.length) { + if (scanned >= WORKSPACE_ROOT_SCAN_MAX_DIRS) { + logger.warn( + `[node] workspace-root scan of ${repoRoot} hit the ${WORKSPACE_ROOT_SCAN_MAX_DIRS}-directory cap; nested workspace roots below it were not considered`, + ); + break; + } + const { dir, depth } = queue[scanned++]!; + const entries = await readDirSorted(dir); + if (entries === null) continue; + if (dir !== repoRoot) { + const names = new Set(entries.filter((e) => e.isFile()).map((e) => e.name)); + let declares = + names.has('pnpm-workspace.yaml') || + names.has('pnpm-workspace.yml') || + names.has('lerna.json'); + if (!declares && names.has('package.json')) { + const manifest = await readJsonFile(path.join(dir, 'package.json')); + declares = manifest?.workspaces !== undefined && manifest.workspaces !== null; + } + if (declares) roots.push(dir); + } + if (depth >= WORKSPACE_ROOT_MAX_DEPTH) continue; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (NON_MEMBER_ROOT_DIRS.has(entry.name.toLowerCase())) continue; + const child = path.join(dir, entry.name); + if (isHardcodedIgnoredDirectoryAtPath(repoRoot, child)) continue; + queue.push({ dir: child, depth: depth + 1 }); + } + } + return roots; } async function readJsonFile(filePath: string): Promise | null> { @@ -327,6 +497,39 @@ async function readYamlPackages(filePath: string): Promise { } } +/** + * Per-repo memo. The TS/JS/Vue scope resolvers and the unresolved-call ledger + * classifier each ask for the same map during one analyze; without this the + * 20k-directory walk ran once per asker (four times on a full run, one of them + * inside the index lock). Invalidated at the start of every `runFullAnalysis` + * so a long-lived server never serves a stale package map across runs. + */ +const workspacePackagesMemo = new Map>(); + +export function invalidateNodeWorkspacePackages(repoRoot: string): void { + workspacePackagesMemo.delete(path.resolve(repoRoot)); +} + +export async function loadNodeWorkspacePackages( + repoRoot: string, +): Promise { + const key = path.resolve(repoRoot); + const cached = workspacePackagesMemo.get(key); + if (cached !== undefined) return cached; + const pending: Promise = loadNodeWorkspacePackagesUncached( + repoRoot, + ).catch((err: unknown) => { + // Evict only OUR entry. If this load was invalidated while in flight and a + // newer load has since been installed under the same key, deleting by key + // alone would evict that one, and every later caller would start another + // full scan instead of joining it. + if (workspacePackagesMemo.get(key) === pending) workspacePackagesMemo.delete(key); + throw err; + }); + workspacePackagesMemo.set(key, pending); + return pending; +} + /** * Collect the `package.json` of every ADMITTED workspace package. * @@ -334,30 +537,29 @@ async function readYamlPackages(filePath: string): Promise { * declaration, so this is far cheaper than the C# namespace scan next door, * which reads every `.cs` file. */ -export async function loadNodeWorkspacePackages( +async function loadNodeWorkspacePackagesUncached( repoRoot: string, ): Promise { const scope = await loadWorkspaceScope(repoRoot); const byName = new Map(); const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + let queueHead = 0; let dirsScanned = 0; - while (queue.length > 0) { + while (queueHead < queue.length) { if (dirsScanned >= SCAN_MAX_DIRS) { logger.warn( `[node] package.json scan of ${repoRoot} hit the ${SCAN_MAX_DIRS}-directory cap; workspace packages below it will not resolve`, ); break; } - const { dir, depth } = queue.shift()!; + const { dir, depth } = queue[queueHead++]!; dirsScanned++; - let entries: import('fs').Dirent[]; - try { - entries = await fs.readdir(dir, { withFileTypes: true }); - } catch { - continue; - } + // Sorted: same-depth name collisions resolve first-wins, and readdir order + // is filesystem-dependent. + const entries = await readDirSorted(dir); + if (entries === null) continue; for (const entry of entries) { if (entry.isDirectory()) { @@ -420,10 +622,34 @@ async function readManifest( const value = parsed[field]; if (typeof value === 'string') push(entries, rebase(value)); } + } + // Entry points that name BUILD OUTPUT (`main: dist/x.js`, `exports: ./dist/…`) + // never match an indexed source file — the package's real source entry has to + // be discovered. keycloak's `@keycloak/keycloak-ui-shared` publishes + // `dist/keycloak-ui-shared.js` and keeps its entry in `vite.config.ts` + // (`build.lib.entry: 'src/main.ts'`); with only the declared fields it had 7 + // resolved calls against ~845 raw. Tried in a fixed order, and ONLY appended + // (declared entries keep precedence): `source` / `publishConfig.source`, the + // vite `lib.entry`, then `src/main` / `src/index`. More than one candidate + // that exists on disk is recorded as ambiguous rather than picked — a wrong + // entry binds every import of the package to the wrong file. + // A package whose `exports` map has no `"."` (only subpaths) refuses the bare + // specifier outright; discovery must not manufacture a `src/index` root for it. + const rootlessExports = declaresExports && rootExports.length === 0; + const discovered = rootlessExports + ? { entries: [], ambiguous: [], allowConventional: false } + : await discoverSourceEntries(parsed, dir, repoRoot); + for (const entry of discovered.entries) push(entries, entry); + if (!declaresExports && discovered.allowConventional) { for (const conventional of ['src/index', 'index', 'lib/index']) { push(entries, joinRepoPath(packageDir, conventional)); } } + if (discovered.ambiguous.length > 0) { + logger.warn( + `[node] package ${name}: ${discovered.ambiguous.length} candidate source entries (${discovered.ambiguous.join(', ')}) — none adopted; declare \`source\` or a single lib entry`, + ); + } const subpathImports = new Map(); collectImports(parsed.imports, subpathImports, rebase); @@ -507,6 +733,195 @@ function collectImports( } } +/** + * Does a declared entry point at build output rather than source? Output + * directories and minified bundles. A plain `.js` is NOT build output on its + * own — `main: "src/index.js"` / `index.js` is a JavaScript package's source, + * and treating every `.js` as output ran discovery for essentially every CJS + * package and could adopt a `src/main` beside the real entry. + */ +function looksLikeBuildOutput(entry: string): boolean { + return /(^|\/)(dist|build|lib|out|esm|cjs|umd)\//.test(entry) || /\.min\.[cm]?js$/.test(entry); +} + +function declaredRootExportStrings(exportsRoot: unknown): string[] { + if (typeof exportsRoot === 'string') return [exportsRoot]; + if (exportsRoot === null || typeof exportsRoot !== 'object') return []; + const dot = (exportsRoot as Record)['.']; + if (typeof dot === 'string') return [dot]; + if (dot === null || typeof dot !== 'object') return []; + return Object.values(dot).filter((v): v is string => typeof v === 'string'); +} + +async function existingStems(repoRoot: string, stems: readonly string[]): Promise { + const present = await Promise.all(stems.map((stem) => stemExists(repoRoot, stem))); + const existing: string[] = []; + for (let i = 0; i < stems.length; i++) { + if (present[i]) push(existing, stems[i]!); + } + return existing; +} + +async function discoverSourceEntries( + parsed: Record, + dir: string, + repoRoot: string, +): Promise<{ entries: string[]; ambiguous: string[]; allowConventional: boolean }> { + const packageDir = repoRelativeDir(repoRoot, dir); + const rebase = (raw: string): string => joinRepoPath(packageDir, stripEntryPrefixes(raw)); + const declared: string[] = declaredRootExportStrings(parsed.exports); + for (const field of ['module', 'main']) { + const value = parsed[field]; + if (typeof value === 'string') declared.push(value); + } + // Only when every declared entry is build output (or nothing is declared and + // the conventional stems are absent) does discovery run at all. + if (declared.length > 0 && !declared.every(looksLikeBuildOutput)) + return { entries: [], ambiguous: [], allowConventional: true }; + + const candidates: string[] = []; + const source = parsed.source; + if (typeof source === 'string') candidates.push(rebase(source)); + const publishConfig = parsed.publishConfig; + if (publishConfig !== null && typeof publishConfig === 'object') { + const ps = (publishConfig as Record).source; + if (typeof ps === 'string') candidates.push(rebase(ps)); + } + let hasViteConfig = false; + for (const cfg of VITE_CONFIG_FILES) { + let text: string; + try { + text = await fs.readFile(path.join(dir, cfg), 'utf-8'); + } catch (err) { + if (isMissingPath(err)) continue; + // Present but unreadable: Vite still selected this filename. + hasViteConfig = true; + break; + } + hasViteConfig = true; + try { + const entry = await staticViteEntry(text); + if (entry !== null) push(candidates, rebase(entry)); + } catch { + // Parse/load failure is not a missing file — do not fall through to a leftover sibling. + } + break; + } + let existing = await existingStems(repoRoot, candidates); + // A config we cannot establish is not evidence for a conventional entry. + if (existing.length === 0 && !hasViteConfig) { + existing = await existingStems( + repoRoot, + ['src/main', 'src/index'].map((conventional) => joinRepoPath(packageDir, conventional)), + ); + } + const allowConventional = !hasViteConfig && candidates.length === 0; + if (existing.length > 1) return { entries: [], ambiguous: existing, allowConventional }; + return { entries: existing, ambiguous: [], allowConventional }; +} + +/** Read a literal exported build.lib.entry without evaluating repository code. */ +async function staticViteEntry(text: string): Promise { + // Most manifests need no Vite discovery. Keep native grammars off that path. + const [{ default: TreeSitter }, { default: grammar }, { parseSourceSafe }] = await Promise.all([ + import('tree-sitter'), + import('tree-sitter-typescript'), + import('../../tree-sitter/safe-parse.js'), + ]); + const parser = new TreeSitter(); + parser.setLanguage(grammar.typescript); + const tree = parseSourceSafe(parser, text); + if (tree.rootNode.hasError) return null; + const exports = tree.rootNode.namedChildren.filter( + (node) => node.type === 'export_statement' && node.children.some((c) => c.type === 'default'), + ); + if (exports.length !== 1) return null; + let config = unwrapViteDefineConfig(tree, exports[0]!.childForFieldName('value')); + if (config === null) return null; + for (const key of ['build', 'lib', 'entry']) config = staticObjectProperty(config, key); + return staticStringValue(config); +} + +/** Unwrap `defineConfig({...})` from Vite; refuse local or re-exported helpers. */ +function unwrapViteDefineConfig( + tree: { rootNode: Parser.SyntaxNode }, + config: Parser.SyntaxNode | null, +): Parser.SyntaxNode | null { + if (config?.type !== 'call_expression') return config; + if (config.childForFieldName('function')?.text !== 'defineConfig') return null; + // A locally defined helper need not preserve its argument like Vite does. + if ( + tree.rootNode + .descendantsOfType(['function_declaration', 'variable_declarator']) + .some((node) => node.childForFieldName('name')?.text === 'defineConfig') + ) + return null; + for (const statement of tree.rootNode.namedChildren) { + if (statement.type !== 'import_statement') continue; + if (!statement.descendantsOfType('identifier').some((n) => n.text === 'defineConfig')) continue; + if (staticStringValue(statement.childForFieldName('source')) !== 'vite') return null; + const identityImport = statement.descendantsOfType('import_specifier').some((specifier) => { + const imported = specifier.childForFieldName('name')?.text; + const local = specifier.childForFieldName('alias')?.text ?? imported; + return imported === 'defineConfig' && local === 'defineConfig'; + }); + if (!identityImport) return null; + } + const args = config + .childForFieldName('arguments') + ?.namedChildren.filter((n) => n.type !== 'comment'); + if (args?.length !== 1) return null; + return args[0]!; +} + +function staticStringValue(node: Parser.SyntaxNode | null): string | null { + if (node?.type !== 'string' || node.namedChildren.some((n) => n.type === 'escape_sequence')) + return null; + return node.text.slice(1, -1); +} + +/** Reject computed keys, spreads and duplicate properties that could override a value. */ +function staticObjectProperty( + node: Parser.SyntaxNode | null, + key: string, +): Parser.SyntaxNode | null { + if (node?.type !== 'object') return null; + let value: Parser.SyntaxNode | null = null; + for (const member of node.namedChildren) { + if (member.type === 'comment') continue; + if (member.type !== 'pair') return null; + const name = member.childForFieldName('key'); + if (name?.type !== 'property_identifier' && name?.type !== 'string') return null; + if (name.type === 'string') { + const property = staticStringValue(name); + if (property === null) return null; + if (property !== key) continue; + } else if (name.text !== key) { + continue; + } + if (value !== null) return null; + value = member.childForFieldName('value'); + } + return value; +} + +/** A repo-relative stem exists as a source file (with any TS/JS extension). */ +// The root is threaded explicitly: a module-level "current root" clobbered +// under two concurrent scans and turned an ambiguity refusal into a confident +// wrong entry (the second repo's root made one candidate "not exist"). +async function stemExists(repoRoot: string, stem: string): Promise { + const abs = path.join(repoRoot, stem); + for (const ext of ['', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs']) { + try { + const st = await fs.stat(abs + ext); + if (st.isFile()) return true; + } catch { + /* try next */ + } + } + return false; +} + /** `"./src/index.ts"` -> `"src/index"`; leaves an extension-less path alone. */ function stripEntryPrefixes(entry: string): string { const withoutDot = entry.replace(/^\.\//, '').replace(/^\//, ''); diff --git a/gitnexus/src/core/ingestion/language-config.ts b/gitnexus/src/core/ingestion/language-config.ts index 5d8fc9061..390e6a32f 100644 --- a/gitnexus/src/core/ingestion/language-config.ts +++ b/gitnexus/src/core/ingestion/language-config.ts @@ -5,6 +5,7 @@ import path from 'path'; import type { CsharpStructureLineScanner } from './languages/csharp/namespace-siblings.js'; import { isDev } from './utils/env.js'; +import { isHardcodedIgnoredDirectoryAtPath } from '../../config/ignore-service.js'; import { mapConcurrent } from '../../lib/utils.js'; import { logger } from '../logger.js'; @@ -182,13 +183,24 @@ export interface SwiftPackageConfig { /** Zig package config parsed from build.zig.zon and the root build.zig */ export interface ZigBuildZonConfig { /** - * Map of dependency name -> the raw `.path = "..."` value, exactly as - * written in build.zig.zon (relative to the repo root, and possibly - * escaping it: `../local_dep`). Consumers normalize — see - * `normalizeZigDepPath` below, which rejects absolute - * and repo-escaping values. `.url`-based deps cannot be resolved to a - * repo-local file (they unpack into a build cache outside the repo) and so - * are not included here. + * Map of dependency name -> the dep's directory, in one of two spellings + * depending on which package this config describes: + * + * - ROOT package (`pkg === ''`): the raw `.path = "..."` value, exactly as + * written in build.zig.zon (relative to the repo root, and possibly + * escaping it: `../local_dep`). This is what `parseZigBuildZon` promises + * and what its tests pin. + * - NESTED package: repo-relative and already normalized, because a nested + * package's `.path` is written relative to ITS directory and means + * nothing against the repo-relative keys consumers match on + * (`packages/app`'s `../core` is stored as `packages/core`). A dep + * escaping the REPO root is dropped rather than stored. + * + * Either spelling is safe to hand to `normalizeZigDepPath` below — it rejects + * absolute and repo-escaping values and is idempotent on an already + * normalized one, which is what `resolveZigImportInternal` relies on. + * `.url`-based deps cannot be resolved to a repo-local file (they unpack into + * a build cache outside the repo) and so are not included here. */ pathDeps: Map; /** @@ -228,6 +240,29 @@ export interface ZigBuildZonConfig { buildModules?: readonly ZigBuildModule[]; } +/** + * One Zig build package: the directory whose `build.zig` / `build.zig.zon` + * declare the config, and that config with every path REPO-relative. + * + * A Zig module's import table is declared by the `build.zig` of the package it + * belongs to, so a repo holding several packages holds several import tables — + * the same shape a TypeScript monorepo has with a `tsconfig.json` per package. + */ +export interface ZigPackageScope { + /** Repo-relative directory the package governs (`''` for the repo root). */ + readonly dir: string; + readonly config: ZigBuildZonConfig; +} + +/** + * Every Zig build package in the repo, indexed so the nearest one to a file + * wins — the `TsconfigIndex` analogue, and for the same reason. + */ +export interface ZigWorkspaceIndex { + /** Deepest-first, so the first `dir` that prefixes a file path governs it. */ + readonly packages: readonly ZigPackageScope[]; +} + /** One build module of the root `build.zig` — see `ZigBuildZonConfig.buildModules`. */ export interface ZigBuildModule { /** The `addModule("", …)` name; absent for `createModule` bindings @@ -646,10 +681,21 @@ export async function loadSwiftPackageConfig(repoRoot: string): Promise { +export async function loadZigBuildConfig( + repoRoot: string, + packageDir = '', +): Promise { + // Every path this function returns is REPO-relative, because that is the + // keyspace `allFilePaths` uses. The parsers below answer package-relative, so + // a nested package rebases them through `inPackage`. For the root package + // (`packageDir === ''`) the prefix is empty and every value is byte-identical + // to what this function returned before nested packages existed. + const pkg = packageDir === '' ? '' : `${packageDir}/`; + const inPackage = (relToPackage: string): string => `${pkg}${relToPackage}`; + const packageFile = (name: string): string => path.join(repoRoot, packageDir, name); let config: ZigBuildZonConfig | null = null; try { - const raw = await fs.readFile(path.join(repoRoot, 'build.zig.zon'), 'utf-8'); + const raw = await fs.readFile(packageFile('build.zig.zon'), 'utf-8'); config = parseZigBuildZon(raw); } catch { // No zon (or unreadable): the root build.zig may still declare modules. @@ -661,9 +707,11 @@ export async function loadZigBuildConfig(repoRoot: string): Promise | undefined; let rootBuildZig: string | null = null; try { - rootBuildZig = await fs.readFile(path.join(repoRoot, 'build.zig'), 'utf-8'); + rootBuildZig = await fs.readFile(packageFile('build.zig'), 'utf-8'); const parsed = parseZigRootModules(rootBuildZig); - if (parsed.size > 0) rootModules = parsed; + if (parsed.size > 0) { + rootModules = new Map(Array.from(parsed, ([name, root]) => [name, inPackage(root)])); + } } catch { // No root build.zig — nothing to declare. } @@ -671,7 +719,7 @@ export async function loadZigBuildConfig(repoRoot: string): Promise>(); + // A nested package's `.path` values are written relative to ITS directory, so + // they are rebased here and stored repo-relative; `resolveZigImportInternal` + // then reads them through the same `normalizeZigDepPath`, which is idempotent + // on an already-normalized value. A dep that escapes the REPO root (not merely + // the package) resolves to nothing and is dropped. The root package keeps its + // raw spelling, which is what `parseZigBuildZon` promises and its tests pin. + const pathDeps = pkg === '' ? config.pathDeps : new Map(); for (const [depName, depPath] of config.pathDeps) { - const rel = normalizeZigDepPath(depPath); + // Asked of the value AS WRITTEN, before the package prefix goes on: an + // absolute `.path` points outside the repository whichever package declared + // it, and prefixing hides that from `normalizeZigDepPath`. See + // `isAbsoluteZigDepPath`. + if (isAbsoluteZigDepPath(depPath)) continue; + const rel = normalizeZigDepPath(`${pkg}${depPath}`); if (rel === null) continue; + if (pkg !== '') pathDeps.set(depName, rel); let buildZig: string; try { buildZig = await fs.readFile(path.join(repoRoot, rel, 'build.zig'), 'utf-8'); @@ -706,15 +767,177 @@ export async function loadZigBuildConfig(repoRoot: string): Promise 0) depModules.set(depName, named); } - const buildModules = rootBuildZig === null ? [] : parseZigBuildModules(rootBuildZig, depModules); + const buildModules = + rootBuildZig === null + ? [] + : rebaseZigBuildModules( + parseZigBuildModules(rootBuildZig, depModules), + inPackage, + depModules, + ); return { ...config, + pathDeps, ...(moduleRoots.size > 0 ? { moduleRoots } : {}), ...(rootModules ? { rootModules } : {}), ...(buildModules.length > 0 ? { buildModules } : {}), }; } +/** + * Rebase a package's own build modules to repo-relative paths. + * + * `parseZigBuildModules` answers package-relative for everything it read out of + * the `build.zig` it was handed, with one exception: an alias resolved through + * `depModules` (`addImport("api", dep.module("core"))`) is already repo-relative, + * because `depModules` was built that way. Prefixing that a second time would + * point the alias at a path no file has. The already-repo-relative values are + * therefore identified by membership in `depModules`, not guessed at from their + * shape. + */ +function rebaseZigBuildModules( + modules: readonly ZigBuildModule[], + inPackage: (relToPackage: string) => string, + depModules?: ReadonlyMap>, +): ZigBuildModule[] { + if (inPackage('') === '') return [...modules]; + const fromDep = new Set(); + for (const named of depModules?.values() ?? []) + for (const root of named.values()) fromDep.add(root); + return modules.map((mod) => ({ + ...(mod.name !== undefined ? { name: mod.name } : {}), + root: inPackage(mod.root), + imports: new Map( + Array.from(mod.imports, ([alias, root]) => [ + alias, + fromDep.has(root) ? root : inPackage(root), + ]), + ), + })); +} + +/** Bounds for the package walk, mirroring the tsconfig scan. */ +const ZIG_SCAN_MAX_DIRS = 20_000; +const ZIG_SCAN_MAX_DEPTH = 24; + +/** + * The Zig build package governing `filePath` — the nearest one at or above it. + * + * A Zig module's import table is declared by the `build.zig` of the package the + * file belongs to, so the nearest enclosing package is the faithful reading of + * `@import("name")` at that site, exactly as `tsconfigFor` reads a non-relative + * specifier against the nearest enclosing project. + * + * There is deliberately NO fall-through to an enclosing package when the nearest + * one does not bind the name. Falling through is how a vendored dependency's + * `@import("config")` silently resolved to the outer repo's `config` module — + * the same failure `loadTsconfigIndex` documents for a package whose own + * tsconfig declares no `baseUrl`, and the same failure the per-module import + * tables in `resolveZigImportInternal` already exist to prevent one level down. + */ +export function zigPackageFor( + index: ZigWorkspaceIndex | null | undefined, + filePath: string, +): ZigBuildZonConfig | null { + if (index === null || index === undefined) return null; + for (const scope of index.packages) { + if (scope.dir === '') return scope.config; + if (filePath.startsWith(`${scope.dir}/`)) return scope.config; + } + return null; +} + +/** + * Load every Zig build package in the repo, nearest-first. + * + * Called with no `packageDir` — which is how every call site read it before + * this function existed — `loadZigBuildConfig` reads the ROOT `build.zig` / + * `build.zig.zon` and nothing else. That is the whole configuration of a + * single-package repo and none of the configuration of a monorepo: a repo + * laying its packages out as `packages//build.zig` has no root build + * files at all, so the loader answers `null` and EVERY bare + * `@import("")` in it goes unresolved — cross-file resolution silently + * degrades to relative imports only. Measured on a two-package fixture: + * `config = null`, `@import("core")` → `null`. + * + * The loader itself is not root-bound any more: this function is what supplies + * it a `packageDir`, one per package below. + * + * So the packages are discovered the way tsconfigs are (`findTsconfigFiles`): + * one bounded breadth-first walk that skips the hardcoded ignore set, then + * deepest-first ordering so `zigPackageFor` can take the first match. + * + * Called from `ScopeResolver.loadResolutionConfig`, which the orchestrator runs + * once per LANGUAGE workspace pass — so the walk happens only for repos that + * actually contain Zig. `loadImportConfigs`, which runs unconditionally for + * every repo, keeps calling `loadZigBuildConfig` for the root package alone; + * that is the same split TypeScript already has between the cheap + * `loadTsconfigPaths` and the repo-walking `loadTsconfigIndex`. + */ +export async function loadZigWorkspaceIndex(repoRoot: string): Promise { + const dirs = await findZigPackageDirs(repoRoot); + if (dirs.length === 0) return null; + const packages: ZigPackageScope[] = []; + for (const dir of dirs) { + const config = await loadZigBuildConfig(repoRoot, dir); + // A `build.zig` that declares no module and no path dep contributes nothing + // a lookup could answer with. Keeping it as an empty scope would be worse + // than dropping it: it would shadow an enclosing package that DOES declare + // the name, and answer nothing in its place. + if (config !== null) packages.push({ dir, config }); + } + if (packages.length === 0) return null; + // Deepest first, so `zigPackageFor` takes the most specific package rather + // than whichever the walk reached first. + packages.sort((a, b) => b.dir.length - a.dir.length || a.dir.localeCompare(b.dir)); + return { packages }; +} + +/** Repo-relative directories holding a `build.zig` and/or a `build.zig.zon`. */ +async function findZigPackageDirs(repoRoot: string): Promise { + const found: string[] = []; + const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + // A HEAD INDEX rather than `queue.shift()`. The queue is pushed to while it is + // drained, which keeps the array in a mode where `shift()` memmoves the whole + // remainder instead of taking V8's left-trimming fast path — so the walk is + // quadratic in the frontier, and `ZIG_SCAN_MAX_DIRS` is the bound on how bad + // that gets. Measured at that bound (20,000 dequeues): 53 ms at fan-out 4 and + // 81 ms at fan-out 20, against 0.8 ms here — 66-106x, paid before any config + // is read. Memory is unchanged: entries were already retained by the pushes, + // `shift()` only dropped the head. + let queueHead = 0; + let dirsScanned = 0; + + while (queueHead < queue.length && dirsScanned < ZIG_SCAN_MAX_DIRS) { + const { dir, depth } = queue[queueHead++]!; + dirsScanned++; + let entries: import('fs').Dirent[]; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + continue; + } + let isPackage = false; + for (const entry of entries) { + if (entry.isDirectory()) { + const childDir = path.join(dir, entry.name); + if (isHardcodedIgnoredDirectoryAtPath(repoRoot, childDir)) continue; + if (depth < ZIG_SCAN_MAX_DEPTH) queue.push({ dir: childDir, depth: depth + 1 }); + continue; + } + if (!entry.isFile()) continue; + // Either marker declares a package: a `build.zig` with no zon still names + // modules, and a zon with no build.zig still names path deps. + if (entry.name === 'build.zig' || entry.name === 'build.zig.zon') isPackage = true; + } + if (isPackage) { + const rel = path.relative(repoRoot, dir).split(path.sep).join('/'); + found.push(rel === '.' || rel === '' ? '' : rel); + } + } + return found; +} + /** * Normalize a `.path` value from build.zig.zon into a repo-relative form. * Returns null for paths that escape the repo root (start with `..`) or @@ -722,13 +945,38 @@ export async function loadZigBuildConfig(repoRoot: string): Promise & TCanonical; } +export interface ProviderSemanticNode { + readonly id: string; + readonly label: NodeLabel; + readonly properties: { + readonly name: string; + readonly filePath: string; + readonly startLine: number; + readonly endLine: number; + readonly language: SupportedLanguages; + readonly isExported: boolean; + readonly qualifiedName?: string; + readonly parameterCount?: number; + readonly requiredParameterCount?: number; + readonly parameterTypes?: readonly string[]; + readonly parameterTypeClasses?: readonly ParameterTypeClass[]; + readonly returnType?: string; + readonly declaredType?: string; + readonly visibility?: string; + readonly isStatic?: boolean; + readonly isReadonly?: boolean; + readonly [key: string]: unknown; + }; +} + +export type ProviderSemanticRelationshipType = Extract< + RelationshipType, + 'DECLARES' | 'DEFINES' | 'HAS_METHOD' | 'HAS_PROPERTY' +>; + +export interface ProviderSemanticRelationship { + readonly id: string; + readonly sourceId: string; + readonly targetId: string; + readonly type: ProviderSemanticRelationshipType; + readonly confidence: number; + readonly reason: string; +} + +export interface ProviderSemanticSymbol { + readonly filePath: string; + readonly name: string; + readonly nodeId: string; + readonly type: NodeLabel; + readonly qualifiedName?: string; + readonly parameterCount?: number; + readonly requiredParameterCount?: number; + readonly parameterTypes?: readonly string[]; + readonly parameterTypeClasses?: readonly ParameterTypeClass[]; + readonly returnType?: string; + readonly declaredType?: string; + readonly ownerId?: string; + readonly visibility?: string; + readonly isStatic?: boolean; + readonly isReadonly?: boolean; +} + +export interface ProviderSemanticGraph { + readonly nodes: readonly ProviderSemanticNode[]; + readonly relationships: readonly ProviderSemanticRelationship[]; + readonly symbols: readonly ProviderSemanticSymbol[]; +} + // ── Strategy tag types ───────────────────────────────────────────────────── // NOTE: `MroStrategy` is defined in `gitnexus-shared` and re-exported above // so `core/ingestion/model/resolve.ts` can consume it without importing from @@ -230,6 +292,29 @@ interface LanguageProviderConfig { */ readonly runtimeSymbolStrategy?: RuntimeSymbolStrategy; + /** + * Optional content-based language classifier. The filename detector remains + * the default source of truth; this hook lets a provider claim ambiguous + * files only when the source text carries language-specific evidence. + * + * Used for extensions shared by several languages, where mapping the suffix + * globally would steal files from an existing provider. Implementations must + * be deterministic and conservative: false negatives are acceptable, false + * positives change which parser and resolver consumes the file. + * + * Default: undefined (provider never overrides filename detection). + */ + readonly classifyFileContent?: (filePath: string, sourceText: string) => boolean; + + /** + * Cheap path-only prefilter for `classifyFileContent`. When supplied, callers + * can avoid loading source text for files this provider would never claim. + * + * Default: undefined (only callers that already have content invoke + * `classifyFileContent`). + */ + readonly shouldClassifyFileContent?: (filePath: string) => boolean; + // ── Core (required) ─────────────────────────────────────────────── /** Type extraction: declarations, initializers, for-loop bindings */ readonly typeConfig: LanguageTypeConfig; @@ -643,6 +728,23 @@ interface LanguageProviderConfig { repo: RepoConstants, ) => string | null; + /** + * Optional provider-owned semantic graph extraction for languages whose + * stable symbol identities cannot be represented by the generic query + * pipeline's `(label, filePath, qualifiedName)` rule. + * + * Runs in the parse worker after tree-sitter has parsed the file and after + * `extractParsedFile` has produced the scope-resolution artifact. The hook is + * deterministic and AST-based: it receives the already-parsed tree and must + * return plain graph nodes/relationships/symbol-table rows. Existing + * providers leave it undefined, preserving the generic capture path exactly. + */ + readonly extractSemanticGraph?: ( + tree: Parser.Tree, + filePath: string, + sourceText: string, + ) => ProviderSemanticGraph; + // ── Noise filtering ──────────────────────────────────────────────── /** Built-in/stdlib names that should be filtered from the call graph for this language. * Default: undefined (no language-specific filtering). */ diff --git a/gitnexus/src/core/ingestion/languages/cobol/copy-target.ts b/gitnexus/src/core/ingestion/languages/cobol/copy-target.ts new file mode 100644 index 000000000..7b0bde4c4 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/cobol/copy-target.ts @@ -0,0 +1,114 @@ +/** + * Shared COBOL `COPY` target resolution (#2967 / #2908). + * + * Both the census (`cobolScopeResolver.resolveImportTarget`) and the live + * regex processor (`resolveCopy` → `cobol-copy` IMPORTS) must answer the + * same path for the same `(name, fromFile, allFilePaths)`. Greening only + * the census would leave analyze fabricating an IMPORTS edge onto a vendor + * decoy the compiler would never have searched. + * + * Well-known copybook directory segments (`copybooks`, `COPYBOOKS`, `cpy`, + * `copy`) plus the importer's own directory are the preferred class. When + * the file set contains at least one of those well-known segments, a `COPY` + * name resolves only inside that class (first in Set-iteration order). + * `vendor/EXTERNAL.cpy` therefore misses while `copybooks/CUSTREC.cpy` hits. + * + * When the file set has no such directory, fail-open to today's two-tier + * first-wins basename index — shops whose copybooks *are* the tree have + * nothing to prefer. The importer directory is *not* enough on its own to + * leave fail-open (`{vendor/EXTERNAL.cpy, src/PROG.cbl}` must still answer + * `vendor/EXTERNAL.cpy`). + * + * The two-tier index is still memoized on the `allFilePaths` Set identity + * (#2908). Preferred-class filtering happens at pick time from the stored + * per-name arrays, so a pass still scans the set once. + */ +import path from 'node:path'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; + +const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); +const COBOL_SOURCE_EXTENSIONS = new Set(['.cbl', '.cob', '.cobol']); + +/** Path-component names that mark a conventional copybook directory. */ +const PREFERRED_DIR_NAMES = new Set(['copybooks', 'COPYBOOKS', 'cpy', 'copy']); + +interface CobolCopyIndex { + /** `.cpy` / `.copybook` files — tier 1, Set-iteration order per stem. */ + readonly copybooks: ReadonlyMap; + /** `.cbl` / `.cob` / `.cobol` files — tier 2, Set-iteration order per stem. */ + readonly sources: ReadonlyMap; + /** True iff any file path has a well-known copybook directory segment. */ + readonly hasPreferredDir: boolean; +} + +function pathHasPreferredDir(fp: string): boolean { + const parts = fp.split('/'); + for (let i = 0; i < parts.length - 1; i++) { + if (PREFERRED_DIR_NAMES.has(parts[i])) return true; + } + return false; +} + +function pushUnique(map: Map, key: string, fp: string): void { + const list = map.get(key); + if (list === undefined) { + map.set(key, [fp]); + return; + } + list.push(fp); +} + +const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { + const copybooks = new Map(); + const sources = new Map(); + let hasPreferredDir = false; + for (const fp of allFilePaths) { + const extRaw = path.extname(fp); + const extLower = extRaw.toLowerCase(); + const tier = COPYBOOK_EXTENSIONS.has(extLower) + ? copybooks + : COBOL_SOURCE_EXTENSIONS.has(extLower) + ? sources + : undefined; + if (tier === undefined) continue; + if (!hasPreferredDir && pathHasPreferredDir(fp)) hasPreferredDir = true; + const basename = path.basename(fp, extRaw).toUpperCase(); + pushUnique(tier, basename, fp); + } + return { copybooks, sources, hasPreferredDir }; +}); + +function pickCopyPath( + paths: readonly string[] | undefined, + fromFile: string, + hasPreferredDir: boolean, +): string | null { + if (paths === undefined || paths.length === 0) return null; + if (!hasPreferredDir) return paths[0] ?? null; + const fromDir = path.dirname(fromFile); + for (const fp of paths) { + if (pathHasPreferredDir(fp) || path.dirname(fp) === fromDir) return fp; + } + return null; +} + +/** + * Resolve a COBOL `COPY` member name against the workspace file set. + * + * `fromFile` is the importing program (census) or the program currently + * being expanded (processor). It is used only to include the importer's + * directory in the preferred class when a well-known copybook dir exists. + */ +export function resolveCobolCopyTarget( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, +): string | null { + const upper = targetRaw.toUpperCase(); + const index = getCobolCopyIndex(allFilePaths); + return ( + pickCopyPath(index.copybooks.get(upper), fromFile, index.hasPreferredDir) ?? + pickCopyPath(index.sources.get(upper), fromFile, index.hasPreferredDir) ?? + null + ); +} diff --git a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts index cfea8f39e..3d9e0eea6 100644 --- a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts @@ -9,74 +9,12 @@ * Reference: `languages/python/scope-resolver.ts`. */ -import path from 'node:path'; import type { ParsedFile } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; -import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; import { cobolProvider } from '../cobol.js'; - -// Copybook file extensions for COPY name resolution -const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); -// COBOL source files, searched only after every copybook has missed. -const COBOL_SOURCE_EXTENSIONS = new Set(['.cbl', '.cob', '.cobol']); - -/** - * Uppercased-basename → first file carrying it, one map PER TIER, memoized on - * the `allFilePaths` Set identity (#2908). - * - * `resolveImportTarget` used to run two full workspace scans per `COPY` — one - * for the copybook tier, one for the source tier — each calling `path.extname` - * + `path.basename` + `toUpperCase` on every entry. A `COPY` of a member that - * lives outside the repo (the common case: vendor and system copybooks) missed - * in both, so both scans always ran to completion, making resolution - * O(copies × files). The orchestrator passes the SAME Set to every import in a - * pass (`pipeline/run.ts` builds it once), so a `WeakMap` keyed on that Set - * turns the scans into one build per run. - * - * Two tiers rather than one map is the tie-break, not a stylistic choice: a - * `.cpy`/`.copybook` hit beats a `.cbl`/`.cob`/`.cobol` hit even when the source - * file comes FIRST in Set-iteration order, which is exactly what collapsing the - * tiers into a single first-wins map would silently discard. Within a tier the - * first file in Set-iteration order wins, mirroring the `return` on first match - * in the scans this replaces. - * - * The per-file key is derived with the same `path.extname(fp).toLowerCase()` → - * `path.basename(fp, ext)` → `toUpperCase()` sequence the scans used, including - * its quirk: `path.basename` strips the suffix only on an exact, case-sensitive - * match, so `Foo.CPY` indexes under `FOO.CPY` rather than `FOO`. Node's `path` - * stays in the loop for the same reason — on POSIX it does not treat `\` as a - * separator, and hand-rolled slicing on `/` would start resolving backslash - * paths the scans never resolved. - */ -interface CobolCopyIndex { - /** `.cpy` / `.copybook` files — tier 1. */ - readonly copybooks: ReadonlyMap; - /** `.cbl` / `.cob` / `.cobol` files — tier 2. */ - readonly sources: ReadonlyMap; -} - -const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { - const copybooks = new Map(); - const sources = new Map(); - // One pass builds both tiers: the two scans walked the same files and - // classified each by the same extension test. - for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - const tier = COPYBOOK_EXTENSIONS.has(ext) - ? copybooks - : COBOL_SOURCE_EXTENSIONS.has(ext) - ? sources - : undefined; - if (tier === undefined) continue; - const basename = path.basename(fp, ext).toUpperCase(); - // First in Set-iteration order wins, as the scans' first-match `return` did. - if (!tier.has(basename)) tier.set(basename, fp); - } - - return { copybooks, sources }; -}); +import { resolveCobolCopyTarget } from './copy-target.js'; const cobolScopeResolver: ScopeResolver = { language: SupportedLanguages.Cobol, @@ -84,11 +22,9 @@ const cobolScopeResolver: ScopeResolver = { importEdgeReason: 'cobol-scope: copy', // ── Resolve COPY bookname to file path ───────────────────────────── - resolveImportTarget: (targetRaw, _fromFile, allFilePaths) => { - const upper = targetRaw.toUpperCase(); - const index = getCobolCopyIndex(allFilePaths); - // Copybooks first, then COBOL sources — the tier order IS the tie-break. - return index.copybooks.get(upper) ?? index.sources.get(upper) ?? null; + // Shared with the regex processor's resolveCopy (#2967 lockstep). + resolveImportTarget: (targetRaw, fromFile, allFilePaths) => { + return resolveCobolCopyTarget(targetRaw, fromFile, allFilePaths); }, // COBOL has no binding-merge rules beyond the default (local-first-then-imports). diff --git a/gitnexus/src/core/ingestion/languages/dart/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/dart/name-fallback-visibility.ts new file mode 100644 index 000000000..562311bb8 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/dart/name-fallback-visibility.ts @@ -0,0 +1,77 @@ +/** + * Dart's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Dart's privacy is LIBRARY-scoped and marked in the identifier itself: a name + * beginning with `_` is visible only inside its own library and cannot be + * imported by any spelling. So a `_`-prefixed candidate in another file is an + * impossible call, not an unlikely one. + * + * The exact boundary is `part` / `part of` — one library spanning several + * files, with `_` names shared between them — and the extractor does not + * surface `part` directives yet (the Dart query captures only `library_import`; + * nothing in `languages/dart/` reads `part`). Without them a cross-file `_` + * candidate is UNDECIDABLE, not impossible: refusing on "different file" would + * delete real edges on Flutter's dominant generated-code idiom (`factory + * Foo.fromJson(j) => _$FooFromJson(j)` calls into `foo.g.dart`, a `part` beside + * it), and refusing on "different directory" is wrong too — a `part` URI is a + * relative URI and legally traverses directories (`part '../shared/gen.dart';`). + * An earlier version refused the cross-directory case as "no `part` layout can + * make this legal"; that claim was false, so the hook now REFUSES NOTHING and + * every cross-file `_` candidate stays a LABELED edge (0.5 / + * `global-name-fallback`), which is the honest answer until `part` is + * extracted. A caller that names the candidate's file in a directive is + * recognized already, for the day the extractor surfaces `part` as an import + * target; at that point "not the same library" becomes decidable and the + * refusal can return. + * + * Public names are left to the labeled-edge path. Dart does require an import + * for a cross-library public name, but the fallback exists partly to recover + * edges where the import chain was not reconstructed, and refusing every + * cross-file public call would delete real edges to buy a rule the `_` marker + * already gives for free. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { + modulePathReaches, + stripExtension, +} from '../../scope-resolution/utils/name-fallback-visibility.js'; + +/** Dart privacy marker: a leading underscore on the declared identifier. Read + * from the last `qualifiedName` segment, so `_Foo.bar` is public `bar` on a + * private class and `Foo._bar` is the private member. */ +function isPrivateDartName(candidate: SymbolDefinition): boolean { + const qualified = candidate.qualifiedName ?? ''; + const dot = qualified.lastIndexOf('.'); + const simple = dot === -1 ? qualified : qualified.slice(dot + 1); + return simple.startsWith('_'); +} + +/** + * Are these two files parts of one library? True when the caller names the + * candidate's file in a `part` / `part of` directive, which the extractor + * surfaces as an ordinary import target. + */ +function sharesLibrary(callerParsed: ParsedFile, candidateFilePath: string): boolean { + const candidateModule = stripExtension(candidateFilePath); + for (const imp of callerParsed.parsedImports) { + if (modulePathReaches(stripExtension(imp.targetRaw), candidateModule)) return true; + } + return false; +} + +export function dartIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + if (!isPrivateDartName(ctx.candidate)) return true; + // A directive naming the candidate's file is positive evidence of one library. + if (sharesLibrary(ctx.callerParsed, ctx.candidate.filePath)) return true; + // Any other file may be a `part` of the caller's library — a sibling or, via + // a relative `part` URI, a file in another directory. Undecidable without + // `part` extraction (see the header), so allowed as a labeled guess, never + // refused. + return true; +} diff --git a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts index 76bec5d74..bc46ad452 100644 --- a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts @@ -41,6 +41,7 @@ import { decodeMarker } from '../../utils/heritage-marker.js'; import { typeApplicationArguments } from '../../utils/template-arguments.js'; import type { HeritageTypeArgumentSink } from '../../scope-resolution/utils/generic-instantiation.js'; import { expandDartWildcardNames } from './expand-wildcards.js'; +import { dartIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; interface ClassDefRef { readonly graphId: string; @@ -233,5 +234,6 @@ export const dartScopeResolver: ScopeResolver = { // No `new`: bare `Foo()` resolves to the type; with cross-file imports the // callee is reachable workspace-wide. allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: dartIsGlobalNameFallbackPlausible, constructorCallTargetsClass: true, }; diff --git a/gitnexus/src/core/ingestion/languages/go/captures.ts b/gitnexus/src/core/ingestion/languages/go/captures.ts index c1a8cf23c..97fe71d83 100644 --- a/gitnexus/src/core/ingestion/languages/go/captures.ts +++ b/gitnexus/src/core/ingestion/languages/go/captures.ts @@ -508,11 +508,24 @@ function normalizeGenericConstructorCapture( if (referenceNode !== undefined && referenceNode.type === 'generic_type') { const base = referenceNode.childForFieldName('type'); if (base !== null) { - grouped['@reference.name'] = syntheticCapture( - '@reference.name', - base, - extractSimpleTypeNameText(base), - ); + const simple = extractSimpleTypeNameText(base); + // Keep the written `pkg.Box[T]` spelling so constructor resolution + // can tell a package-qualified type from a bare unique-name guess. + // `Box[T]` is also `text !== simple` (`Box[T]` vs `Box`) — that is + // not a qualifier. Only a `qualified_type` base (`pkg.Box`) is. + if ( + grouped['@reference.qualified-name'] === undefined && + base.type === 'qualified_type' && + referenceNode.text.length > 0 && + referenceNode.text !== simple + ) { + grouped['@reference.qualified-name'] = syntheticCapture( + '@reference.qualified-name', + referenceNode, + referenceNode.text, + ); + } + grouped['@reference.name'] = syntheticCapture('@reference.name', base, simple); } } } diff --git a/gitnexus/src/core/ingestion/languages/go/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/go/name-fallback-visibility.ts new file mode 100644 index 000000000..02b968e7b --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/go/name-fallback-visibility.ts @@ -0,0 +1,126 @@ +/** + * Go's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Go's visibility rules are unusually decidable from a file path and an + * identifier, which is why this is the language the guard is most complete for: + * + * 1. A package IS a directory. Two files in the same directory see each + * other's identifiers with no import and no qualification, so a same- + * directory candidate is always plausible. + * 2. An identifier is exported iff it begins with an upper-case letter. An + * UNEXPORTED identifier is invisible outside its own package — no import + * makes it reachable, so a cross-directory candidate with a lower-case + * initial is not unlikely, it is IMPOSSIBLE. + * 3. A bare exported identifier from another package requires a DOT import. + * Ordinary, alias, and blank imports never introduce a bare callable. + * + * Rule 2 is the one that matters most in practice: `uniqueHelperXyz` defined + * once in package `a` used to acquire a caller in package `b` purely because + * the name was unique in the repo, and the resulting edge was published as + * `import-resolved` — a caller Go itself would reject. + * + * Methods require a receiver and never qualify for this bare-name tier. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { + modulePathReaches, + directoryOf, +} from '../../scope-resolution/utils/name-fallback-visibility.js'; +import { inferGoPackageName } from './package-clause.js'; + +/** + * Compare actual package clauses, not a guessed `_test` suffix convention: + * `foo_test` can itself be a production package name. Missing source remains + * undecidable, while the test-file boundary is always available from the path. + */ +function classifyGoFile( + filePath: string, + sourceTextOf: ((p: string) => string | undefined) | undefined, +): { isTest: boolean; declared: string | undefined } { + const isTest = filePath.endsWith('_test.go'); + const text = sourceTextOf?.(filePath); + return { + isTest, + declared: text === undefined ? undefined : (inferGoPackageName(text) ?? undefined), + }; +} + +/** Go export rule: an upper-case initial, by Unicode letter case. */ +function isExportedGoName(name: string): boolean { + return /^\p{Lu}/u.test(name); +} + +/** + * The simple identifier a Go declaration contributes to its package scope: the + * last segment of `qualifiedName`, which is `Type.method` for a method and the + * bare identifier for a function. Either way the LAST segment is the identifier + * whose case decides export. + */ +function goSimpleName(candidate: SymbolDefinition): string { + const qualified = candidate.qualifiedName ?? ''; + const dot = qualified.lastIndexOf('.'); + return dot === -1 ? qualified : qualified.slice(dot + 1); +} + +export function goIsGlobalNameFallbackPlausible(ctx: { + readonly sourceTextOf?: (filePath: string) => string | undefined; + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly site?: { readonly rawQualifiedName?: string }; +}): boolean { + // A PATH-QUALIFIED call (`pkg.Export()`) reached this tier because the + // qualifier could not be followed. Qualification removes the bare-name + // "only a dot import introduces this identifier" rule — not export or + // test-package visibility. `other.private()` is still impossible. + const isPackageQualified = ctx.site?.rawQualifiedName !== undefined; + // Methods require a receiver, even within their own package or a dot import. + if (ctx.candidate.type === 'Method' || ctx.candidate.qualifiedName?.includes('.')) return false; + const callerDir = directoryOf(ctx.callerParsed.filePath); + const candidateDir = directoryOf(ctx.candidate.filePath); + // Same directory is NOT the same package (rule 1, refined): a `_test.go` + // file may declare `foo_test`, and test-only declarations are invisible to + // non-test files. Apply the split `package-siblings.ts` uses for the + // confident tier, so the heuristic tier cannot reopen what it closed. + if (callerDir === candidateDir) { + const caller = classifyGoFile(ctx.callerParsed.filePath, ctx.sourceTextOf); + const cand = classifyGoFile(ctx.candidate.filePath, ctx.sourceTextOf); + // Non-test files never see test-only declarations. + if (cand.isTest && !caller.isTest) return false; + // Different clauses require an explicit dot import, even in one directory. + // Fall through to the exported/import checks for external test packages. + // Missing source leaves package identity undecidable. + if ( + caller.declared === undefined || + cand.declared === undefined || + caller.declared === cand.declared + ) + return true; + } + + // Different package — and a `_test.go` file's + // declarations are compiled only into ITS OWN package's test binary. No other + // package, test or not, can see them, exported or not. Decidable from the + // path alone, so it comes before every exception below (the module-root + // exception in particular used to accept a root `helper_test.go` export). + if (classifyGoFile(ctx.candidate.filePath, ctx.sourceTextOf).isTest) return false; + + const simpleName = goSimpleName(ctx.candidate); + // No identifier to read the case of — an unanswered question, not a refusal. + if (simpleName === '') return true; + // Unexported across a package boundary (rule 2): no import can reach it. + if (!isExportedGoName(simpleName)) return false; + if (isPackageQualified) return true; + + // Only a dot import introduces a bare name. A candidate in the module ROOT package has an empty + // directory, which `modulePathReaches` cannot align against any import path + // (the root package is imported by the module path alone, which the repo- + // relative layout does not carry). That is an unanswered question, not a + // refusal — a dot import is plausible even if its path cannot be aligned. + return ctx.callerParsed.parsedImports.some( + (imp) => + imp.kind === 'wildcard' && + (candidateDir === '' || modulePathReaches(imp.targetRaw, candidateDir)), + ); +} diff --git a/gitnexus/src/core/ingestion/languages/go/package-siblings.ts b/gitnexus/src/core/ingestion/languages/go/package-siblings.ts index bb5c4cab8..4efe57f82 100644 --- a/gitnexus/src/core/ingestion/languages/go/package-siblings.ts +++ b/gitnexus/src/core/ingestion/languages/go/package-siblings.ts @@ -4,92 +4,189 @@ import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexe import { expandGoDotImports } from './expand-wildcards.js'; import { goPackageDir, inferGoPackageName } from './package-clause.js'; +function isGoTestFile(filePath: string): boolean { + return filePath.endsWith('_test.go'); +} + /** - * O(n²×d) where n = files per package, d = defs per file. - * Acceptable for V1 since Go packages are typically small (< 20 files). - * Future optimization: build a name→def inverted index per package to reduce - * to O(n×d). + * The package a `_test.go` file's clause belongs to, given the package names + * the directory's NON-test files declare. + * + * `package foo_test` is the external-test convention ONLY when the directory's + * real package is `foo`; the `_test` suffix is otherwise a legal identifier + * (`package foo_test` in a directory whose non-test files also say `foo_test`). + * Stripping it unconditionally keyed such a package's own internal tests as + * external tests of a non-existent `foo`, so they saw no sibling at all — a + * resolution miss on every same-package call. Strip only when the stripped + * name is what the non-test siblings declare; with no non-test sibling to ask + * (a test-only directory) the convention is assumed, as before. + */ +function testFilePackageOf( + declared: string, + nonTestPackagesInDir: ReadonlySet | undefined, +): { readonly pkg: string; readonly external: boolean } { + if (!declared.endsWith('_test') || declared.length <= '_test'.length) { + return { pkg: declared, external: false }; + } + const stripped = declared.slice(0, -'_test'.length); + if (nonTestPackagesInDir !== undefined && nonTestPackagesInDir.has(declared)) { + return { pkg: declared, external: false }; + } + if (nonTestPackagesInDir === undefined || nonTestPackagesInDir.has(stripped)) { + return { pkg: stripped, external: true }; + } + // Neither name is declared by a non-test sibling: keep the clause as written. + return { pkg: declared, external: false }; +} + +interface IndexedDef { + readonly filePath: string; + readonly ref: BindingRef; +} + +/** name → defs, in package file order. */ +type NameIndex = Map; + +function defBareName(def: SymbolDefinition): string { + return def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; +} + +function appendToIndex( + index: NameIndex, + filePath: string, + defs: readonly SymbolDefinition[], +): void { + for (const def of defs) { + const name = defBareName(def); + if (name === '') continue; + const list = index.get(name) ?? []; + list.push({ filePath, ref: { def, origin: 'namespace' } }); + index.set(name, list); + } +} + +function publishIndex( + augmentations: Map>, + index: NameIndex, + receiverPath: string, + receiverModule: ScopeId, +): void { + if (index.size === 0) return; + let scopeBindings = augmentations.get(receiverModule); + if (scopeBindings === undefined) { + scopeBindings = new Map(); + augmentations.set(receiverModule, scopeBindings); + } + for (const [name, entries] of index) { + let bucket = scopeBindings.get(name); + const seen = + bucket === undefined ? new Set() : new Set(bucket.map((b) => b.def.nodeId)); + for (const entry of entries) { + if (entry.filePath === receiverPath) continue; + if (seen.has(entry.ref.def.nodeId)) continue; + if (bucket === undefined) { + bucket = []; + scopeBindings.set(name, bucket); + } + bucket.push(entry.ref); + seen.add(entry.ref.def.nodeId); + } + } +} + +/** + * Publish same-package sibling bindings, including `_test.go` files. + * + * Internal tests (`package foo`) see production and other internal-test names. + * External tests (`package foo_test`) get no bare-name bindings across that + * partition — qualified `foo.X` and `import .` stay on the import resolver. + * Non-test files never see test-only helpers (`go build` does not compile them). + * + * Per-package name→def indexes are built in O(n×d). Each receiver walks only + * the partitions it can see, so defs are not re-scanned against every sibling + * file. Binding refs are allocated once and reused across receivers. */ export function populateGoPackageSiblings( parsedFiles: readonly ParsedFile[], indexes: ScopeResolutionIndexes, ctx: { readonly fileContents: ReadonlyMap }, ): void { - // 0. Filter out test files — Go _test.go files should not contribute - // same-package sibling bindings to non-test files. - const nonTestFiles = parsedFiles.filter((f) => !f.filePath.endsWith('_test.go')); - // 1. Expand dot imports first so subsequent same-package sibling - // augmentation can also see dot-imported names. - expandGoDotImports(nonTestFiles, indexes); + // augmentation can also see dot-imported names. Test files dot-import too. + expandGoDotImports(parsedFiles, indexes); // 2. Group files by package directory plus package name. Go package // identity is directory-scoped; repeated `package main` directories // must not see each other's unqualified names. - const packageByFile = new Map(); - for (const parsed of nonTestFiles) { - // Same derivation as `populateGoWorkspaceOwners` — one shared resolver, so - // the two passes cannot disagree about a file's package (#2837). The - // no-clause case is reported there; warning twice for one fact would be - // noise. - const pkgName = inferGoPackageName(ctx.fileContents.get(parsed.filePath) ?? ''); - if (pkgName !== null) { - packageByFile.set(parsed.filePath, `${goPackageDir(parsed.filePath)}\0${pkgName}`); - } + // + // `_test.go` files join the INTERNAL package's bucket (a `foo_test` + // external test is keyed by `foo`, its `external` flag marking the + // partition so no bare-name bindings cross it), so one bucket holds + // everything the test binary compiles together, and the visibility + // rules below decide who sees whom. + interface SiblingFile { + readonly filePath: string; + readonly defs: readonly SymbolDefinition[]; + readonly isTest: boolean; + readonly external: boolean; + } + const filesByPackage = new Map(); + // Same derivation as `populateGoWorkspaceOwners` — one shared resolver, so + // the two passes cannot disagree about a file's package (#2837). The + // no-clause case is reported there; warning twice for one fact would be + // noise. + const declaredByFile = new Map(); + const nonTestPackagesByDir = new Map>(); + for (const parsed of parsedFiles) { + const declared = inferGoPackageName(ctx.fileContents.get(parsed.filePath) ?? ''); + if (declared === null) continue; + declaredByFile.set(parsed.filePath, declared); + if (isGoTestFile(parsed.filePath)) continue; + const dir = goPackageDir(parsed.filePath); + const names = nonTestPackagesByDir.get(dir) ?? new Set(); + names.add(declared); + nonTestPackagesByDir.set(dir, names); + } + for (const parsed of parsedFiles) { + const declared = declaredByFile.get(parsed.filePath); + if (declared === undefined) continue; + const isTest = isGoTestFile(parsed.filePath); + const dir = goPackageDir(parsed.filePath); + const { pkg, external } = isTest + ? testFilePackageOf(declared, nonTestPackagesByDir.get(dir)) + : { pkg: declared, external: false }; + const key = `${dir}\0${pkg}`; + const list = filesByPackage.get(key) ?? []; + list.push({ filePath: parsed.filePath, defs: parsed.localDefs, isTest, external }); + filesByPackage.set(key, list); } - const filesByPackage = new Map(); - for (const parsed of nonTestFiles) { - const pkgName = packageByFile.get(parsed.filePath); - if (pkgName === undefined) continue; - const list = filesByPackage.get(pkgName) ?? []; - list.push({ filePath: parsed.filePath, defs: [...parsed.localDefs] }); - filesByPackage.set(pkgName, list); - } - - // 2. Use bindingAugmentations channel per I8 + // 3. Use bindingAugmentations channel per I8. Same-package files see ALL + // sibling names (exported and unexported). Cross-package visibility is + // the import resolver's job. const augmentations = indexes.bindingAugmentations as Map>; - for (const [, siblings] of filesByPackage) { - for (const target of siblings) { - const targetModule = indexes.moduleScopes.byFilePath.get(target.filePath); - if (targetModule === undefined) continue; - - for (const receiver of siblings) { - if (receiver.filePath === target.filePath) continue; // no self-reference - const receiverModule = indexes.moduleScopes.byFilePath.get(receiver.filePath); - if (receiverModule === undefined) continue; - - for (const def of target.defs) { - // Go: same-package sibling files can see ALL names (both - // exported/uppercase and unexported/lowercase). Only cross- - // package visibility requires uppercase first letter. - const name = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; - if (name === '') continue; - - const bucket = getAugmentationBucket(augmentations, receiverModule, name); - if (bucket.some((b) => b.def.nodeId === def.nodeId)) continue; - bucket.push({ def, origin: 'namespace' }); - } + for (const siblings of filesByPackage.values()) { + if (siblings.length < 2) continue; + const production: NameIndex = new Map(); + const internalTest: NameIndex = new Map(); + const external: NameIndex = new Map(); + const receivers: { file: SiblingFile; module: ScopeId }[] = []; + for (const file of siblings) { + const module = indexes.moduleScopes.byFilePath.get(file.filePath); + if (module === undefined) continue; + receivers.push({ file, module }); + if (file.external) appendToIndex(external, file.filePath, file.defs); + else if (file.isTest) appendToIndex(internalTest, file.filePath, file.defs); + else appendToIndex(production, file.filePath, file.defs); + } + for (const { file, module } of receivers) { + if (file.external) { + publishIndex(augmentations, external, file.filePath, module); + continue; } + publishIndex(augmentations, production, file.filePath, module); + if (file.isTest) publishIndex(augmentations, internalTest, file.filePath, module); } } } - -function getAugmentationBucket( - augmentations: Map>, - scopeId: ScopeId, - name: string, -): BindingRef[] { - let scopeBindings = augmentations.get(scopeId); - if (scopeBindings === undefined) { - scopeBindings = new Map(); - augmentations.set(scopeId, scopeBindings); - } - let bucketArr = scopeBindings.get(name); - if (bucketArr === undefined) { - bucketArr = []; - scopeBindings.set(name, bucketArr); - } - return bucketArr; -} diff --git a/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts index ff891d079..7c03e10f6 100644 --- a/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/go/scope-resolver.ts @@ -16,6 +16,7 @@ import { detectGoInterfaceImplementations } from './interface-impls.js'; import { populateGoRangeBindings } from './range-binding.js'; import { expandGoWildcardNames } from './expand-wildcards.js'; import { goMapValueType } from './interpret.js'; +import { goIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; /** Slice `[]T` and array `[N]T` / `[...]T` → the element spelling. Hoisted — * a literal inside the hook would mint a fresh RegExp per folded subscript. */ @@ -85,6 +86,7 @@ export const goScopeResolver: ScopeResolver = { hoistTypeBindingsToModule: true, propagatesReturnTypesAcrossImports: true, allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: goIsGlobalNameFallbackPlausible, populateNamespaceSiblings: populateGoPackageSiblings, mirrorNamespaceTypeBindings: mirrorGoNamespaceTypeBindings, diff --git a/gitnexus/src/core/ingestion/languages/index.ts b/gitnexus/src/core/ingestion/languages/index.ts index d36c70d9f..b10822a6f 100644 --- a/gitnexus/src/core/ingestion/languages/index.ts +++ b/gitnexus/src/core/ingestion/languages/index.ts @@ -8,7 +8,11 @@ * 4. Run `tsc --noEmit` to verify */ -import { SupportedLanguages, isBladeTemplateFilename } from 'gitnexus-shared'; +import { + SupportedLanguages, + getLanguageFromFilename, + isBladeTemplateFilename, +} from 'gitnexus-shared'; import type { LanguageProvider } from '../language-provider.js'; import { typescriptProvider, javascriptProvider } from './typescript.js'; @@ -19,6 +23,7 @@ import { goProvider } from './go.js'; import { rustProvider } from './rust.js'; import { csharpProvider } from './csharp.js'; import { cProvider, cppProvider } from './c-cpp.js'; +import { objectiveCProvider } from './objective-c.js'; import { phpProvider } from './php.js'; import { rubyProvider } from './ruby.js'; import { swiftProvider } from './swift.js'; @@ -38,6 +43,7 @@ export const providers = { [SupportedLanguages.CSharp]: csharpProvider, [SupportedLanguages.C]: cProvider, [SupportedLanguages.CPlusPlus]: cppProvider, + [SupportedLanguages.ObjectiveC]: objectiveCProvider, [SupportedLanguages.PHP]: phpProvider, [SupportedLanguages.Ruby]: rubyProvider, [SupportedLanguages.Swift]: swiftProvider, @@ -70,3 +76,39 @@ export function getProviderForFile(filePath: string): LanguageProvider | null { const basename = filePath.slice(filePath.lastIndexOf('/') + 1); return extensionMap.get(ext) ?? extensionMap.get(basename) ?? null; } + +/** Return the provider whose content classifier confidently claims this file. */ +export function getProviderForFileContent( + filePath: string, + content: string, +): LanguageProvider | null { + if (isBladeTemplateFilename(filePath)) return null; + + for (const provider of Object.values(providers)) { + if (provider.classifyFileContent?.(filePath, content) === true) return provider; + } + const byExtension = getProviderForFile(filePath); + if (byExtension) return byExtension; + // Filename-only languages (Rakefile, Gemfile, …) live in + // getLanguageFromFilename, not provider.extensions. + const filenameLang = getLanguageFromFilename(filePath); + return filenameLang === null ? null : providers[filenameLang]; +} + +/** True when at least one provider wants source text before language bucketing. */ +export function needsContentLanguageClassification(filePath: string): boolean { + if (isBladeTemplateFilename(filePath)) return false; + return Object.values(providers).some( + (provider) => + provider.classifyFileContent !== undefined && + provider.shouldClassifyFileContent?.(filePath) === true, + ); +} + +/** Return the effective language for a file, optionally using source content. */ +export function getLanguageForFileContent( + filePath: string, + content: string, +): SupportedLanguages | null { + return getProviderForFileContent(filePath, content)?.id ?? null; +} diff --git a/gitnexus/src/core/ingestion/languages/java/captures.ts b/gitnexus/src/core/ingestion/languages/java/captures.ts index 5d31aed4a..b227d549c 100644 --- a/gitnexus/src/core/ingestion/languages/java/captures.ts +++ b/gitnexus/src/core/ingestion/languages/java/captures.ts @@ -357,6 +357,26 @@ export function emitJavaScopeCaptures( } } + // Qualified constructor `new pkg.Foo()` already binds the simple tail as + // `@reference.name` (F35). Copy the scoped spelling onto + // `@reference.qualified-name` so constructor resolution can treat it as a + // written qualifier instead of a bare unique-name guess (C# does the same). + const ctorQualifiedNode = nodeMap['@reference.call.constructor.qualified']; + const ctorQualifiedText = grouped['@reference.call.constructor.qualified']?.text; + if ( + ctorQualifiedNode !== undefined && + ctorQualifiedText !== undefined && + ctorQualifiedText.length > 0 && + ctorQualifiedText !== grouped['@reference.name']?.text && + grouped['@reference.qualified-name'] === undefined + ) { + grouped['@reference.qualified-name'] = syntheticCapture( + '@reference.qualified-name', + ctorQualifiedNode, + ctorQualifiedText, + ); + } + // Synthesize `@reference.arity` on every callsite. const callTag = ( ['@reference.call.free', '@reference.call.member', '@reference.call.constructor'] as const diff --git a/gitnexus/src/core/ingestion/languages/javascript/captures.ts b/gitnexus/src/core/ingestion/languages/javascript/captures.ts index d3ba94791..b16cb2ba1 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/captures.ts @@ -36,6 +36,7 @@ import { syntheticCapture, type SyntaxNode, } from '../../utils/ast-helpers.js'; +import { collectEsmExportEvidence, esmExportVerdict } from '../../ts-js-export-marker.js'; import { splitImportStatement } from '../typescript/import-decomposer.js'; import { getJsParser, getJsScopeQuery, jsCachedTreeMatchesGrammar } from './query.js'; import { computeTsArityMetadata } from '../typescript/arity-metadata.js'; @@ -983,6 +984,8 @@ export function emitJsScopeCaptures( } const rawMatches = getJsScopeQuery(filePath).matches(tree.rootNode); + // Export evidence, read once per file (see `ts-js-export-marker.ts`). + const exportEvidence = collectEsmExportEvidence(tree.rootNode, filePath); const out: CaptureMatch[] = []; for (const m of rawMatches) { @@ -1124,6 +1127,13 @@ export function emitJsScopeCaptures( fnNode, deriveDefaultExportHocName(filePath), ); + // This declaration's name is synthetic, so the later query-name + // marker cannot see it. The HOC predicate already proves the export. + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + fnNode, + 'true', + ); } } @@ -1207,6 +1217,20 @@ export function emitJsScopeCaptures( // non-call match, an absent receiver, or a chain with no nameable base // all leave `grouped` untouched. synthesizeReceiverChainCapture(grouped, groupedNodes['@reference.receiver']); + // `@declaration.is-exported`: a verdict for every declaration the file's + // export surface can decide (see `ts-js-export-marker.ts`); nothing where + // it cannot, because absence is the honest answer there. + const declNameNode = groupedNodes['@declaration.name']; + if (exportEvidence !== undefined && declNameNode !== undefined) { + const verdict = esmExportVerdict(declNameNode, exportEvidence); + if (verdict !== undefined) { + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + declNameNode, + verdict ? 'true' : 'false', + ); + } + } out.push(grouped); // Synthesize `this` receiver type-bindings on class member functions. diff --git a/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts index 8d74798ee..280fd9929 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/scope-resolver.ts @@ -97,6 +97,12 @@ const javascriptScopeResolver: ScopeResolver = { // explicit imports at the call site. Workspace-wide unique-name fallback // recovers these edges. allowGlobalFreeCallFallback: true, + + // Same ECMAScript `export *` exclusivity as TypeScript: a name declared by + // two wildcard sources is refused, not guessed. + exclusiveWildcardReexports: true, + // Same ECMAScript rule as TypeScript: named imports bind module-level declarations only. + namedImportsBindTopLevelOnly: true, }; export { javascriptScopeResolver }; diff --git a/gitnexus/src/core/ingestion/languages/objective-c.ts b/gitnexus/src/core/ingestion/languages/objective-c.ts new file mode 100644 index 000000000..8e7f3e2b4 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c.ts @@ -0,0 +1,343 @@ +import path from 'path'; +import { + SupportedLanguages, + type CaptureMatch, + type ParsedImport, + type ParsedTypeBinding, +} from 'gitnexus-shared'; +import Parser from 'tree-sitter'; +import { defineLanguage } from '../language-provider.js'; +import type { ImportResolverFn } from '../import-resolvers/types.js'; +import { getLanguageGrammar } from '../../tree-sitter/parser-loader.js'; +import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; +import { assertCloneable } from '../workers/clone-safety.js'; +import { preprocessObjectiveCMacroMarkers } from './objective-c/macro-marker-preprocess.js'; +import { + buildObjectiveCSemanticGraph, + buildObjectiveCScopeCaptures, + collectObjectiveCCaptureSideChannel, + collectObjectiveCFacts, + parseObjCType, + setObjectiveCFileFacts, +} from './objective-c/facts.js'; + +const OBJECTIVE_C_SCOPE_QUERY = `((translation_unit) @objc.root)`; + +const EMPTY_TYPE_CONFIG = { + declarationNodeTypes: new Set(), + extractDeclaration: () => null, + extractParameter: () => null, +}; + +const noImportResolution: ImportResolverFn = () => null; + +function normalizedExt(filePath: string): string { + return path.extname(filePath).toLowerCase(); +} + +function isObjectiveCSourcePath(filePath: string): boolean { + const ext = normalizedExt(filePath); + return ext === '.m' || ext === '.mm'; +} + +function isHeaderPath(filePath: string): boolean { + return normalizedExt(filePath) === '.h'; +} + +const OBJECTIVE_C_HEADER_NODE_TYPES = new Set([ + 'class_declaration', + 'class_interface', + 'class_implementation', + 'compatibility_alias_declaration', + 'module_import', + 'protocol_declaration', + // Embedding chunks a Method node's snippet, not the enclosing @interface. + // tree-sitter-objc only emits these for +/- declarators, not C++ functions. + 'method_declaration', + 'method_definition', +]); + +const OBJECTIVE_C_HEADER_DIRECTIVES = [ + '@class', + '@compatibility_alias', + '@import', + '@implementation', + '@interface', + '@protocol', +]; + +function hasObjectiveCHeaderSyntax(sourceText: string): boolean { + try { + const tree = parseObjectiveCSource(sourceText); + const stack: Parser.SyntaxNode[] = [tree.rootNode]; + while (stack.length > 0) { + const node = stack.pop(); + if (node === undefined) continue; + if (OBJECTIVE_C_HEADER_NODE_TYPES.has(node.type)) return true; + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child !== null) stack.push(child); + } + } + } catch { + // Keep unambiguous Objective-C headers on the normal unavailable-parser path. + return hasObjectiveCHeaderDirective(sourceText) || hasObjectiveCMethodDeclarator(sourceText); + } + // Bare method snippets (`- (void)run;`) are not `method_declaration` at + // translation-unit scope; the grammar only emits that under @interface. + return hasObjectiveCMethodDeclarator(sourceText); +} + +/** + * Line-leading `+/- (Type)selector` used when the tree has no @interface and + * when the grammar cannot load. `-(x);` does not match: `)` is not followed + * by a selector identifier. + */ +function hasObjectiveCMethodDeclarator(sourceText: string): boolean { + return /(?:^|[\r\n])[ \t]*[+-][ \t]*\([^;\n]+\)[ \t]*[A-Za-z_]/.test( + sourceTextWithoutComments(sourceText), + ); +} + +/** Replace comments with spaces so line-leading method probes see only code. */ +function sourceTextWithoutComments(sourceText: string): string { + const chars = sourceText.split(''); + let index = 0; + let state: 'code' | 'line-comment' | 'block-comment' | 'single-quote' | 'double-quote' = 'code'; + + while (index < chars.length) { + const current = chars[index]; + const next = chars[index + 1]; + + if (state === 'line-comment') { + if (current === '\n' || current === '\r') state = 'code'; + else chars[index] = ' '; + index++; + continue; + } + if (state === 'block-comment') { + if (current === '*' && next === '/') { + chars[index] = ' '; + chars[index + 1] = ' '; + state = 'code'; + index += 2; + } else { + if (current !== '\n' && current !== '\r') chars[index] = ' '; + index++; + } + continue; + } + if (state === 'single-quote' || state === 'double-quote') { + if (current === '\\') { + index += 2; + } else if ( + (state === 'single-quote' && current === "'") || + (state === 'double-quote' && current === '"') + ) { + state = 'code'; + index++; + } else { + index++; + } + continue; + } + + if (current === '/' && next === '/') { + chars[index] = ' '; + chars[index + 1] = ' '; + state = 'line-comment'; + index += 2; + continue; + } + if (current === '/' && next === '*') { + chars[index] = ' '; + chars[index + 1] = ' '; + state = 'block-comment'; + index += 2; + continue; + } + if (current === "'") { + state = 'single-quote'; + index++; + continue; + } + if (current === '"') { + state = 'double-quote'; + index++; + continue; + } + index++; + } + + return chars.join(''); +} + +function hasObjectiveCHeaderDirective(sourceText: string): boolean { + let index = 0; + let state: 'code' | 'line-comment' | 'block-comment' | 'single-quote' | 'double-quote' = 'code'; + + while (index < sourceText.length) { + const current = sourceText[index]; + const next = sourceText[index + 1]; + + if (state === 'line-comment') { + if (current === '\n' || current === '\r') state = 'code'; + index++; + continue; + } + if (state === 'block-comment') { + if (current === '*' && next === '/') { + state = 'code'; + index += 2; + } else { + index++; + } + continue; + } + if (state === 'single-quote' || state === 'double-quote') { + if (current === '\\') { + index += 2; + } else if ( + (state === 'single-quote' && current === "'") || + (state === 'double-quote' && current === '"') + ) { + state = 'code'; + index++; + } else { + index++; + } + continue; + } + + if (current === '/' && next === '/') { + state = 'line-comment'; + index += 2; + continue; + } + if (current === '/' && next === '*') { + state = 'block-comment'; + index += 2; + continue; + } + if (current === "'") { + state = 'single-quote'; + index++; + continue; + } + if (current === '"') { + state = 'double-quote'; + index++; + continue; + } + if (current === '@') { + const directive = OBJECTIVE_C_HEADER_DIRECTIVES.find((candidate) => + sourceText.startsWith(candidate, index), + ); + if (directive !== undefined && !isIdentifierCharacter(sourceText[index + directive.length])) { + return true; + } + } + index++; + } + return false; +} + +function isIdentifierCharacter(character: string | undefined): boolean { + return character !== undefined && /[A-Za-z0-9_]/.test(character); +} + +export function classifyObjectiveCFileContent(filePath: string, sourceText: string): boolean { + if (isObjectiveCSourcePath(filePath)) return true; + if (!isHeaderPath(filePath)) return false; + return hasObjectiveCHeaderSyntax(sourceText); +} + +function parseObjectiveCSource(sourceText: string): Parser.Tree { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(SupportedLanguages.ObjectiveC)); + return parseSourceSafe( + parser, + preprocessObjectiveCMacroMarkers(sourceText), + undefined, + undefined, + 'Objective-C source', + ); +} + +function treeFromCachedOrSource(cachedTree: unknown, sourceText: string): Parser.Tree { + if (cachedTree !== undefined && looksLikeTree(cachedTree)) return cachedTree; + return parseObjectiveCSource(sourceText); +} + +function looksLikeTree(value: unknown): value is Parser.Tree { + return ( + value !== null && + typeof value === 'object' && + 'rootNode' in value && + (value as { rootNode?: unknown }).rootNode !== undefined + ); +} + +function interpretObjectiveCImport(captures: CaptureMatch): ParsedImport | null { + const source = captures['@import.source']; + if (source === undefined || source.text.trim().length === 0) return null; + const targetRaw = source.text.trim(); + const kind = captures['@import.kind']?.text.trim(); + const isSystemHeader = targetRaw.startsWith('<') && targetRaw.endsWith('>'); + return { + kind: 'side-effect', + // Scope resolution needs to distinguish a quoted header path from a bare + // @import module name and an angle-bracket system header. The latter stays + // wrapped so the Objective-C resolver can fail closed instead of resolving + // a framework header to a same-named local file. + targetRaw: + isSystemHeader || kind === 'module' || targetRaw.startsWith('./') + ? targetRaw + : `./${targetRaw}`, + }; +} + +function interpretObjectiveCTypeBinding(captures: CaptureMatch): ParsedTypeBinding | null { + const name = captures['@type-binding.name']; + const type = captures['@type-binding.type']; + if (name === undefined || type === undefined) return null; + const parsed = parseObjCType(type.text); + return { + boundName: name.text, + rawTypeName: parsed?.name ?? parsed?.raw ?? type.text, + declaredSpelling: type.text, + source: 'annotation', + }; +} + +export const objectiveCProvider = defineLanguage({ + id: SupportedLanguages.ObjectiveC, + extensions: ['.m', '.mm'], + treeSitterQueries: OBJECTIVE_C_SCOPE_QUERY, + typeConfig: EMPTY_TYPE_CONFIG, + exportChecker: () => true, + importResolver: noImportResolution, + classifyFileContent: classifyObjectiveCFileContent, + shouldClassifyFileContent: isHeaderPath, + preprocessSource: preprocessObjectiveCMacroMarkers, + importsExecuteWhereWritten: false, + + emitScopeCaptures: (sourceText, filePath, cachedTree): readonly CaptureMatch[] => { + const tree = treeFromCachedOrSource(cachedTree, sourceText); + const facts = collectObjectiveCFacts(tree, filePath); + setObjectiveCFileFacts(facts); + return buildObjectiveCScopeCaptures(facts, tree.rootNode); + }, + + collectCaptureSideChannel: (filePath) => + assertCloneable(collectObjectiveCCaptureSideChannel(filePath)), + + interpretImport: interpretObjectiveCImport, + interpretTypeBinding: interpretObjectiveCTypeBinding, + + extractSemanticGraph: (tree, filePath) => { + const facts = collectObjectiveCFacts(tree, filePath); + setObjectiveCFileFacts(facts); + return buildObjectiveCSemanticGraph(facts); + }, +}); diff --git a/gitnexus/src/core/ingestion/languages/objective-c/analysis-features.ts b/gitnexus/src/core/ingestion/languages/objective-c/analysis-features.ts new file mode 100644 index 000000000..7f2546162 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/analysis-features.ts @@ -0,0 +1,27 @@ +import type { AnalysisFeatureDescriptor } from '../../../analysis-features.js'; +import { + OBJECTIVE_C_GRAMMAR_PACKAGE, + OBJECTIVE_C_GRAMMAR_VERSION, + OBJECTIVE_C_PROVIDER_VERSION, +} from './facts.js'; + +function isObjectiveCProviderCandidatePath(filePath: string): boolean { + const normalized = filePath.replaceAll('\\', '/').toLowerCase(); + return normalized.endsWith('.m') || normalized.endsWith('.mm') || normalized.endsWith('.h'); +} + +/** + * Durable metadata stamp for Objective-C semantic indexing. The feature id + * carries provider and grammar versions verbatim so a semantic identity/edge + * change records the exact producer in index metadata and forces a full rebuild. + * + * `.h` is included only as a path-level rebuild predicate; content classification + * still decides whether a header is actually parsed as Objective-C. + */ +export const OBJECTIVE_C_PROVIDER_FEATURE: AnalysisFeatureDescriptor = { + id: + `objective-c.provider-${OBJECTIVE_C_PROVIDER_VERSION}.` + + `${OBJECTIVE_C_GRAMMAR_PACKAGE}-${OBJECTIVE_C_GRAMMAR_VERSION}`, + version: 1, + appliesTo: (filePaths) => filePaths.some(isObjectiveCProviderCandidatePath), +}; diff --git a/gitnexus/src/core/ingestion/languages/objective-c/compilation-unit-siblings.ts b/gitnexus/src/core/ingestion/languages/objective-c/compilation-unit-siblings.ts new file mode 100644 index 000000000..5d00f7efd --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/compilation-unit-siblings.ts @@ -0,0 +1,138 @@ +/** + * Header / implementation visibility for `populateNamespaceSiblings`. + * + * Objective-C has no Java package or C# namespace. The compilation unit is + * the workspace analog: `Foo.h` + `Foo.m` in the same directory, and files + * that declare or implement the same class / protocol, see each other's + * top-level defs without a second `#import` of a sibling they already share. + * Bindings go through the append-only `bindingAugmentations` channel + * (`origin: 'namespace'`), matching Swift / Go. + */ + +import type { BindingRef, ParsedFile, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { + isInternalObjectiveCFunctionDef, + objectiveCFactsFromParsedFiles, + type ObjCContainerFact, +} from './facts.js'; + +export function populateObjectiveCCompilationUnitSiblings( + parsedFiles: readonly ParsedFile[], + indexes: ScopeResolutionIndexes, +): void { + const groups = new Map(); + const byPath = new Map(); + for (const parsed of parsedFiles) { + byPath.set(parsed.filePath, parsed); + const stemKey = compilationUnitStemKey(parsed.filePath); + if (stemKey !== undefined) appendGroup(groups, stemKey, parsed); + } + + const internalFunctionIds = new Set(); + for (const fact of objectiveCFactsFromParsedFiles(parsedFiles)) { + const parsed = byPath.get(fact.filePath); + if (parsed !== undefined) { + for (const container of fact.containers) { + appendGroup(groups, `type:${typeVisibilityKey(container)}`, parsed); + } + } + for (const fn of fact.functions) { + if (fn.linkage === 'internal') internalFunctionIds.add(fn.nodeId); + } + } + + const augmentations = indexes.bindingAugmentations as Map>; + for (const group of groups.values()) { + if (group.length < 2) continue; + const unique = uniqueFiles(group); + if (unique.length < 2) continue; + for (const receiver of unique) { + const receiverModule = indexes.moduleScopes.byFilePath.get(receiver.filePath); + if (receiverModule === undefined) continue; + for (const source of unique) { + if (source.filePath === receiver.filePath) continue; + for (const def of source.localDefs as SymbolDefinition[]) { + if (internalFunctionIds.has(def.nodeId) || isInternalObjectiveCFunctionDef(def)) { + continue; + } + const name = siblingBindingName(def); + if (name === '') continue; + const bucket = getAugmentationBucket(augmentations, receiverModule, name); + if (bucket.some((binding) => binding.def.nodeId === def.nodeId)) continue; + bucket.push({ def, origin: 'namespace' }); + } + } + } + } +} + +function compilationUnitStemKey(filePath: string): string | undefined { + const normalized = filePath.replaceAll('\\', '/'); + const slash = normalized.lastIndexOf('/'); + const base = slash === -1 ? normalized : normalized.slice(slash + 1); + const ext = extensionOf(base); + if (ext !== '.h' && ext !== '.m' && ext !== '.mm') return undefined; + const dir = slash === -1 ? '' : normalized.slice(0, slash); + return `stem:${dir}/${base.slice(0, -ext.length)}`; +} + +/** Class/category/extension files share one group; a same-named protocol does not. */ +function typeVisibilityKey(container: ObjCContainerFact): string { + if (container.kind === 'category' || container.kind === 'extension') { + return `class:${container.hostClass ?? container.name}`; + } + return `${container.kind}:${container.name}`; +} + +function siblingBindingName(def: SymbolDefinition): string { + const qualified = def.qualifiedName ?? ''; + if (qualified.startsWith('objc:')) { + const colon = qualified.lastIndexOf(':'); + return colon === -1 ? qualified : qualified.slice(colon + 1); + } + return qualified.split('.').pop() ?? qualified; +} + +function extensionOf(fileName: string): string { + const dot = fileName.lastIndexOf('.'); + return dot === -1 ? '' : fileName.slice(dot).toLowerCase(); +} + +function appendGroup(groups: Map, key: string, parsed: ParsedFile): void { + const bucket = groups.get(key); + if (bucket === undefined) { + groups.set(key, [parsed]); + return; + } + bucket.push(parsed); +} + +function uniqueFiles(group: readonly ParsedFile[]): ParsedFile[] { + const seen = new Set(); + const out: ParsedFile[] = []; + for (const parsed of group) { + if (seen.has(parsed.filePath)) continue; + seen.add(parsed.filePath); + out.push(parsed); + } + return out; +} + +function getAugmentationBucket( + augmentations: Map>, + scopeId: ScopeId, + name: string, +): BindingRef[] { + let scopeBindings = augmentations.get(scopeId); + if (scopeBindings === undefined) { + scopeBindings = new Map(); + augmentations.set(scopeId, scopeBindings); + } + let bucket = scopeBindings.get(name); + if (bucket === undefined) { + bucket = []; + scopeBindings.set(name, bucket); + } + return bucket; +} diff --git a/gitnexus/src/core/ingestion/languages/objective-c/facts.ts b/gitnexus/src/core/ingestion/languages/objective-c/facts.ts new file mode 100644 index 000000000..d0ae21bef --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/facts.ts @@ -0,0 +1,1653 @@ +import type { Capture, CaptureMatch, NodeLabel, ParsedFile } from 'gitnexus-shared'; +import { SupportedLanguages } from 'gitnexus-shared'; +import type Parser from 'tree-sitter'; +import { generateId } from '../../../../lib/utils.js'; +import type { + ProviderSemanticGraph, + ProviderSemanticNode, + ProviderSemanticRelationship, + ProviderSemanticSymbol, +} from '../../language-provider.js'; +import { nodeToCapture, walkNamedTree, type SyntaxNode } from '../../utils/ast-helpers.js'; + +export const OBJECTIVE_C_PROVIDER_VERSION = '0.1.8'; +export const OBJECTIVE_C_GRAMMAR_PACKAGE = 'tree-sitter-objc'; +export const OBJECTIVE_C_GRAMMAR_VERSION = '3.0.2'; + +export type ObjCMethodKind = '-' | '+'; +export type ObjCContainerKind = 'class' | 'protocol' | 'category' | 'extension'; + +export interface ObjCTypeInfo { + readonly kind: 'class' | 'protocol' | 'dynamic' | 'class-object' | 'unknown'; + readonly name?: string; + readonly raw: string; +} + +export interface ObjCContainerFact { + readonly kind: ObjCContainerKind; + readonly declarationRole: 'interface' | 'implementation'; + readonly name: string; + readonly qualifiedName: string; + readonly nodeId: string; + readonly label: 'Class' | 'Protocol' | 'Category'; + readonly filePath: string; + readonly startLine: number; + readonly endLine: number; + readonly superclass?: string; + readonly protocols: readonly string[]; + readonly hostClass?: string; + readonly categoryName?: string; +} + +export interface ObjCMethodFact { + readonly name: string; + readonly selector: string; + readonly methodKind: ObjCMethodKind; + readonly ownerQualifiedName: string; + readonly ownerName: string; + readonly ownerKind: ObjCContainerKind; + readonly hostClass?: string; + readonly qualifiedName: string; + readonly nodeId: string; + readonly filePath: string; + readonly startLine: number; + readonly endLine: number; + readonly declarationRole: 'declaration' | 'implementation'; + readonly returnType?: string; + readonly parameterTypes: readonly string[]; + readonly parameterNames: readonly string[]; +} + +export interface ObjCMemberFact { + readonly kind: 'property' | 'ivar'; + readonly name: string; + readonly qualifiedName: string; + readonly nodeId: string; + readonly ownerQualifiedName: string; + readonly ownerName: string; + readonly ownerKind: ObjCContainerKind; + readonly ownerLabel: 'Class' | 'Protocol' | 'Category'; + readonly hostClass?: string; + readonly declaredType?: string; + readonly filePath: string; + readonly startLine: number; + readonly endLine: number; +} + +export interface ObjCFunctionFact { + readonly name: string; + readonly linkage: 'external' | 'internal'; + readonly qualifiedName: string; + readonly nodeId: string; + readonly filePath: string; + readonly startLine: number; + readonly endLine: number; + readonly returnType?: string; + readonly parameterTypes: readonly string[]; +} + +export interface ObjCImportFact { + readonly kind: 'import' | 'include' | 'module'; + readonly raw: string; + readonly targetRaw: string; + readonly filePath: string; + readonly startLine: number; + readonly endLine: number; +} + +export interface ObjCMessageFact { + readonly selector: string; + readonly receiverText: string; + readonly receiverKind: + | 'self' + | 'super' + | 'class' + | 'local' + | 'property' + | 'ivar' + | 'dynamic' + | 'unknown'; + readonly receiverType?: ObjCTypeInfo; + /** Member name retained when its type must be resolved across files. */ + readonly receiverMemberName?: string; + readonly sourceMethodQualifiedName: string; + readonly sourceMethodId: string; + readonly sourceOwnerQualifiedName: string; + readonly sourceOwnerName: string; + readonly sourceMethodKind: ObjCMethodKind; + readonly filePath: string; + readonly startLine: number; + readonly startCol: number; +} + +export interface ObjCUnresolvedMessageFact { + readonly selector: string; + readonly receiverText: string; + readonly reason: string; + readonly sourceMethodQualifiedName: string; + readonly sourceMethodId: string; + readonly filePath: string; + readonly startLine: number; + readonly startCol: number; +} + +export interface ObjCFileFacts { + readonly providerVersion: string; + readonly grammarPackage: string; + readonly grammarVersion: string; + readonly filePath: string; + readonly containers: readonly ObjCContainerFact[]; + readonly methods: readonly ObjCMethodFact[]; + readonly members: readonly ObjCMemberFact[]; + readonly functions: readonly ObjCFunctionFact[]; + readonly imports: readonly ObjCImportFact[]; + readonly messages: readonly ObjCMessageFact[]; + readonly unresolvedMessages: readonly ObjCUnresolvedMessageFact[]; +} + +export interface ObjCCaptureSideChannel { + readonly kind: 'objective-c'; + readonly facts: ObjCFileFacts; +} + +const factsByFile = new Map(); + +export function setObjectiveCFileFacts(facts: ObjCFileFacts): void { + factsByFile.set(facts.filePath, facts); +} + +export function getObjectiveCFileFacts(filePath: string): ObjCFileFacts | undefined { + return factsByFile.get(filePath); +} + +/** Drop process-local facts before a new workspace pass (Java/Kotlin analog). */ +export function clearObjectiveCFileFacts(): void { + factsByFile.clear(); +} + +export function collectObjectiveCCaptureSideChannel( + filePath: string, +): ObjCCaptureSideChannel | undefined { + const facts = factsByFile.get(filePath); + return facts === undefined ? undefined : { kind: 'objective-c', facts }; +} + +export function applyObjectiveCCaptureSideChannel(parsed: ParsedFile): void { + const payload = parsed.captureSideChannel; + if (!isObjectiveCSideChannel(payload)) return; + factsByFile.set(payload.facts.filePath, payload.facts); +} + +export function objectiveCFactsFromParsedFiles( + parsedFiles: readonly ParsedFile[], +): ObjCFileFacts[] { + const facts: ObjCFileFacts[] = []; + const seen = new Set(); + for (const parsed of parsedFiles) { + const payload = parsed.captureSideChannel; + const fromPayload = isObjectiveCSideChannel(payload) ? payload.facts : undefined; + const fact = fromPayload ?? factsByFile.get(parsed.filePath); + if (fact === undefined || seen.has(fact.filePath)) continue; + seen.add(fact.filePath); + facts.push(fact); + } + return facts; +} + +function isObjectiveCSideChannel(value: unknown): value is ObjCCaptureSideChannel { + if (value === null || typeof value !== 'object') return false; + const record = value as Record; + return record.kind === 'objective-c' && record.facts !== null && typeof record.facts === 'object'; +} + +export const objcClassQualifiedName = (name: string): string => `objc:class:${name}`; +export const objcProtocolQualifiedName = (name: string): string => `objc:protocol:${name}`; +export const objcCategoryQualifiedName = (hostClass: string, categoryName: string): string => + `objc:category:${hostClass}:${categoryName}`; +export const objcMethodQualifiedName = ( + ownerQualifiedName: string, + methodKind: ObjCMethodKind, + selector: string, +): string => `objc:method:${ownerQualifiedName}:${methodKind}:${selector}`; +export const objcPropertyQualifiedName = (ownerQualifiedName: string, name: string): string => + `objc:property:${ownerQualifiedName}:${name}`; +export const objcIvarQualifiedName = (ownerQualifiedName: string, name: string): string => + `objc:ivar:${ownerQualifiedName}:${name}`; +export const objcFunctionQualifiedName = ( + name: string, + linkage: 'external' | 'internal' = 'external', + filePath?: string, +): string => + linkage === 'internal' && filePath !== undefined + ? `objc:function:static:${filePath}:${name}` + : `objc:function:${name}`; + +/** True for C functions this provider models as translation-unit local. */ +export function isInternalObjectiveCFunctionDef(def: { + readonly nodeId?: string; + readonly qualifiedName?: string; +}): boolean { + const qualifiedName = def.qualifiedName ?? ''; + const nodeId = def.nodeId ?? ''; + return ( + qualifiedName.startsWith('objc:function:static:') || nodeId.includes('objc:function:static:') + ); +} +export const objcUnresolvedMessageQualifiedName = ( + filePath: string, + startLine: number, + startCol: number, + selector: string, +): string => `objc:unresolved:${filePath}:${startLine}:${startCol}:${selector}`; + +const graphNodeId = (label: NodeLabel, qualifiedName: string): string => + generateId(label, qualifiedName); + +function ownerLabel(kind: ObjCContainerKind): 'Class' | 'Protocol' | 'Category' { + return kind === 'class' ? 'Class' : kind === 'protocol' ? 'Protocol' : 'Category'; +} + +const range = (node: SyntaxNode) => ({ + startLine: node.startPosition.row, + endLine: node.endPosition.row, +}); + +function directNamedChildren(node: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child !== null) out.push(child); + } + return out; +} + +function directChildren(node: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i); + if (child !== null) out.push(child); + } + return out; +} + +function directIdentifiers(node: SyntaxNode): SyntaxNode[] { + return directNamedChildren(node).filter((child) => child.type === 'identifier'); +} + +function logicalTopLevelNodes(root: SyntaxNode): SyntaxNode[] { + const nodes: SyntaxNode[] = []; + const visit = (node: SyntaxNode): void => { + for (const child of directNamedChildren(node)) { + if ( + child.type === 'preproc_if' || + child.type === 'preproc_ifdef' || + child.type === 'preproc_else' || + child.type === 'preproc_elif' + ) { + visit(child); + } else { + nodes.push(child); + } + } + }; + visit(root); + return nodes; +} + +function containerMemberNodes(containerNode: SyntaxNode): SyntaxNode[] { + const members: SyntaxNode[] = []; + const visit = (node: SyntaxNode): void => { + for (const child of directNamedChildren(node)) { + if (child.type === 'qualified_protocol_interface_declaration') { + visit(child); + } else { + members.push(child); + } + } + }; + visit(containerNode); + return members; +} + +function stripQuotes(raw: string): string { + const trimmed = raw.trim(); + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith('<') && trimmed.endsWith('>')) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function isSystemHeaderSpelling(raw: string): boolean { + const trimmed = raw.trim(); + return trimmed.startsWith('<') && trimmed.endsWith('>'); +} + +function cleanType(raw: string | undefined): string | undefined { + if (raw === undefined) return undefined; + const text = raw + .replace(/^\(/, '') + .replace(/\)$/, '') + .replace(/\b(?:nullable|nonnull|__nullable|__nonnull|_Nullable|_Nonnull|const)\b/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (text.length === 0) return undefined; + return text; +} + +export function parseObjCType(raw: string | undefined): ObjCTypeInfo | undefined { + const text = cleanType(raw); + if (text === undefined) return undefined; + if (text === 'id' || text === 'instancetype') return { kind: 'dynamic', raw: text }; + if (text === 'Class') return { kind: 'class-object', raw: text }; + + const protocolMatch = text.match(/^id\s*<\s*([A-Za-z_][A-Za-z0-9_]*)\s*>/); + if (protocolMatch !== null) { + return { kind: 'protocol', name: protocolMatch[1], raw: text }; + } + + const classMatch = text.match(/([A-Za-z_][A-Za-z0-9_]*)\s*(?:<[^>]+>)?\s*\*?$/); + if (classMatch !== null) { + return { kind: 'class', name: classMatch[1], raw: text }; + } + return { kind: 'unknown', raw: text }; +} + +function methodKind(node: SyntaxNode): ObjCMethodKind { + const first = node.child(0)?.text; + return first === '+' ? '+' : '-'; +} + +function methodTypeText(node: SyntaxNode): string | undefined { + const typeNode = directNamedChildren(node).find((child) => child.type === 'method_type'); + return cleanType(typeNode?.text); +} + +function methodSelector(node: SyntaxNode): string { + const children = directChildren(node); + const keywordDeclarators = children.filter((child) => child.type === 'keyword_declarator'); + if (keywordDeclarators.length > 0) { + return keywordDeclarators + .map((declarator) => directIdentifiers(declarator)[0]?.text) + .filter((name): name is string => name !== undefined) + .map((name) => `${name}:`) + .join(''); + } + + const pieces: string[] = []; + for (let i = 0; i < children.length; i++) { + const child = children[i]; + if (child.type !== 'identifier') continue; + const next = children[i + 1]; + if (next?.type === 'method_parameter') { + pieces.push(`${child.text}:`); + continue; + } + if (pieces.length === 0) pieces.push(child.text); + } + return pieces.join(''); +} + +function messageSelector(node: SyntaxNode): string { + const children = directChildren(node); + const pieces: string[] = []; + for (let i = 2; i < children.length; i++) { + const child = children[i]; + if (child.type !== 'identifier' && child.type !== 'field_identifier') continue; + const next = children[i + 1]; + if (next?.text === ':') pieces.push(`${child.text}:`); + else if (pieces.length === 0 && next?.text === ']') pieces.push(child.text); + } + return pieces.join(''); +} + +function methodParameterInfo(node: SyntaxNode): { + parameterTypes: string[]; + parameterNames: string[]; + typeBindings: Map; +} { + const parameterTypes: string[] = []; + const parameterNames: string[] = []; + const typeBindings = new Map(); + for (const child of directNamedChildren(node)) { + if (child.type !== 'method_parameter' && child.type !== 'keyword_declarator') continue; + const named = directNamedChildren(child); + const typeNode = named.find((n) => n.type === 'method_type'); + const nameNode = [...named].reverse().find((n) => n.type === 'identifier'); + const typeText = cleanType(typeNode?.text); + if (typeText !== undefined) parameterTypes.push(typeText); + if (nameNode !== undefined) { + parameterNames.push(nameNode.text); + const parsed = parseObjCType(typeText); + if (parsed !== undefined) typeBindings.set(nameNode.text, parsed); + } + } + return { parameterTypes, parameterNames, typeBindings }; +} + +function propertyInfos(node: SyntaxNode): Array<{ name: string; type?: string }> { + const structDecl = directNamedChildren(node).find((child) => child.type === 'struct_declaration'); + if (structDecl === undefined) return []; + return declarationNamesAndType(structDecl); +} + +function declarationNameAndType(node: SyntaxNode): { name: string; type?: string } | null { + return declarationNamesAndType(node)[0] ?? null; +} + +function declarationNamesAndType(node: SyntaxNode): Array<{ name: string; type?: string }> { + const type = cleanType(firstTypeNode(node)?.text); + return declaratorNames(node).map((name) => (type !== undefined ? { name, type } : { name })); +} + +function declaratorNames(node: SyntaxNode): string[] { + const names: string[] = []; + collectDeclaratorNames(node, names); + return names; +} + +function collectDeclaratorNames(node: SyntaxNode, out: string[]): void { + if (node.type === 'identifier') { + out.push(node.text); + return; + } + if ( + node.type === 'init_declarator' || + node.type === 'pointer_declarator' || + node.type === 'array_declarator' || + node.type === 'function_declarator' || + node.type === 'parenthesized_declarator' || + node.type === 'struct_declarator' + ) { + for (const child of directNamedChildren(node)) collectDeclaratorNames(child, out); + return; + } + + for (const child of directNamedChildren(node)) { + if ( + child.type === 'identifier' || + child.type === 'init_declarator' || + child.type === 'pointer_declarator' || + child.type === 'array_declarator' || + child.type === 'function_declarator' || + child.type === 'parenthesized_declarator' || + child.type === 'struct_declarator' || + child.type === 'struct_declaration' + ) { + collectDeclaratorNames(child, out); + } + } +} + +function declaratorName(node: SyntaxNode): string | undefined { + if (node.type === 'identifier') return node.text; + if (node.type === 'init_declarator') { + for (const child of directNamedChildren(node)) { + const name = declaratorName(child); + if (name !== undefined) return name; + } + return undefined; + } + if ( + node.type === 'pointer_declarator' || + node.type === 'array_declarator' || + node.type === 'function_declarator' || + node.type === 'parenthesized_declarator' || + node.type === 'struct_declarator' + ) { + for (const child of directNamedChildren(node)) { + const name = declaratorName(child); + if (name !== undefined) return name; + } + return undefined; + } + + for (const child of directNamedChildren(node)) { + if ( + child.type === 'identifier' || + child.type === 'init_declarator' || + child.type === 'pointer_declarator' || + child.type === 'array_declarator' || + child.type === 'function_declarator' || + child.type === 'parenthesized_declarator' || + child.type === 'struct_declarator' || + child.type === 'struct_declaration' + ) { + const name = declaratorName(child); + if (name !== undefined) return name; + } + } + return undefined; +} + +function firstTypeNode(node: SyntaxNode): SyntaxNode | undefined { + if ( + node.type === 'type_identifier' || + node.type === 'primitive_type' || + node.type === 'typedefed_specifier' + ) { + return node; + } + for (const child of directNamedChildren(node)) { + const found = firstTypeNode(child); + if (found !== undefined) return found; + } + return undefined; +} + +function functionInfos(node: SyntaxNode): Array<{ + name: string; + linkage: 'external' | 'internal'; + returnType?: string; + parameterTypes: string[]; +}> { + const declarators: SyntaxNode[] = []; + const collect = (current: SyntaxNode): void => { + for (const child of directNamedChildren(current)) { + if (child.type === 'function_declarator') { + declarators.push(child); + continue; + } + collect(child); + } + }; + collect(node); + + const returnType = cleanType( + directNamedChildren(node).find( + (child) => + child.type === 'type_identifier' || + child.type === 'primitive_type' || + child.type === 'typedefed_specifier', + )?.text, + ); + const linkage = directNamedChildren(node).some( + (child) => child.type === 'storage_class_specifier' && child.text === 'static', + ) + ? 'internal' + : 'external'; + + return declarators.flatMap((declarator) => { + // `int (*callback)(int)` is a function pointer declaration, not a callable + // definition. A pointer return (`int *func(void)`) remains a C function. + if (declarator.childForFieldName('declarator')?.type === 'parenthesized_declarator') { + return []; + } + const name = declaratorName(declarator); + if (name === undefined) return []; + const parameterTypes: string[] = []; + walkNamedTree(declarator, (child) => { + if (child.type !== 'parameter_declaration') return; + const typeNode = directNamedChildren(child).find( + (n) => + n.type === 'type_identifier' || + n.type === 'primitive_type' || + n.type === 'typedefed_specifier', + ); + const typeText = cleanType(typeNode?.text); + if (typeText !== undefined) parameterTypes.push(typeText); + }); + return [{ name, linkage, returnType, parameterTypes }]; + }); +} + +function directProtocolNames(node: SyntaxNode): string[] { + const out: string[] = []; + for (const child of directNamedChildren(node)) { + if (child.type === 'protocol_reference_list') { + out.push(...directIdentifiers(child).map((id) => id.text)); + } else if (child.type === 'parameterized_arguments') { + for (const named of directNamedChildren(child)) { + if (named.type !== 'type_name') continue; + const id = directNamedChildren(named).find((n) => n.type === 'type_identifier'); + if (id !== undefined) out.push(id.text); + } + } + } + return Array.from(new Set(out)); +} + +function parseContainer(node: SyntaxNode, filePath: string): ObjCContainerFact | null { + if ( + node.type !== 'class_interface' && + node.type !== 'class_implementation' && + node.type !== 'protocol_declaration' + ) { + return null; + } + const ids = directIdentifiers(node); + const first = ids[0]; + if (first === undefined) return null; + const { startLine, endLine } = range(node); + if (node.type === 'protocol_declaration') { + const qualifiedName = objcProtocolQualifiedName(first.text); + return { + kind: 'protocol', + declarationRole: 'interface', + name: first.text, + qualifiedName, + nodeId: graphNodeId('Protocol', qualifiedName), + label: 'Protocol', + filePath, + startLine, + endLine, + protocols: directProtocolNames(node).filter((name) => name !== first.text), + }; + } + + const children = directChildren(node); + const openParenIndex = children.findIndex((child) => child.text === '('); + if (openParenIndex !== -1) { + const categoryIdentifier = children + .slice(openParenIndex + 1) + .find((child) => child.type === 'identifier'); + const categoryName = categoryIdentifier?.text ?? '__extension__'; + const kind: ObjCContainerKind = categoryIdentifier === undefined ? 'extension' : 'category'; + const qualifiedName = objcCategoryQualifiedName(first.text, categoryName); + return { + kind, + declarationRole: node.type === 'class_interface' ? 'interface' : 'implementation', + name: `${first.text} (${categoryIdentifier?.text ?? ''})`, + qualifiedName, + nodeId: graphNodeId('Category', qualifiedName), + label: 'Category', + filePath, + startLine, + endLine, + protocols: directProtocolNames(node), + hostClass: first.text, + categoryName, + }; + } + + const colonIndex = children.findIndex((child) => child.text === ':'); + const superclass = + colonIndex !== -1 + ? children.slice(colonIndex + 1).find((child) => child.type === 'identifier')?.text + : undefined; + const qualifiedName = objcClassQualifiedName(first.text); + return { + kind: 'class', + declarationRole: node.type === 'class_interface' ? 'interface' : 'implementation', + name: first.text, + qualifiedName, + nodeId: graphNodeId('Class', qualifiedName), + label: 'Class', + filePath, + startLine, + endLine, + ...(superclass !== undefined ? { superclass } : {}), + protocols: directProtocolNames(node), + }; +} + +interface ObjCBinding { + readonly name: string; + readonly type: ObjCTypeInfo; + readonly declarationIndex: number; + readonly scopeStartIndex: number; + readonly scopeEndIndex: number; +} + +function enclosingCompoundScope(node: SyntaxNode, methodBody: SyntaxNode): SyntaxNode { + let current = node.parent; + while (current !== null && current !== methodBody) { + if (current.type === 'compound_statement') return current; + current = current.parent; + } + return methodBody; +} + +function collectMethodBindings( + methodNode: SyntaxNode, + parameterTypes: ReadonlyMap, +): ObjCBinding[] { + const methodBody = directNamedChildren(methodNode).find( + (child) => child.type === 'compound_statement', + ); + if (methodBody === undefined) return []; + + const bindings: ObjCBinding[] = [...parameterTypes].map(([name, type]) => ({ + name, + type, + declarationIndex: methodBody.startIndex, + scopeStartIndex: methodBody.startIndex, + scopeEndIndex: methodBody.endIndex, + })); + walkNamedTree(methodBody, (node) => { + if (node.type !== 'declaration') return; + const info = declarationNameAndType(node); + if (info === null) return; + const type = parseObjCType(info.type); + if (type === undefined) return; + const scope = enclosingCompoundScope(node, methodBody); + bindings.push({ + name: info.name, + type, + declarationIndex: node.endIndex, + scopeStartIndex: scope.startIndex, + scopeEndIndex: scope.endIndex, + }); + }); + return bindings; +} + +function bindingAt( + bindings: readonly ObjCBinding[], + name: string, + sourceIndex: number, +): ObjCTypeInfo | undefined { + const matches = bindings.filter( + (binding) => + binding.name === name && + binding.declarationIndex <= sourceIndex && + binding.scopeStartIndex <= sourceIndex && + sourceIndex <= binding.scopeEndIndex, + ); + matches.sort((left, right) => { + const leftScope = left.scopeEndIndex - left.scopeStartIndex; + const rightScope = right.scopeEndIndex - right.scopeStartIndex; + return leftScope - rightScope || right.declarationIndex - left.declarationIndex; + }); + return matches[0]?.type; +} + +function isReflectionSelector(selector: string, receiverText: string): boolean { + return ( + selector.startsWith('performSelector:') || + selector === 'methodForSelector:' || + selector === 'forwardInvocation:' || + receiverText === 'NSInvocation' + ); +} + +function isPascalCaseLike(text: string): boolean { + if (!/^[A-Z][A-Za-z0-9]*$/.test(text)) return false; + return text.length === 1 || /[a-z0-9]/.test(text.slice(1)); +} + +function messageReceiver(node: SyntaxNode): SyntaxNode | null { + return directNamedChildren(node)[0] ?? null; +} + +function macroName(node: SyntaxNode): string | undefined { + return node.childForFieldName('name')?.text ?? directIdentifiers(node)[0]?.text; +} + +function classifyReceiver( + receiver: SyntaxNode, + selector: string, + method: ObjCMethodFact, + methodBindings: readonly ObjCBinding[], + membersByOwner: ReadonlyMap>, + macroNames: ReadonlySet, + classNames: ReadonlySet, +): { + kind: ObjCMessageFact['receiverKind']; + type?: ObjCTypeInfo; + memberName?: string; + unresolvedReason?: string; +} { + const text = receiver.text; + if (isReflectionSelector(selector, text)) { + return { + kind: 'dynamic', + unresolvedReason: 'reflection selector dispatch is dynamic', + }; + } + const receiverMacroName = + receiver.type === 'call_expression' + ? receiver.childForFieldName('function')?.text + : receiver.type === 'identifier' + ? text + : undefined; + if (receiverMacroName !== undefined && macroNames.has(receiverMacroName)) { + return { + kind: 'dynamic', + type: { kind: 'dynamic', raw: text }, + unresolvedReason: `macro receiver ${receiverMacroName} is dynamic`, + }; + } + if (receiver.type === 'identifier') { + if (text === 'self') return { kind: 'self' }; + if (text === 'super') return { kind: 'super' }; + const bound = bindingAt(methodBindings, text, receiver.startIndex); + if (bound !== undefined) { + if (bound.kind === 'dynamic' || bound.kind === 'class-object') { + return { + kind: 'dynamic', + type: bound, + unresolvedReason: `${bound.raw} receiver is dynamic`, + }; + } + return { kind: 'local', type: bound }; + } + const hostOwner = + method.hostClass !== undefined + ? objcClassQualifiedName(method.hostClass) + : method.ownerQualifiedName; + const memberType = membersByOwner.get(hostOwner)?.get(text); + if (memberType !== undefined) return { kind: 'ivar', type: memberType }; + if (classNames.has(text) || isPascalCaseLike(text)) { + return { kind: 'class', type: { kind: 'class', name: text, raw: text } }; + } + return { kind: 'unknown', memberName: text }; + } + + if (receiver.type === 'field_expression') { + const parts = directNamedChildren(receiver); + const base = parts[0]; + const field = parts[1]; + if (base?.text === 'self' && field !== undefined) { + const hostOwner = + method.hostClass !== undefined + ? objcClassQualifiedName(method.hostClass) + : method.ownerQualifiedName; + const memberType = membersByOwner.get(hostOwner)?.get(field.text); + if (memberType !== undefined) return { kind: 'property', type: memberType }; + return { kind: 'property', memberName: field.text }; + } + } + return { + kind: 'unknown', + unresolvedReason: 'receiver expression is not statically typed', + }; +} + +function collectMethodMessages( + methodNode: SyntaxNode, + method: ObjCMethodFact, + parameterTypes: ReadonlyMap, + membersByOwner: ReadonlyMap>, + macroNames: ReadonlySet, + classNames: ReadonlySet, + messages: ObjCMessageFact[], + unresolvedMessages: ObjCUnresolvedMessageFact[], +): void { + const methodBindings = collectMethodBindings(methodNode, parameterTypes); + walkNamedTree(methodNode, (messageNode) => { + if (messageNode.type !== 'message_expression') return; + const receiver = messageReceiver(messageNode); + if (receiver === null) return; + const selector = messageSelector(messageNode); + if (selector.length === 0) return; + const classified = classifyReceiver( + receiver, + selector, + method, + methodBindings, + membersByOwner, + macroNames, + classNames, + ); + const fact: ObjCMessageFact = { + selector, + receiverText: receiver.text, + receiverKind: classified.kind, + ...(classified.type !== undefined ? { receiverType: classified.type } : {}), + ...(classified.memberName !== undefined ? { receiverMemberName: classified.memberName } : {}), + sourceMethodQualifiedName: method.qualifiedName, + sourceMethodId: method.nodeId, + sourceOwnerQualifiedName: method.ownerQualifiedName, + sourceOwnerName: method.ownerName, + sourceMethodKind: method.methodKind, + filePath: method.filePath, + startLine: messageNode.startPosition.row, + startCol: messageNode.startPosition.column, + }; + if (classified.unresolvedReason !== undefined) { + unresolvedMessages.push({ + selector, + receiverText: receiver.text, + reason: classified.unresolvedReason, + sourceMethodQualifiedName: method.qualifiedName, + sourceMethodId: method.nodeId, + filePath: method.filePath, + startLine: messageNode.startPosition.row, + startCol: messageNode.startPosition.column, + }); + } + messages.push(fact); + }); +} + +export function collectObjectiveCFacts(tree: Parser.Tree, filePath: string): ObjCFileFacts { + const containers: ObjCContainerFact[] = []; + const methods: ObjCMethodFact[] = []; + const members: ObjCMemberFact[] = []; + const functions: ObjCFunctionFact[] = []; + const imports: ObjCImportFact[] = []; + const messages: ObjCMessageFact[] = []; + const unresolvedMessages: ObjCUnresolvedMessageFact[] = []; + const membersByOwner = new Map>(); + const macroNames = new Set(); + const classNames = new Set(); + + const addFunctionFact = (node: SyntaxNode): void => { + for (const info of functionInfos(node)) { + const qualifiedName = objcFunctionQualifiedName(info.name, info.linkage, filePath); + const { startLine, endLine } = range(node); + functions.push({ + name: info.name, + linkage: info.linkage, + qualifiedName, + nodeId: graphNodeId('Function', qualifiedName), + filePath, + startLine, + endLine, + ...(info.returnType !== undefined ? { returnType: info.returnType } : {}), + parameterTypes: info.parameterTypes, + }); + } + }; + + walkNamedTree(tree.rootNode, (node) => { + if (node.type !== 'preproc_def' && node.type !== 'preproc_function_def') return; + const name = macroName(node); + if (name !== undefined) macroNames.add(name); + }); + + walkNamedTree(tree.rootNode, (node) => { + const container = parseContainer(node, filePath); + if (container?.kind === 'class') classNames.add(container.name); + }); + + const addMemberType = ( + ownerQualifiedName: string, + name: string, + typeText: string | undefined, + ) => { + const typeInfo = parseObjCType(typeText); + if (typeInfo === undefined) return; + let byName = membersByOwner.get(ownerQualifiedName); + if (byName === undefined) { + byName = new Map(); + membersByOwner.set(ownerQualifiedName, byName); + } + byName.set(name, typeInfo); + }; + + const collectContainerMemberTypes = ( + containerNode: Parser.SyntaxNode, + container: ObjCContainerFact, + ): void => { + for (const inner of containerMemberNodes(containerNode)) { + if (inner.type === 'property_declaration') { + for (const prop of propertyInfos(inner)) { + addMemberType(container.qualifiedName, prop.name, prop.type); + if (container.hostClass !== undefined) { + addMemberType(objcClassQualifiedName(container.hostClass), prop.name, prop.type); + } + } + } else if (inner.type === 'instance_variables') { + walkNamedTree(inner, (ivarNode) => { + if (ivarNode.type !== 'instance_variable') return; + for (const ivar of declarationNamesAndType(ivarNode)) { + addMemberType(container.qualifiedName, ivar.name, ivar.type); + if (container.hostClass !== undefined) { + addMemberType(objcClassQualifiedName(container.hostClass), ivar.name, ivar.type); + } + } + }); + } + } + }; + + const topLevelNodes = logicalTopLevelNodes(tree.rootNode); + for (const child of topLevelNodes) { + const container = parseContainer(child, filePath); + if (container !== null) collectContainerMemberTypes(child, container); + } + + for (const child of topLevelNodes) { + if (child.type === 'preproc_include') { + const rawNode = directNamedChildren(child).find( + (n) => n.type === 'string_literal' || n.type === 'system_lib_string', + ); + if (rawNode !== undefined) { + const raw = rawNode.text; + const directive = child.text.trimStart().startsWith('#include') ? 'include' : 'import'; + const { startLine, endLine } = range(child); + imports.push({ + kind: directive, + raw, + targetRaw: stripQuotes(raw), + filePath, + startLine, + endLine, + }); + } + continue; + } + if (child.type === 'module_import') { + const moduleName = directIdentifiers(child) + .map((n) => n.text) + .join('.'); + const { startLine, endLine } = range(child); + imports.push({ + kind: 'module', + raw: moduleName, + targetRaw: moduleName, + filePath, + startLine, + endLine, + }); + continue; + } + + const container = parseContainer(child, filePath); + if (container !== null) { + containers.push(container); + for (const inner of containerMemberNodes(child)) { + if (inner.type === 'method_declaration' || inner.type === 'method_definition') { + const selector = methodSelector(inner); + const { parameterTypes, parameterNames, typeBindings } = methodParameterInfo(inner); + const kind = methodKind(inner); + const qualifiedName = objcMethodQualifiedName(container.qualifiedName, kind, selector); + const { startLine, endLine } = range(inner); + const method: ObjCMethodFact = { + name: selector, + selector, + methodKind: kind, + ownerQualifiedName: container.qualifiedName, + ownerName: container.name, + ownerKind: container.kind, + ...(container.hostClass !== undefined ? { hostClass: container.hostClass } : {}), + qualifiedName, + nodeId: graphNodeId('Method', qualifiedName), + filePath, + startLine, + endLine, + declarationRole: inner.type === 'method_definition' ? 'implementation' : 'declaration', + ...(methodTypeText(inner) !== undefined ? { returnType: methodTypeText(inner) } : {}), + parameterTypes, + parameterNames, + }; + methods.push(method); + + collectMethodMessages( + inner, + method, + typeBindings, + membersByOwner, + macroNames, + classNames, + messages, + unresolvedMessages, + ); + } else if (inner.type === 'property_declaration') { + const { startLine, endLine } = range(inner); + for (const prop of propertyInfos(inner)) { + const qualifiedName = objcPropertyQualifiedName(container.qualifiedName, prop.name); + members.push({ + kind: 'property', + name: prop.name, + qualifiedName, + nodeId: graphNodeId('Property', qualifiedName), + ownerQualifiedName: container.qualifiedName, + ownerName: container.name, + ownerKind: container.kind, + ownerLabel: ownerLabel(container.kind), + ...(container.hostClass !== undefined ? { hostClass: container.hostClass } : {}), + ...(prop.type !== undefined ? { declaredType: prop.type } : {}), + filePath, + startLine, + endLine, + }); + } + } else if (inner.type === 'instance_variables') { + walkNamedTree(inner, (ivarNode) => { + if (ivarNode.type !== 'instance_variable') return; + const { startLine, endLine } = range(ivarNode); + for (const ivar of declarationNamesAndType(ivarNode)) { + const qualifiedName = objcIvarQualifiedName(container.qualifiedName, ivar.name); + members.push({ + kind: 'ivar', + name: ivar.name, + qualifiedName, + nodeId: graphNodeId('Variable', qualifiedName), + ownerQualifiedName: container.qualifiedName, + ownerName: container.name, + ownerKind: container.kind, + ownerLabel: ownerLabel(container.kind), + ...(container.hostClass !== undefined ? { hostClass: container.hostClass } : {}), + ...(ivar.type !== undefined ? { declaredType: ivar.type } : {}), + filePath, + startLine, + endLine, + }); + } + }); + } else if (inner.type === 'implementation_definition') { + const functionNode = directNamedChildren(inner).find( + (node) => node.type === 'function_definition', + ); + if (functionNode !== undefined) { + addFunctionFact(functionNode); + continue; + } + const methodNode = directNamedChildren(inner).find((n) => n.type === 'method_definition'); + if (methodNode !== undefined) { + const selector = methodSelector(methodNode); + const { parameterTypes, parameterNames, typeBindings } = + methodParameterInfo(methodNode); + const kind = methodKind(methodNode); + const qualifiedName = objcMethodQualifiedName(container.qualifiedName, kind, selector); + const { startLine, endLine } = range(methodNode); + const method: ObjCMethodFact = { + name: selector, + selector, + methodKind: kind, + ownerQualifiedName: container.qualifiedName, + ownerName: container.name, + ownerKind: container.kind, + ...(container.hostClass !== undefined ? { hostClass: container.hostClass } : {}), + qualifiedName, + nodeId: graphNodeId('Method', qualifiedName), + filePath, + startLine, + endLine, + declarationRole: 'implementation', + ...(methodTypeText(methodNode) !== undefined + ? { returnType: methodTypeText(methodNode) } + : {}), + parameterTypes, + parameterNames, + }; + methods.push(method); + collectMethodMessages( + methodNode, + method, + typeBindings, + membersByOwner, + macroNames, + classNames, + messages, + unresolvedMessages, + ); + } + } + } + continue; + } + + if (child.type === 'function_definition' || child.type === 'declaration') { + addFunctionFact(child); + } + } + + return { + providerVersion: OBJECTIVE_C_PROVIDER_VERSION, + grammarPackage: OBJECTIVE_C_GRAMMAR_PACKAGE, + grammarVersion: OBJECTIVE_C_GRAMMAR_VERSION, + filePath, + containers, + methods, + members, + functions, + imports, + messages, + unresolvedMessages, + }; +} + +function semanticNode( + label: NodeLabel, + id: string, + name: string, + qualifiedName: string, + filePath: string, + startLine: number, + endLine: number, + extras: Record = {}, +): ProviderSemanticNode { + return { + id, + label, + properties: { + name, + filePath, + startLine, + endLine, + language: SupportedLanguages.ObjectiveC, + isExported: true, + qualifiedName, + objectiveCProviderVersion: OBJECTIVE_C_PROVIDER_VERSION, + objectiveCGrammar: `${OBJECTIVE_C_GRAMMAR_PACKAGE}@${OBJECTIVE_C_GRAMMAR_VERSION}`, + ...extras, + }, + }; +} + +function relationship( + type: 'DEFINES' | 'DECLARES' | 'HAS_METHOD' | 'HAS_PROPERTY', + sourceId: string, + targetId: string, + reason: string, +): ProviderSemanticRelationship { + return { + id: generateId(type, `${sourceId}->${targetId}:${reason}`), + sourceId, + targetId, + type, + confidence: 1, + reason, + }; +} + +function implementationEvidenceNode( + qualifiedName: string, + name: string, + filePath: string, + startLine: number, + endLine: number, + extras: Record, +): ProviderSemanticNode { + return semanticNode( + 'CodeElement', + graphNodeId('CodeElement', qualifiedName), + name, + qualifiedName, + filePath, + startLine, + endLine, + { + objectiveCKind: 'implementation-evidence', + ...extras, + }, + ); +} + +export function buildObjectiveCSemanticGraph(facts: ObjCFileFacts): ProviderSemanticGraph { + const nodes: ProviderSemanticNode[] = []; + const relationships: ProviderSemanticRelationship[] = []; + const symbols: ProviderSemanticSymbol[] = []; + const fileId = generateId('File', facts.filePath); + + for (const container of facts.containers) { + nodes.push( + semanticNode( + container.label, + container.nodeId, + container.name, + container.qualifiedName, + facts.filePath, + container.startLine, + container.endLine, + { + objectiveCKind: container.kind, + declarationRole: container.declarationRole, + ...(container.superclass !== undefined ? { superclass: container.superclass } : {}), + ...(container.protocols.length > 0 ? { protocols: [...container.protocols] } : {}), + ...(container.hostClass !== undefined ? { hostClass: container.hostClass } : {}), + ...(container.categoryName !== undefined ? { categoryName: container.categoryName } : {}), + }, + ), + ); + relationships.push(relationship('DEFINES', fileId, container.nodeId, 'objc-definition')); + if (container.declarationRole === 'implementation') { + const qualifiedName = `objc:implementation:${container.qualifiedName}:${facts.filePath}:${container.startLine}`; + const evidenceId = graphNodeId('CodeElement', qualifiedName); + nodes.push( + implementationEvidenceNode( + qualifiedName, + `@implementation ${container.name}`, + facts.filePath, + container.startLine, + container.endLine, + { + implementationKind: container.kind, + targetQualifiedName: container.qualifiedName, + }, + ), + ); + relationships.push( + relationship('DEFINES', fileId, evidenceId, 'objc: implementation evidence'), + ); + relationships.push( + relationship( + 'DECLARES', + evidenceId, + container.nodeId, + 'objc: implementation of merged symbol', + ), + ); + } + symbols.push({ + filePath: facts.filePath, + name: container.name, + nodeId: container.nodeId, + type: container.label, + qualifiedName: container.qualifiedName, + }); + } + + for (const method of facts.methods) { + nodes.push( + semanticNode( + 'Method', + method.nodeId, + method.selector, + method.qualifiedName, + facts.filePath, + method.startLine, + method.endLine, + { + selector: method.selector, + methodKind: method.methodKind, + objectiveCOwner: method.ownerQualifiedName, + objectiveCOwnerName: method.ownerName, + description: `${method.ownerName} ${method.methodKind}${method.selector}`, + declarationRole: method.declarationRole, + parameterCount: method.parameterTypes.length, + parameterTypes: [...method.parameterTypes], + ...(method.returnType !== undefined ? { returnType: method.returnType } : {}), + ...(method.hostClass !== undefined ? { hostClass: method.hostClass } : {}), + }, + ), + ); + const ownerId = graphNodeId( + method.ownerKind === 'class' + ? 'Class' + : method.ownerKind === 'protocol' + ? 'Protocol' + : 'Category', + method.ownerQualifiedName, + ); + relationships.push(relationship('HAS_METHOD', ownerId, method.nodeId, 'objc-owner-method')); + if (method.declarationRole === 'implementation') { + const qualifiedName = `objc:method-implementation:${method.qualifiedName}:${facts.filePath}:${method.startLine}`; + const evidenceId = graphNodeId('CodeElement', qualifiedName); + nodes.push( + implementationEvidenceNode( + qualifiedName, + `${method.methodKind}[${method.ownerName} ${method.selector}] implementation`, + facts.filePath, + method.startLine, + method.endLine, + { + implementationKind: 'method', + targetQualifiedName: method.qualifiedName, + selector: method.selector, + methodKind: method.methodKind, + objectiveCOwner: method.ownerQualifiedName, + }, + ), + ); + relationships.push( + relationship('DEFINES', fileId, evidenceId, 'objc: implementation evidence'), + ); + relationships.push( + relationship( + 'DECLARES', + evidenceId, + method.nodeId, + 'objc: implementation of merged symbol', + ), + ); + } + if (method.hostClass !== undefined) { + relationships.push( + relationship( + 'HAS_METHOD', + graphNodeId('Class', objcClassQualifiedName(method.hostClass)), + method.nodeId, + 'objc-category-host-method', + ), + ); + } + symbols.push({ + filePath: facts.filePath, + name: method.selector, + nodeId: method.nodeId, + type: 'Method', + qualifiedName: method.qualifiedName, + parameterCount: method.parameterTypes.length, + requiredParameterCount: method.parameterTypes.length, + parameterTypes: [...method.parameterTypes], + ...(method.returnType !== undefined ? { returnType: method.returnType } : {}), + ownerId, + isStatic: method.methodKind === '+', + }); + } + + for (const member of facts.members) { + const label: NodeLabel = member.kind === 'property' ? 'Property' : 'Variable'; + nodes.push( + semanticNode( + label, + member.nodeId, + member.name, + member.qualifiedName, + facts.filePath, + member.startLine, + member.endLine, + { + objectiveCKind: member.kind, + ...(member.declaredType !== undefined ? { declaredType: member.declaredType } : {}), + }, + ), + ); + relationships.push( + relationship( + member.kind === 'property' ? 'HAS_PROPERTY' : 'HAS_PROPERTY', + graphNodeId(member.ownerLabel, member.ownerQualifiedName), + member.nodeId, + `objc-${member.kind}`, + ), + ); + if (member.hostClass !== undefined) { + relationships.push( + relationship( + 'HAS_PROPERTY', + graphNodeId('Class', objcClassQualifiedName(member.hostClass)), + member.nodeId, + `objc-category-host-${member.kind}`, + ), + ); + } + symbols.push({ + filePath: facts.filePath, + name: member.name, + nodeId: member.nodeId, + type: label, + qualifiedName: member.qualifiedName, + ...(member.declaredType !== undefined ? { declaredType: member.declaredType } : {}), + ownerId: graphNodeId(member.ownerLabel, member.ownerQualifiedName), + }); + } + + for (const fn of facts.functions) { + nodes.push( + semanticNode( + 'Function', + fn.nodeId, + fn.name, + fn.qualifiedName, + facts.filePath, + fn.startLine, + fn.endLine, + { + parameterCount: fn.parameterTypes.length, + parameterTypes: [...fn.parameterTypes], + linkage: fn.linkage, + ...(fn.returnType !== undefined ? { returnType: fn.returnType } : {}), + }, + ), + ); + relationships.push(relationship('DEFINES', fileId, fn.nodeId, 'objc-c-function')); + symbols.push({ + filePath: facts.filePath, + name: fn.name, + nodeId: fn.nodeId, + type: 'Function', + qualifiedName: fn.qualifiedName, + parameterCount: fn.parameterTypes.length, + parameterTypes: [...fn.parameterTypes], + ...(fn.returnType !== undefined ? { returnType: fn.returnType } : {}), + }); + } + + for (const imp of facts.imports) { + const qualifiedName = `objc:import:${facts.filePath}:${imp.startLine}:${imp.kind}:${imp.targetRaw}`; + const id = graphNodeId('Import', qualifiedName); + nodes.push( + semanticNode( + 'Import', + id, + imp.targetRaw, + qualifiedName, + facts.filePath, + imp.startLine, + imp.endLine, + { + objectiveCKind: 'import', + importKind: imp.kind, + raw: imp.raw, + targetRaw: imp.targetRaw, + }, + ), + ); + relationships.push(relationship('DEFINES', fileId, id, 'objc-import')); + symbols.push({ + filePath: facts.filePath, + name: imp.targetRaw, + nodeId: id, + type: 'Import', + qualifiedName, + }); + } + + for (const unresolved of facts.unresolvedMessages) { + const qn = objcUnresolvedMessageQualifiedName( + facts.filePath, + unresolved.startLine, + unresolved.startCol, + unresolved.selector, + ); + const id = graphNodeId('CodeElement', qn); + nodes.push( + semanticNode( + 'CodeElement', + id, + `[${unresolved.receiverText} ${unresolved.selector}] unresolved: ${unresolved.reason}`, + qn, + facts.filePath, + unresolved.startLine, + unresolved.startLine, + { + objectiveCKind: 'unresolved-message', + selector: unresolved.selector, + receiver: unresolved.receiverText, + resolution: 'unresolved', + }, + ), + ); + relationships.push( + relationship('DEFINES', fileId, id, `objc-unresolved-message: ${unresolved.reason}`), + ); + symbols.push({ + filePath: facts.filePath, + name: `[${unresolved.receiverText} ${unresolved.selector}]`, + nodeId: id, + type: 'CodeElement', + qualifiedName: qn, + }); + } + + return { nodes, relationships, symbols }; +} + +function captureAt(name: string, text: string, startLine: number, endLine: number): Capture { + return { + name, + text, + range: { + startLine: startLine + 1, + startCol: 0, + endLine: endLine + 1, + endCol: 0, + }, + }; +} + +export function buildObjectiveCScopeCaptures( + facts: ObjCFileFacts, + root: SyntaxNode, +): readonly CaptureMatch[] { + const captures: CaptureMatch[] = [ + { + '@scope.module': nodeToCapture('@scope.module', root), + }, + ]; + + const seenDeclarations = new Set(); + for (const container of facts.containers) { + const kind = declarationCaptureKind(container.kind); + if (kind === undefined) continue; + if (!seenDeclarations.add(`${kind}:${container.qualifiedName}`)) continue; + captures.push( + declarationMatch( + kind, + container.kind === 'category' || container.kind === 'extension' + ? (container.categoryName ?? container.name) + : container.name, + container.qualifiedName, + container.startLine, + container.endLine, + ), + ); + } + + for (const fn of facts.functions) { + if (!seenDeclarations.add(`function:${fn.qualifiedName}`)) continue; + captures.push( + declarationMatch('function', fn.name, fn.qualifiedName, fn.startLine, fn.endLine), + ); + } + + for (const imp of facts.imports) { + const anchor = captureAt('@import.statement', imp.raw, imp.startLine, imp.endLine); + const sourceText = isSystemHeaderSpelling(imp.raw) ? imp.raw : imp.targetRaw; + captures.push({ + '@import.statement': anchor, + '@import.source': { + ...anchor, + name: '@import.source', + text: sourceText, + }, + '@import.name': { ...anchor, name: '@import.name', text: sourceText }, + '@import.kind': { ...anchor, name: '@import.kind', text: imp.kind }, + }); + } + + return captures; +} + +function declarationCaptureKind( + kind: ObjCContainerKind, +): 'class' | 'protocol' | 'category' | undefined { + if (kind === 'extension') return undefined; + return kind; +} + +function declarationMatch( + kind: 'class' | 'protocol' | 'category' | 'function', + name: string, + qualifiedName: string, + startLine: number, + endLine: number, +): CaptureMatch { + const anchor = captureAt(`@declaration.${kind}`, name, startLine, endLine); + return { + [`@declaration.${kind}`]: anchor, + '@declaration.name': { ...anchor, name: '@declaration.name', text: name }, + '@declaration.qualified_name': { + ...anchor, + name: '@declaration.qualified_name', + text: qualifiedName, + }, + }; +} diff --git a/gitnexus/src/core/ingestion/languages/objective-c/import-target.ts b/gitnexus/src/core/ingestion/languages/objective-c/import-target.ts new file mode 100644 index 000000000..fb493e466 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/import-target.ts @@ -0,0 +1,161 @@ +/** + * Resolve an Objective-C `#import` / `#include` / `@import` to a workspace file. + * + * Quoted imports keep the suffix-index first-path tie-break used before the + * workspace config existed. Angle-bracket imports never suffix-match the + * source tree — they only hit in-repo frameworks and HEADER_SEARCH_PATHS — + * so `` stays unresolved even when a decoy + * `Foundation.h` exists. `@import` names resolve only through a scanned + * module map. + */ + +import path from 'path'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; +import { + coerceObjectiveCResolutionConfig, + normalizeRepoPath, + type ObjectiveCResolutionConfig, +} from './resolution-config.js'; + +interface ObjectiveCImportIndex { + readonly filePathSet: ReadonlySet; + readonly suffixMatches: ReadonlyMap< + string, + { readonly filePath: string; readonly order: number } + >; +} + +/** + * Per-pass memo of `allFilePaths ∪ scanned headers`. Identity is load-bearing + * for `getObjectiveCImportIndex` (same rule as C/C++ `augmentedFilePathsFor`). + * This memo stays private to Objective-C — sharing it with C would hand each + * language the other's suffix index. + */ +const augmentedFilePathsFor = perFileSet((allFilePaths: ReadonlySet) => + perFileSet((headerPaths: ReadonlySet): ReadonlySet => { + const set = new Set(allFilePaths); + for (const header of headerPaths) set.add(header); + return set; + }), +); + +const getObjectiveCImportIndex = perFileSet( + (allFilePaths: ReadonlySet): ObjectiveCImportIndex => { + const filePaths = [...allFilePaths]; + const suffixMatches = new Map(); + for (const [order, filePath] of filePaths.entries()) { + const segments = normalizeRepoPath(filePath).split('/'); + for (let start = 1; start < segments.length; start++) { + const suffix = segments.slice(start).join('/'); + if (!suffixMatches.has(suffix)) suffixMatches.set(suffix, { filePath, order }); + } + } + return { filePathSet: new Set(filePaths), suffixMatches }; + }, +); + +export function resolveObjectiveCImportTarget( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, + resolutionConfig?: unknown, +): string | null { + const target = targetRaw.trim(); + if (target.length === 0) return null; + + const config = coerceObjectiveCResolutionConfig(resolutionConfig); + const searchPaths = + config !== undefined && config.headers.size > 0 + ? augmentedFilePathsFor(allFilePaths)(config.headers) + : allFilePaths; + + if (target.startsWith('<') && target.endsWith('>')) { + return resolveAngleImport(target.slice(1, -1), searchPaths, config); + } + + const looksLikeFileImport = + target.startsWith('.') || target.includes('/') || path.posix.extname(target).length > 0; + if (!looksLikeFileImport) { + return resolveModuleImport(target, searchPaths, config); + } + + return findImportCandidate(target, fromFile, getObjectiveCImportIndex(searchPaths), config); +} + +function resolveAngleImport( + inner: string, + searchPaths: ReadonlySet, + config: ObjectiveCResolutionConfig | undefined, +): string | null { + const normalized = normalizeRepoPath(inner); + if (normalized.length === 0 || config === undefined) return null; + + const slash = normalized.indexOf('/'); + if (slash > 0) { + const framework = normalized.slice(0, slash); + const rest = normalized.slice(slash + 1); + const headersDir = config.frameworks.get(framework); + if (headersDir !== undefined) { + const framed = normalizeRepoPath(`${headersDir}/${rest}`); + if (searchPaths.has(framed)) return framed; + } + } + + for (const searchPath of config.headerSearchPaths) { + const candidate = normalizeRepoPath(`${searchPath}/${normalized}`); + if (searchPaths.has(candidate)) return candidate; + } + return null; +} + +function resolveModuleImport( + moduleName: string, + searchPaths: ReadonlySet, + config: ObjectiveCResolutionConfig | undefined, +): string | null { + if (config === undefined) return null; + const exact = config.modules.get(moduleName); + if (exact !== undefined && searchPaths.has(exact)) return exact; + const root = moduleName.split('.')[0]; + if (root === undefined || root === moduleName) return null; + const fallback = config.modules.get(root); + return fallback !== undefined && searchPaths.has(fallback) ? fallback : null; +} + +function findImportCandidate( + targetRaw: string, + fromFile: string, + importIndex: ObjectiveCImportIndex, + config: ObjectiveCResolutionConfig | undefined, +): string | null { + const normalizedTarget = normalizeRepoPath(targetRaw); + const fromDir = normalizeRepoPath(path.posix.dirname(normalizeRepoPath(fromFile))); + const spelledCandidates = new Set([ + normalizeRepoPath(path.posix.join(fromDir, normalizedTarget)), + normalizedTarget, + ]); + if (config !== undefined) { + for (const searchPath of [...config.userHeaderSearchPaths, ...config.headerSearchPaths]) { + spelledCandidates.add(normalizeRepoPath(`${searchPath}/${normalizedTarget}`)); + } + } + const ext = path.posix.extname(normalizedTarget); + if (ext.length === 0) { + for (const base of [...spelledCandidates]) { + spelledCandidates.add(`${base}.h`); + spelledCandidates.add(`${base}.m`); + spelledCandidates.add(`${base}.mm`); + } + } + for (const candidate of spelledCandidates) { + if (importIndex.filePathSet.has(candidate)) return candidate; + } + let suffixMatch: { readonly filePath: string; readonly order: number } | undefined; + for (const candidate of spelledCandidates) { + const match = importIndex.suffixMatches.get(candidate); + if (match !== undefined && (suffixMatch === undefined || match.order < suffixMatch.order)) { + suffixMatch = match; + } + } + return suffixMatch?.filePath ?? null; +} diff --git a/gitnexus/src/core/ingestion/languages/objective-c/macro-marker-preprocess.ts b/gitnexus/src/core/ingestion/languages/objective-c/macro-marker-preprocess.ts new file mode 100644 index 000000000..feec0d8a2 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/macro-marker-preprocess.ts @@ -0,0 +1,283 @@ +/** + * Normalize bare file-scope Objective-C macro markers before parsing. + * + * Headers commonly use macro pairs such as `RCT_EXTERN_C_BEGIN` and + * `RCT_EXTERN_C_END` around C declarations. tree-sitter-objc does not expand + * those macros; a bare invocation can put the parser into error recovery and + * hide every Objective-C declaration that follows it. These markers do not + * contribute syntax on their own, so we replace only the narrow, generic form + * with spaces before parsing. + * + * This is deliberately not macro expansion or a framework-specific allowlist: + * a candidate must be a whole, all-caps identifier at file scope. Function-like + * macros, directives, statements, strings, and comments remain untouched. + * Replacement preserves UTF-16 length and line endings exactly. Candidates are + * ASCII-only, so their byte offsets are preserved as well. + */ + +interface ScanState { + inBlockComment: boolean; + inLineCommentContinuation: boolean; + inCodeLineContinuation: boolean; + inPreprocessorDirective: boolean; + quote: '"' | "'" | undefined; + braceDepth: number; + parenDepth: number; + objcDeclDepth: number; +} + +function isPreprocessorWhitespace(code: number): boolean { + // C preprocessing whitespace includes space, horizontal tab, vertical tab, + // and form feed. Newlines are split before this scanner sees a line. + return code === 0x20 || code === 0x09 || code === 0x0b || code === 0x0c; +} + +function isBareMarkerIdentifier(line: string): boolean { + let index = 0; + while (index < line.length && isPreprocessorWhitespace(line.charCodeAt(index))) index++; + + const identifierStart = index; + const first = line.charCodeAt(index); + if (!((first >= 0x41 && first <= 0x5a) || first === 0x5f)) return false; + + let hasUppercaseLetter = false; + while (index < line.length) { + const code = line.charCodeAt(index); + if (code >= 0x41 && code <= 0x5a) { + hasUppercaseLetter = true; + index++; + continue; + } + if ((code >= 0x30 && code <= 0x39) || code === 0x5f) { + index++; + continue; + } + break; + } + if (index === identifierStart || !hasUppercaseLetter) return false; + + while (index < line.length && isPreprocessorWhitespace(line.charCodeAt(index))) index++; + return index === line.length; +} + +function isIdentifierContinue(line: string, index: number): boolean { + if (index >= line.length) return false; + const code = line.charCodeAt(index); + return ( + (code >= 0x41 && code <= 0x5a) || + (code >= 0x61 && code <= 0x7a) || + (code >= 0x30 && code <= 0x39) || + code === 0x5f + ); +} + +function leadingObjCDeclKeyword(line: string, inBlockComment = false): 'begin' | 'end' | null { + let index = 0; + if (inBlockComment) { + const blockCommentEnd = line.indexOf('*/'); + if (blockCommentEnd < 0) return null; + index = blockCommentEnd + 2; + } + while (index < line.length) { + while (index < line.length && isPreprocessorWhitespace(line.charCodeAt(index))) index++; + if (line.startsWith('//', index)) return null; + if (line.startsWith('/*', index)) { + const blockCommentEnd = line.indexOf('*/', index + 2); + if (blockCommentEnd < 0) return null; + index = blockCommentEnd + 2; + continue; + } + break; + } + if (line.startsWith('@end', index) && !isIdentifierContinue(line, index + 4)) return 'end'; + for (const keyword of ['@interface', '@protocol', '@implementation'] as const) { + if (line.startsWith(keyword, index) && !isIdentifierContinue(line, index + keyword.length)) { + return 'begin'; + } + } + return null; +} + +function hasEscapedLineEnding(line: string): boolean { + let trailingBackslashes = 0; + for (let index = line.length - 1; index >= 0 && line.charCodeAt(index) === 0x5c; index--) { + trailingBackslashes++; + } + return trailingBackslashes % 2 === 1; +} + +function startsPreprocessorDirective(line: string, state: ScanState): boolean { + let index = 0; + let inBlockComment = state.inBlockComment; + + while (index < line.length) { + if (inBlockComment) { + const blockCommentEnd = line.indexOf('*/', index); + if (blockCommentEnd < 0) return false; + inBlockComment = false; + index = blockCommentEnd + 2; + continue; + } + + while (index < line.length && isPreprocessorWhitespace(line.charCodeAt(index))) index++; + if (line.startsWith('/*', index)) { + inBlockComment = true; + index += 2; + continue; + } + if (line.startsWith('//', index)) return false; + if (line.charCodeAt(index) !== 0x23) return false; + + state.inBlockComment = false; + return true; + } + + return false; +} + +function scanBalancedDelimiters(code: number, state: ScanState): void { + if (code === 0x28) state.parenDepth++; + else if (code === 0x29) state.parenDepth = Math.max(0, state.parenDepth - 1); + else if (code === 0x7b) state.braceDepth++; + else if (code === 0x7d) state.braceDepth = Math.max(0, state.braceDepth - 1); +} + +function scanLineBody( + line: string, + state: ScanState, + startIndex: number, + trackDelimiters: boolean, +): void { + for (let index = startIndex; index < line.length; index++) { + const code = line.charCodeAt(index); + const next = line.charCodeAt(index + 1); + + if (state.inBlockComment) { + if (code === 0x2a && next === 0x2f) { + state.inBlockComment = false; + index++; + } + continue; + } + + if (state.quote !== undefined) { + if (code === 0x5c) { + index++; + } else if (line[index] === state.quote) { + state.quote = undefined; + } + continue; + } + + if (code === 0x2f && next === 0x2f) { + state.inLineCommentContinuation = hasEscapedLineEnding(line); + return; + } + if (code === 0x2f && next === 0x2a) { + state.inBlockComment = true; + index++; + continue; + } + if (code === 0x22 || code === 0x27) { + state.quote = line[index] as '"' | "'"; + continue; + } + // Directive replacement text is tokens, not C scopes. `{` in + // `#define WRAP {` must not pin braceDepth for later file-scope markers. + if (trackDelimiters) scanBalancedDelimiters(code, state); + } +} + +function scanDirectiveLine(line: string, state: ScanState): void { + const continued = hasEscapedLineEnding(line); + scanLineBody(line, state, 0, false); + state.inPreprocessorDirective = continued; + // A quote opened in replacement text does not survive past the directive. + if (!continued) state.quote = undefined; +} + +function scanLine(line: string, state: ScanState): void { + if (state.inLineCommentContinuation) { + state.inLineCommentContinuation = hasEscapedLineEnding(line); + // A `// ... \` splice keeps the next physical line in the comment, but the + // directive ends unless that line is itself backslash-continued. + if (state.inPreprocessorDirective) { + state.inPreprocessorDirective = state.inLineCommentContinuation; + } + return; + } + if (state.inPreprocessorDirective) { + scanDirectiveLine(line, state); + return; + } + if (startsPreprocessorDirective(line, state)) { + // Directives can open a block comment (`#define X /*`) that continues + // onto the next physical line. Scan comment/quote state only. + scanDirectiveLine(line, state); + return; + } + + if (state.inCodeLineContinuation) { + scanLineBody(line, state, 0, true); + if (state.quote !== undefined && !hasEscapedLineEnding(line)) state.quote = undefined; + state.inCodeLineContinuation = + !state.inLineCommentContinuation && !state.inBlockComment && hasEscapedLineEnding(line); + return; + } + + const keyword = leadingObjCDeclKeyword(line, state.inBlockComment); + if (keyword === 'begin') state.objcDeclDepth++; + else if (keyword === 'end') state.objcDeclDepth = Math.max(0, state.objcDeclDepth - 1); + + scanLineBody(line, state, 0, true); + + if (state.quote !== undefined && !hasEscapedLineEnding(line)) state.quote = undefined; + // Line-comment `\` continuation is not a C splice; it must not suppress + // the next physical line's file-scope marker test. + state.inCodeLineContinuation = + !state.inLineCommentContinuation && !state.inBlockComment && hasEscapedLineEnding(line); +} + +/** + * Elide bare, file-scope macro markers while preserving source positions. + * + * `_filePath` is accepted for the LanguageProvider hook signature. The + * transform is based only on source syntax and deliberately has no framework + * or repository-specific configuration. + */ +export function preprocessObjectiveCMacroMarkers(source: string, _filePath?: string): string { + const state: ScanState = { + inBlockComment: false, + inLineCommentContinuation: false, + inCodeLineContinuation: false, + inPreprocessorDirective: false, + quote: undefined, + braceDepth: 0, + parenDepth: 0, + objcDeclDepth: 0, + }; + const segments = source.split(/(\r\n|\n|\r)/); + let changed = false; + + for (let index = 0; index < segments.length; index += 2) { + const line = segments[index]; + if ( + !state.inBlockComment && + !state.inLineCommentContinuation && + !state.inCodeLineContinuation && + !state.inPreprocessorDirective && + state.quote === undefined && + state.braceDepth === 0 && + state.parenDepth === 0 && + state.objcDeclDepth === 0 && + isBareMarkerIdentifier(line) + ) { + segments[index] = ' '.repeat(line.length); + changed = true; + continue; + } + scanLine(line, state); + } + + return changed ? segments.join('') : source; +} diff --git a/gitnexus/src/core/ingestion/languages/objective-c/resolution-config.ts b/gitnexus/src/core/ingestion/languages/objective-c/resolution-config.ts new file mode 100644 index 000000000..2a7869fec --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/resolution-config.ts @@ -0,0 +1,323 @@ +/** + * Per-workspace Objective-C import config — the analog of TypeScript's + * tsconfig, C#'s csproj scan, and C/C++'s header scan. + * + * Loaded once per analyze pass via `objectiveCScopeResolver.loadResolutionConfig` + * and threaded into `resolveObjectiveCImportTarget`. `.h` files are usually + * classified as C++ unless they contain Objective-C syntax, so the ObjC + * resolver's `allFilePaths` can miss headers that `#import` / `#include` + * still name. This scan is what makes those paths visible. + */ + +import { readdirSync, readFileSync, type Dirent } from 'fs'; +import { join, relative } from 'path'; +import { clearObjectiveCFileFacts } from './facts.js'; + +const HEADER_EXTENSIONS = new Set(['.h']); +const MODULE_MAP_NAMES = new Set(['module.modulemap', 'module.map']); + +export interface ObjectiveCResolutionConfig { + /** Repo-relative `.h` paths discovered on disk (C/C++ header-scan analog). */ + readonly headers: ReadonlySet; + /** + * xcconfig `HEADER_SEARCH_PATHS`. Used for both quoted and angle-bracket + * imports — clang's system/project include path. + */ + readonly headerSearchPaths: readonly string[]; + /** + * xcconfig `USER_HEADER_SEARCH_PATHS` plus in-repo `Headers` / `include` + * roots. Quoted `#import "…"` only. Angle-bracket imports must not use + * these, or `` would bind a local decoy. + */ + readonly userHeaderSearchPaths: readonly string[]; + /** Framework name → repo-relative `*.framework/Headers` directory. */ + readonly frameworks: ReadonlyMap; + /** `@import` module name → umbrella / first header path. */ + readonly modules: ReadonlyMap; +} + +export function emptyObjectiveCResolutionConfig(): ObjectiveCResolutionConfig { + return { + headers: new Set(), + headerSearchPaths: [], + userHeaderSearchPaths: [], + frameworks: new Map(), + modules: new Map(), + }; +} + +export function coerceObjectiveCResolutionConfig( + value: unknown, +): ObjectiveCResolutionConfig | undefined { + if (value == null) return undefined; + if (value instanceof Set) { + return { + headers: value as ReadonlySet, + headerSearchPaths: [], + userHeaderSearchPaths: [], + frameworks: new Map(), + modules: new Map(), + }; + } + if (typeof value !== 'object') return undefined; + const record = value as Partial; + if (record.headers === undefined) return undefined; + return { + headers: record.headers, + headerSearchPaths: record.headerSearchPaths ?? [], + userHeaderSearchPaths: record.userHeaderSearchPaths ?? [], + frameworks: record.frameworks ?? new Map(), + modules: record.modules ?? new Map(), + }; +} + +export function loadObjectiveCResolutionConfig(repoPath: string): ObjectiveCResolutionConfig { + // Worker capture facts are process-local and outlive a single analysis in + // server mode. This hook runs once before each Objective-C workspace pass, + // before ParsedFile side channels are restored for the current files. + clearObjectiveCFileFacts(); + return scanObjectiveCWorkspace(repoPath); +} + +export function scanObjectiveCWorkspace(repoPath: string): ObjectiveCResolutionConfig { + const headers = new Set(); + const frameworks = new Map(); + const moduleMapPaths: string[] = []; + const xcconfigPaths: string[] = []; + const implicitSearchRoots = new Set(); + + walk(repoPath, repoPath, { + headers, + frameworks, + moduleMapPaths, + xcconfigPaths, + implicitSearchRoots, + }); + + const xcconfig = { headerSearchPaths: [] as string[], userHeaderSearchPaths: [] as string[] }; + for (const filePath of xcconfigPaths) { + const parsed = parseXcconfigSearchPaths(readText(join(repoPath, filePath))); + xcconfig.headerSearchPaths.push(...parsed.headerSearchPaths); + xcconfig.userHeaderSearchPaths.push(...parsed.userHeaderSearchPaths); + } + const headerSearchPaths = uniquePaths(xcconfig.headerSearchPaths); + const userHeaderSearchPaths = uniquePaths([ + ...implicitSearchRoots, + ...xcconfig.userHeaderSearchPaths, + ]); + + const modules = new Map(); + for (const mapPath of moduleMapPaths) { + for (const [name, header] of parseModuleMap( + readText(join(repoPath, mapPath)), + dirnamePosix(mapPath), + )) { + if (!modules.has(name)) modules.set(name, header); + } + } + + return { headers, headerSearchPaths, userHeaderSearchPaths, frameworks, modules }; +} + +interface ScanSink { + readonly headers: Set; + readonly frameworks: Map; + readonly moduleMapPaths: string[]; + readonly xcconfigPaths: string[]; + readonly implicitSearchRoots: Set; +} + +function walk(dir: string, root: string, sink: ScanSink): void { + let entries: Dirent[]; + try { + entries = readdirSync(dir, { withFileTypes: true, encoding: 'utf8' }); + } catch { + return; + } + for (const entry of entries) { + const name = entry.name; + const full = join(dir, name); + if (entry.isDirectory()) { + if (shouldSkipDirectory(name)) continue; + const relativeDir = normalizeRepoPath(relative(root, full)); + if (name.endsWith('.framework')) { + const frameworkName = name.slice(0, -'.framework'.length); + if (frameworkName.length > 0 && !sink.frameworks.has(frameworkName)) { + sink.frameworks.set(frameworkName, `${relativeDir}/Headers`); + } + } + if (name === 'Headers' || name === 'include') { + sink.implicitSearchRoots.add(relativeDir); + } + walk(full, root, sink); + } else if (entry.isFile()) { + const relativePath = normalizeRepoPath(relative(root, full)); + const ext = name.slice(name.lastIndexOf('.')); + if (HEADER_EXTENSIONS.has(ext)) sink.headers.add(relativePath); + if (MODULE_MAP_NAMES.has(name)) sink.moduleMapPaths.push(relativePath); + if (ext === '.xcconfig') sink.xcconfigPaths.push(relativePath); + } + } +} + +function shouldSkipDirectory(name: string): boolean { + return ( + name === 'node_modules' || + name === '.git' || + name === 'vendor' || + name === 'dist' || + name === 'build' || + name === 'out' || + name === 'target' || + name === '_build' || + name === '.next' || + name === 'DerivedData' || + name === 'xcuserdata' || + name === '.build' || + name === 'Pods' || + name === 'Carthage' || + name.startsWith('cmake-build') + ); +} + +export function parseXcconfigSearchPaths(source: string): { + readonly headerSearchPaths: string[]; + readonly userHeaderSearchPaths: string[]; +} { + const headerSearchPaths: string[] = []; + const userHeaderSearchPaths: string[] = []; + const joined = source.replace(/\\\r?\n/g, ' '); + for (const rawLine of joined.split(/\r?\n/)) { + const line = rawLine.replace(/\/\/.*$/, '').trim(); + if (line.length === 0) continue; + const match = /^(HEADER_SEARCH_PATHS|USER_HEADER_SEARCH_PATHS)\s*=\s*(.*)$/.exec(line); + if (match === null) continue; + const sink = + match[1] === 'USER_HEADER_SEARCH_PATHS' ? userHeaderSearchPaths : headerSearchPaths; + for (const token of tokenizeXcconfigValue(match[2] ?? '')) { + const resolved = expandXcconfigPath(token); + if (resolved !== undefined) sink.push(resolved); + } + } + return { headerSearchPaths, userHeaderSearchPaths }; +} + +function tokenizeXcconfigValue(value: string): string[] { + const tokens: string[] = []; + let current = ''; + let quoted = false; + for (let i = 0; i < value.length; i++) { + const ch = value[i]; + if (ch === '"') { + quoted = !quoted; + continue; + } + if (!quoted && /\s/.test(ch ?? '')) { + if (current.length > 0) { + tokens.push(current); + current = ''; + } + continue; + } + current += ch; + } + if (current.length > 0) tokens.push(current); + return tokens; +} + +function expandXcconfigPath(token: string): string | undefined { + if (token === '$(inherited)' || token === '$inherited') return undefined; + if (/\$\((SDKROOT|DEVELOPER_DIR|PLATFORM_DIR|TOOLCHAIN_DIR)\)/.test(token)) return undefined; + const expanded = token + .replace(/\$\((SRCROOT|PROJECT_DIR)\)\/?/g, '') + .replace(/\/\*\*?$/, '') + .replace(/\\/g, '/') + .replace(/^\.\//, ''); + if (expanded.startsWith('$') || expanded.startsWith('/')) return undefined; + return collapseRepoPath(expanded); +} + +export function parseModuleMap( + source: string, + mapDir: string, +): ReadonlyArray { + const modules: Array = []; + const blocks = /(?:framework\s+)?module\s+([A-Za-z_][\w.]*)\s*\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}/g; + for (const match of source.matchAll(blocks)) { + const name = match[1]; + const body = match[2] ?? ''; + if (name === undefined || name === '*') continue; + const header = + /umbrella\s+header\s+"([^"]+)"/.exec(body)?.[1] ?? + /umbrella\s+"([^"]+)"/.exec(body)?.[1] ?? + /header\s+"([^"]+)"/.exec(body)?.[1]; + if (header === undefined) continue; + modules.push([name, collapseRepoPath(resolveModuleMapHeader(mapDir, header))]); + } + return modules; +} + +function readText(filePath: string): string { + try { + return readFileSync(filePath, 'utf8'); + } catch { + return ''; + } +} + +function dirnamePosix(filePath: string): string { + const normalized = normalizeRepoPath(filePath); + const slash = normalized.lastIndexOf('/'); + return slash === -1 ? '' : normalized.slice(0, slash); +} + +function uniquePaths(paths: readonly string[]): string[] { + const seen = new Set(); + const out: string[] = []; + for (const path of paths) { + const normalized = normalizeRepoPath(path); + if (normalized.length === 0 || seen.has(normalized)) continue; + seen.add(normalized); + out.push(normalized); + } + return out; +} + +export function normalizeRepoPath(value: string): string { + return value.replaceAll('\\', '/').replace(/^\.\//, ''); +} + +/** Collapse `.` / `..` in scanned config paths. Import targets stay raw. */ +export function collapseRepoPath(value: string): string { + const parts = normalizeRepoPath(value).split('/'); + const out: string[] = []; + for (const part of parts) { + if (part === '' || part === '.') continue; + if (part === '..') { + out.pop(); + continue; + } + out.push(part); + } + return out.join('/'); +} + +function resolveModuleMapHeader(mapDir: string, header: string): string { + const direct = collapseRepoPath(joinPosix(mapDir, header)); + const frameworkRoot = frameworkRootFromModulesDir(mapDir); + if (frameworkRoot === undefined) return direct; + const base = header.slice(header.replaceAll('\\', '/').lastIndexOf('/') + 1); + return collapseRepoPath(joinPosix(`${frameworkRoot}/Headers`, base)); +} + +function frameworkRootFromModulesDir(mapDir: string): string | undefined { + if (mapDir === 'Modules') return undefined; + if (!mapDir.endsWith('/Modules')) return undefined; + const parent = mapDir.slice(0, -'/Modules'.length); + return parent.endsWith('.framework') ? parent : undefined; +} + +function joinPosix(left: string, right: string): string { + if (left.length === 0) return right; + return `${left}/${right}`; +} diff --git a/gitnexus/src/core/ingestion/languages/objective-c/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/objective-c/scope-resolver.ts new file mode 100644 index 000000000..7ae8d5d12 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/objective-c/scope-resolver.ts @@ -0,0 +1,827 @@ +import { + SupportedLanguages, + type ParsedFile, + type SymbolDefinition, + type Callsite, +} from 'gitnexus-shared'; +import type { RelationshipType } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import { generateId } from '../../../../lib/utils.js'; +import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; +import { objectiveCProvider } from '../objective-c.js'; +import { populateObjectiveCCompilationUnitSiblings } from './compilation-unit-siblings.js'; +import { resolveObjectiveCImportTarget } from './import-target.js'; +import { loadObjectiveCResolutionConfig } from './resolution-config.js'; +import { + applyObjectiveCCaptureSideChannel, + isInternalObjectiveCFunctionDef, + objcClassQualifiedName, + objcProtocolQualifiedName, + objcUnresolvedMessageQualifiedName, + objectiveCFactsFromParsedFiles, + type ObjCContainerFact, + type ObjCFileFacts, + type ObjCMemberFact, + type ObjCMessageFact, + type ObjCMethodFact, + type ObjCTypeInfo, + parseObjCType, +} from './facts.js'; + +interface ObjCWorkspaceFacts { + readonly containersByQualifiedName: ReadonlyMap; + readonly classByName: ReadonlyMap; + readonly protocolsByName: ReadonlyMap; + readonly methodsByDispatchOwner: ReadonlyMap; + readonly methodsByExactOwner: ReadonlyMap; + readonly memberTypesByOwner: ReadonlyMap>; + readonly classProtocols: ReadonlyMap>; + readonly protocolParents: ReadonlyMap>; + readonly superclassByClass: ReadonlyMap; + /** Class names that conform to each protocol, including inherited protocols. */ + readonly classesByProtocol: ReadonlyMap; + /** + * Per-(protocol, selector) implementer methods. Filled on first query so + * N messages to the same protocol do not rescan the class set. + */ + readonly protocolImplementationBySelector: Map; + /** + * Shared implementer-evidence nodes already written. The implementer + * USES fan-out is emitted once; later messages only add the + * source→shared hop (C# workspaceTypeBindings analog). + */ + readonly emittedProtocolCandidateSets: Set; +} + +export const objectiveCScopeResolver: ScopeResolver = { + language: SupportedLanguages.ObjectiveC, + languageProvider: objectiveCProvider, + importEdgeReason: 'objective-c-scope: import', + + loadResolutionConfig: (repoPath) => loadObjectiveCResolutionConfig(repoPath), + + resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => + resolveObjectiveCImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig), + + mergeBindings: (existing, incoming) => [...existing, ...incoming], + + arityCompatibility: (callsite: Callsite, def: SymbolDefinition) => { + if (callsite.arity === undefined || def.parameterCount === undefined) return 'unknown'; + return callsite.arity === def.parameterCount ? 'compatible' : 'incompatible'; + }, + + buildMro: () => new Map(), + + applyCaptureSideChannel: applyObjectiveCCaptureSideChannel, + populateOwners: (parsed: ParsedFile) => populateClassOwnedMembers(parsed), + populateNamespaceSiblings: populateObjectiveCCompilationUnitSiblings, + isFileLocalDef: (def: SymbolDefinition) => isInternalObjectiveCFunctionDef(def), + isSuperReceiver: (receiverText) => receiverText.trim() === 'super', + + fieldFallbackOnMethodLookup: false, + propagatesReturnTypesAcrossImports: false, + collapseMemberCallsByCallerTarget: true, + + emitPostResolutionEdges(graph, parsedFiles) { + const facts = objectiveCFactsFromParsedFiles(parsedFiles); + if (facts.length === 0) return; + const workspace = buildObjectiveCWorkspaceFacts(facts); + + for (const fact of facts) { + emitObjectiveCHeritageEdges(graph, fact, workspace); + emitObjectiveCCategoryEdges(graph, fact, workspace); + emitObjectiveCImplementationEvidence(graph, fact); + emitObjectiveCUnresolvedMessageEvidence(graph, fact); + emitObjectiveCMessageEdges(graph, fact, workspace); + } + }, +}; + +function graphNodeId(label: string, qualifiedName: string): string { + return generateId(label, qualifiedName); +} + +function relationshipId( + type: RelationshipType, + sourceId: string, + targetId: string, + reason: string, +): string { + return generateId(type, `${sourceId}->${targetId}:${reason}`); +} + +function addRelationship( + graph: KnowledgeGraph, + type: RelationshipType, + sourceId: string, + targetId: string, + reason: string, + confidence = 0.9, +): void { + graph.addRelationship({ + id: relationshipId(type, sourceId, targetId, reason), + sourceId, + targetId, + type, + confidence, + reason, + }); +} + +function labelForContainer(container: ObjCContainerFact): 'Class' | 'Protocol' | 'Category' { + return container.label; +} + +function buildObjectiveCWorkspaceFacts(facts: readonly ObjCFileFacts[]): ObjCWorkspaceFacts { + const containersByQualifiedName = new Map(); + const classByName = new Map(); + const protocolsByName = new Map(); + const methodsByExactOwner = new Map(); + const methodsByDispatchOwner = new Map(); + const memberTypesByOwner = new Map>(); + const classProtocols = new Map>(); + const protocolParents = new Map>(); + const superclassByClass = new Map(); + + for (const fileFact of facts) { + for (const container of fileFact.containers) { + const existing = containersByQualifiedName.get(container.qualifiedName); + containersByQualifiedName.set( + container.qualifiedName, + mergeContainerFacts(existing, container), + ); + if (container.kind === 'class') { + classByName.set(container.name, container); + if (container.superclass !== undefined) + superclassByClass.set(container.name, container.superclass); + if (container.protocols.length > 0) { + let protocols = classProtocols.get(container.name); + if (protocols === undefined) { + protocols = new Set(); + classProtocols.set(container.name, protocols); + } + for (const protocol of container.protocols) protocols.add(protocol); + } + } else if (container.kind === 'protocol') { + protocolsByName.set(container.name, container); + if (container.protocols.length > 0) { + let parents = protocolParents.get(container.name); + if (parents === undefined) { + parents = new Set(); + protocolParents.set(container.name, parents); + } + for (const protocol of container.protocols) parents.add(protocol); + } + } else if (container.hostClass !== undefined) { + if (container.protocols.length > 0) { + let protocols = classProtocols.get(container.hostClass); + if (protocols === undefined) { + protocols = new Set(); + classProtocols.set(container.hostClass, protocols); + } + for (const protocol of container.protocols) protocols.add(protocol); + } + } + } + + for (const method of fileFact.methods) { + appendMap(methodsByExactOwner, method.ownerQualifiedName, method); + appendMap(methodsByDispatchOwner, method.ownerQualifiedName, method); + if (method.hostClass !== undefined) { + appendMap(methodsByDispatchOwner, objcClassQualifiedName(method.hostClass), method); + } + } + + for (const member of fileFact.members) { + addMemberType(memberTypesByOwner, member.ownerQualifiedName, member); + if (member.hostClass !== undefined) { + addMemberType(memberTypesByOwner, objcClassQualifiedName(member.hostClass), member); + } + } + } + + const hierarchyWorkspace = { protocolParents }; + const classNames = new Set([...classByName.keys(), ...classProtocols.keys()]); + const classesByProtocol = new Map(); + for (const className of classNames) { + const seenClasses = new Set(); + let currentClass: string | undefined = className; + while (currentClass !== undefined && !seenClasses.has(currentClass)) { + seenClasses.add(currentClass); + const directProtocols = classProtocols.get(currentClass); + if (directProtocols !== undefined) { + for (const directProtocol of directProtocols) { + for (const protocolName of protocolHierarchy(hierarchyWorkspace, directProtocol)) { + let implementers = classesByProtocol.get(protocolName); + if (implementers === undefined) { + implementers = []; + classesByProtocol.set(protocolName, implementers); + } + implementers.push(className); + } + } + } + currentClass = superclassByClass.get(currentClass); + } + } + const classesByProtocolFrozen = new Map(); + for (const [protocolName, implementers] of classesByProtocol) { + classesByProtocolFrozen.set( + protocolName, + Object.freeze([...new Set(implementers)].sort((left, right) => left.localeCompare(right))), + ); + } + + return { + containersByQualifiedName, + classByName, + protocolsByName, + methodsByDispatchOwner, + methodsByExactOwner, + memberTypesByOwner, + classProtocols, + protocolParents, + superclassByClass, + classesByProtocol: classesByProtocolFrozen, + protocolImplementationBySelector: new Map(), + emittedProtocolCandidateSets: new Set(), + }; +} + +function addMemberType( + memberTypesByOwner: Map>, + ownerQualifiedName: string, + member: ObjCMemberFact, +): void { + const type = parseObjCType(member.declaredType); + if (type === undefined) return; + let types = memberTypesByOwner.get(ownerQualifiedName); + if (types === undefined) { + types = new Map(); + memberTypesByOwner.set(ownerQualifiedName, types); + } + types.set(member.name, type); +} + +function mergeContainerFacts( + existing: ObjCContainerFact | undefined, + incoming: ObjCContainerFact, +): ObjCContainerFact { + if (existing === undefined) return incoming; + const protocols = Array.from(new Set([...existing.protocols, ...incoming.protocols])).sort(); + return { + ...existing, + declarationRole: + existing.declarationRole === 'implementation' || incoming.declarationRole === 'implementation' + ? 'implementation' + : 'interface', + startLine: Math.min(existing.startLine, incoming.startLine), + endLine: Math.max(existing.endLine, incoming.endLine), + ...(existing.superclass !== undefined || incoming.superclass !== undefined + ? { superclass: existing.superclass ?? incoming.superclass } + : {}), + protocols, + }; +} + +function appendMap(map: Map, key: K, value: V): void { + const existing = map.get(key); + if (existing === undefined) map.set(key, [value]); + else existing.push(value); +} + +function emitObjectiveCHeritageEdges( + graph: KnowledgeGraph, + facts: ObjCFileFacts, + workspace: ObjCWorkspaceFacts, +): void { + for (const container of facts.containers) { + const sourceId = graphNodeId(labelForContainer(container), container.qualifiedName); + if (container.kind === 'class' && container.superclass !== undefined) { + const superclass = workspace.classByName.get(container.superclass); + if (superclass !== undefined) { + addRelationship( + graph, + 'EXTENDS', + sourceId, + graphNodeId('Class', superclass.qualifiedName), + 'objc: superclass', + ); + } + } + + const protocolSourceId = + container.hostClass !== undefined + ? graphNodeId('Class', objcClassQualifiedName(container.hostClass)) + : sourceId; + for (const protocolName of container.protocols) { + const protocol = workspace.protocolsByName.get(protocolName); + if (protocol === undefined) continue; + addRelationship( + graph, + 'IMPLEMENTS', + protocolSourceId, + graphNodeId('Protocol', protocol.qualifiedName), + 'objc: protocol conformance', + ); + } + } +} + +function emitObjectiveCCategoryEdges( + graph: KnowledgeGraph, + facts: ObjCFileFacts, + workspace: ObjCWorkspaceFacts, +): void { + for (const container of facts.containers) { + if (container.hostClass === undefined) continue; + if (!workspace.classByName.has(container.hostClass)) continue; + addRelationship( + graph, + 'MEMBER_OF', + graphNodeId('Category', container.qualifiedName), + graphNodeId('Class', objcClassQualifiedName(container.hostClass)), + 'objc: category host class', + ); + } +} + +function emitObjectiveCUnresolvedMessageEvidence( + graph: KnowledgeGraph, + facts: ObjCFileFacts, +): void { + for (const unresolved of facts.unresolvedMessages) { + const evidenceId = graphNodeId( + 'CodeElement', + objcUnresolvedMessageQualifiedName( + facts.filePath, + unresolved.startLine, + unresolved.startCol, + unresolved.selector, + ), + ); + addRelationship( + graph, + 'USES', + unresolved.sourceMethodId, + evidenceId, + `objc-message: unresolved: ${unresolved.reason}`, + 0.5, + ); + } +} + +function emitObjectiveCImplementationEvidence(graph: KnowledgeGraph, facts: ObjCFileFacts): void { + for (const container of facts.containers) { + if (container.declarationRole !== 'implementation') continue; + const targetId = graphNodeId(labelForContainer(container), container.qualifiedName); + emitImplementationEvidence( + graph, + facts.filePath, + targetId, + `@implementation ${container.name}`, + `objc:implementation:${container.qualifiedName}:${facts.filePath}:${container.startLine}`, + container.startLine, + container.endLine, + { + objectiveCKind: 'implementation-evidence', + implementationKind: container.kind, + targetQualifiedName: container.qualifiedName, + }, + ); + } + + for (const method of facts.methods) { + if (method.declarationRole !== 'implementation') continue; + emitImplementationEvidence( + graph, + facts.filePath, + method.nodeId, + `${method.methodKind}[${method.ownerName} ${method.selector}] implementation`, + `objc:method-implementation:${method.qualifiedName}:${facts.filePath}:${method.startLine}`, + method.startLine, + method.endLine, + { + objectiveCKind: 'implementation-evidence', + implementationKind: 'method', + targetQualifiedName: method.qualifiedName, + selector: method.selector, + methodKind: method.methodKind, + objectiveCOwner: method.ownerQualifiedName, + }, + ); + } +} + +function emitImplementationEvidence( + graph: KnowledgeGraph, + filePath: string, + targetId: string, + name: string, + qualifiedName: string, + startLine: number, + endLine: number, + extras: Record, +): void { + const nodeId = graphNodeId('CodeElement', qualifiedName); + graph.addNode({ + id: nodeId, + label: 'CodeElement', + properties: { + name, + qualifiedName, + filePath, + startLine, + endLine, + language: SupportedLanguages.ObjectiveC, + isExported: false, + ...extras, + }, + }); + addRelationship( + graph, + 'DEFINES', + graphNodeId('File', filePath), + nodeId, + 'objc: implementation evidence', + 1, + ); + addRelationship(graph, 'DECLARES', nodeId, targetId, 'objc: implementation of merged symbol', 1); +} + +function emitObjectiveCMessageEdges( + graph: KnowledgeGraph, + facts: ObjCFileFacts, + workspace: ObjCWorkspaceFacts, +): void { + for (const message of facts.messages) { + const targets = resolveMessageTargets(message, workspace); + if (targets.kind === 'none') continue; + if (targets.kind === 'category-collision') { + emitCategoryDispatchEvidence(graph, facts, message, targets.candidates); + continue; + } + if (targets.kind === 'protocol') { + emitProtocolMessageEvidence( + graph, + facts, + message, + targets.protocolName, + targets.candidates, + workspace, + ); + } + for (const target of targets.methods) { + if (graph.getNode(target.nodeId) === undefined) continue; + addRelationship( + graph, + 'CALLS', + message.sourceMethodId, + target.nodeId, + targets.kind === 'protocol' + ? 'objc-message: protocol receiver' + : `objc-message: ${message.receiverKind} receiver`, + targets.kind === 'protocol' ? 0.8 : 0.9, + ); + } + } +} + +type MessageTargets = + | { readonly kind: 'none'; readonly methods: readonly ObjCMethodFact[] } + | { readonly kind: 'direct'; readonly methods: readonly ObjCMethodFact[] } + | { readonly kind: 'category-collision'; readonly candidates: readonly ObjCMethodFact[] } + | { + readonly kind: 'protocol'; + readonly protocolName: string; + readonly methods: readonly ObjCMethodFact[]; + readonly candidates: readonly ObjCMethodFact[]; + }; + +function messageTargetsFromDispatchLookup(lookup: DispatchMethodLookup): MessageTargets { + return lookup.categoryCollision + ? { kind: 'category-collision', candidates: lookup.methods } + : { kind: 'direct', methods: lookup.methods }; +} + +function resolveMessageTargets( + message: ObjCMessageFact, + workspace: ObjCWorkspaceFacts, +): MessageTargets { + if (message.receiverKind === 'dynamic') { + return { kind: 'none', methods: [] }; + } + + if (message.receiverKind === 'class') { + const className = message.receiverType?.name ?? message.receiverText; + return messageTargetsFromDispatchLookup( + findDispatchMethods(workspace, className, '+', message.selector), + ); + } + + if (message.receiverKind === 'self') { + const owner = workspace.containersByQualifiedName.get(message.sourceOwnerQualifiedName); + const className = owner?.hostClass ?? owner?.name ?? message.sourceOwnerName; + if (owner?.kind === 'protocol') { + return { + kind: 'direct', + methods: findProtocolMethods( + workspace, + owner.name, + message.sourceMethodKind, + message.selector, + ), + }; + } + return messageTargetsFromDispatchLookup( + findDispatchMethods(workspace, className, message.sourceMethodKind, message.selector), + ); + } + + if (message.receiverKind === 'super') { + const owner = workspace.containersByQualifiedName.get(message.sourceOwnerQualifiedName); + const className = owner?.hostClass ?? owner?.name ?? message.sourceOwnerName; + const superclass = workspace.superclassByClass.get(className); + return superclass === undefined + ? { kind: 'none', methods: [] } + : messageTargetsFromDispatchLookup( + findDispatchMethods(workspace, superclass, message.sourceMethodKind, message.selector), + ); + } + + const receiverType = message.receiverType ?? resolveMemberReceiverType(message, workspace); + if (receiverType?.kind === 'dynamic' || receiverType?.kind === 'class-object') { + return { kind: 'none', methods: [] }; + } + if (message.receiverKind === 'unknown' && receiverType === undefined) { + return { kind: 'none', methods: [] }; + } + if (receiverType?.kind === 'class' && receiverType.name !== undefined) { + return messageTargetsFromDispatchLookup( + findDispatchMethods(workspace, receiverType.name, '-', message.selector), + ); + } + + if (receiverType?.kind === 'protocol' && receiverType.name !== undefined) { + const methods = findProtocolMethods(workspace, receiverType.name, '-', message.selector); + const candidates = findProtocolImplementationCandidates( + workspace, + receiverType.name, + message.selector, + ); + return { + kind: 'protocol', + protocolName: receiverType.name, + methods, + candidates, + }; + } + + return { kind: 'none', methods: [] }; +} + +function resolveMemberReceiverType( + message: ObjCMessageFact, + workspace: ObjCWorkspaceFacts, +): ObjCTypeInfo | undefined { + if (message.receiverMemberName === undefined) return undefined; + const owner = workspace.containersByQualifiedName.get(message.sourceOwnerQualifiedName); + let className = owner?.hostClass ?? owner?.name ?? message.sourceOwnerName; + const seen = new Set(); + while (className !== undefined && !seen.has(className)) { + seen.add(className); + const type = workspace.memberTypesByOwner + .get(objcClassQualifiedName(className)) + ?.get(message.receiverMemberName); + if (type !== undefined) return type; + className = workspace.superclassByClass.get(className); + } + return workspace.memberTypesByOwner + .get(message.sourceOwnerQualifiedName) + ?.get(message.receiverMemberName); +} + +function findDispatchMethods( + workspace: ObjCWorkspaceFacts, + className: string, + methodKind: '-' | '+', + selector: string, +): DispatchMethodLookup { + const seen = new Set(); + let currentClass: string | undefined = className; + while (currentClass !== undefined && !seen.has(currentClass)) { + seen.add(currentClass); + const ownerQn = objcClassQualifiedName(currentClass); + const methods = uniqueMethods( + (workspace.methodsByDispatchOwner.get(ownerQn) ?? []).filter( + (method) => method.methodKind === methodKind && method.selector === selector, + ), + ); + if (methods.length > 0) { + return { + methods, + // Named categories can replace a host-class implementation at runtime, + // but image-load order is not statically knowable. Preserve all targets + // as evidence instead of claiming that each is a certain CALLS edge. + categoryCollision: + methods.length > 1 && methods.some((method) => method.ownerKind === 'category'), + }; + } + currentClass = workspace.superclassByClass.get(currentClass); + } + return { methods: [], categoryCollision: false }; +} + +interface DispatchMethodLookup { + readonly methods: readonly ObjCMethodFact[]; + readonly categoryCollision: boolean; +} + +function findExactOwnerMethods( + workspace: ObjCWorkspaceFacts, + ownerQualifiedName: string, + methodKind: '-' | '+', + selector: string, +): readonly ObjCMethodFact[] { + return (workspace.methodsByExactOwner.get(ownerQualifiedName) ?? []).filter( + (method) => method.methodKind === methodKind && method.selector === selector, + ); +} + +function findProtocolMethods( + workspace: ObjCWorkspaceFacts, + protocolName: string, + methodKind: '-' | '+', + selector: string, +): readonly ObjCMethodFact[] { + for (const name of protocolHierarchy(workspace, protocolName)) { + const methods = findExactOwnerMethods( + workspace, + objcProtocolQualifiedName(name), + methodKind, + selector, + ); + if (methods.length > 0) return uniqueMethods(methods); + } + return []; +} + +function protocolHierarchy( + workspace: Pick, + protocolName: string, +): readonly string[] { + const seen = new Set(); + const pending = [protocolName]; + const hierarchy: string[] = []; + while (pending.length > 0) { + const current = pending.shift(); + if (current === undefined || seen.has(current)) continue; + seen.add(current); + hierarchy.push(current); + const parents = workspace.protocolParents.get(current); + if (parents === undefined) continue; + pending.push(...[...parents].sort()); + } + return hierarchy; +} + +function uniqueMethods(methods: readonly ObjCMethodFact[]): readonly ObjCMethodFact[] { + return [...new Map(methods.map((method) => [method.nodeId, method])).values()].sort( + (left, right) => left.qualifiedName.localeCompare(right.qualifiedName), + ); +} + +function findProtocolImplementationCandidates( + workspace: ObjCWorkspaceFacts, + protocolName: string, + selector: string, +): readonly ObjCMethodFact[] { + const cacheKey = `${protocolName}\0${selector}`; + const cached = workspace.protocolImplementationBySelector.get(cacheKey); + if (cached !== undefined) return cached; + + const out: ObjCMethodFact[] = []; + for (const className of workspace.classesByProtocol.get(protocolName) ?? []) { + out.push(...findDispatchMethods(workspace, className, '-', selector).methods); + } + const result = uniqueMethods(out); + workspace.protocolImplementationBySelector.set(cacheKey, result); + return result; +} + +function emitCategoryDispatchEvidence( + graph: KnowledgeGraph, + facts: ObjCFileFacts, + message: ObjCMessageFact, + candidates: readonly ObjCMethodFact[], +): void { + if (candidates.length === 0) return; + const qualifiedName = + `objc:category-dispatch-candidates:${facts.filePath}:` + + `${message.startLine}:${message.startCol}:${message.selector}`; + const nodeId = graphNodeId('CodeElement', qualifiedName); + graph.addNode({ + id: nodeId, + label: 'CodeElement', + properties: { + name: `[${message.receiverText} ${message.selector}] category dispatch candidates`, + qualifiedName, + filePath: facts.filePath, + startLine: message.startLine, + endLine: message.startLine, + language: SupportedLanguages.ObjectiveC, + isExported: false, + objectiveCKind: 'category-dispatch-candidates', + }, + }); + addRelationship( + graph, + 'DEFINES', + graphNodeId('File', facts.filePath), + nodeId, + `objc: category dispatch candidate evidence: ${message.selector}`, + 1, + ); + addRelationship( + graph, + 'USES', + message.sourceMethodId, + nodeId, + `objc-message: category dispatch candidates: ${message.selector}`, + 0.7, + ); + for (const candidate of candidates) { + if (graph.getNode(candidate.nodeId) === undefined) continue; + addRelationship( + graph, + 'USES', + nodeId, + candidate.nodeId, + `objc-category-dispatch-candidate: ${message.selector}`, + 0.5, + ); + } +} + +function emitProtocolMessageEvidence( + graph: KnowledgeGraph, + facts: ObjCFileFacts, + message: ObjCMessageFact, + protocolName: string, + candidates: readonly ObjCMethodFact[], + workspace: ObjCWorkspaceFacts, +): void { + if (candidates.length === 0) return; + const setKey = `${protocolName}\0${message.selector}`; + const qualifiedName = `objc:protocol-candidates:${protocolName}:${message.selector}`; + const nodeId = graphNodeId('CodeElement', qualifiedName); + const protocol = workspace.protocolsByName.get(protocolName); + const evidenceFilePath = protocol?.filePath ?? facts.filePath; + + if (!workspace.emittedProtocolCandidateSets.has(setKey)) { + workspace.emittedProtocolCandidateSets.add(setKey); + graph.addNode({ + id: nodeId, + label: 'CodeElement', + properties: { + name: `[id<${protocolName}> ${message.selector}] protocol implementers`, + qualifiedName, + filePath: evidenceFilePath, + startLine: protocol?.startLine ?? message.startLine, + endLine: protocol?.endLine ?? message.startLine, + language: SupportedLanguages.ObjectiveC, + isExported: false, + objectiveCKind: 'protocol-implementers', + }, + }); + addRelationship( + graph, + 'DEFINES', + graphNodeId('File', evidenceFilePath), + nodeId, + `objc: protocol receiver candidate evidence: ${protocolName} ${message.selector}`, + 1, + ); + for (const candidate of candidates) { + if (graph.getNode(candidate.nodeId) === undefined) continue; + addRelationship( + graph, + 'USES', + nodeId, + candidate.nodeId, + `objc-protocol-candidate: ${protocolName} ${message.selector}`, + 0.5, + ); + } + } + + addRelationship( + graph, + 'USES', + message.sourceMethodId, + nodeId, + `objc-message: protocol receiver candidates: ${protocolName} ${message.selector}`, + 0.7, + ); +} diff --git a/gitnexus/src/core/ingestion/languages/ruby/import-target.ts b/gitnexus/src/core/ingestion/languages/ruby/import-target.ts index 62d6fca3d..eae1f56e3 100644 --- a/gitnexus/src/core/ingestion/languages/ruby/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/ruby/import-target.ts @@ -3,13 +3,19 @@ * * Ruby import resolution rules: * - `require_relative './foo'` → resolve relative to the importing file's dir - * - `require 'foo'` → suffix-match via the existing Ruby import resolver - * - External gems → null (unresolvable within the repo) + * - `require 'foo'` → use scoped gem metadata before legacy suffix matching + * - Known local gems → resolve only within their declared load roots + * - Known external gems → null, even if an unrelated repo file suffix matches */ import { resolveRubyImportInternal } from '../../import-resolvers/ruby.js'; import { getWorkspaceFileIndex } from '../../import-resolvers/workspace-file-index.js'; import { isHeritageMarker } from '../../utils/heritage-marker.js'; +import { + findRubyResolutionScope, + type RubyResolutionConfig, + type RubyResolutionScope, +} from './resolution-config.js'; export interface RubyResolveContext { readonly fromFile: string; @@ -26,16 +32,17 @@ export interface RubyResolveContext { * against the importing file's directory, trying `.rb` and `/index.rb` * suffixes. * - * For bare requires (gem-style like `'json'`, `'serializable'`), delegates - * to the existing `resolveRubyImportInternal` which uses suffix matching. - * - * Returns `null` for external gems that have no matching file in the repo. + * For bare requires, scoped manifest metadata takes precedence: known local + * gems resolve only within their declared load roots (a miss returns `null`), + * and known external gem prefixes return `null` even if a repo suffix matches. + * Only requires without matching gem evidence delegate to the existing + * `resolveRubyImportInternal` suffix matcher. */ export function resolveRubyImportTarget( targetRaw: string, fromFile: string, allFilePaths: ReadonlySet, - _resolutionConfig?: unknown, + resolutionConfig?: unknown, ): string | readonly string[] | null { if (!targetRaw) return null; if (isHeritageMarker(targetRaw)) return null; @@ -51,7 +58,16 @@ export function resolveRubyImportTarget( return resolved; } - // ── require: bare/gem-style suffix matching ───────────────────────── + // ── require: scoped gem evidence before repository-wide fallback ──── + const config = resolutionConfig as RubyResolutionConfig | null | undefined; + const scope = + config === null || config === undefined ? undefined : findRubyResolutionScope(config, fromFile); + if (scope !== undefined) { + const localGemTarget = resolveLocalGemTarget(targetRaw, scope, allFilePaths); + if (localGemTarget !== undefined) return localGemTarget; + if (matchesRequirePrefix(targetRaw, scope.externalRequirePrefixes)) return null; + } + return resolveBare(targetRaw, allFilePaths); } @@ -95,6 +111,60 @@ function resolveRelative( return null; } +/** + * Yield the require stem, then each slash-delimited ancestor. This makes both + * local-root and external-gem lookup O(require path depth), not O(gem count). + * A trailing `.rb` is stripped first so `require 'my_engine.rb'` matches the + * configured prefix `my_engine`, matching Ruby's optional-suffix require. + */ +function requirePrefixCandidates(targetRaw: string): readonly string[] { + const stem = targetRaw.endsWith('.rb') ? targetRaw.slice(0, -3) : targetRaw; + if (stem.length === 0) return []; + const candidates = [stem]; + let slash = stem.lastIndexOf('/'); + while (slash !== -1) { + candidates.push(stem.slice(0, slash)); + slash = stem.lastIndexOf('/', slash - 1); + } + return candidates; +} + +function matchesRequirePrefix(targetRaw: string, prefixes: ReadonlySet): boolean { + return requirePrefixCandidates(targetRaw).some((candidate) => prefixes.has(candidate)); +} + +/** + * Resolve a path/gemspec-backed gem against its declared Ruby load roots. + * `undefined` means no local-gem prefix matched; `null` means one matched but + * none of its declared load roots contained the target. + */ +function resolveLocalGemTarget( + targetRaw: string, + scope: RubyResolutionScope, + allFilePaths: ReadonlySet, +): string | null | undefined { + for (const prefix of requirePrefixCandidates(targetRaw)) { + const loadRoots = scope.localLoadRootsByPrefix.get(prefix); + if (loadRoots === undefined) continue; + + for (const loadRoot of loadRoots) { + const base = loadRoot ? `${loadRoot}/${targetRaw}` : targetRaw; + if (base.endsWith('.rb')) { + if (allFilePaths.has(base)) return base; + continue; + } + const rbFile = `${base}.rb`; + if (allFilePaths.has(rbFile)) return rbFile; + } + + // The prefix is owned by a known local gem. If its declared roots do not + // contain the target, repository-wide suffix matching would fabricate an + // edge to an unrelated file. + return null; + } + return undefined; +} + /** * Resolve a bare require path (`'serializable'`, `'json'`, `'net/http'`) * via suffix matching using the existing Ruby import resolver. diff --git a/gitnexus/src/core/ingestion/languages/ruby/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/ruby/name-fallback-visibility.ts new file mode 100644 index 000000000..4b225c96f --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/ruby/name-fallback-visibility.ts @@ -0,0 +1,196 @@ +/** + * Ruby's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Ruby keeps the labeled fallback deliberately: with autoload (Rails' zeitwerk, + * `ActiveSupport::Dependencies`) a file genuinely can call a method whose + * defining file it never requires, so "no require" is NOT evidence of + * impossibility the way it is in Go or Rust. + * + * What IS decidable is namespacing — but only for CLASS bodies. A method + * defined inside a `class` is not callable as a bare `helper()` from another + * file unless that file names the class somehow (a receiver `Ns.helper`, a + * subclass declaration, an `include`, a `require`). A method defined inside a + * `module` body is different in kind: modules exist to be mixed in, and Rails + * mixes them in for you — every `*Helper` module is included into views and + * controllers by the framework, and concerns arrive through `included do` + * hooks — so a bare `format_money()` in a view legitimately reaches + * `module ApplicationHelper` with no `include`, `require`, or constant + * anywhere in the caller. Refusing that would delete real edges on exactly the + * codebase shape Ruby's fallback exists to serve. So module-owned methods stay + * a LABELED guess, and the refusal targets CLASS-owned methods whose class the + * caller never names. A top-level method (`ownerId === undefined`) is left + * alone, since that is the shape autoload actually delivers. When the owner + * cannot be found or typed (no `parsedFileOf`, owner outside the file set), + * the question is unanswered and the edge is allowed. + * + * "Never names" is read from the caller's own text-visible signals — its + * `require`/`include`/`extend` targets, and any reference site spelling the + * namespace's constant. If any of them mentions the namespace, the call is + * plausible and the labeled edge stands. `require` paths are snake_case + * (`billing/invoice_service`) while constants are CamelCase + * (`Billing::InvoiceService`), so the comparison normalizes both sides — + * without that the `require` branch never matched anything. + * + * One more thing a caller file can do without naming the class: INHERIT its + * way to it. `class UsersController < AdminController` reaches every method + * `ApplicationController` defines while naming only `AdminController`, and an + * `include Concern` reaches whatever that concern includes in turn. Neither + * chain is decidable from one file, so a caller with ANY inheritance or mixin + * surface (an `inherits` site, an `include`/`extend`/`prepend` marker) is + * treated as plausible. So is a caller file that DEFINES a module: a module's + * methods run against whatever class includes the module, and call that class's + * methods bare — `module PostGuardian; def can_see?; is_staff? ...` reaches + * `class Guardian#is_staff?` because Guardian includes PostGuardian, a fact the + * module file never states. What remains refused is the shape Ruby itself + * rejects: a bare call to a class's method from a file that inherits nothing, + * mixes in nothing, defines no module, and never spells the class. Measured on + * discourse@3f71fa15c that is ~1000 refusals, and the sample is what the rule + * promises: RSpec `before`/`after`/`subject` guessed to serializer methods of + * the same name, `Gemfile`'s `gem` to `Plugin::Instance#gem`, `routes.rb`'s + * `get` to `Draft#get` — fabricated callers, every one. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { moduleSegments } from '../../scope-resolution/utils/name-fallback-visibility.js'; +import { HERITAGE_MARKER_PREFIX } from '../../utils/heritage-marker.js'; + +/** + * The namespace segments a candidate's qualified name declares, minus the + * method itself. `Billing::Invoice#total` / `Billing.Invoice.total` → the + * `Billing`, `Invoice` constants a caller would have to name. + */ +function namespaceConstantsOf(candidate: SymbolDefinition): readonly string[] { + const qualified = candidate.qualifiedName; + if (qualified === undefined || qualified === '') return []; + const segments = qualified + .split(/::|\.|#/) + .map((s) => s.trim()) + .filter((s) => s !== ''); + // Drop the trailing method name; what remains is the namespace chain. + const namespace = segments.slice(0, -1); + // Only CONSTANTS name a Ruby namespace (upper-case initial). A lower-case + // segment is a receiver expression, not a namespace a caller can mention. + return namespace.filter((s) => /^[A-Z]/.test(s)); +} + +/** + * `billing/invoice_service` vs `InvoiceService`: a path segment names a + * constant when, with underscores removed, the two are equal case-insensitively. + * Zeitwerk's own inflection rule, minus acronym overrides — over-matching here + * only loses a refusal (the edge stays, labeled), which is the safe direction. + */ +function pathSegmentNamesConstant(segment: string, constant: string): boolean { + return segment.replace(/_/g, '').toLowerCase() === constant.toLowerCase(); +} + +function requireReachesConstant(targetRaw: string, constant: string): boolean { + for (const segment of moduleSegments(targetRaw)) { + if (pathSegmentNamesConstant(segment, constant)) return true; + } + return false; +} + +/** + * The declared kind of the candidate's owner, read from the candidate's own + * file. Ruby labels `class` bodies `Class` and `module` bodies `Trait` + * (`query.ts`: "module (labeled Trait for class-like registry lookup)"). + * `undefined` when the owner cannot be found — an unanswered question. + */ +function ownerLabelOf( + candidate: SymbolDefinition, + parsedFileOf: ((filePath: string) => ParsedFile | undefined) | undefined, +): string | undefined { + const ownerId = candidate.ownerId; + if (ownerId === undefined || parsedFileOf === undefined) return undefined; + const owner = parsedFileOf(candidate.filePath)?.localDefs.find((d) => d.nodeId === ownerId); + return owner?.type; +} + +/** + * Calls that rebind `self` for the duration of a block: inside + * `service.instance_eval do … end` a bare `helper()` is dispatched on + * `service`, so it legitimately reaches a class-owned method the caller file + * never names. Detected on the caller's SOURCE TEXT, not the call site — the + * site does not know which block encloses it — so any file that uses one of + * these forms keeps its class-owned guesses LABELED rather than refused. Coarse + * in the safe direction: it loses refusals in that file, never an edge. + */ +const SELF_REBINDING_CALL = + /\b(?:instance_eval|instance_exec|class_eval|class_exec|module_eval|module_exec)\b/; + +export function rubyIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly parsedFileOf?: (filePath: string) => ParsedFile | undefined; + readonly sourceTextOf?: (filePath: string) => string | undefined; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + // Top-level method — the autoload shape the fallback exists for. + if (ctx.candidate.ownerId === undefined) return true; + // A self-rebinding block anywhere in the caller makes "the class is never + // named here" no proof of impossibility (see `SELF_REBINDING_CALL`). The + // pipeline always supplies the source; a missing text is an unanswered + // question and keeps the labeled edge as well. + const text = ctx.sourceTextOf?.(ctx.callerParsed.filePath); + if (text === undefined || SELF_REBINDING_CALL.test(text)) return true; + // Only a CLASS body makes a bare cross-file call impossible without naming + // it (see the header). A module owner, or an owner we cannot type, is not a + // refusal. + if (ownerLabelOf(ctx.candidate, ctx.parsedFileOf) !== 'Class') return true; + + const constants = namespaceConstantsOf(ctx.candidate); + // Owned but with no nameable namespace (an anonymous or lower-cased owner): + // nothing to check, so do not refuse on an unanswered question. + if (constants.length === 0) return true; + + // Any inheritance or mixin surface in the caller can reach the class + // transitively (see the header) — not decidable here, so not refused. + for (const site of ctx.callerParsed.referenceSites) { + if (site.kind === 'inherits') return true; + } + for (const imp of ctx.callerParsed.parsedImports) { + if (imp.targetRaw.startsWith(HERITAGE_MARKER_PREFIX)) return true; + } + // A file that defines a module is a mixin whose methods run inside some + // including class (see the header). Ruby labels `module` bodies `Trait`. + for (const def of ctx.callerParsed.localDefs) { + if (def.type === 'Trait') return true; + // A file that REOPENS `class Invoice` names the owner even without an + // inherits site or a reference capture — the declaration itself is the + // mention. + if (def.type === 'Class') { + const declared = (def.qualifiedName ?? '') + .split(/::|\.|#/) + .map((s) => s.trim()) + .filter((s) => s !== '') + .at(-1); + if (declared !== undefined && constants.includes(declared)) return true; + } + } + + for (const imp of ctx.callerParsed.parsedImports) { + for (const constant of constants) { + if (requireReachesConstant(imp.targetRaw, constant)) return true; + // `include Billing::Invoice` arrives as an import whose LOCAL name is the + // constant rather than a path. Not every variant carries one. + if ('localName' in imp && imp.localName === constant) return true; + } + } + // A bare mention of the constant anywhere in the caller (`Billing::Invoice`, + // `Invoice.new`) is enough to make the namespace present in this file. The + // qualified spelling is matched SEGMENT-wise on `::` / `.`: `InvoiceService` + // is not a mention of `Invoice`, and a substring test made it one. + for (const site of ctx.callerParsed.referenceSites) { + for (const constant of constants) { + if (site.name === constant) return true; + if ( + site.rawQualifiedName !== undefined && + site.rawQualifiedName.split(/::|\./).some((segment) => segment === constant) + ) { + return true; + } + } + } + return false; +} diff --git a/gitnexus/src/core/ingestion/languages/ruby/resolution-config.ts b/gitnexus/src/core/ingestion/languages/ruby/resolution-config.ts new file mode 100644 index 000000000..52c213af3 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/ruby/resolution-config.ts @@ -0,0 +1,380 @@ +import { readdirSync, readFileSync, type Dirent } from 'node:fs'; +import { isAbsolute, join, relative, resolve } from 'node:path'; + +export interface RubyResolutionScope { + /** Require prefixes provided by gems whose source is outside this repository. */ + readonly externalRequirePrefixes: ReadonlySet; + /** Require prefix -> repository-relative Ruby load roots for local gems. */ + readonly localLoadRootsByPrefix: ReadonlyMap; +} + +export interface RubyResolutionConfig { + /** Manifest directory (repository-relative POSIX path) -> dependency scope. */ + readonly scopesByDirectory: ReadonlyMap; +} + +interface MutableRubyResolutionScope { + readonly externalRequirePrefixes: Set; + readonly localLoadRootsByPrefix: Map>; +} + +interface GemspecMetadata { + readonly name: string | null; + readonly requirePaths: readonly string[]; +} + +const MAX_VISITED_DIRECTORIES = 20_000; +const SKIP_DIRECTORIES = new Set([ + '.git', + '.gitnexus', + 'node_modules', + 'vendor', + 'dist', + 'build', + 'coverage', +]); + +const GEMFILE_DEPENDENCY = /^\s*gem\s*(?:\(\s*)?(['"])([A-Za-z0-9_.-]+)\1(?.*)$/; +const GEMSPEC_DEPENDENCY = + /^\s*(?:[A-Za-z_]\w*\.)?(?:add_dependency|add_runtime_dependency|add_development_dependency)\s*(?:\(\s*)?(['"])([A-Za-z0-9_.-]+)\1/; +const GEMSPEC_NAME = /^\s*(?:[A-Za-z_]\w*\.)?name\s*=\s*(['"])([A-Za-z0-9_.-]+)\1/; +const GEMSPEC_REQUIRE_PATHS = /^\s*(?:[A-Za-z_]\w*\.)?require_paths\s*=\s*\[([^\]]*)\]/; +const QUOTED_LITERAL = /(['"])([^'"]+)\1/g; +const LOCKFILE_SECTION = /^([A-Z][A-Z ]*)\s*$/; +const LOCKFILE_REMOTE = /^ {2}remote:\s*(.+?)\s*$/; +const LOCKFILE_SPECS_HEADER = /^ {2}specs:\s*$/; +const LOCKFILE_SPEC = /^ {4}([A-Za-z0-9_.-]+) \(/; + +function createMutableScope(): MutableRubyResolutionScope { + return { + externalRequirePrefixes: new Set(), + localLoadRootsByPrefix: new Map>(), + }; +} + +function normalizeRepoPath(filePath: string): string { + return filePath.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, ''); +} + +function repoRelativePath(repoPath: string, candidate: string): string | null { + const rel = relative(resolve(repoPath), resolve(candidate)); + if (rel === '') return ''; + if (rel === '..' || rel.startsWith(`..\\`) || rel.startsWith('../') || isAbsolute(rel)) { + return null; + } + return normalizeRepoPath(rel); +} + +function requirePrefixes(gemName: string, requireAs?: string): readonly string[] { + const prefixes = new Set([gemName]); + if (gemName.includes('-')) prefixes.add(gemName.replaceAll('-', '/')); + if (requireAs !== undefined && requireAs.length > 0) prefixes.add(requireAs); + return [...prefixes]; +} + +function literalOption(options: string, name: string): string | undefined { + const modern = new RegExp(`(?:^|[,\\s])${name}\\s*:\\s*(['"])([^'"]+)\\1`).exec(options); + if (modern !== null) return modern[2]; + const hashRocket = new RegExp(`(?:^|[,\\s]):${name}\\s*=>\\s*(['"])([^'"]+)\\1`).exec(options); + return hashRocket?.[2]; +} + +function hasOption(options: string, name: string): boolean { + return ( + new RegExp(`(?:^|[,\\s])${name}\\s*:`).test(options) || + new RegExp(`(?:^|[,\\s]):${name}\\s*=>`).test(options) + ); +} + +function addExternalGem( + scope: MutableRubyResolutionScope, + gemName: string, + requireAs?: string, +): void { + for (const prefix of requirePrefixes(gemName, requireAs)) { + if (!scope.localLoadRootsByPrefix.has(prefix)) scope.externalRequirePrefixes.add(prefix); + } +} + +function addLocalGem( + scope: MutableRubyResolutionScope, + gemName: string, + loadRoots: readonly string[], + requireAs?: string, +): void { + for (const prefix of requirePrefixes(gemName, requireAs)) { + scope.externalRequirePrefixes.delete(prefix); + let roots = scope.localLoadRootsByPrefix.get(prefix); + if (roots === undefined) { + roots = new Set(); + scope.localLoadRootsByPrefix.set(prefix, roots); + } + for (const loadRoot of loadRoots) roots.add(loadRoot); + } +} + +function parseGemspecMetadata(contents: string): GemspecMetadata { + let name: string | null = null; + let requirePaths: string[] | null = null; + + for (const line of contents.split(/\r?\n/)) { + const nameMatch = GEMSPEC_NAME.exec(line); + if (nameMatch !== null) name = nameMatch[2]; + + const requirePathsMatch = GEMSPEC_REQUIRE_PATHS.exec(line); + if (requirePathsMatch === null) continue; + const paths: string[] = []; + for (const match of requirePathsMatch[1].matchAll(QUOTED_LITERAL)) paths.push(match[2]); + if (paths.length > 0) requirePaths = paths; + } + + return { name, requirePaths: requirePaths ?? ['lib'] }; +} + +function loadLocalGemRoots( + repoPath: string, + gemRoot: string, + expectedGemName: string, +): readonly string[] | null { + const gemRootRelative = repoRelativePath(repoPath, gemRoot); + if (gemRootRelative === null) return null; + + let requirePaths: readonly string[] = ['lib']; + try { + const gemspecs = readdirSync(gemRoot, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith('.gemspec')) + .map((entry) => join(gemRoot, entry.name)) + .sort(); + for (const gemspec of gemspecs) { + const metadata = parseGemspecMetadata(readFileSync(gemspec, 'utf8')); + if (metadata.name === expectedGemName) { + requirePaths = metadata.requirePaths; + break; + } + } + } catch { + // An unreadable local gem stays local; the conventional lib root is a + // bounded best effort, and resolution falls through if no file matches. + } + + return requirePaths + .map((requirePath) => repoRelativePath(repoPath, resolve(gemRoot, requirePath))) + .filter((loadRoot): loadRoot is string => loadRoot !== null); +} + +function addGemfileDependencies( + contents: string, + manifestDirectory: string, + repoPath: string, + scope: MutableRubyResolutionScope, +): void { + for (const line of contents.split(/\r?\n/)) { + const match = GEMFILE_DEPENDENCY.exec(line); + if (match === null) continue; + + const gemName = match[2]; + const options = match.groups?.options ?? ''; + const requireAs = literalOption(options, 'require'); + const localPath = literalOption(options, 'path'); + + if (localPath !== undefined) { + const loadRoots = loadLocalGemRoots(repoPath, resolve(manifestDirectory, localPath), gemName); + if (loadRoots === null) addExternalGem(scope, gemName, requireAs); + else addLocalGem(scope, gemName, loadRoots, requireAs); + continue; + } + + // A dynamic path expression cannot be classified without evaluating Ruby. + // Fail open instead of deleting a potentially real in-repo import edge. + if (hasOption(options, 'path')) continue; + + addExternalGem(scope, gemName, requireAs); + } +} + +function addGemspecDependencies( + contents: string, + gemspecDirectory: string, + repoPath: string, + scope: MutableRubyResolutionScope, +): void { + const metadata = parseGemspecMetadata(contents); + if (metadata.name !== null) { + if (repoRelativePath(repoPath, gemspecDirectory) !== null) { + const loadRoots = metadata.requirePaths + .map((requirePath) => repoRelativePath(repoPath, resolve(gemspecDirectory, requirePath))) + .filter((loadRoot): loadRoot is string => loadRoot !== null); + addLocalGem(scope, metadata.name, loadRoots); + } + } + + for (const line of contents.split(/\r?\n/)) { + const match = GEMSPEC_DEPENDENCY.exec(line); + if (match !== null) addExternalGem(scope, match[2]); + } +} + +function addLockedDependencies( + contents: string, + lockfileDirectory: string, + repoPath: string, + scope: MutableRubyResolutionScope, +): void { + let section = ''; + let remote: string | null = null; + let inSpecs = false; + + for (const line of contents.split(/\r?\n/)) { + const sectionMatch = LOCKFILE_SECTION.exec(line); + if (sectionMatch !== null) { + section = sectionMatch[1]; + remote = null; + inSpecs = false; + continue; + } + + const remoteMatch = LOCKFILE_REMOTE.exec(line); + if (remoteMatch !== null) { + remote = remoteMatch[1]; + continue; + } + + if (LOCKFILE_SPECS_HEADER.test(line)) { + inSpecs = true; + continue; + } + if (!inSpecs) continue; + + const specMatch = LOCKFILE_SPEC.exec(line); + if (specMatch === null) continue; + const gemName = specMatch[1]; + + if (section === 'GEM' || section === 'GIT') { + addExternalGem(scope, gemName); + continue; + } + + if (section !== 'PATH' && section !== 'GEMSPEC') continue; + if (remote === null) continue; + + const loadRoots = loadLocalGemRoots(repoPath, resolve(lockfileDirectory, remote), gemName); + if (loadRoots === null) addExternalGem(scope, gemName); + else addLocalGem(scope, gemName, loadRoots); + } +} + +function freezeScope(scope: MutableRubyResolutionScope): RubyResolutionScope { + const localLoadRootsByPrefix = new Map(); + for (const [prefix, roots] of scope.localLoadRootsByPrefix) { + localLoadRootsByPrefix.set(prefix, [...roots].sort()); + } + return { + externalRequirePrefixes: new Set(scope.externalRequirePrefixes), + localLoadRootsByPrefix, + }; +} + +/** + * Select the nearest manifest directory that owns `fromFile`. + * + * Walking ancestors makes lookup O(path depth), independent of how many + * sibling Gemfiles a monorepo contains. + */ +export function findRubyResolutionScope( + config: RubyResolutionConfig, + fromFile: string, +): RubyResolutionScope | undefined { + const normalized = normalizeRepoPath(fromFile); + let directory = normalized.includes('/') ? normalized.slice(0, normalized.lastIndexOf('/')) : ''; + + for (;;) { + const scope = config.scopesByDirectory.get(directory); + if (scope !== undefined) return scope; + if (directory === '') return undefined; + const slash = directory.lastIndexOf('/'); + directory = slash === -1 ? '' : directory.slice(0, slash); + } +} + +/** + * Statically collect Ruby dependency sources without evaluating Gemfile or + * gemspec code. Scopes stay separate by manifest directory so sibling projects + * in a monorepo cannot suppress one another's local imports. Lockfiles + * contribute remote GEM/GIT specs and local PATH/GEMSPEC load roots only when + * an adjacent Gemfile or gemspec establishes a Bundler/RubyGems project. + * + * No declarative manifest means no safe gate, so return null and preserve the + * resolver's existing fail-open behavior for loose script directories. + */ +export function loadRubyResolutionConfig(repoPath: string): RubyResolutionConfig | null { + const pending = [repoPath]; + const manifestsByDirectory = new Map(); + const lockfilesByDirectory = new Map(); + let visitedDirectories = 0; + + while (pending.length > 0 && visitedDirectories < MAX_VISITED_DIRECTORIES) { + const directory = pending.pop(); + if (directory === undefined) break; + visitedDirectories++; + + let entries: Dirent[]; + try { + entries = readdirSync(directory, { withFileTypes: true }).sort((left, right) => + left.name.localeCompare(right.name), + ); + } catch { + continue; + } + + for (const entry of entries) { + if (entry.isDirectory() && !SKIP_DIRECTORIES.has(entry.name)) { + pending.push(join(directory, entry.name)); + } else if (entry.isFile() && (entry.name === 'Gemfile' || entry.name.endsWith('.gemspec'))) { + const manifests = manifestsByDirectory.get(directory) ?? []; + manifests.push(join(directory, entry.name)); + manifestsByDirectory.set(directory, manifests); + } else if (entry.isFile() && entry.name === 'Gemfile.lock') { + lockfilesByDirectory.set(directory, join(directory, entry.name)); + } + } + } + + if (manifestsByDirectory.size === 0) return null; + + const mutableScopes = new Map(); + for (const [directory, manifests] of [...manifestsByDirectory].sort(([left], [right]) => + left.localeCompare(right), + )) { + const directoryRelative = repoRelativePath(repoPath, directory); + if (directoryRelative === null) continue; + const scope = createMutableScope(); + mutableScopes.set(directoryRelative, scope); + + for (const manifest of manifests.sort()) { + try { + const contents = readFileSync(manifest, 'utf8'); + if (manifest.endsWith('.gemspec')) { + addGemspecDependencies(contents, directory, repoPath, scope); + } else { + addGemfileDependencies(contents, directory, repoPath, scope); + } + } catch { + // A partially readable scope remains fail-open for unknown gems. + } + } + + const lockfile = lockfilesByDirectory.get(directory); + if (lockfile === undefined) continue; + try { + addLockedDependencies(readFileSync(lockfile, 'utf8'), directory, repoPath, scope); + } catch { + // Direct declarations still provide bounded evidence when the lock is unreadable. + } + } + + const scopesByDirectory = new Map(); + for (const [directory, scope] of mutableScopes) { + scopesByDirectory.set(directory, freezeScope(scope)); + } + return { scopesByDirectory }; +} diff --git a/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts index 160bec94e..2ce047dbd 100644 --- a/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/ruby/scope-resolver.ts @@ -10,6 +10,8 @@ import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-l import type { KnowledgeGraph } from '../../../graph/types.js'; import { generateId } from '../../../../lib/utils.js'; import { decodeMarker } from '../../utils/heritage-marker.js'; +import { rubyIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; +import { loadRubyResolutionConfig } from './resolution-config.js'; /** * #1991: resolve a BARE mixin reference (`include Loggable`) to a nested module by @@ -268,6 +270,8 @@ export const rubyScopeResolver: ScopeResolver = { resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => resolveRubyImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig), + loadResolutionConfig: (repoPath) => loadRubyResolutionConfig(repoPath), + expandsWildcardTo: (targetModuleScope, parsedFiles) => expandRubyWildcardNames(targetModuleScope, parsedFiles), @@ -287,4 +291,5 @@ export const rubyScopeResolver: ScopeResolver = { fieldFallbackOnMethodLookup: true, propagatesReturnTypesAcrossImports: true, allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: rubyIsGlobalNameFallbackPlausible, }; diff --git a/gitnexus/src/core/ingestion/languages/rust/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/rust/name-fallback-visibility.ts new file mode 100644 index 000000000..1afdb900f --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/rust/name-fallback-visibility.ts @@ -0,0 +1,183 @@ +/** + * Rust's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Rust has NO ambient namespace. A bare `helper()` resolves only against names + * in scope, and for an item declared in another module the only way in is a + * `use` path (or a fully-qualified `crate::a::b::helper()` call, which is not a + * bare free call and never reaches this tier — it carries a qualified name and + * is resolved earlier by `resolveQualifiedFreeCall`). + * + * This hook checks import evidence, not Rust item visibility. A child module + * can use private ancestor items, and visibility restrictions such as + * `pub(crate)` require more context than `SymbolDefinition` carries. A matching + * import therefore keeps a labeled guess rather than proving accessibility. + * + * Module paths are matched against the candidate's FILE path (extension + * stripped, and `mod`/`lib`/`main` stem dropped, since `a/b/mod.rs` IS module + * `a::b`). `crate::` names the root; `self::` and `super::` resolve relative + * to the caller's module before comparison. + */ + +import type { ParsedFile, Scope, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { + modulePathReaches, + stripExtension, +} from '../../scope-resolution/utils/name-fallback-visibility.js'; + +/** File-stems that name their PARENT directory as the module, not themselves. */ +const RUST_DIRECTORY_MODULE_STEMS: ReadonlySet = new Set(['mod', 'lib', 'main']); + +/** Directories that hold a crate's root and contribute no module segment, so + * `src/net/http.rs` is module `net::http` and not `src::net::http`. */ +const RUST_CRATE_ROOT_DIRS: ReadonlySet = new Set(['src', 'tests', 'benches', 'examples']); + +/** Path prefixes of a `use` that name a root rather than a module segment. */ +const RUST_USE_ROOT_PREFIXES: ReadonlySet = new Set(['crate', '$crate']); + +// One scope lookup per immutable parsed-file snapshot, not per fallback site. +// Weak keys release both the snapshot and its index at the end of ingestion. +const scopeLookupByFile = new WeakMap>(); + +/** + * The module path a Rust file provides, as a `/`-joined path. + * + * Two normalizations, both needed for a file path and a `use` path to line up + * on their trailing segments: the `mod`/`lib`/`main` stem names its parent + * directory, and a leading crate-root directory (`src/`) is not a module. + */ +function rustModulePathOf(filePath: string): string { + const withoutExtension = stripExtension(filePath); + const segments = withoutExtension.split('/').filter((s) => s !== ''); + const stem = segments[segments.length - 1]; + if (stem !== undefined && RUST_DIRECTORY_MODULE_STEMS.has(stem)) segments.pop(); + // Only the crate-root folder is non-semantic. A nested `src/` is module + // `src` (`src/src/helper.rs` → `crate::src::helper`), not another root. + if (segments.length > 0 && RUST_CRATE_ROOT_DIRS.has(segments[0]!)) segments.shift(); + return segments.join('/'); +} + +/** + * Directory that owns the crate-root folder (`src`/`tests`/…). Empty when the + * file sits at the analyzed root (`src/lib.rs`) so a single-crate tree is not + * refused on a missing workspace prefix. + */ +function rustCrateRootOf(filePath: string): string { + const segments = stripExtension(filePath) + .split('/') + .filter((s) => s !== ''); + const stem = segments[segments.length - 1]; + if (stem !== undefined && RUST_DIRECTORY_MODULE_STEMS.has(stem)) segments.pop(); + const rootIdx = segments.findIndex((s) => RUST_CRATE_ROOT_DIRS.has(s)); + if (rootIdx <= 0) return ''; + return segments.slice(0, rootIdx).join('/'); +} + +/** Resolve explicit relative prefixes against the caller's module path. */ +function rustUsePathOf(targetRaw: string, callerFilePath: string): string { + const segments = targetRaw.split('::').filter((s) => s !== ''); + if (segments[0] === 'self' || segments[0] === 'super') { + const base = rustModulePathOf(callerFilePath).split('/').filter(Boolean); + if (segments[0] === 'self') segments.shift(); + while (segments[0] === 'super') { + base.pop(); + segments.shift(); + } + return [...base, ...segments].join('::'); + } + while (segments.length > 0 && RUST_USE_ROOT_PREFIXES.has(segments[0]!)) segments.shift(); + return segments.join('::'); +} + +export function rustIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly site: { + readonly name: string; + readonly rawQualifiedName?: string; + readonly inScope?: ScopeId; + }; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + // A PATH-QUALIFIED call (`User::new(...)`, `crate::a::helper()`) reaches this + // tier when the qualifier could not be followed, carrying only its tail name. + // It is not a bare-name guess: the source named the path, so the module rule + // below would refuse an edge the code spells out. + if (ctx.site.rawQualifiedName !== undefined) return true; + + const candidateModule = rustModulePathOf(ctx.candidate.filePath); + // A candidate whose file maps to no module path (a crate root reduced to '') + // is not something this rule can speak about; allow the labeled edge rather + // than refuse on an unanswered question. + if (candidateModule === '') return true; + + const candidateName = rustSimpleNameOf(ctx.candidate); + // Imports are lexical evidence, not a file-wide allowlist. Legacy/synthetic + // imports without a scope receipt retain the previous conservative behavior. + let visibleScopes: Set | undefined; + if (ctx.site.inScope !== undefined) { + let scopes = scopeLookupByFile.get(ctx.callerParsed); + if (scopes === undefined) { + scopes = new Map(ctx.callerParsed.scopes.map((scope) => [scope.id, scope])); + scopeLookupByFile.set(ctx.callerParsed, scopes); + } + visibleScopes = new Set(); + let current: ScopeId | null = ctx.site.inScope; + while (current !== null && !visibleScopes.has(current)) { + visibleScopes.add(current); + const scope = scopes.get(current); + if (scope === undefined || scope.kind === 'Namespace') break; + current = scope.parent; + } + } + for (const imp of ctx.callerParsed.parsedImports) { + if ( + imp.declaredAtScope !== undefined && + visibleScopes !== undefined && + !visibleScopes.has(imp.declaredAtScope) + ) + continue; + // `crate::` is the caller's crate. A same trailing module in another + // workspace crate is a different item and cannot authorize the guess. + if (imp.targetRaw === 'crate' || imp.targetRaw.startsWith('crate::')) { + const callerRoot = rustCrateRootOf(ctx.callerParsed.filePath); + const candidateRoot = rustCrateRootOf(ctx.candidate.filePath); + if (callerRoot !== '' && candidateRoot !== '' && callerRoot !== candidateRoot) continue; + } + const usePath = rustUsePathOf(imp.targetRaw, ctx.callerParsed.filePath); + // Only a glob introduces every bare item of a module. A named import must + // match both the candidate's original name and the call's local spelling. + if (imp.kind === 'wildcard') { + if (modulePathReaches(usePath, candidateModule)) return true; + continue; + } + if (!('localName' in imp) || imp.localName !== ctx.site.name) continue; + // Otherwise the path names ONE item inside a module (`use crate::a::other`). + // Its PARENT is the candidate's module only if that item IS the candidate: + // importing `other` says nothing about a `helper` in `a`, and the bare + // parent-path match used to accept every item of `a` on its strength. + // An alias authorizes only the local spelling checked above. + if (importedNameOf(imp) !== candidateName) continue; + if (modulePathReaches(usePath, candidateModule)) return true; + const parent = usePath.slice(0, Math.max(0, usePath.lastIndexOf('::'))); + if (parent !== '' && modulePathReaches(parent, candidateModule)) return true; + } + return false; +} + +/** The identifier a `use` binds, as written at its source (`importedName`). */ +function importedNameOf(imp: ParsedFile['parsedImports'][number]): string | undefined { + return 'importedName' in imp ? imp.importedName : undefined; +} + +/** + * The identifier a Rust declaration contributes to its module: the FIRST + * segment of `qualifiedName` after any module prefix — `User` for `User.new` + * (an associated function is reached through its type, so it is the type the + * `use` must name), the bare name for a free function. + */ +function rustSimpleNameOf(candidate: SymbolDefinition): string { + const qualified = candidate.qualifiedName ?? ''; + const segments = qualified.split(/::|\./).filter((s) => s !== ''); + return segments[0] ?? ''; +} diff --git a/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts index bea53d9fd..71fb697db 100644 --- a/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/rust/scope-resolver.ts @@ -19,6 +19,7 @@ import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-l import type { HeritageTypeArgumentSink } from '../../scope-resolution/utils/generic-instantiation.js'; import type { KnowledgeGraph } from '../../../graph/types.js'; import { generateId } from '../../../../lib/utils.js'; +import { rustIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; /** * Emit Rust `S IMPLEMENTS T` edges from `impl T for S` trait implementations. @@ -192,4 +193,6 @@ export const rustScopeResolver: ScopeResolver = { hoistTypeBindingsToModule: true, propagatesReturnTypesAcrossImports: true, allowGlobalFreeCallFallback: true, + importsBindAtLexicalScope: true, + isGlobalNameFallbackPlausible: rustIsGlobalNameFallbackPlausible, }; diff --git a/gitnexus/src/core/ingestion/languages/swift/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/languages/swift/name-fallback-visibility.ts new file mode 100644 index 000000000..916465d94 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/swift/name-fallback-visibility.ts @@ -0,0 +1,67 @@ +/** + * Swift's veto on the global-name fallback — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. + * + * Swift's default access level is `internal`: visible throughout the MODULE and + * nowhere else. Swift needs no per-file import inside a module, which is why + * the global fallback is enabled for it at all — but that whole-module + * visibility stops hard at the module boundary. A candidate in a DIFFERENT + * module is reachable only if the caller wrote `import `, and even + * then only if the declaration is `public`. + * + * A module is approximated by its source directory, the layout every Swift + * package configuration discovers: `Sources//…` (also under `Package/`). + * `src//…` is also recognized by the package configuration loader. + * Files outside these layouts have unknown module identity. + * + * The `private` / `fileprivate` half of the rule is NOT implemented, because + * neither marker is recoverable from the parse model this hook sees — + * `SymbolDefinition` carries no access level and `ParsedFile` no modifiers. + * Those candidates keep the labeled low-confidence edge, which is the + * "cannot decide, so do not refuse" direction the hook contract asks for. + */ + +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { modulePathReaches } from '../../scope-resolution/utils/name-fallback-visibility.js'; + +/** Keep aligned with loadSwiftPackageConfig: other layouts are unconfigured. */ +const SWIFT_TARGET_ROOTS: ReadonlySet = new Set(['Sources', 'src']); + +/** + * The module (target) a Swift file belongs to. + * + * `Sources/Core/User.swift` → `Core`. Arbitrary Xcode folders are not module + * boundaries; without a recognized target layout leave the answer unknown. + */ +function swiftModuleOf(filePath: string): string { + const segments = filePath.split('/').filter((s) => s !== ''); + for (let i = 0; i < segments.length - 2; i++) { + if (SWIFT_TARGET_ROOTS.has(segments[i]!)) { + // Keep the path through the target so `Sources/Core` and + // `Package/Sources/Core` stay distinct modules. `import Core` still + // reaches either via trailing-segment containment. + return segments.slice(0, i + 2).join('/'); + } + } + return ''; +} + +export function swiftIsGlobalNameFallbackPlausible(ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; +}): boolean { + if (ctx.candidate.filePath === ctx.callerParsed.filePath) return true; + + const callerModule = swiftModuleOf(ctx.callerParsed.filePath); + const candidateModule = swiftModuleOf(ctx.candidate.filePath); + // Same module: whole-module `internal` visibility, no import needed. + if (callerModule === candidateModule) return true; + // A file the layout heuristic cannot place is not something this rule can + // speak about — allow rather than refuse on an unanswered question. + if (callerModule === '' || candidateModule === '') return true; + + for (const imp of ctx.callerParsed.parsedImports) { + if (modulePathReaches(imp.targetRaw, candidateModule)) return true; + } + return false; +} diff --git a/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts index 6d71117e4..83a8e6ce7 100644 --- a/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/swift/scope-resolver.ts @@ -66,6 +66,7 @@ import { mirrorSwiftSiblingTypeBindings, type SwiftResolveContext, } from './index.js'; +import { swiftIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; const ZERO_RANGE = { startLine: 0, startCol: 0, endLine: 0, endCol: 0 } as const; @@ -136,6 +137,7 @@ const swiftScopeResolver: ScopeResolver = { // global free-call fallback (as Python/Go/Ruby/COBOL do for the same // no-`new` constructor + cross-file free-call shape). allowGlobalFreeCallFallback: true, + isGlobalNameFallbackPlausible: swiftIsGlobalNameFallbackPlausible, // Swift's call graph models `Type(...)` as a reference to the type // itself, not its `init` — both the legacy DAG and this test suite link diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 4cf95f9c9..1cdaacea2 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -385,7 +385,7 @@ const tsScopeOwnsReceivers = (match: CaptureMatch): ReadonlySet | undefi export const typescriptProvider = defineLanguage({ id: SupportedLanguages.TypeScript, - extensions: ['.ts', '.tsx'], + extensions: ['.ts', '.tsx', '.mts', '.cts'], entryPointPatterns: [/^use[A-Z]/], astFrameworkPatterns: [ { diff --git a/gitnexus/src/core/ingestion/languages/typescript/captures.ts b/gitnexus/src/core/ingestion/languages/typescript/captures.ts index 2e3a76290..955b70a25 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/captures.ts @@ -32,6 +32,7 @@ import { syntheticCapture, type SyntaxNode, } from '../../utils/ast-helpers.js'; +import { collectEsmExportEvidence, esmExportVerdict } from '../../ts-js-export-marker.js'; import { splitImportStatement } from './import-decomposer.js'; import { getTsParser, getTsScopeQuery, tsCachedTreeMatchesGrammar } from './query.js'; import { recordCacheHit, recordCacheMiss } from './cache-stats.js'; @@ -388,6 +389,8 @@ export function emitTsScopeCaptures( } const rawMatches = getTsScopeQuery(filePath).matches(tree.rootNode); + // Export evidence, read once per file (see `ts-js-export-marker.ts`). + const exportEvidence = collectEsmExportEvidence(tree.rootNode, filePath); const out: CaptureMatch[] = []; for (const m of rawMatches) { @@ -561,6 +564,13 @@ export function emitTsScopeCaptures( fnNode, deriveDefaultExportHocName(filePath), ); + // This declaration's name is synthetic, so the later query-name + // marker cannot see it. The HOC predicate already proves the export. + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + fnNode, + 'true', + ); } } @@ -660,6 +670,20 @@ export function emitTsScopeCaptures( // instead of re-parsing the receiver's source text. Self-gating: a // non-call match, an absent receiver, or a chain with no nameable base // all leave `grouped` untouched. + // `@declaration.is-exported`: a verdict for every declaration the file's + // export surface can decide (see `ts-js-export-marker.ts`); nothing where + // it cannot, because absence is the honest answer there. + const declNameNode = groupedNodes['@declaration.name']; + if (exportEvidence !== undefined && declNameNode !== undefined) { + const verdict = esmExportVerdict(declNameNode, exportEvidence); + if (verdict !== undefined) { + grouped['@declaration.is-exported'] = syntheticCapture( + '@declaration.is-exported', + declNameNode, + verdict ? 'true' : 'false', + ); + } + } synthesizeReceiverChainCapture(grouped, groupedNodes['@reference.receiver']); out.push(grouped); diff --git a/gitnexus/src/core/ingestion/languages/typescript/query.ts b/gitnexus/src/core/ingestion/languages/typescript/query.ts index babdb273b..ce1afcadd 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/query.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/query.ts @@ -1346,7 +1346,13 @@ export const TYPESCRIPT_SCOPE_QUERY = ` * (\`...\`) emit; the closing tag is intentionally NOT captured — * each JSX element should emit exactly one CALLS edge per use site. */ -const TSX_JSX_QUERY_SUFFIX = ` +/** + * Exported alongside `TYPESCRIPT_SCOPE_QUERY` so + * `value-ref-dispatchability.test.ts` checks the whole query a `.tsx` file is + * analyzed with. Checking the base alone would miss a `value-ref` rule added + * here. Not part of the provider surface — use `getTsScopeQuery`. + */ +export const TSX_JSX_QUERY_SUFFIX = ` ;; ((jsx_self_closing_element name: (identifier) @reference.name) @reference.call.free diff --git a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts index bc2af4bcb..b0d12a3e8 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts @@ -132,6 +132,15 @@ const typescriptScopeResolver: ScopeResolver = { fieldFallbackOnMethodLookup: false, propagatesReturnTypesAcrossImports: true, + // ECMAScript: `export * from './a'; export * from './b'` with a name declared + // in both exports NEITHER — the finalize pass refuses the binding instead of + // taking the first-listed source (see `exclusiveWildcardReexports`). + exclusiveWildcardReexports: true, + // `import { x }` never reaches a class member: named imports and named + // re-exports bind to module-level declarations only (a class method sharing + // a name with a top-level value must not win the callable preference). + namedImportsBindTopLevelOnly: true, + // TypeScript uses `.values()` / `.keys()` method-call syntax for collection // views -- no property-style accessors like C#'s `Dictionary.Values` -- // so `elementTypeOf` answers only the `index` route and lets the regular diff --git a/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts b/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts index 0e9871c64..30da1a677 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/tsconfig.ts @@ -278,10 +278,11 @@ async function firstReadableConfig(base: string): Promise { async function findTsconfigFiles(repoRoot: string): Promise { const found: string[] = []; const queue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + let queueHead = 0; let dirsScanned = 0; - while (queue.length > 0 && dirsScanned < SCAN_MAX_DIRS) { - const { dir, depth } = queue.shift()!; + while (queueHead < queue.length && dirsScanned < SCAN_MAX_DIRS) { + const { dir, depth } = queue[queueHead++]!; dirsScanned++; let entries: import('fs').Dirent[]; try { @@ -317,8 +318,8 @@ function parseJsonc(raw: string): Record { return JSON.parse(withoutComments) as Record; } -function repoRelative(repoRoot: string, absDir: string): string { - const rel = path.relative(repoRoot, absDir).split(path.sep).join('/'); +function repoRelative(repoRoot: string, absDir: string, pathApi: typeof path = path): string { + const rel = pathApi.relative(repoRoot, absDir).split(pathApi.sep).join('/'); return rel === '.' || rel === '' ? '' : rel; } @@ -328,11 +329,34 @@ function repoRelative(repoRoot: string, absDir: string): string { * `path.resolve` swallows the wildcard into a path segment, so it is stripped * before resolving and re-appended after — the `*` is a substitution marker, * not a directory named `*`. + * + * `pathApi` is injectable so the win32 separator branch is unit-testable from a + * POSIX runner; production callers always use the platform-bound `path`. */ -function rebaseTarget(repoRoot: string, absTarget: string): string { +function rebaseTarget(repoRoot: string, absTarget: string, pathApi: typeof path = path): string { // `/repo/src/*` must come back as `src/*`, not `src*`: stripping only the // star leaves a trailing slash that `path.relative` then eats. - const suffix = absTarget.endsWith('/*') ? '/*' : absTarget.endsWith('*') ? '*' : ''; - const base = suffix === '' ? absTarget : absTarget.slice(0, -suffix.length); - return `${repoRelative(repoRoot, base)}${suffix}`; + // + // The target arrives from `path.resolve`, so on Windows it is `C:\repo\src\*` + // and an `endsWith('/*')` check never matches. It fell through to the bare + // `*` branch, `path.relative` ate the trailing backslash, and every alias + // target came back as `src*` — which `substituteStar` turns into `srclib/x`, + // so nothing an alias reached was ever resolved on Windows. Normalise the + // separator before looking at the suffix. + const normalized = absTarget.split(pathApi.sep).join('/'); + const suffix = normalized.endsWith('/*') ? '/*' : normalized.endsWith('*') ? '*' : ''; + const base = suffix === '' ? normalized : normalized.slice(0, -suffix.length); + const prefix = repoRelative(repoRoot, base, pathApi); + // A target naming the repo ROOT (`"*": ["./*"]` under `baseUrl: "."`) leaves + // an empty prefix, and `${''}${'/*'}` is `/*`. `substituteStar` turns that + // into `/lib/date`, but `resolveFile` matches repo-relative keys and never + // strips a leading slash, so `lib/date.ts` misses and the alias goes + // external. The bare `*` is the encoding that substitutes correctly — and it + // is what the pre-#3203 Windows path emitted by accident, so this keeps the + // separator fix from narrowing what already resolved there. + if (prefix === '' && suffix === '/*') return '*'; + return `${prefix}${suffix}`; } + +/** Test seam for {@link rebaseTarget} (see `test/unit/tsconfig-rebase-target.test.ts`). */ +export const _rebaseTargetForTests = rebaseTarget; diff --git a/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts index d2f37781f..d0b0cfaed 100644 --- a/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/vue/scope-resolver.ts @@ -117,6 +117,9 @@ const vueScopeResolver: ScopeResolver = { // Vue uses explicit imports for all external symbols; no global free- // call fallback needed (would produce spurious edges for built-ins). allowGlobalFreeCallFallback: false, + // Vue SFC scripts are TypeScript/JavaScript: a named import binds a module-level + // declaration, never a class member (see the TS resolver). + namedImportsBindTopLevelOnly: true, /** * Expand the scope-resolution file universe for Vue by performing a diff --git a/gitnexus/src/core/ingestion/languages/zig/query.ts b/gitnexus/src/core/ingestion/languages/zig/query.ts index 9aa8e106a..0b4540417 100644 --- a/gitnexus/src/core/ingestion/languages/zig/query.ts +++ b/gitnexus/src/core/ingestion/languages/zig/query.ts @@ -20,7 +20,13 @@ import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.j * container and import bindings — `emitZigScopeCaptures` filters those * groups out so a name binds exactly once. */ -const ZIG_SCOPE_QUERY = ` +/** + * Exported for `value-ref-dispatchability.test.ts`, which reads every language's + * scope query to enforce the keyed/unkeyed partition that + * `callableValueReferenceBoundaries`' dispatch exclusion depends on. Not part of + * the provider surface — nothing else should import it. + */ +export const ZIG_SCOPE_QUERY = ` ;; Scopes (source_file) @scope.module (struct_declaration) @scope.class @@ -359,6 +365,60 @@ const ZIG_SCOPE_QUERY = ` "const" . (identifier) @type-binding.name (call_expression) @type-binding.type .) @type-binding.alias +;; References — VALUE positions (#3399): a callable named where a value is +;; expected rather than where a callee is. Zig's JS bridge is built entirely +;; out of this shape — +;; +;; pub const namespaceURI = bridge.accessor(Element.getNamespaceUri, null, .{}); +;; +;; — 2,047 such declarations across 257 files in lightpanda-io/browser, the +;; project's whole JS↔Zig surface, and NONE of them reached the graph: Zig +;; emitted no \`value-ref\` capture at all, so a public DOM accessor's only +;; recorded callers were the two internal ones and \`impact\` called that \`exact\`. +;; +;; These become reference-class USES edges through the existing +;; \`mapReferenceKindToEdgeType\` mapping — a registration is not an invocation +;; (Kythe \`ref\` vs \`ref/call\`; Joern \`METHOD_REF\`) — resolved by the +;; property-dispatch pass, which keeps ONLY callable targets. That callable gate +;; is what makes these deliberately broad rules safe: \`js.Bridge(Element)\` and +;; \`register(count)\` match too, and emit nothing, exactly as TypeScript's +;; \`{ port: DEFAULT_PORT }\` does. +;; +;; No \`@reference.property-key\` is attached: Zig has no object-literal key to +;; dispatch through, so these register a reference and never synthesize CALLS. +;; The terminal invoke — \`Accessor.init\` → a struct field → \`Factory.zig\`'s +;; \`inline for\`/\`@typeInfo\` → \`@call(.auto, func, args)\` — needs comptime +;; evaluation and is deliberately NOT modelled; \`impact\` reports the shortfall +;; as \`epistemic: "lower-bound"\` instead of pretending to certainty. + +;; Call ARGUMENTS. In tree-sitter-zig arguments are direct children of +;; \`call_expression\`, NOT wrapped in an \`arguments\` node (only builtins have +;; one), so the callee has to be consumed explicitly by \`function:\` — without +;; that binding the same rule also matches the callee of \`foo(bar)\` and mints a +;; USES edge duplicating the call. +(call_expression + function: (_) + (identifier) @reference.name @reference.value-ref) + +;; Qualified argument — \`bridge.accessor(Element.getNamespaceUri, …)\`. The +;; RECEIVER is captured alongside the member so the site carries the owner it +;; was written with; \`@reference.name\` stays the member, which is the name the +;; scope walk resolves. +(call_expression + function: (_) + (field_expression + object: (_) @reference.receiver + member: (identifier) @reference.name) @reference.value-ref) + +;; Const binding initialiser — \`pub const defaultHandler = onReset;\`. Both +;; anchors are load-bearing: the leading \`.\` pins the bound name to the first +;; named child (see the declaration rules above), and the trailing \`.\` keeps the +;; initializer as the LAST child, so a \`const x: T = y\` annotation shape cannot +;; put the TYPE in value position. +(variable_declaration + "const" . (identifier) + (identifier) @reference.name @reference.value-ref .) + ;; References — free calls: foo(...) (call_expression function: (identifier) @reference.name) @reference.call.free diff --git a/gitnexus/src/core/ingestion/languages/zig/range-binding.ts b/gitnexus/src/core/ingestion/languages/zig/range-binding.ts index a0186242f..408386bdf 100644 --- a/gitnexus/src/core/ingestion/languages/zig/range-binding.ts +++ b/gitnexus/src/core/ingestion/languages/zig/range-binding.ts @@ -17,6 +17,14 @@ * * Runs after `propagateImportedReturnTypes`, so return bindings hoisted from * other files are visible when a subject is a call. + * + * This hook also carries `bindZigThisAliases` (`this-alias-bindings.ts`), which + * is a different job — binding `const Self = @This();` to its container — but + * needs the same thing this pass is already paying for: the file's parsed tree, + * post-finalize. Giving it a pass of its own would re-parse every Zig file in + * the repo whenever the tree cache is cold. That is the whole reason it is + * here; the two do not otherwise interact, for the reason recorded at the call + * site. */ import type { ParsedFile, Scope, ScopeId, TypeRef } from 'gitnexus-shared'; @@ -32,6 +40,7 @@ import { isClassLike, } from '../../scope-resolution/scope/walkers.js'; import { isZigKeywordDeclaration, zigUnwrapValue } from './captures.js'; +import { bindZigThisAliases } from './this-alias-bindings.js'; import { normalizeZigTypeName } from './interpret.js'; type ZigTree = ReturnType['parse']>; @@ -89,6 +98,23 @@ export function populateZigRangeBindings( } const scopes = parsed.scopes; if (scopes.length === 0) continue; + + // `const Self = @This();` — bind the alias to its container. It sits in this + // loop for ONE reason: the tree. A pass of its own would re-parse every Zig + // file in the repo whenever the tree cache is cold. + // + // Its position relative to the payload walk below is NOT load-bearing, and + // saying otherwise would be wrong in a checkable way: the payload walk types + // a subject through `findReceiverTypeBinding`, which reads `typeBindings` + // and the namespace/workspace type channels — never `bindingAugmentations`, + // where this writes. Measured on `for (Self.items) |it|`: `it` is bound + // neither before nor after. Nor is that a gap this should close by also + // writing a typeBinding — NO container name has one, the file stem included, + // so a payload subject written `Type.member` resolves for no spelling at + // all, and giving the alias an entry would make it behave unlike the very + // container it names. + bindZigThisAliases(parsed, tree.rootNode, indexes); + const resolver = new ZigSubjectTypeResolver(scopes, indexes, classScopeByDefId); // Pre-order: an outer payload is bound before an inner construct reads it diff --git a/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts index 86afcfbf2..675f6e1a8 100644 --- a/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/zig/scope-resolver.ts @@ -5,8 +5,9 @@ * Thin wiring: Zig has no inheritance (default MRO linearization over an * empty heritage set), no `super`, and is statically typed (field-fallback * heuristic off per the contract guidance). Import resolution reuses the - * same `resolveZigImportInternal` the legacy import-resolver config wraps, - * with `build.zig.zon` `.path` deps threaded through `loadResolutionConfig`. + * same `resolveZigImportInternal` the legacy import-resolver config wraps, with + * the repo's Zig build packages threaded through `loadResolutionConfig` and the + * one governing each file selected per-file by `zigPackageFor`. */ import type { ParsedFile } from 'gitnexus-shared'; @@ -14,7 +15,11 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; -import { loadZigBuildConfig, type ZigBuildZonConfig } from '../../language-config.js'; +import { + loadZigWorkspaceIndex, + zigPackageFor, + type ZigWorkspaceIndex, +} from '../../language-config.js'; import { resolveZigImportInternal } from '../../import-resolvers/zig.js'; import { zigProvider } from '../zig.js'; import { expandZigWildcardNames, zigArityCompatibility, zigMergeBindings } from './index.js'; @@ -46,14 +51,23 @@ export const zigScopeResolver: ScopeResolver = { // import; a one-hop split at the last dot resolved none of them. resolveNamespaceChains: true, - loadResolutionConfig: (repoPath: string) => loadZigBuildConfig(repoPath), + // The whole workspace, not just the root package. A Zig module's import table + // is declared by the `build.zig` of the package the file belongs to, so a repo + // laying its packages out as `packages//build.zig` has as many import + // tables as packages — and reading only the root one leaves every bare + // `@import("")` in such a repo unresolved. Same shape, same reason, as + // `loadTsconfigIndex` for a TypeScript monorepo. + loadResolutionConfig: (repoPath: string) => loadZigWorkspaceIndex(repoPath), resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => resolveZigImportInternal( fromFile, targetRaw, allFilePaths, - (resolutionConfig as ZigBuildZonConfig | null | undefined) ?? null, + // The package governing THIS file — `zigPackageFor` is the `tsconfigFor` + // analogue. `resolveZigImportInternal` is handed one package's config and + // is unchanged by this: which config it receives is the only difference. + zigPackageFor(resolutionConfig as ZigWorkspaceIndex | null | undefined, fromFile), ), // `pub usingnamespace @import("x.zig");` — target decls become local decls. diff --git a/gitnexus/src/core/ingestion/languages/zig/this-alias-bindings.ts b/gitnexus/src/core/ingestion/languages/zig/this-alias-bindings.ts new file mode 100644 index 000000000..8a124bf3e --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/zig/this-alias-bindings.ts @@ -0,0 +1,191 @@ +/** + * Bind a Zig `const X = @This();` alias to the container it names (#3219 + * review round 8). + * + * `@This()` IS the enclosing container, and Zig code says so constantly: the + * single most common spelling is `const Self = @This();`, followed by + * `Alias.member` wherever the explicit form is wanted (`Self.width(self)`, a + * registration `bridge.accessor(Self.width, …)`, a nested type `Self.Node`). + * A compiler resolves `Self` and the container's own name to the same type. + * This index did not, for one specific reason: + * + * - A container-level alias (`const Self = @This();` inside `struct {…}`) + * mints a `Variable` beside the `Struct`, so `Self` binds to a VALUE. Every + * class-like lookup filters on `isClassLike` and walks straight past it. + * - A FILE-level alias in a file-as-struct mints nothing at all — + * `isZigFileThisAlias` suppresses the Const deliberately, so it cannot + * shadow the type for `x: *Page`. The name is then bound to nothing, while + * the container itself is bound under the FILE STEM. + * + * So `Self.width` resolved only when the alias happened to be spelled like its + * container. Measured on the three Zig corpora on hand: 73 of ghostty's 185 + * `@This()` files, 93 of tigerbeetle's 94 and 8 of mach's 42 spell it + * differently, carrying 302 `Alias.member` references between them, 96 of those + * calls. Those were not wrong edges — they were no edges, and a caller list + * missing them is exactly the false confidence #3399 is about. + * + * WHAT THIS ADDS, precisely: one binding of the alias NAME to the container's + * own definition, appended to `indexes.bindingAugmentations` (the sanctioned + * post-finalize channel, invariant I8 — `indexes.bindings` is frozen). The + * augmentation channel is consulted by `lookupBindingsAt` only AFTER a scope's + * own `Scope.bindings`, so this can never outrank a real local declaration: the + * `Variable` the container-level alias already mints still answers first for + * anything that wants a value, and the container answers for anything that + * wants a type. Nothing is replaced and nothing is removed. + * + * WHAT IT DELIBERATELY DOES NOT DO. Only aliases declared DIRECTLY in a + * container body or at file level are bound, which is the same set + * `collectZigThisAliases` recognizes for the type-rewrite path. A function-local + * `const Self = @This();` also names the enclosing container, but it belongs in + * that function's scope, not the container's, and binding it here would make it + * visible to sibling functions that never declared it. + */ + +import type { ParsedFile, Scope, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import type { BindingRef } from 'gitnexus-shared'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { isClassLike } from '../../scope-resolution/scope/walkers.js'; +import { isZigKeywordDeclaration, ZIG_CONTAINER_TYPES } from './captures.js'; + +/** `const X = @This();` — the declaration shape, wherever it sits. */ +function thisAliasName(node: SyntaxNode): string | undefined { + if (node.type !== 'variable_declaration' || !isZigKeywordDeclaration(node)) return undefined; + const named = node.namedChildren.filter((c): c is SyntaxNode => c !== null); + if (named.length !== 2 || named[0]!.type !== 'identifier') return undefined; + const value = named[1]!; + if (value.type !== 'builtin_function' || value.namedChild(0)?.text !== '@This') return undefined; + return named[0]!.text; +} + +/** The class-like definition a Class scope is the body of. */ +function containerDefOf(scope: Scope): SymbolDefinition | undefined { + return scope.ownedDefs.find((d) => isClassLike(d.type)); +} + +/** + * The Class scope of the FILE-AS-STRUCT, if this file is one. + * + * Identified by range rather than by name: it is the only Class scope that both + * hangs directly off the module scope AND spans the same lines as it. A + * namespace-only file with a top-level `pub const Counter = struct {…}` also + * produces a Class scope under the module scope, and it also carries the file + * stem when the file is `Counter.zig` — matching on the name would bind that + * file's `@This()` alias to `Counter`, which is not what `@This()` means there. + */ +function fileStructScope(scopes: readonly Scope[], moduleScope: Scope): Scope | undefined { + return scopes.find( + (s) => + s.kind === 'Class' && + s.parent === moduleScope.id && + s.range.startLine === moduleScope.range.startLine && + s.range.endLine === moduleScope.range.endLine, + ); +} + +/** + * Innermost Class scope containing `node`. + * + * Compared on (line, column) rather than line alone, because a Zig container + * can nest inside another on ONE line — `const A = struct { const S = @This(); + * const B = struct { const T = @This(); }; };` gives both scopes the same start + * line, and picking between them by line would be a coin toss that binds an + * alias to the wrong container. A wrong edge is the one outcome this whole + * change set treats as worse than no edge. + */ +function enclosingClassScope(scopes: readonly Scope[], node: SyntaxNode): Scope | undefined { + const line = node.startPosition.row + 1; + const column = node.startPosition.column; + const startsAtOrBefore = (l: number, c: number): boolean => + l < line || (l === line && c <= column); + const endsAtOrAfter = (l: number, c: number): boolean => l > line || (l === line && c >= column); + let best: Scope | undefined; + for (const s of scopes) { + if (s.kind !== 'Class') continue; + if (!startsAtOrBefore(s.range.startLine, s.range.startCol)) continue; + if (!endsAtOrAfter(s.range.endLine, s.range.endCol)) continue; + if ( + best === undefined || + s.range.startLine > best.range.startLine || + (s.range.startLine === best.range.startLine && s.range.startCol > best.range.startCol) + ) { + best = s; + } + } + return best; +} + +/** + * Append `alias -> def` at `scopeId`, skipping the append when THIS def is + * already bound there. Idempotence, not precedence: a duplicate would be + * harmless for lookup (`lookupBindingsAt` dedupes by `def.nodeId`) but would + * make the channel's contents depend on how many times the hook ran. Other + * bindings under the same name are left alone — the channel is append-only per + * I8, and the whole point of the augmentation tier is that a scope's own + * `Scope.bindings` are consulted first and still win. + */ +function appendBinding( + indexes: ScopeResolutionIndexes, + scopeId: ScopeId, + alias: string, + def: SymbolDefinition, +): void { + const channel = indexes.bindingAugmentations as Map>; + let byName = channel.get(scopeId); + if (byName === undefined) { + byName = new Map(); + channel.set(scopeId, byName); + } + const bucket = byName.get(alias); + if (bucket === undefined) { + byName.set(alias, [{ def, origin: 'local' }]); + return; + } + if (bucket.some((ref) => ref.def.nodeId === def.nodeId)) return; + bucket.push({ def, origin: 'local' }); +} + +/** + * Bind every `@This()` alias declared in `parsed` at container or file level. + * + * Called per file from `populateZigRangeBindings`, which already holds the + * parsed tree — a second pass over `parsedFiles` would re-parse every file when + * the tree cache is cold. That is the only thing the two share. Ordering + * against the payload bindings that follow does not matter, twice over: the + * payload walk types its subjects through `findReceiverTypeBinding`, which + * never reads the channel written here, and a `@This()` alias is a + * container-private name no other file can import, so nothing outside this file + * reads it either. + */ +export function bindZigThisAliases( + parsed: ParsedFile, + root: SyntaxNode, + indexes: ScopeResolutionIndexes, +): void { + const moduleScope = parsed.scopes.find((s) => s.kind === 'Module'); + if (moduleScope === undefined) return; + const fileStruct = fileStructScope(parsed.scopes, moduleScope); + + const visit = (node: SyntaxNode): void => { + const alias = thisAliasName(node); + if (alias !== undefined) { + const parent = node.parent; + if (parent?.type === 'source_file') { + // The file-as-struct's own name. Bound at the MODULE scope, which is + // where the container is already bound under the file stem, so the + // alias and the stem are visible to exactly the same sites. + const def = fileStruct === undefined ? undefined : containerDefOf(fileStruct); + if (def !== undefined) appendBinding(indexes, moduleScope.id, alias, def); + } else if (parent !== null && ZIG_CONTAINER_TYPES.has(parent.type)) { + const scope = enclosingClassScope(parsed.scopes, node); + const def = scope === undefined ? undefined : containerDefOf(scope); + if (scope !== undefined && def !== undefined) appendBinding(indexes, scope.id, alias, def); + } + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child !== null) visit(child); + } + }; + visit(root); +} diff --git a/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts b/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts index 040c058be..11c13b259 100644 --- a/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts +++ b/gitnexus/src/core/ingestion/languages/zig/workspace-static-gating.ts @@ -1,6 +1,6 @@ import type { ParsedFile, ReferenceSite } from 'gitnexus-shared'; import { getTreeSitterBufferSize } from '../../constants.js'; -import type { ZigBuildZonConfig } from '../../language-config.js'; +import { zigPackageFor, type ZigWorkspaceIndex } from '../../language-config.js'; import { resolveZigImportInternal } from '../../import-resolvers/zig.js'; import { buildZigBoolConstMap, @@ -51,7 +51,11 @@ export function populateZigWorkspaceStaticGating( tree, parsed.filePath, knownPaths, - ctx.resolutionConfig as ZigBuildZonConfig | null | undefined, + // `resolutionConfig` is the whole Zig workspace; the package governing + // THIS file is what `resolveZigImportInternal` takes. Selected here rather + // than hoisted out of the loop because the answer is per-file: two files + // of this pass can belong to different packages. + zigPackageFor(ctx.resolutionConfig as ZigWorkspaceIndex | null | undefined, parsed.filePath), ); if (aliases.size === 0) continue; const ranges = collectZigStaticGatedRanges( @@ -76,7 +80,7 @@ function collectImportAliases( tree: ZigTree, fromFile: string, knownPaths: ReadonlySet, - resolutionConfig?: ZigBuildZonConfig | null, + packageConfig: ReturnType, ): ZigImportAliasMap { const candidates = new Map(); const declarationCounts = new Map(); @@ -91,7 +95,7 @@ function collectImportAliases( const raw = builtin?.descendantsOfType('string').at(0)?.text; if (raw === undefined) continue; const specifier = raw.replace(/^['"]|['"]$/g, ''); - const target = resolveZigImportInternal(fromFile, specifier, knownPaths, resolutionConfig); + const target = resolveZigImportInternal(fromFile, specifier, knownPaths, packageConfig); if (target !== null) candidates.set(binding, target); } diff --git a/gitnexus/src/core/ingestion/model/registration-table.ts b/gitnexus/src/core/ingestion/model/registration-table.ts index e9e85eeb0..718079a95 100644 --- a/gitnexus/src/core/ingestion/model/registration-table.ts +++ b/gitnexus/src/core/ingestion/model/registration-table.ts @@ -15,11 +15,11 @@ * └── symbols (SymbolTable) ← owns fileIndex + callableByName, * calls dispatch() in add() * - * ## Behavior groups (5 hooks, 13 table entries) + * ## Behavior groups (4 hooks, 12 table entries) * * | Group | NodeLabel values | Hook | Skip callable? | * |---------------|---------------------------------------------------|--------------|----------------| - * | class-like | Class, Struct, Interface, Enum, Record, Trait | classLikeHook | no | + * | class-like | Class, Protocol, Category, Struct, Interface, Enum, Record, Trait | classLikeHook | no | * | method-like | Method, Constructor | methodHook | no | * | property | Property | propertyHook | YES | * | impl-block | Impl | implHook | no | @@ -95,7 +95,7 @@ export interface RegistrationTableDeps { * registry (if any) receives the symbol write during `SymbolTable.add()`: * * - `dispatch` — owner-scoped registry write via the dispatch table - * (Class/Struct/Interface/Enum/Record/Trait → types.registerClass, + * (Class/Protocol/Category/Struct/Interface/Enum/Record/Trait → types.registerClass, * Method/Constructor → methods.register, * Property → fields.register, * Impl → types.registerImpl) @@ -143,6 +143,8 @@ export type LabelBehavior = 'dispatch' | 'callable-only' | 'inert'; const LABEL_BEHAVIOR = { // dispatch — owner-scoped registry writes Class: 'dispatch', + Protocol: 'dispatch', + Category: 'dispatch', Struct: 'dispatch', Interface: 'dispatch', Enum: 'dispatch', @@ -271,8 +273,8 @@ export const createRegistrationTable = ( ): Map => { const { types, methods, fields } = deps; - // Hook 1: class-like — Class, Struct, Interface, Enum, Record, Trait. - // Shared reference — six table entries point at this one closure. + // Hook 1: class-like — Class, Protocol, Category, Struct, Interface, Enum, Record, Trait. + // Shared reference — eight table entries point at this one closure. const classLikeHook: RegistrationHook = (name, def) => { const qualifiedKey = def.qualifiedName ?? name; types.registerClass(name, qualifiedKey, def); @@ -311,11 +313,13 @@ export const createRegistrationTable = ( // classified as 'dispatch'. This is the compile-time twin of the // runtime taxonomy — no drift possible. const dispatchByLabel = { - // class-like — six labels share the single `classLikeHook` closure, + // class-like — eight labels share the single `classLikeHook` closure, // kept in lockstep with `CLASS_TYPES_TUPLE` via the // `Record` cross-invariant on // `LABEL_BEHAVIOR`. Class: classLikeHook, + Protocol: classLikeHook, + Category: classLikeHook, Struct: classLikeHook, Interface: classLikeHook, Enum: classLikeHook, diff --git a/gitnexus/src/core/ingestion/model/symbol-table.ts b/gitnexus/src/core/ingestion/model/symbol-table.ts index d7330a093..58165bacf 100644 --- a/gitnexus/src/core/ingestion/model/symbol-table.ts +++ b/gitnexus/src/core/ingestion/model/symbol-table.ts @@ -56,6 +56,8 @@ import type { NodeLabel, ParameterTypeClass, SymbolDefinition } from 'gitnexus-s */ export const CLASS_TYPES_TUPLE = [ 'Class', + 'Protocol', + 'Category', 'Struct', 'Interface', 'Enum', diff --git a/gitnexus/src/core/ingestion/model/type-registry.ts b/gitnexus/src/core/ingestion/model/type-registry.ts index 135e61dfd..c23a36046 100644 --- a/gitnexus/src/core/ingestion/model/type-registry.ts +++ b/gitnexus/src/core/ingestion/model/type-registry.ts @@ -16,7 +16,7 @@ const EMPTY: readonly SymbolDefinition[] = Object.freeze([]); export interface TypeRegistry { /** - * Look up class-like definitions (Class, Struct, Interface, Enum, Record, Trait) + * Look up class-like definitions (Class, Protocol, Category, Struct, Interface, Enum, Record, Trait) * by simple name. Returns all matching definitions across files * (e.g. partial classes). Returned array is a view into the live * internal index — do not mutate. diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 4798c1f9f..8aafd23a7 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -1,7 +1,7 @@ import type { NodeLabel } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; import type { SymbolTableWriter } from './model/index.js'; -import { getLanguageFromFilename } from 'gitnexus-shared'; +import { getLanguageForFileContent } from './languages/index.js'; import { accumulateExportedTypesFromParsedNode, type ExportedTypeMap } from './call-processor.js'; @@ -260,7 +260,7 @@ export const dispatchChunkParseRound = async ( const dispatchGroups: DispatchGroup[] = groups.map((group) => { const items: ParseWorkerInput[] = []; for (const file of group.items) { - const lang = getLanguageFromFilename(file.path); + const lang = getLanguageForFileContent(file.path, file.content); if (lang) items.push({ path: file.path, content: file.content }); } return { items, chunkHash: group.chunkHash }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 18542637d..443c13289 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -61,7 +61,13 @@ import { createParserForLanguage, } from '../../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; -import { getProvider, getProviderForFile, providers } from '../languages/index.js'; +import { + getProvider, + getProviderForFile, + needsContentLanguageClassification, + providers, +} from '../languages/index.js'; +import { classifyContentLanguages } from '../content-language-classification.js'; import { SCOPE_RESOLVERS } from '../scope-resolution/pipeline/registry.js'; import { DATA_ROUTE_TABLE_SOURCE } from '../route-extractors/data-route-table.js'; import type Parser from 'tree-sitter'; @@ -497,6 +503,14 @@ export async function runChunkedParseAndResolve( * cache analyze run can skip the dominant `extractParsedFile` cost * (otherwise ~58s on a 1000-file repo). */ parsedFiles: import('gitnexus-shared').ParsedFile[]; + /** + * Content-derived language decisions for extension-ambiguous files. + * + * The source text used to classify these paths is released before parsing + * begins. Scope resolution consumes this snapshot instead of reading the + * same files again just to repeat classification. + */ + contentLanguageByPath: ReadonlyMap; /** Repo-wide harvested constants, already prepared per provider. See * `ParseOutput.moduleConstants` for why this leaves the parse phase. */ moduleConstants: ReadonlyMap; @@ -507,15 +521,27 @@ export async function runChunkedParseAndResolve( const model = createSemanticModel(); const symbolTable = model.symbols; + const contentClassifiedPaths = scannedFiles + .map((file) => file.path) + .filter(needsContentLanguageClassification); + const contentLanguageByPath = + contentClassifiedPaths.length > 0 + ? await classifyContentLanguages(repoPath, contentClassifiedPaths) + : new Map(); + const languageForScannedFile = (file: (typeof scannedFiles)[number]) => { + return contentLanguageByPath.has(file.path) + ? (contentLanguageByPath.get(file.path) ?? null) + : getLanguageFromFilename(file.path); + }; const parseableScanned = scannedFiles.filter((f) => { - const lang = getLanguageFromFilename(f.path); + const lang = languageForScannedFile(f); return lang && isLanguageAvailable(lang); }); // Warn about files skipped due to unavailable parsers const skippedByLang = new Map(); for (const f of scannedFiles) { - const lang = getLanguageFromFilename(f.path); + const lang = languageForScannedFile(f); const provider = lang === null ? undefined : getProvider(lang); if (lang && provider?.parseStrategy !== 'standalone' && !isLanguageAvailable(lang)) { skippedByLang.set(lang, (skippedByLang.get(lang) || 0) + 1); @@ -1879,6 +1905,7 @@ export async function runChunkedParseAndResolve( // cache: when the file's ParsedFile is here, scope-resolution skips its own // `extractParsedFile` call. parsedFiles: allParsedFiles, + contentLanguageByPath, // Repo-wide, file-path-keyed constants, already through each provider's // `prepareRouteConstants` hook. Empty when no provider harvests constants // for the languages in this repo. diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index b47ceafdf..503f3be75 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -20,7 +20,7 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; import { getPhaseOutput } from './types.js'; import type { StructureOutput } from './structure.js'; import type { BindingAccumulator } from '../binding-accumulator.js'; -import type { ParsedFile } from 'gitnexus-shared'; +import type { ParsedFile, SupportedLanguages } from 'gitnexus-shared'; import type { ExtractedFetchCall, ExtractedRoute, @@ -83,6 +83,12 @@ export interface ParseOutput { * costing ~58s on a 1000-file repo). */ readonly parsedFiles: readonly ParsedFile[]; + /** + * Content-derived language decisions for files such as `.h` whose extension + * is ambiguous. Scope resolution reuses this result rather than rereading + * source solely to classify it. + */ + readonly contentLanguageByPath: ReadonlyMap; /** * Repo-wide string constants harvested by the providers that declare * `extractModuleConstants`, keyed by file path and already through each diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 8ee94e265..d81b7116e 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -16,6 +16,8 @@ */ import { createKnowledgeGraph } from '../graph/graph.js'; +import type { KnowledgeGraph } from '../graph/types.js'; +import { GLOBAL_NAME_FALLBACK_REASON } from '../graph/edge-reasons.js'; import { GraphEmitSink, type GraphEmitManifest } from '../lbug/graph-emit-sink.js'; import { type PipelineProgress } from 'gitnexus-shared'; import { PipelineResult } from '../../types/pipeline.js'; @@ -410,6 +412,11 @@ export const runPipelineFromRepo = async ( graphEmitSink?.close(); } + // Resolved-call index for the name-fallback census: read through the SINK, + // whose field-wise scan includes every streamed edge, not through `graph`, + // which under streaming holds none of them. + const resolvedCalleeNamesByCaller = collectResolvedCalleeNames(graphEmitSink ?? graph, graph); + // Extract final results for the PipelineResult contract const { totalFiles, @@ -473,6 +480,7 @@ export const runPipelineFromRepo = async ( communityResult, processResult, resolutionOutcomes, + resolvedCalleeNamesByCaller, undecidedSatisfaction, usedWorkerPool, reparsedFileCount, @@ -518,3 +526,29 @@ export const runPipelineFromRepo = async ( return result; }; + +/** + * Caller node id → the simple names of every callee it has a CALLS edge to. + * + * `edges` may be the streaming sink or the raw graph; `nodes` is always the raw + * graph, which holds every node in both modes (only relationships stream). One + * O(E) field-wise pass, allocation-free per edge except for the per-caller set. + */ +export function collectResolvedCalleeNames( + edges: Pick, + nodes: Pick, +): ReadonlyMap> { + const out = new Map>(); + edges.forEachRelationshipFields((sourceId, targetId, type, _confidence, reason) => { + if (type !== 'CALLS' || reason === GLOBAL_NAME_FALLBACK_REASON) return; + const name = nodes.getNode(targetId)?.properties.name; + if (typeof name !== 'string' || name === '') return; + let names = out.get(sourceId); + if (names === undefined) { + names = new Set(); + out.set(sourceId, names); + } + names.add(name); + }); + return out; +} diff --git a/gitnexus/src/core/ingestion/process-processor.ts b/gitnexus/src/core/ingestion/process-processor.ts index 67804d268..6a7cc4a10 100644 --- a/gitnexus/src/core/ingestion/process-processor.ts +++ b/gitnexus/src/core/ingestion/process-processor.ts @@ -10,8 +10,9 @@ * Processes help agents understand how features work through the codebase. */ -import type { GraphNode, NodeLabel } from 'gitnexus-shared'; +import type { GraphNode, NodeLabel, RelationshipType } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; +import { isHeuristicEdgeReason } from '../graph/edge-reasons.js'; import { CommunityMembership } from './community-processor.js'; import { calculateEntryPointScore, isTestFile } from './entry-point-scoring.js'; import { SupportedLanguages } from 'gitnexus-shared'; @@ -436,12 +437,28 @@ type AdjacencyList = Map; */ const MIN_TRACE_CONFIDENCE = 0.5; +/** + * True when an edge may seed or extend a traced flow. + * + * The confidence floor alone is not sufficient: the global-name fallback emits + * at exactly `MIN_TRACE_CONFIDENCE`, so a `<` comparison admits every one of + * its guesses. A flow assembled from name guesses reads as a real execution + * path through code that may never call each other, so the reason is checked + * too — see `graph/edge-reasons.ts`. + */ +const isTraceableCallsEdge = ( + type: RelationshipType, + confidence: number, + reason: string, +): boolean => + type === 'CALLS' && confidence >= MIN_TRACE_CONFIDENCE && !isHeuristicEdgeReason(reason); + const buildCallsGraph = (graph: KnowledgeGraph): AdjacencyList => { const adj = new Map(); - // Field-wise scan (#2680) — whole-graph walk, four fields, no object needed. - graph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { - if (type !== 'CALLS' || confidence < MIN_TRACE_CONFIDENCE) return; + // Field-wise scan (#2680) — whole-graph walk, five fields, no object needed. + graph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { + if (!isTraceableCallsEdge(type, confidence, reason)) return; const existing = adj.get(sourceId); if (existing === undefined) adj.set(sourceId, [targetId]); else existing.push(targetId); @@ -453,8 +470,8 @@ const buildCallsGraph = (graph: KnowledgeGraph): AdjacencyList => { const buildReverseCallsGraph = (graph: KnowledgeGraph): AdjacencyList => { const adj = new Map(); - graph.forEachRelationshipFields((sourceId, targetId, type, confidence) => { - if (type !== 'CALLS' || confidence < MIN_TRACE_CONFIDENCE) return; + graph.forEachRelationshipFields((sourceId, targetId, type, confidence, reason) => { + if (!isTraceableCallsEdge(type, confidence, reason)) return; const existing = adj.get(targetId); if (existing === undefined) adj.set(targetId, [sourceId]); else existing.push(sourceId); diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index 31f66b896..898282b5e 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -707,6 +707,10 @@ function buildDefFromDeclarationMatch( const isExplicit = parseBooleanCapture(match['@declaration.is-explicit']); const isDeleted = parseBooleanCapture(match['@declaration.is-deleted']); const isSynthetic = parseBooleanCapture(match['@declaration.is-synthetic']); + // Tri-state on purpose: only a producer that saw the file's export surface + // emits the marker, and both `true` and `false` are verdicts (see + // `SymbolDefinition.isExported`). Absent stays absent. + const isExported = parseBooleanCapture(match['@declaration.is-exported']); return { nodeId: makeDefId(filePath, anchor.range, type, nameCap.text), @@ -725,6 +729,7 @@ function buildDefFromDeclarationMatch( ...(isExplicit === true ? { isExplicit: true } : {}), ...(isDeleted === true ? { isDeleted: true } : {}), ...(isSynthetic === true ? { isSynthetic: true } : {}), + ...(isExported !== undefined ? { isExported } : {}), }; } @@ -847,6 +852,10 @@ function normalizeNodeLabel(kindStr: string): SymbolDefinition['type'] | undefin switch (kindStr.toLowerCase()) { case 'class': return 'Class'; + case 'protocol': + return 'Protocol'; + case 'category': + return 'Category'; case 'interface': return 'Interface'; case 'enum': @@ -1033,8 +1042,7 @@ function pass3CollectImports( if (provider.interpretImport === undefined) return; // Hoisted: the capability is a property of the language, identical for every // match in the file. A provider that declares its imports do not execute - // where they are written (C/C++ `#include`, Rust `use`, COBOL `COPY`) skips - // the position walk entirely — position cannot defer something that never + // where they are written skips the execution-deferral walk — position cannot defer something that never // runs, and marking one deferred would hide a real cycle. Absent reads as // `true`, so an undeclared provider is unchanged. See // `LanguageProvider.importsExecuteWhereWritten`. @@ -1045,14 +1053,22 @@ function pass3CollectImports( const parsed = provider.interpretImport(match); if (parsed === null) continue; // The statement's own position, resolved to the innermost scope holding - // it. An unlocatable anchor leaves the import unmarked, which reads as + // it. Provenance is retained independently of execution timing. An + // unlocatable anchor leaves the import unmarked, which reads as // "runs at initialization" — the fail-safe direction, since it can only // make `check --cycles` over-report. - const inScopeId = positionCanDefer - ? positionIndex.atPosition(filePath, anchor.range.startLine, anchor.range.startCol) - : undefined; - const deferred = inScopeId !== undefined && runsOnlyWhenCalled(scopeTree, inScopeId); - parsedImports.push(deferred ? { ...parsed, runsOnlyWhenCalled: true } : parsed); + const inScopeId = positionIndex.atPosition( + filePath, + anchor.range.startLine, + anchor.range.startCol, + ); + const deferred = + positionCanDefer && inScopeId !== undefined && runsOnlyWhenCalled(scopeTree, inScopeId); + parsedImports.push({ + ...parsed, + ...(inScopeId !== undefined ? { declaredAtScope: inScopeId } : {}), + ...(deferred ? { runsOnlyWhenCalled: true } : {}), + }); } } diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index e364b0227..d6da6d4d9 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -82,13 +82,14 @@ * `handledSites` IFF a `tryEmitEdge` call returned `true` for it. * Sites a pass touched but couldn't resolve do NOT get marked — * they still get a chance from the shared resolver. Exception: - * the free-call fallback marks the site unconditionally after - * attempting emission (even on dedup-collapse), because the - * per-(caller, target) collapse semantics require multiple call - * sites in the same caller body not produce multiple edges. - * `preEmitInheritanceEdges` also pre-marks every `inherits` site so - * the generic bridge cannot remap class heritage into method-owned - * EXTENDS edges via `resolveCallerGraphId`. + * the free-call fallback marks the site after it decides the site, + * including when it emits no edge — dedup-collapse, a visibility + * veto (`fallback-refused`), or a selected callable that is deleted + * / invisible. The later generic pass must not recreate an edge + * that pass just refused. `preEmitInheritanceEdges` also pre-marks + * every `inherits` site so the generic bridge cannot remap class + * heritage into method-owned EXTENDS edges via + * `resolveCallerGraphId`. * * - **I3 — `propagateImportedReturnTypes` mutation timing + ordering.** * The pass mutates `Scope.typeBindings` (a plain `new Map(...)` from @@ -248,8 +249,8 @@ * ## Semantic-model source of truth * * `ParsedFile` (from `gitnexus-shared/src/scope-resolution/parsed-file.ts`) - * is the single semantic model consumed by both the legacy DAG and the - * scope-resolution pipeline. Scope-resolution passes MUST NOT build a + * is the single semantic model consumed by the scope-resolution pipeline. + * Scope-resolution passes MUST NOT build a * parallel parse representation; if a pass needs AST-level facts that * `ParsedFile` doesn't expose, it should reuse the orchestrator's * `treeCache` (see `RunScopeResolutionInput.treeCache`) rather than @@ -257,23 +258,20 @@ * * ## Same-graph guarantee * - * Edges emitted by `runScopeResolution` and edges emitted by the legacy - * DAG are indistinguishable to downstream consumers: + * All language resolvers emit edges through `runScopeResolution` using the + * shared graph contract: * - Node identity: same `generateId(...)` helper, same qualified-name * keyspace, same File/Folder/Method/Class node labels. * - Edge vocabulary: `'import-resolved' | 'global' | 'local-call' | - * 'same-file' | 'interface-dispatch' | 'read' | 'write'` — both - * paths emit the same reasons (see - * `gitnexus/src/core/ingestion/call-processor.ts` for the legacy - * emitter and `passes/receiver-bound-calls.ts` / + * 'same-file' | 'interface-dispatch' | 'read' | 'write' | + * 'global-name-fallback'` (see `passes/receiver-bound-calls.ts` / * `passes/free-call-fallback.ts` for the scope-resolution emitters). - * - Overload disambiguation: both paths use + * - Overload disambiguation: resolvers use * `generateId('Method', ...)` suffixed with `parameterTypes` when a * method has overloads — see `graph-bridge/ids.ts`. * - * The CI parity workflow (`.github/workflows/ci-scope-parity.yml`) - * runs both paths on every migrated language's fixture corpus and - * fails if the graph outputs diverge. + * Resolver fixture coverage lives in `ci-tests.yml` (the separate + * `ci-scope-parity.yml` gate was removed in RING4-1 #942). * * Plan that introduced most of these invariants: * `docs/plans/2026-04-20-001-refactor-emit-pipeline-generalization-plan.md`. @@ -865,6 +863,88 @@ export interface ScopeResolver { */ readonly allowGlobalFreeCallFallback?: boolean; + /** Opt in to binding imports at their extracted lexical scope, rather than + * publishing every file's imports at module scope. */ + readonly importsBindAtLexicalScope?: boolean; + + /** + * Two `wildcard` re-exports that both DECLARE a name make it AMBIGUOUS in + * this language — ECMAScript `export *` semantics, where the module simply + * does not export the name and any binding is a guess. Opt-in: for a + * language whose wildcard import is `#include`, `require` or a package + * fan-out, the same name in two files is an overload set or a redeclaration, + * and refusing it would delete real edges (C++ arity-narrowed overloads + * across two headers). Forwarded to `FinalizeHooks.wildcardCollisionIsAmbiguous`. + */ + readonly exclusiveWildcardReexports?: boolean; + + /** + * A named import or named re-export can only bind to a MODULE-LEVEL + * declaration of the target file — ECMAScript semantics, where + * `import { x }` never reaches a class member. Opt-in: languages that bind + * module-level members by bare name (static members, module functions) + * leave it off. Forwarded to `FinalizeHooks.namedImportsBindTopLevelOnly`. + */ + readonly namedImportsBindTopLevelOnly?: boolean; + + /** + * Veto for a single `allowGlobalFreeCallFallback` guess. + * + * The fallback picks a callable because its SIMPLE NAME is unique in the + * workspace — it consults no import and no scope chain. For most languages a + * large share of those guesses are not merely unlikely but IMPOSSIBLE: Go + * cannot call an unexported identifier from another package, ESM cannot see a + * name it did not import, Rust cannot reach an item with no `use` path. This + * hook is where a language states those rules, so the shared pass can drop + * the edge instead of publishing a guess that the language forbids. + * + * Return `false` to REFUSE (no edge, recorded as `fallback-refused`). Return + * `true`, or leave the hook undefined, to emit the labeled + * `global-name-fallback` edge. **Only answer `false` when the call is + * impossible, not when it is merely unproven** — a wrongly-refused candidate + * is a lost real edge, whereas a wrongly-allowed one is at least labeled and + * excluded from flows. + * + * Deliberately NOT folded into `isCallableVisibleFromCaller`: that hook also + * gates precise dispatch paths (implicit-this, member calls), so a rule + * written for the name-guess tier would silently suppress resolved edges too. + * + * `parsedFileOf` reaches the CANDIDATE's parse result — a language whose rule + * depends on declaration-side facts not carried by `SymbolDefinition` (for + * example Rust's `pub` form) needs it. `isExported` is the tri-state export + * marker when the language supplies one; this hook still needs the parse for + * visibility facts that marker does not represent. It returns `undefined` + * for a path outside this pass's file set; treat that as "cannot decide" + * and allow. + */ + readonly isGlobalNameFallbackPlausible?: (ctx: { + readonly callerParsed: ParsedFile; + readonly candidate: SymbolDefinition; + readonly parsedFileOf: (filePath: string) => ParsedFile | undefined; + /** + * Raw source of any parsed file, for languages whose visibility rule needs + * a declaration the parse does not carry (Go's package clause: a test file's + * `package foo_test` is a different package from its directory's `foo`). + * Absent when the pipeline has no contents at hand; hooks must then answer + * from paths alone and, when undecidable, allow. + */ + readonly sourceTextOf?: (filePath: string) => string | undefined; + /** + * The call site, so a language can tell a BARE name from a PATH-QUALIFIED + * one. Both reach this tier — a qualified call whose qualifier the resolver + * could not follow falls through to the bare-name search with only its tail + * name — but they are not the same claim. Rust's `User::new(...)` named its + * type in source, so refusing it for lacking a `use` of the module would + * delete an edge the source spells out. + */ + readonly site: { + readonly name: string; + readonly rawQualifiedName?: string; + /** Lexical call-site scope; absent only for legacy/synthetic hook callers. */ + readonly inScope?: ScopeId; + }; + }) => boolean; + /** * In this language every `Method` belongs to a class instance, so a * FREE (receiver-less) call may resolve to a `Method` only when the diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts index 55becdcb6..48673f027 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/ids.ts @@ -87,6 +87,8 @@ export const CALLER_ANCHOR_LABELS: ReadonlySet = new Set([ 'Constructor', 'Module', 'Class', + 'Protocol', + 'Category', 'Interface', 'Struct', 'Enum', @@ -503,9 +505,25 @@ function resolveDefGraphIdUncached( if (qualifiedHit !== undefined) return qualifiedHit; } const simpleName = qn.lastIndexOf('.') === -1 ? qn : qn.slice(qn.lastIndexOf('.') + 1); + // FAIL CLOSED before the label-agnostic simple key when a FUNCTION-LOCAL + // callable of this name exists in the file. The guards above cover a def + // that is itself a callable; a NON-callable def (`export const selected = + // factory()`, a `Variable` with no graph node of its own) used to fall + // through here and alias onto `wrapper.selected`, the function-local one + // (#3182 review). Whatever the def's label, a bare name matching a local + // callable is the aliasing this key cannot tell apart, and a missing edge + // is the correct failure direction. + for (const localLabel of LOCAL_CALLABLE_LABELS) { + if (nodeLookup.get(localNameKey(filePath, localLabel, simpleName)) !== undefined) { + return undefined; + } + } return nodeLookup.get(simpleKey(filePath, simpleName)); } +/** Labels the structure phase registers function-local declarations under. */ +const LOCAL_CALLABLE_LABELS: readonly NodeLabel[] = ['Function', 'Method']; + /** Derive the simple (unqualified) name of a def from its `qualifiedName`. */ export function simpleQualifiedName(def: SymbolDefinition): string | undefined { const q = def.qualifiedName; diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts index fe7f4506b..73506aad6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/node-lookup.ts @@ -280,6 +280,8 @@ export const LINKABLE_LABELS: ReadonlySet = new Set([ // targets and need the same def→graph bridge. 'Module', 'Class', + 'Protocol', + 'Category', 'Interface', 'Struct', 'Enum', diff --git a/gitnexus/src/core/ingestion/scope-resolution/name-fallback-summary.ts b/gitnexus/src/core/ingestion/scope-resolution/name-fallback-summary.ts new file mode 100644 index 000000000..e96d9d5c7 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/name-fallback-summary.ts @@ -0,0 +1,212 @@ +/** + * Per-language census of global-name fallback decisions: how many call sites + * used a unique-name guess, and how many candidates visibility rules refused. + * Sites are counted before edge coalescing: a precisely bound site can prove + * the same caller/target dependency, so this is not a count of heuristic edges. + * + * Both halves are needed and neither is meaningful alone. A guess count with no + * refusal count cannot distinguish a language with genuinely few impossible + * candidates from one whose hook is missing; a refusal count with no guess count + * cannot distinguish a working guard from one that rejects everything. The pair + * is what makes the guard auditable on a real repository, which is the whole + * point of recording it — before this, guessed edges were emitted with the same + * reason and confidence as import-resolved ones and the number was unknowable. + * + * Structural sibling of `unresolved-receivers.ts`'s summary, and persisted the + * same way (`RepoMeta.nameFallbackEdges`). + */ + +import { getLanguageFromFilename } from 'gitnexus-shared'; +import { logger } from '../../logger.js'; +import type { ResolutionOutcome } from './resolution-outcome.js'; + +/** + * Distinct caller-file/callee-name pairs per language, from the caller→callee-name index the pipeline + * builds while its streaming sink is live (so it is complete under `--force`), + * bucketed by the CALLER's file language. Gives `NameFallbackSummary.byLanguage` + * its denominator: a guess count is only readable as a share of the calls. + */ +export function countCallsByLanguage( + index: ReadonlyMap> | undefined, + nodes: { getNode(id: string): { properties: Record } | undefined } | undefined, +): Readonly> | undefined { + if (index === undefined || nodes === undefined) return undefined; + const counts = new Map(); + const namesByFile = new Map>(); + for (const [callerId, callees] of index) { + const filePath = nodes.getNode(callerId)?.properties?.filePath; + if (typeof filePath !== 'string') continue; + let names = namesByFile.get(filePath); + if (names === undefined) { + names = new Set(); + namesByFile.set(filePath, names); + } + for (const name of callees) names.add(name); + } + for (const [filePath, names] of namesByFile) { + const language = getLanguageFromFilename(filePath) ?? 'unknown'; + counts.set(language, (counts.get(language) ?? 0) + names.size); + } + if (counts.size === 0) return undefined; + const out: Record = {}; + for (const [language, count] of [...counts.entries()].sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + )) { + out[language] = count; + } + return out; +} + +/** Unattributed bucket for a pass that recorded no language. */ +const UNKNOWN_LANGUAGE = 'unknown'; + +export interface NameFallbackLanguageCounts { + /** Call sites resolved by a unique-name guess, before edge coalescing. */ + readonly guessed: number; + /** + * Distinct (caller file, callee name) pairs among those sites — the unit + * `callsByLanguage` counts in, so `guessedPairs / callsByLanguage[lang]` is a + * ratio bounded by 1. Absent on a summary persisted before this field existed. + */ + readonly guessedPairs?: number; + /** Candidates this language's `isGlobalNameFallbackPlausible` hook refused. */ + readonly refused: number; +} + +export interface NameFallbackSummary { + /** Language → guessed/refused counts. Languages with neither are absent. */ + readonly byLanguage: Readonly>; + /** Guessed call sites, repo-wide. */ + readonly totalGuessed: number; + /** Distinct (caller file, callee name) pairs among the guessed sites. */ + readonly distinctGuessedPairs?: number; + readonly totalRefused: number; + /** + * Barrel names refused because two `export *` sources both declared them + * (`reexport-ambiguous`). Not a guess and not per-language — a name the + * finalize pass declined to bind at all — but it belongs in the same census: + * it is the other place the resolver used to publish an arbitrary winner as + * `import-resolved`. + */ + readonly totalAmbiguousReexports: number; + /** + * The refused barrel names themselves (`file:name`), sorted, capped at + * `MAX_AMBIGUOUS_NAMES`. A count alone cannot say whether a refusal landed on + * a name anyone calls; the list can be joined against the ledger's `byName`. + */ + readonly ambiguousReexportNames?: readonly string[]; + /** + * Distinct caller-file/callee-name pairs per language (through any path), so `guessedPairs` can be + * read as a SHARE of a language's call graph rather than a bare count. Absent + * when the caller did not supply the totals. + */ + readonly callsByLanguage?: Readonly>; +} + +/** Bound on the persisted ambiguous-name list; the total beside it stays exact. */ +export const MAX_AMBIGUOUS_NAMES = 200; + +/** + * Build the summary, or `undefined` when the run neither guessed nor refused — + * a repository with no opt-in language stores no key at all, so the artifact + * stays absent rather than recording a row of zeroes. + */ +export function summarizeNameFallback( + outcomes: readonly ResolutionOutcome[], + callsByLanguage?: Readonly>, +): NameFallbackSummary | undefined { + const guessed = new Map(); + const guessedPairsByLanguage = new Map(); + const refused = new Map(); + const ambiguousNames = new Set(); + // Two units, both kept. `guessed` counts call SITES, not final graph edges: + // a precise site may prove the same dependency during edge coalescing. + // Preserve the unit earlier summaries hold. `guessedPairs` dedupes by (caller file, + // callee name), the unit `callsByLanguage` is counted in: ten guessed `foo()` + // calls in one file are one pair against a denominator that counts `foo` + // once, so the guessy RATIO uses pairs and is bounded by 1. Changing the unit + // of `guessed` itself silently read as a large improvement across engines. + const guessedPairs = new Set(); + let totalGuessed = 0; + let totalRefused = 0; + let totalAmbiguousReexports = 0; + + for (const outcome of outcomes) { + if (outcome.kind === 'fallback-guessed') { + const language = outcome.language ?? UNKNOWN_LANGUAGE; + guessed.set(language, (guessed.get(language) ?? 0) + 1); + totalGuessed++; + const pair = `${outcome.filePath}\u0000${outcome.name}`; + if (!guessedPairs.has(pair)) { + guessedPairs.add(pair); + guessedPairsByLanguage.set(language, (guessedPairsByLanguage.get(language) ?? 0) + 1); + } + } else if (outcome.kind === 'fallback-refused') { + const language = outcome.language ?? UNKNOWN_LANGUAGE; + refused.set(language, (refused.get(language) ?? 0) + 1); + totalRefused++; + } else if (outcome.kind === 'reexport-ambiguous') { + totalAmbiguousReexports++; + ambiguousNames.add(`${outcome.filePath}:${outcome.name}`); + } + } + if (totalGuessed === 0 && totalRefused === 0 && totalAmbiguousReexports === 0) return undefined; + + const byLanguage: Record = {}; + for (const language of new Set([...guessed.keys(), ...refused.keys()])) { + byLanguage[language] = { + guessed: guessed.get(language) ?? 0, + guessedPairs: guessedPairsByLanguage.get(language) ?? 0, + refused: refused.get(language) ?? 0, + }; + } + const sortedNames = [...ambiguousNames].sort(); + return { + byLanguage, + totalGuessed, + distinctGuessedPairs: guessedPairs.size, + totalRefused, + totalAmbiguousReexports, + ...(sortedNames.length > 0 + ? { ambiguousReexportNames: sortedNames.slice(0, MAX_AMBIGUOUS_NAMES) } + : {}), + ...(callsByLanguage !== undefined ? { callsByLanguage } : {}), + }; +} + +/** + * One-line readout for the analyze summary. `undefined` when there is nothing to + * report, so a run on a repository with no opt-in language prints no line. + */ +export function formatNameFallbackSummary( + summary: NameFallbackSummary | undefined, +): string | undefined { + if (summary === undefined) return undefined; + const perLanguage = Object.entries(summary.byLanguage) + .sort(([, a], [, b]) => b.guessed + b.refused - (a.guessed + a.refused)) + .map(([language, counts]) => `${language} ${counts.guessed}/${counts.refused}`) + .join(', '); + const ambiguous = + summary.totalAmbiguousReexports > 0 + ? `; ${summary.totalAmbiguousReexports} barrel name(s) refused as ambiguous \`export *\`` + : ''; + const languages = perLanguage === '' ? 'none' : perLanguage; + const pairs = + summary.distinctGuessedPairs !== undefined + ? ` (${summary.distinctGuessedPairs} distinct caller-file/name pairs)` + : ''; + const siteUnit = summary.totalGuessed === 1 ? 'call site' : 'call sites'; + return `name-fallback resolution: ${summary.totalGuessed} ${siteUnit}${pairs}, ${summary.totalRefused} refused as impossible (guessed/refused by language: ${languages})${ambiguous}`; +} + +/** + * Print the readout as part of the analyze summary. Unconditional (not behind a + * debug env var, unlike the receiver-drop diagnostic): a reader deciding how far + * to trust this index's call graph needs to know how much of it is guessed, and + * a number nobody sees is the state this work exists to end. + */ +export function logNameFallbackSummary(summary: NameFallbackSummary | undefined): void { + const line = formatNameFallbackSummary(summary); + if (line === undefined) return; + logger.info(line); +} diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 0457767b2..7dcc792f3 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -36,6 +36,7 @@ import type { ResolutionOutcomeRecorder, ResolutionSuppressionReason, } from '../resolution-outcome.js'; +import { GLOBAL_NAME_FALLBACK_REASON } from '../../../graph/edge-reasons.js'; import { resolveCallerGraphId, resolveDefGraphId } from '../graph-bridge/ids.js'; import type { CalleeIdSink } from '../graph-bridge/callee-id-sink.js'; import { @@ -43,7 +44,9 @@ import { findAllCallableBindingsInScope, findCallableBindingInScope, findCallableBindingsAndAdlBlocker, + findClassBindingInScope, findEnclosingClassDef, + isClassFileImportGrounded, resolveInheritanceBaseInScope, type CallableBindingCandidate, } from '../scope/walkers.js'; @@ -64,6 +67,15 @@ export function emitFreeCallFallback( workspaceIndex: WorkspaceResolutionIndex, options: { readonly allowGlobalFallback?: boolean; + /** Language whose pass this is, carried onto the fallback outcome records + * so the analyze summary can report guesses/refusals PER LANGUAGE. A + * repo-wide total hides which language's rules are the loose ones. */ + readonly language?: string; + /** Per-language veto on a name guess — see + * `ScopeResolver.isGlobalNameFallbackPlausible`. */ + readonly isGlobalNameFallbackPlausible?: ScopeResolver['isGlobalNameFallbackPlausible']; + /** Raw source lookup handed to `isGlobalNameFallbackPlausible` (optional). */ + readonly sourceTextOf?: (filePath: string) => string | undefined; /** When true, `Type(...)` constructor calls link to the Class def * itself rather than its explicit Constructor. Swift opts in. */ readonly constructorCallTargetsClass?: boolean; @@ -138,6 +150,14 @@ export function emitFreeCallFallback( let allFilePathsMemo: ReadonlySet | undefined; const allFilePaths = (): ReadonlySet => (allFilePathsMemo ??= new Set(parsedFiles.map((p) => p.filePath))); + // Candidate-side parse lookup for `isGlobalNameFallbackPlausible`. Built + // lazily and once, on the same terms as `allFilePaths` above: a language + // without the hook never pays for the index. + let parsedByPathMemo: ReadonlyMap | undefined; + const parsedFileByPath = (): ((filePath: string) => ParsedFile | undefined) => { + parsedByPathMemo ??= new Map(parsedFiles.map((p) => [p.filePath, p])); + return (filePath) => parsedByPathMemo!.get(filePath); + }; // Per-pass memo of pickUniqueGlobalCallable's post-filter candidate list, // keyed (simpleName, callerFilePath). Only created when no per-caller // visibility filter applies (the list is then a pure function of name+file — @@ -183,7 +203,17 @@ export function emitFreeCallFallback( }; for (const parsed of parsedFiles) { - type PendingRel = { rel: Parameters[0]; gatedAll: boolean }; + type PendingRel = { + rel: Parameters[0]; + gatedAll: boolean; + /** + * The confidence/reason a PRECISELY resolved site (a real binding, not a + * unique-name guess) proved for this edge; `undefined` while every site + * collapsed into it so far was a guess. Decided at flush, not by the + * first site the walk met. + */ + precise: { confidence: number; reason: string } | undefined; + }; const pending = new Map(); const bindingCandidatesByScope = options.freeCallsRequireInstanceOwnership === true @@ -199,13 +229,48 @@ export function emitFreeCallFallback( // to the Class node itself (implicit constructor). Legacy emits // the same two targets; see test expectations. let fnDef: SymbolDefinition | undefined; + // Guess flag starts here so the constructor unique-class pick can mark + // the same class of guess as `pickUniqueGlobalCallable` below. The + // veto/label path keys off this flag; declaring it after that pick left + // constructor-form unique-name hits labeled `import-resolved` at 0.85. + let fnDefFromGlobalNameFallback = false; + // Language visibility vetoes the constructed TYPE (Class/Struct), not a + // Constructor child — Go refuses any `qualifiedName` containing `.`. + let globalFallbackVetoTarget: SymbolDefinition | undefined; if (site.callForm === 'constructor') { - const classDef = resolveInheritanceBaseInScope( + // Lexical / written-qualifier first. A bare unique type with no + // import/#include evidence is a guess (JS `new UniqueWidget()`, + // Go unexported `uniqueWidget{}`). A unique type whose file is + // imported, whose import resolved to this def, or whose name + // was imported is precise — C++ `#include "user.h"` and Rust + // `use`/`pub use` often mint no lexical class binding. An + // imported sibling file of a different name is not evidence. + let classDef = resolveInheritanceBaseInScope( site.inScope, site.name, scopes, site.rawQualifiedName, + undefined, + { uniqueQualifiedNameFallback: site.rawQualifiedName !== undefined }, ); + if (classDef === undefined) { + classDef = + findClassBindingInScope(site.inScope, site.name, scopes, undefined, { + uniqueQualifiedNameFallback: true, + }) ?? + (options.allowGlobalFallback === true + ? pickUniqueGlobalClass(site.name, globalClassesBySimpleName) + : undefined); + if ( + classDef !== undefined && + classDef.type !== 'Interface' && + site.rawQualifiedName === undefined && + !isClassFileImportGrounded(site.inScope, classDef, scopes, site.name) + ) { + fnDefFromGlobalNameFallback = true; + globalFallbackVetoTarget = classDef; + } + } if (classDef !== undefined && classDef.type !== 'Interface') { // Most languages link `Type(...)` to the explicit Constructor def // when one exists (else the Class). Languages that model the call @@ -215,21 +280,6 @@ export function emitFreeCallFallback( options.constructorCallTargetsClass === true ? classDef : pickConstructorOrClass(classDef, workspaceIndex, scopes, site.arity); - } else if (options.allowGlobalFallback === true) { - // The constructed type may live in a sibling/imported file that is - // not in the call-site's lexical scope-chain bindings. Fall back to - // a unique workspace-wide Class def by simple name (gated on the - // same global-fallback opt-in as free calls). Then target the - // Class or its Constructor per the language's preference. - const globalClass = pickUniqueGlobalClass(site.name, globalClassesBySimpleName); - if (globalClass !== undefined) { - fnDef = - globalClass.type === 'Interface' - ? undefined - : options.constructorCallTargetsClass === true - ? globalClass - : pickConstructorOrClass(globalClass, workspaceIndex, scopes, site.arity); - } } } // Module-qualified free call (`mod::fn()`): the source named the module @@ -543,10 +593,17 @@ export function emitFreeCallFallback( } } } - // V1: pickUniqueGlobalCallable ignores import context — resolves to any - // globally-unique callable. False cross-package edges are possible when - // the caller does not import the target package. Same-package calls are - // usually caught by nearest-scope lookup before reaching here. + // Name-guess tier: pickUniqueGlobalCallable consults no import context — + // it resolves to any globally-unique callable. Same-package calls are + // usually caught by nearest-scope lookup before reaching here, so what + // lands in this tier is disproportionately cross-module, and a + // cross-module name match is a guess. + // + // Two things make that honest rather than a lie. The language's + // `isGlobalNameFallbackPlausible` hook refuses candidates its own + // visibility rules forbid (below), and every edge that survives is + // emitted with `GLOBAL_NAME_FALLBACK_REASON` at 0.5 rather than + // masquerading as `import-resolved` at 0.85 (see the emit site). if (fnDef === undefined && options.allowGlobalFallback === true) { fnDef = pickUniqueGlobalCallable( site.name, @@ -570,6 +627,53 @@ export function emitFreeCallFallback( scopeDefsCache, options.conversionOnlyArgTypePrefixes, ); + fnDefFromGlobalNameFallback = fnDef !== undefined; + if (fnDef !== undefined) globalFallbackVetoTarget = fnDef; + } + if (fnDefFromGlobalNameFallback && fnDef !== undefined) { + // An explicit named import cannot bind a declaration proven private + // to another module. In particular, a rejected named import must not + // reappear as a name guess to a class member or nested function. + // Unknown export evidence (e.g. dynamic module exports) keeps the + // existing fallback behavior. + const vetoCandidate = globalFallbackVetoTarget ?? fnDef; + const importsPrivateDeclaration = + vetoCandidate.filePath !== parsed.filePath && + vetoCandidate.isExported === false && + parsed.parsedImports.some( + (imported) => + (imported.kind === 'named' || imported.kind === 'alias') && + imported.localName === site.name, + ); + if ( + importsPrivateDeclaration || + options.isGlobalNameFallbackPlausible?.({ + callerParsed: parsed, + candidate: vetoCandidate, + parsedFileOf: parsedFileByPath(), + sourceTextOf: options.sourceTextOf, + site: { + name: site.name, + rawQualifiedName: site.rawQualifiedName, + inScope: site.inScope, + }, + }) === false + ) { + // The language proved this call impossible. Mark the site handled so + // `emit-references` does not substitute its own looser guess for the + // edge we just refused — the point is no edge, not a different one. + options.recordResolutionOutcome?.({ + kind: 'fallback-refused', + candidateId: fnDef.nodeId, + language: options.language, + phase: 'free-call-fallback', + filePath: parsed.filePath, + name: site.name, + range: site.atRange, + }); + handledSites.add(siteKey(parsed.filePath, site)); + continue; + } } if (fnDef === undefined) continue; if (fnDef.isDeleted === true) { @@ -617,40 +721,78 @@ export function emitFreeCallFallback( site.atRange.startCol, tgtGraphId, ); + if (fnDefFromGlobalNameFallback) { + options.recordResolutionOutcome?.({ + kind: 'fallback-guessed', + targetId: fnDef.nodeId, + language: options.language, + phase: 'free-call-fallback', + filePath: parsed.filePath, + name: site.name, + range: site.atRange, + }); + } const relId = `rel:CALLS:${callerGraphId}->${tgtGraphId}`; // One edge per (caller, callee): `staticGated` is the AND over every site // that collapses into it, so a callee reached from one live site and one // dead site stays live whichever site the walk meets first. Emission is // deferred to the end of this file's sites for that reason. + const preciseHere = fnDefFromGlobalNameFallback + ? undefined + : { + confidence: 0.85, + // Match legacy DAG's reason convention so consumers that + // assert `reason === 'import-resolved'` keep working. The + // construction-site marker is opt-in for the same reason. + reason: constructionSiteReason( + fnDef.filePath !== parsed.filePath ? 'import-resolved' : 'local-call', + site, + options.markConstructionSites, + ), + }; const pendingRel = pending.get(relId); if (pendingRel !== undefined) { if (site.staticGated !== true) pendingRel.gatedAll = false; + // The edge's label is decided at flush time from EVERY site that + // collapsed into it, not from whichever the walk met first. One site + // resolved through a real binding PROVES the dependency; a guessed + // site for the same pair is then redundant evidence, not a taint. + if (pendingRel.precise === undefined) pendingRel.precise = preciseHere; continue; } if (seen.has(relId)) continue; seen.add(relId); pending.set(relId, { gatedAll: site.staticGated === true, + precise: preciseHere, rel: { id: relId, sourceId: callerGraphId, targetId: tgtGraphId, type: 'CALLS', - confidence: 0.85, - // Match legacy DAG's reason convention so consumers that - // assert `reason === 'import-resolved'` keep working. The - // construction-site marker is opt-in for the same reason. - reason: constructionSiteReason( - fnDef.filePath !== parsed.filePath ? 'import-resolved' : 'local-call', - site, - options.markConstructionSites, - ), + // Guess values as placeholders; decided at flush from `precise`. + confidence: 0.5, + reason: GLOBAL_NAME_FALLBACK_REASON, }, }); emitted++; } - for (const { rel, gatedAll } of pending.values()) { - graph.addRelationship(gatedAll ? { ...rel, staticGated: true } : rel); + for (const { rel, gatedAll, precise } of pending.values()) { + // A name guess is not an import resolution and must not be spelled like + // one. It used to be emitted at 0.85 / `'import-resolved'`, which made + // it indistinguishable from an edge a real import produced — so every + // consumer that wanted to discount guesses had no field to do it with. + // 0.5 is the deliberate "coin flip" value, and the reason is what the + // process/community walks and the MCP tools actually key on, because + // 0.5 sits exactly ON their thresholds (see graph/edge-reasons.ts). + // An edge is a guess only when EVERY site that collapsed into it was one; + // a single precisely resolved site proves it, whatever order the walk + // met the sites in. Independent of `gatedAll`. + const labeled = + precise !== undefined + ? { ...rel, confidence: precise.confidence, reason: precise.reason } + : rel; + graph.addRelationship(gatedAll ? { ...labeled, staticGated: true } : labeled); } } return emitted; diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts index 157348c5b..0736bb53c 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/imported-return-types.ts @@ -120,19 +120,10 @@ export function followChainPostFinalize( * typeBindings — the in-extractor pass-4 ran before propagation and * missed any chain whose terminal lived in a foreign file. * - * Scope-chain concern (verified 2026-04-21): `pythonImportOwningScope` - * documents that function-local `from x import y` binds `y` to the - * inner function scope, which would make a module-only write miss - * non-module importers. In practice `finalize-algorithm` hoists those - * bindings into `indexes.bindings[moduleScope]` regardless of where - * the `import` statement appears — the integration fixture - * `python-function-local-import-chain` exercises a chained - * receiver-bound call `u = get_user(); u.save()` inside a function - * body and emits the expected `do_work → User.save` edge. The - * module-scope write is sufficient today. If finalize routing ever - * changes to honor the hook's per-scope contract, this pass must - * iterate `indexes.bindings` over every scope and mirror into the - * binding-owning scope's `typeBindings`, not just the module's. + * Finalize can retain lexical import ownership. Mirror into each binding's + * owning scope, never hoist local import-derived types into the file scope. + * Module ordering remains SCC-based; non-module chains are followed after + * all import bindings have been mirrored. */ export function propagateImportedReturnTypes( parsedFiles: readonly ParsedFile[], @@ -140,6 +131,7 @@ export function propagateImportedReturnTypes( index: WorkspaceResolutionIndex, ): void { const moduleScopeByFile = index.moduleScopeByFile; + const scopesByFile = new Map(parsedFiles.map((file) => [file.filePath, file.scopes])); // Walk SCCs in reverse-topological order (`indexes.sccs` is leaves- // first per `tarjanSccs`). For each file we mirror import bindings @@ -167,42 +159,44 @@ export function propagateImportedReturnTypes( // import-derived typeBinding mirror. Both helpers fast-path when // no augmentations exist for the scope, so the common case is // allocation-free. See I8. - for (const localName of namesAtScope(importerModule.id, indexes)) { - // Skip if importer already has a typeBinding for this name — - // an explicit local annotation must win over import-derived. - if (importerModule.typeBindings.has(localName)) continue; + for (const importerScope of scopesByFile.get(filePath) ?? [importerModule]) { + for (const localName of namesAtScope(importerScope.id, indexes)) { + // Skip if importer already has a typeBinding for this name — + // an explicit local annotation must win over import-derived. + if (importerScope.typeBindings.has(localName)) continue; - const refs = lookupBindingsAt(importerModule.id, localName, indexes); - for (const ref of refs) { - if (ref.origin !== 'import' && ref.origin !== 'reexport' && ref.origin !== 'wildcard') - continue; - const sourceModule = moduleScopeByFile.get(ref.def.filePath); - if (sourceModule === undefined) continue; + const refs = lookupBindingsAt(importerScope.id, localName, indexes); + for (const ref of refs) { + if (ref.origin !== 'import' && ref.origin !== 'reexport' && ref.origin !== 'wildcard') + continue; + const sourceModule = moduleScopeByFile.get(ref.def.filePath); + if (sourceModule === undefined) continue; - // The source file's typeBinding is keyed by the def's simple - // name (e.g. `get_user`), not the importer's local alias. - const qn = ref.def.qualifiedName; - if (qn === undefined) continue; - const dot = qn.lastIndexOf('.'); - const sourceName = dot === -1 ? qn : qn.slice(dot + 1); + // The source file's typeBinding is keyed by the def's simple + // name (e.g. `get_user`), not the importer's local alias. + const qn = ref.def.qualifiedName; + if (qn === undefined) continue; + const dot = qn.lastIndexOf('.'); + const sourceName = dot === -1 ? qn : qn.slice(dot + 1); - const sourceTypeRef = sourceModule.typeBindings.get(sourceName); - if (sourceTypeRef === undefined) continue; + const sourceTypeRef = sourceModule.typeBindings.get(sourceName); + if (sourceTypeRef === undefined) continue; - // Chain-follow inside the source module so we mirror the - // terminal type, not an intermediate intra-source reference. - const terminal = followChainPostFinalize(sourceTypeRef, sourceModule.id, indexes); + // Chain-follow inside the source module so we mirror the + // terminal type, not an intermediate intra-source reference. + const terminal = followChainPostFinalize(sourceTypeRef, sourceModule.id, indexes); - // Mutating typeBindings is safe because draftToScope - // produced a non-frozen Map (Contract Invariant I3/I8). - (importerModule.typeBindings as Map).set(localName, terminal); - // First-write-wins for the local alias: if the same - // `localName` was registered multiple times via - // `mergeBindings` (rare; happens with conflicting - // re-exports), only the first ref with a usable - // typeBinding source is mirrored. Conflict resolution - // among multiple sources is the merger's job, not ours. - break; + // Mutating typeBindings is safe because draftToScope + // produced a non-frozen Map (Contract Invariant I3/I8). + (importerScope.typeBindings as Map).set(localName, terminal); + // First-write-wins for the local alias: if the same + // `localName` was registered multiple times via + // `mergeBindings` (rare; happens with conflicting + // re-exports), only the first ref with a usable + // typeBinding source is mirrored. Conflict resolution + // among multiple sources is the merger's job, not ours. + break; + } } } diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts index fef9ed29d..6a23ca94d 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/property-dispatch.ts @@ -17,9 +17,11 @@ * registries only consult pre-finalize local bindings — imported names live * in finalized bindings (the same reason free calls need * `emitFreeCallFallback`) — so `resolveReferenceSites` skips `value-ref` - * sites and this pass resolves them post-finalize via - * `findCallableBindingInScope` (Function/Method/Constructor only — the - * callable gate that keeps `{ port: DEFAULT_PORT }` from emitting anything). + * sites and this pass resolves them post-finalize (see + * `resolveValueRefTarget` — Function/Method/Constructor only, the callable + * gate that keeps `{ port: DEFAULT_PORT }` from emitting anything, and + * receiver-aware so a qualified reference binds the owner it was written + * with). * * Precision posture (mirrors `emitInterfaceDispatchFor`): * - reason `'property-dispatch'` keeps synthesized CALLS auditable; @@ -36,13 +38,29 @@ * emits the capture participates) and generic member-call sites. */ -import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import type { + BindingRef, + ParsedFile, + ReferenceSite, + ScopeId, + SymbolDefinition, +} from 'gitnexus-shared'; import type { KnowledgeGraph } from '../../../graph/types.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; import { tryEmitEdge, type CalleeIdCaptureCtx } from '../graph-bridge/edges.js'; import type { GraphNodeLookup } from '../graph-bridge/node-lookup.js'; import type { CalleeIdSink } from '../graph-bridge/callee-id-sink.js'; -import { findCallableBindingInScope } from '../scope/walkers.js'; +import { + findCallableBindingInScope, + findClassBindingInScope, + findOwnedMember, + isNamespaceNameShadowed, + isOwnerNameShadowedBySomethingElse, + lookupBindingsAt, +} from '../scope/walkers.js'; +import { VALUE_REF_EDGE_REASON } from '../value-ref-edges.js'; +import type { SemanticModel } from '../../model/semantic-model.js'; +import { CALL_TARGET_TYPES } from '../../model/symbol-table.js'; /** * Keys registered by more than this many distinct functions are skipped — @@ -65,12 +83,282 @@ export const MAX_PROPERTY_DISPATCH_FANOUT = (() => { /** Below the 0.85 resolved baseline; same discount idea as interface-dispatch. */ export const PROPERTY_DISPATCH_CONFIDENCE = 0.7; +/** + * Resolve a `value-ref` site to the callable it names. + * + * Two shapes, and the difference matters: + * + * - BARE (`{ handler: onClick }`, `register(onTick)`) — the name is resolved + * up the lexical chain, which is what an unqualified name means. Be exact + * about what that walk does, because it is not a plain lexical lookup: + * `findCallableBindingInScope` applies the callable predicate WHILE walking, + * so a scope binding the name to a parameter or a local contributes nothing + * and the walk continues outward. A nearer non-callable binding is stepped + * over — the same shape the qualified path guards against below, unguarded + * here. Pre-existing (#2437) and out of scope for #3399; reachable in JS/TS + * (`function outer(handler) { return { h: handler }; }` beside a top-level + * `function handler`), and narrow in Zig, which rejects a local shadowing a + * container declaration. + * + * - QUALIFIED (`bridge.accessor(Element.getNamespaceUri, …)`) — the source + * WROTE the owner, so the lexical chain is the wrong instrument. It gives + * local bindings precedence (`walkScopeChain`), so a nested container + * holding its own `getNamespaceUri` answers first and the registration is + * attached to a DIFFERENT function than the one written. That is a wrong + * edge, not a missing one — strictly worse for a tool whose value is that + * its edges can be trusted — and it is reachable in any language that + * allows a same-named callable in a nested container, Zig included. + * + * So a qualified site resolves through its receiver: name the owner, then take + * the member off that owner. An owner is either a MODULE or a CLASS-like + * container — `utils.compare` and `Element.getNamespaceUri` are the same shape + * written against the two kinds of namespace a language has, and the member-call + * path already resolves both (receiver-bound-calls Case 1 / Case 2). Both are + * tried here, MODULE FIRST: see `findNamespaceValueRefTarget` for why a module + * receiver cannot simply be declined, and the comment on the call below for why + * the container channel must not go first. + * + * If neither channel names the owner, or the owner is named but owns no such + * callable, this DECLINES rather than falling back to the lexical walk. + * + * Be precise about what declining costs, because it is more than one reference: + * a site that emits NO edge leaves no evidence for `impact`'s value-reference + * probe to read, so the target keeps `epistemic: "exact"` — silence, not a + * hedge. That is why the module channel above exists rather than being waved + * through as "just a decline". What remains declined is the case where the + * written receiver names nothing this index knows at all (a Zig `@This()` alias + * whose name differs from its container's, an owner from outside the workspace): + * there the alternative is not a hedge either, it is a confident edge to a + * lexically-nearer function that the source did not name, and a wrong edge is + * strictly worse than a missing one for a tool whose value is that its edges can + * be trusted. + * + * `CALL_TARGET_TYPES`, not a hand-rolled label set: `findOwnedMember` also + * answers with FIELDS, and a field named like the member would otherwise + * register as if it were the callable. + * + * Exported for `bench/value-ref-resolution/measure.mjs`, which gates both the + * resolved-target SET and the per-site cost of the four channels above across a + * 4x file-count step. It is the per-site half of the pass, so timing it in + * isolation is what makes a workspace-size dependency visible; timing + * `emitPropertyDispatchCalls` would fold that signal into edge emission, and + * re-implementing the channel order in the bench would pin the bench's idea of + * this function rather than this function. + */ +export function resolveValueRefTarget( + site: ReferenceSite, + filePath: string, + scopes: ScopeResolutionIndexes, + model: SemanticModel, + publishesImportedNames: boolean, +): SymbolDefinition | undefined { + const receiverName = site.explicitReceiver?.name; + if (receiverName === undefined) { + return findCallableBindingInScope(site.inScope, site.name, scopes); + } + // NAMESPACE FIRST, and the order is load-bearing. `findClassBindingInScope` + // does not stop at the scope chain: when its `isClassLike` walk misses — and a + // namespace handle binds a Module, so it always misses — it falls back to + // `scopes.qualifiedNames`, a WORKSPACE-wide index, and answers with the unique + // def of that name anywhere in the repo. Trying it first therefore lets a + // same-named container in a file this one never imported preempt the `@import` + // this file actually wrote: + // + // const dom_utils = @import("dom_utils.zig"); // namespace-only module + // … bridge.accessor(dom_utils.compare, …) // → decoy.zig's compare + // + // The shadow guard below cannot catch it: the import binds at MODULE scope, + // which the guard treats as the floor. An import written in this file is the + // strongest statement about what the name means here, so it outranks a global + // guess — and when the handle is not an import of this file, this answers + // nothing and the container channel runs exactly as before. + const viaNamespace = findNamespaceValueRefTarget( + site, + filePath, + receiverName, + scopes, + publishesImportedNames, + ); + if (viaNamespace !== undefined) { + // `'owned'` is NOT "no answer" — it is "this receiver is a namespace handle + // this file wrote, and it names no callable member". The two must not be + // conflated, because falling through from the second one reaches + // `findClassBindingInScope`, whose miss path answers from the WORKSPACE-wide + // qualified-name index: a same-named container in a file this one never + // imported then supplies the member the written module does not have. That + // is a confident edge into an unrelated file, and the owner-shadow guard + // below does not stop it — a plain `const utils = @import("utils.zig");` + // records a namespace IMPORT EDGE, not a module-scope binding, so the guard + // sees nothing bound under the name and reads the container as unshadowed. + // Verified with a fixture rather than argued: `dom_utils.onlyOnDecoy`, where + // `dom_utils.zig` has no such member and `decoy.zig` declares a same-named + // struct that does, minted `JsApi → onlyOnDecoy` before this line existed. + // + // The file said which module it meant. If that module does not expose the + // name as a callable, the honest answer is no edge. + return viaNamespace === 'owned' ? undefined : viaNamespace; + } + + const owner = findClassBindingInScope(site.inScope, receiverName, scopes); + if (owner !== undefined) { + // The container lookup is a CLASS-ONLY walk: `walkScopeChain` filters by + // `isClassLike`, so it steps over a nearer binding that is a value and keeps + // climbing — and past the scope chain entirely, into a qualified-name + // fallback that answers with the unique workspace definition of the name. + // `fn f(Ticker: u8) { register(Ticker.fire) }` in a file that neither + // declares nor imports `Ticker` therefore resolves to some other file's + // `Ticker` container. That is the wrong-edge failure R1-2 exists to prevent, + // arriving through the class channel instead of the lexical one, and a + // registration pointing at a function the source never named is worse than + // no registration at all. + // + // So the name has to still MEAN that container at this site. The namespace + // channel below asks the same question through `isNamespaceNameShadowed`; + // a container needs the variant that exempts the container ITSELF, because + // `fn make() { const Local = struct {…}; register(Local.go); }` binds the + // name locally to the very def we resolved, and reading that as its own + // shadow would suppress the resolutions this path exists to make. + if (isOwnerNameShadowedBySomethingElse(receiverName, owner, site.inScope, scopes)) + return undefined; + const member = findOwnedMember(owner.nodeId, site.name, model); + if (member === undefined || !CALL_TARGET_TYPES.has(member.type)) return undefined; + return member; + } + return undefined; +} + +/** + * The second kind of owner: a namespace handle. + * + * `const utils = @import("utils.zig"); register(utils.compare);` — `utils` is a + * MODULE, not a class, so `findClassBindingInScope` answers nothing and the + * class path above declines. Declining here would be a silent hole rather than + * a conservative one: no USES edge is emitted, so + * `callableValueReferenceBoundaries` measures a real zero and `impact` on + * `compare` republishes `exact` — the very claim this feature exists to stop + * making. Nothing downstream can hedge on evidence that was never recorded. + * + * So resolve it, through the SAME channel the member-CALL path already trusts + * for `utils.compare()` (receiver-bound-calls Case 1): the file's namespace + * import edges name the target module, and the target module's own local + * module-scope bindings name its members. `utils.compare` and `utils.compare()` + * disagreeing about what `utils` is would be the anomaly. + * + * The same three guards Case 1 applies, for the same reasons: + * - a LOCAL declaration shadowing the handle suppresses the resolution + * (`isNamespaceNameShadowed`) — `fn f(utils: Decoy) { register(utils.compare) }` + * names the parameter's member, and resolving through the import would be a + * wrong edge rather than a missing one; + * - a locally declared member wins, and a name the target file merely IMPORTED + * counts only when the provider says its imports ARE its exports + * (`ScopeResolver.namespaceExportsIncludeImportedNames`). That opt-in is not + * a detail to skip: a Zig HUB — a file made only of re-exports, ghostty's + * `src/terminal/`, tigerbeetle's `stdx` — declares nothing, so requiring a + * local declaration declines every member reached through one. `hub.fn()` + * resolves and `register(hub.fn)` would not, and one name would mean two + * things depending on whether a `(` followed it. In languages that do not + * opt in, a module's imports are not its exports and this stays closed; + * - two distinct defs under one name resolve NOTHING. Never guess a namespace + * member — the whole point of reading the written receiver is precision. + * + * `CALL_TARGET_TYPES` gates the answer for the same reason the class path needs + * it: `utils.DEFAULT_PORT` is a module-scope binding too, and a registration + * table full of constants must keep emitting nothing. + */ +function findNamespaceValueRefTarget( + site: ReferenceSite, + filePath: string, + receiverName: string, + scopes: ScopeResolutionIndexes, + publishesImportedNames: boolean, +): SymbolDefinition | 'owned' | undefined { + const moduleScopeId = scopes.moduleScopes.get(filePath); + if (moduleScopeId === undefined) return undefined; + const targetFiles: string[] = []; + for (const edge of scopes.imports.get(moduleScopeId) ?? []) { + if (edge.kind !== 'namespace' || edge.localName !== receiverName) continue; + if (edge.targetFile === null) continue; + if (!targetFiles.includes(edge.targetFile)) targetFiles.push(edge.targetFile); + } + if (targetFiles.length === 0) return undefined; + if (isNamespaceNameShadowed(receiverName, site.inScope, scopes)) return undefined; + + /** The unique callable `select` finds across every target file, or nothing. */ + const uniqueMember = ( + select: (moduleScope: ScopeId) => readonly BindingRef[], + ): SymbolDefinition | undefined | 'ambiguous' => { + let picked: SymbolDefinition | undefined; + for (const targetFile of targetFiles) { + const targetScopeId = scopes.moduleScopes.get(targetFile); + if (targetScopeId === undefined) continue; + for (const ref of select(targetScopeId)) { + if (!CALL_TARGET_TYPES.has(ref.def.type)) continue; + if (picked !== undefined && picked.nodeId !== ref.def.nodeId) return 'ambiguous'; + picked = ref.def; + } + } + return picked; + }; + + // A locally declared member first — same precedence `findExportedDef` states + // and `walkScopeChain` applies: what the target file DECLARED beats what it + // merely re-published. + const localRefs = (scope: ScopeId): readonly BindingRef[] => + (scopes.bindings.get(scope)?.get(site.name) ?? []).filter((ref) => ref.origin === 'local'); + const local = uniqueMember(localRefs); + if (local === 'ambiguous') return 'owned'; + if (local !== undefined) return local; + if (!publishesImportedNames) return 'owned'; + + // PRECEDENCE IS DECIDED BEFORE THE TYPE GATE, not by it. `uniqueMember` + // applies `CALL_TARGET_TYPES` while it selects, so a target file declaring a + // NON-callable under this name answers `undefined` above and would otherwise + // fall through to the published channel — publishing a re-exported callable + // under a name the module's own declaration owns. `findExportedDef` does not + // do that: it returns any local def it finds and lets its caller's type gate + // reject it, so `findExportedDefIncludingImportedNames` never reaches the + // imported names for a name the file declares. Same rule here, so `x.f` and + // `x.f()` cannot disagree about which module owns the name. + // + // Not reachable through valid Zig today — a container cannot declare a name + // twice, so one target file cannot hold both spellings, and Zig is the only + // provider that sets `namespaceExportsIncludeImportedNames`. It becomes + // reachable the moment a second provider opts in, or a receiver binds more + // than one target file; the guard is one `some` and the alternative failure + // is a confident edge into the wrong module. + const declaredLocally = targetFiles.some((targetFile) => { + const targetScopeId = scopes.moduleScopes.get(targetFile); + return targetScopeId !== undefined && localRefs(targetScopeId).length > 0; + }); + if (declaredLocally) return 'owned'; + + // Then a name the target file publishes but did not declare — the hub case. + // `lookupBindingsAt`, not `scopes.bindings`, because a hub's module scope owns + // no local binding for these names and the finalized/augmented channel is the + // only place they exist; the same read `findExportedDefIncludingImportedNames` + // does for the CALL form. + const published = uniqueMember((scope) => + lookupBindingsAt(scope, site.name, scopes).filter( + (ref) => ref.origin === 'import' || ref.origin === 'namespace' || ref.origin === 'reexport', + ), + ); + return published === 'ambiguous' || published === undefined ? 'owned' : published; +} + export function emitPropertyDispatchCalls( graph: KnowledgeGraph, scopes: ScopeResolutionIndexes, parsedFiles: readonly ParsedFile[], nodeLookup: GraphNodeLookup, + model: SemanticModel, calleeIdSink?: CalleeIdSink, + /** + * `ScopeResolver.namespaceExportsIncludeImportedNames`, forwarded rather than + * re-derived. The pass names no language; it asks the provider the same + * question `receiver-bound-calls` asks before resolving a namespace member, + * so the CALL and the REGISTRATION forms of `hub.fn` cannot disagree. + */ + publishesImportedNames = false, ): { usesEmitted: number; callsEmitted: number; @@ -86,18 +374,16 @@ export function emitPropertyDispatchCalls( for (const parsed of parsedFiles) { for (const site of parsed.referenceSites) { if (site.kind !== 'value-ref') continue; - const def = findCallableBindingInScope(site.inScope, site.name, scopes); + const def = resolveValueRefTarget( + site, + parsed.filePath, + scopes, + model, + publishesImportedNames, + ); if (def === undefined) continue; - const ok = tryEmitEdge( - graph, - scopes, - nodeLookup, - site, - def, - 'scope-resolution: value-ref', - seen, - ); + const ok = tryEmitEdge(graph, scopes, nodeLookup, site, def, VALUE_REF_EDGE_REASON, seen); if (ok) usesEmitted++; if (site.propertyKey === undefined) continue; diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index bb9854b3e..5602dcb8d 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -178,7 +178,7 @@ export const scopeResolutionPhase: PipelinePhase = { logHeapProbe('scopeResolution-enter'); const { scannedFiles } = getPhaseOutput(deps, 'structure'); const parseOutput = getPhaseOutput(deps, 'parse'); - const { model, parsedFiles: workerParsedFiles } = parseOutput; + const { model, parsedFiles: workerParsedFiles, contentLanguageByPath } = parseOutput; const scopeExtractionFailures = new Set(parseOutput.scopeExtractionFailures); // SemanticModel populated during `parse`: scope-resolution consumes // TypeRegistry / MethodRegistry / SymbolTable lookups instead of @@ -258,7 +258,9 @@ export const scopeResolutionPhase: PipelinePhase = { (typeof scannedFiles)[number][] >(); for (const f of scannedFiles) { - const fileLang = getLanguageFromFilename(f.path); + const fileLang = contentLanguageByPath.has(f.path) + ? (contentLanguageByPath.get(f.path) ?? null) + : getLanguageFromFilename(f.path); if (fileLang === null) continue; // Tree-sitter providers require an available grammar. Standalone regex // providers deliberately have none and re-extract on the main thread. diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts index bcfe5a983..ee98e3807 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts @@ -28,6 +28,7 @@ import { swiftScopeResolver } from '../../languages/swift/scope-resolver.js'; import { dartScopeResolver } from '../../languages/dart/scope-resolver.js'; import { vueScopeResolver } from '../../languages/vue/scope-resolver.js'; import { zigScopeResolver } from '../../languages/zig/scope-resolver.js'; +import { objectiveCScopeResolver } from '../../languages/objective-c/scope-resolver.js'; /** Map of `SupportedLanguages` → `ScopeResolver`. The scope-resolution phase * iterates this map directly — every registered resolver runs. This is the @@ -53,4 +54,5 @@ export const SCOPE_RESOLVERS: ReadonlyMap = n [SupportedLanguages.Dart, dartScopeResolver], [SupportedLanguages.Vue, vueScopeResolver], [SupportedLanguages.Zig, zigScopeResolver], + [SupportedLanguages.ObjectiveC, objectiveCScopeResolver], ]); diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index aa30acf91..7cc96e829 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -726,6 +726,7 @@ export function runScopeResolution( const resolutionConfig = input.resolutionConfig; const finalized = finalizeScopeModel(parsedFiles, { hooks: { + importsBindAtLexicalScope: provider.importsBindAtLexicalScope === true, resolveImportTarget: (targetRaw, fromFile, _workspaceIndex, parsedImport) => provider.resolveImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig, { parsedFiles, @@ -737,9 +738,23 @@ export function runScopeResolution( provider.expandsWildcardTo?.(targetModuleScope, parsedFiles) ?? [], mergeBindings: (existing, incoming, scopeId) => provider.mergeBindings(existing, incoming, scopeId), + wildcardCollisionIsAmbiguous: provider.exclusiveWildcardReexports === true, + namedImportsBindTopLevelOnly: provider.namedImportsBindTopLevelOnly === true, }, }); logHeapProbe('sr-post-finalize', `lang=${provider.language}`); + // `export *` collisions the shared finalize refused to bind (WS1 C2). Recorded + // as outcomes so the refusal is auditable next to the name-fallback census — + // a silently unresolved importer is indistinguishable from a resolver gap. + for (const refused of finalized.stats.ambiguousWildcardExports) { + recordResolutionOutcome({ + kind: 'reexport-ambiguous', + candidateIds: refused.candidateDefIds, + phase: 'finalize', + filePath: refused.filePath, + name: refused.name, + }); + } // One store and ONE writer rule for heritage instantiations (#2912), shared by // the pre-pass below and by the language hook further down — a heritage shape // the pre-pass cannot express (Rust `impl T for S`, Dart `implements`) records @@ -1085,6 +1100,12 @@ export function runScopeResolution( workspaceIndex, { allowGlobalFallback: provider.allowGlobalFreeCallFallback === true, + language: provider.language, + isGlobalNameFallbackPlausible: provider.isGlobalNameFallbackPlausible, + sourceTextOf: + provider.isGlobalNameFallbackPlausible !== undefined + ? (filePath: string) => getFileContents().get(filePath) + : undefined, constructorCallTargetsClass: provider.constructorCallTargetsClass === true, markConstructionSites: provider.markConstructionSites === true, isFileLocalDef: provider.isFileLocalDef, @@ -1210,7 +1231,12 @@ export function runScopeResolution( indexes, emitParsedFiles, postHeritageNodeLookup, + readonlyModel, calleeIdAccumulator, + // Same provider hook the receiver-bound pass consults for a namespace + // member (Case 1). Without it a hub module's re-exported callable + // resolves when CALLED and declines when REGISTERED. + provider.namespaceExportsIncludeImportedNames === true, ); if (propertyDispatch.skippedKeys > 0) { // Never drop dispatch coverage silently: a hook table larger than the diff --git a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts index a0e5e0919..8a14460a6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts @@ -108,6 +108,60 @@ export type ResolutionOutcome = * every real codebase and taught readers to ignore it. */ readonly receiverOrigin?: ReceiverOrigin; + } + /** + * The global-name fallback fired: a callable was chosen because its SIMPLE + * NAME is unique in the workspace, with no import or scope chain leading to + * it. A labeled low-confidence edge WAS emitted. + * + * Separate from `resolved` because it is not a resolution, and separate from + * `suppressed` because an edge exists. Counting it is the only way a reader + * can tell how much of a language's call graph rests on name uniqueness — the + * number that was previously invisible because these edges were emitted with + * the same reason and confidence as import-resolved ones. + */ + | { + readonly kind: 'fallback-guessed'; + readonly targetId: string; + readonly language?: string; + readonly phase: string; + readonly filePath: string; + readonly name: string; + readonly range: Range; + } + /** + * A global-name-fallback candidate was REFUSED by the language's plausibility + * hook: the language's own visibility rules make that call impossible, so no + * edge was emitted. + * + * The counterpart of `fallback-guessed`, and the pair is what makes the + * refusal auditable — a refusal count with no guess count cannot distinguish + * "the guard works" from "the guard rejects everything". + */ + | { + readonly kind: 'fallback-refused'; + readonly candidateId: string; + readonly language?: string; + readonly phase: string; + readonly filePath: string; + readonly name: string; + readonly range: Range; + } + /** + * A barrel re-exported `name` through two or more `export *` sources that + * each declare it, so the language names no winner. The shared finalize pass + * REFUSED the binding (see `FinalizeStats.ambiguousWildcardExports`) instead + * of publishing the first-listed source as `import-resolved`; every importer + * of `name` through `filePath` stays unresolved. No `range`: the collision + * belongs to the file's export surface, not to one statement. + */ + | { + readonly kind: 'reexport-ambiguous'; + readonly candidateIds: readonly string[]; + readonly phase: string; + /** The barrel file whose `export *` sources collide. */ + readonly filePath: string; + readonly name: string; }; /** diff --git a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts index 62e996737..54daf1c92 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/scope/walkers.ts @@ -181,6 +181,8 @@ export function namesAtScope(scopeId: ScopeId, scopes: ScopeResolutionIndexes): export function isClassLike(t: string): boolean { return ( t === 'Class' || + t === 'Protocol' || + t === 'Category' || t === 'Interface' || t === 'Struct' || t === 'Record' || @@ -351,6 +353,82 @@ export function isNamespaceNameShadowed( return true; } +/** + * Does something between `inScope` and its module scope bind `name` to + * ANYTHING other than `def`? + * + * `isNamespaceNameShadowed` asks the same question for a namespace handle, + * where any local binding of the name is by definition not the import. A + * CONTAINER receiver needs the extra clause: the container may itself be the + * local declaration (`fn make() { const Local = struct {…}; … Local.go … }`), + * and reading that as its own shadow would suppress exactly the resolutions it + * is meant to permit — the #2723 mistake, one channel over. + * + * So a scope that binds the name answers immediately, and the answer is "not + * shadowed" only when one of that scope's bindings IS `def`. A name bound in a + * nearer scope to something else — a parameter, a local, a type binding — wins + * the lexical race, which is the whole point: `findClassBindingInScope` filters + * the chain by `isClassLike` and therefore cannot see that it lost it. + * + * The MODULE scope is inspected too, unlike `isNamespaceNameShadowed`, and the + * exemption is what makes that safe. That guard stops one rung short because a + * namespace import writes its own name into the module scope and would read as + * its own shadow (#2723); here the owner is compared by identity, so the binding + * that IS the owner exempts itself and only a binding to something ELSE answers + * `true`. Stopping short would leave the exact hole this walk exists to close: + * `findClassBindingInScope` steps over a module-scope binding that is not + * class-like and then answers from a WORKSPACE-wide qualified-name index, so + * `const Gauge = @import("other.zig").SOME_CONST;` in a file that never imports + * `Gauge.zig` would still resolve `Gauge.read` to that file's container. + * `lookupBindingsAt` is used at that scope and only there, because an imported + * alias lives in the finalized channel rather than in `scope.bindings`. + * + * Fail-closed like its sibling: a missing scope or a parent cycle answers + * `true`, because suppressing a resolution costs a missing edge while trusting a + * corrupt chain costs a wrong one. + */ +export function isOwnerNameShadowedBySomethingElse( + name: string, + def: SymbolDefinition, + inScope: ScopeId, + scopes: ScopeResolutionIndexes, +): boolean { + let currentId: ScopeId | null = inScope; + const visited = new Set(); + while (currentId !== null) { + if (visited.has(currentId)) return true; + visited.add(currentId); + const scope = scopes.scopeTree.getScope(currentId); + if (scope === undefined) return true; + if (scope.kind !== 'Object') { + const isModule = scope.kind === 'Module'; + const imported = isModule ? lookupBindingsAt(currentId, name, scopes) : []; + const bindsHere = + scope.bindings.has(name) || + scope.typeBindings.has(name) || + scope.lexicalNames?.has(name) === true || + imported.length > 0 || + scope.ownedDefs.some((d) => { + const qualifiedName = d.qualifiedName; + if (qualifiedName === undefined) return false; + const dot = qualifiedName.lastIndexOf('.'); + return (dot === -1 ? qualifiedName : qualifiedName.slice(dot + 1)) === name; + }); + if (bindsHere) { + if ((scope.bindings.get(name) ?? []).some((b) => b.def.nodeId === def.nodeId)) return false; + if (scope.ownedDefs.some((d) => d.nodeId === def.nodeId)) return false; + if (imported.some((b) => b.def.nodeId === def.nodeId)) return false; + return true; + } + } + // The module scope is the last rung, not a rung to skip: nothing above it + // can shadow a name for this file. + if (scope.kind === 'Module') return false; + currentId = scope.parent; + } + return true; +} + export function findReceiverTypeBinding( startScope: ScopeId, receiverName: string, @@ -431,9 +509,10 @@ export function moduleScopeIdOf( /** * Look up a class-like binding by name in the given scope's chain. * - * "Class-like" covers `Class | Interface | Struct | Record | Enum | - * Trait` via the shared `isClassLike` predicate — every kind that - * collapses to `@scope.class` in the scope-extractor query contract. + * "Class-like" covers `Class | Interface | Struct | Record | Enum | Trait | + * Protocol | Category` via the shared `isClassLike` predicate. Objective-C + * protocol and category definitions are graph-side containers rather than + * `@scope.class` captures. * * Walks the scope chain upward and consults TWO sources at each step: * 1. `scope.bindings` — populated during scope-extraction Pass 2 with @@ -703,6 +782,20 @@ function soleBoundBaseName(bound: string): string | undefined { return base.length === 0 ? undefined : base; } +export type ClassBindingLookup = { + /** + * When false, skip the workspace-unique QualifiedNameIndex hit (and the + * dotted-tail variant). Default true — Go inheritance still needs that + * hit because namespace-style imports often create no scope binding. + * + * Constructor-form free calls pass false so a unique type name is not + * treated as a precise in-scope binding. Those sites belong to + * `pickUniqueGlobalClass`, which labels the edge as a guess and runs + * `isGlobalNameFallbackPlausible`. + */ + readonly uniqueQualifiedNameFallback?: boolean; +}; + export function findClassBindingInScope( startScope: ScopeId, receiverName: string, @@ -719,6 +812,7 @@ export function findClassBindingInScope( * selection. Only receiver-chain base and step resolution passes this. */ stripDecoration?: DecorationStripper, + lookup?: ClassBindingLookup, ): SymbolDefinition | undefined { // A TYPE PARAMETER is not a class, and it is checked before every route below // rather than inside one of them because each route would otherwise reach a @@ -727,7 +821,13 @@ export function findClassBindingInScope( // fallback after stripping. The declaration that introduced the parameter is // the only thing that knows, and it knows for all three. if (bindsTypeParameter(startScope, receiverName, scopes)) { - return resolveThroughTypeParameterBound(startScope, receiverName, scopes, stripDecoration); + return resolveThroughTypeParameterBound( + startScope, + receiverName, + scopes, + stripDecoration, + lookup, + ); } const local = walkScopeChain(startScope, receiverName, scopes, (def) => isClassLike(def.type)); @@ -736,21 +836,25 @@ export function findClassBindingInScope( // Fallback for languages (Go) where namespace-style imports don't // create scope bindings: resolve via QualifiedNameIndex. Only fires // when the scope-chain walk found nothing; single-match wins. - const qnames = scopes.qualifiedNames.get(receiverName); - if (qnames.length === 1) { - const def = scopes.defs.get(qnames[0]!); - if (def !== undefined && isClassLike(def.type)) return def; - } - // Second fallback: dotted names like "models.User" — try the simple - // name (tail after last dot) for languages where defs are indexed by - // simple name (Go). Only when the dotted lookup fails. - if (receiverName.includes('.')) { - const simple = receiverName.slice(receiverName.lastIndexOf('.') + 1); - if (simple.length > 0 && simple !== receiverName) { - const simpleIds = scopes.qualifiedNames.get(simple); - if (simpleIds.length === 1) { - const def = scopes.defs.get(simpleIds[0]!); - if (def !== undefined && isClassLike(def.type)) return def; + // Constructor-form free calls opt out — a unique type name is a guess, + // not proof the type is in scope (see `ClassBindingLookup`). + if (lookup?.uniqueQualifiedNameFallback !== false) { + const qnames = scopes.qualifiedNames.get(receiverName); + if (qnames.length === 1) { + const def = scopes.defs.get(qnames[0]!); + if (def !== undefined && isClassLike(def.type)) return def; + } + // Second fallback: dotted names like "models.User" — try the simple + // name (tail after last dot) for languages where defs are indexed by + // simple name (Go). Only when the dotted lookup fails. + if (receiverName.includes('.')) { + const simple = receiverName.slice(receiverName.lastIndexOf('.') + 1); + if (simple.length > 0 && simple !== receiverName) { + const simpleIds = scopes.qualifiedNames.get(simple); + if (simpleIds.length === 1) { + const def = scopes.defs.get(simpleIds[0]!); + if (def !== undefined && isClassLike(def.type)) return def; + } } } } @@ -796,6 +900,7 @@ function resolveThroughTypeParameterBound( parameterName: string, scopes: ScopeResolutionIndexes, stripDecoration?: DecorationStripper, + lookup?: ClassBindingLookup, ): SymbolDefinition | undefined { const bound = typeParameterAt(startScope, parameterName, scopes)?.bound; if (bound === undefined) return undefined; @@ -803,7 +908,7 @@ function resolveThroughTypeParameterBound( if (baseName === undefined || baseName === parameterName) return undefined; // A bound naming another parameter terminates here rather than recursing. if (bindsTypeParameter(startScope, baseName, scopes)) return undefined; - return findClassBindingInScope(startScope, baseName, scopes, stripDecoration); + return findClassBindingInScope(startScope, baseName, scopes, stripDecoration, lookup); } function normalizeTemplateArgToken(value: string): string { @@ -1204,6 +1309,7 @@ export function resolveInheritanceBaseInScope( scopes: ScopeResolutionIndexes, rawQualifiedName?: string, enclosingClassDef?: SymbolDefinition, + lookup?: ClassBindingLookup, ): SymbolDefinition | undefined { // #1982: when the source wrote a qualified base (`Other::Inner`), resolve it // against the full-path QualifiedNameIndex FIRST, so a same-tail nested base @@ -1223,7 +1329,7 @@ export function resolveInheritanceBaseInScope( if (qualified !== undefined) return qualified; } return ( - findClassBindingInScope(startScope, baseName, scopes) ?? + findClassBindingInScope(startScope, baseName, scopes, undefined, lookup) ?? resolveAmbiguousInheritanceBaseViaImports(startScope, baseName, scopes) ); } @@ -1449,6 +1555,43 @@ function dirnameOf(filePath: string): string { return idx === -1 ? '' : filePath.slice(0, idx); } +/** + * True when the caller can see `classDef` through import evidence, not + * merely because the class name is workspace-unique. + * + * Exact imported file (`#include "user.h"`) and a finalize-resolved + * `targetDefId` (named `use` / `pub use` followed to the def) are + * precise. An import that *names* this class (`targetExportedName`) + * is also precise — a barrel's `targetFile` is the re-exporting module, + * not the defining file. A sibling-file import of a *different* name + * is not: `use crate::a::bar` does not make `Foo` in `a/foo.rs` visible. + * + * Directory-only matching is intentionally omitted. The same-directory + * tier on `resolveAmbiguousInheritanceBaseViaImports` is for picking + * among *ambiguous* names (C# namespace `using`); it is not proof a + * unique constructor type is in scope. + */ +export function isClassFileImportGrounded( + startScope: ScopeId, + classDef: Pick, + scopes: ScopeResolutionIndexes, + importedName: string, +): boolean { + const callerModule = moduleScopeIdOf(startScope, scopes); + const callerFile = scopes.scopeTree.getScope(startScope)?.filePath; + if (callerFile !== undefined && classDef.filePath === callerFile) return true; + if (callerModule === null) return false; + const importEdges = scopes.imports.get(callerModule); + if (importEdges === undefined || importEdges.length === 0) return false; + for (const edge of importEdges) { + if (edge.targetDefId !== undefined && edge.targetDefId === classDef.nodeId) return true; + if (edge.targetFile === null) continue; + if (edge.targetFile === classDef.filePath) return true; + if (importedName.length > 0 && edge.targetExportedName === importedName) return true; + } + return false; +} + /** * Predicate for value-receiver bridge: the labels for which * `reconcileOwnership` registers methods/fields under the def's diff --git a/gitnexus/src/core/ingestion/scope-resolution/utils/name-fallback-visibility.ts b/gitnexus/src/core/ingestion/scope-resolution/utils/name-fallback-visibility.ts new file mode 100644 index 000000000..2c66f1c05 --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/utils/name-fallback-visibility.ts @@ -0,0 +1,97 @@ +/** + * Language-agnostic primitives for `ScopeResolver.isGlobalNameFallbackPlausible` + * implementations. + * + * The hook itself is per-language — the RULES here are not. What every + * implementation needs is the same small set of path arithmetic: which + * directory a file sits in, and whether a module path a caller wrote can name + * a given file or directory. Those questions are about paths, not about any + * language, so they live in shared code (see AGENTS.md: shared ingestion must + * not name languages) and the language files supply only the semantics. + */ + +import type { ParsedFile } from 'gitnexus-shared'; + +/** POSIX-style parent directory. `''` for a file at the repo root. */ +export function directoryOf(filePath: string): string { + const slash = filePath.lastIndexOf('/'); + return slash === -1 ? '' : filePath.slice(0, slash); +} + +/** + * Split a module path into segments, accepting the three separators languages + * spell module nesting with: `/` (Go, Node), `::` (Rust, C++) and `.` (JVM, + * Python). Empty segments and `.` are dropped, so a leading `./` contributes + * nothing. Named root prefixes are NOT stripped here — `crate::a` yields + * `['crate', 'a']`; a language whose paths carry one (Rust's `crate::` / + * `super::`) removes it before calling, see `rustUsePathOf`. + * + * `.` is only treated as a separator when the path contains no `/`: a Node + * specifier like `./util/parse.js` must not split on the extension dot. + */ +export function moduleSegments(modulePath: string): readonly string[] { + const bySlashOrColon = modulePath.split(/\/|::/).filter((s) => s !== '' && s !== '.'); + if (modulePath.includes('/')) return bySlashOrColon; + return bySlashOrColon.flatMap((s) => s.split('.').filter((p) => p !== '')); +} + +/** + * Does a module path a caller wrote reach `targetPath`? + * + * True when either side's segments are a SUFFIX of the other's. Both directions + * are needed and neither alone is sufficient: + * + * - The written path is usually longer than the repo-relative one, because it + * carries a module/package prefix that is not a directory + * (`github.com/org/svc/internal/models` → `internal/models`). + * - The repo-relative path is longer when the manifest that defines the module + * root sits in a subdirectory of the analyzed tree (`svc/go.mod`, so + * `svc/internal/models` is written `mod/internal/models`). + * + * So the match is on ALIGNED TRAILING SEGMENTS, and it succeeds when either the + * shorter side is fully contained in the longer, or at least two segments align. + * Both conditions are needed: full containment covers a one-segment package + * (`mod/models` reaching `models`), while the two-segment floor covers the case + * where neither side contains the other because each carries a different root + * (`mod/internal/models` vs `svc/internal/models`). A SINGLE aligned segment + * with neither side contained is not enough — `handlers` appearing at the end of + * two unrelated trees says nothing. + * + * Tolerance is the SAFE direction here: this predicate is consulted to decide + * whether to REFUSE an edge, so over-matching loses a refusal (the edge stays, + * labeled and excluded from flows) while under-matching loses a real edge. + */ +export function modulePathReaches(writtenPath: string, targetPath: string): boolean { + const written = moduleSegments(writtenPath); + const target = moduleSegments(targetPath); + if (written.length === 0 || target.length === 0) return false; + const shorter = Math.min(written.length, target.length); + let aligned = 0; + while ( + aligned < shorter && + written[written.length - 1 - aligned] === target[target.length - 1 - aligned] + ) { + aligned++; + } + return aligned === shorter || aligned >= 2; +} + +/** Every module path this file's import statements named, in source order. */ +function importedModulePaths(parsed: ParsedFile): readonly string[] { + return parsed.parsedImports.map((imp) => imp.targetRaw); +} + +/** True when any of the caller's imports reaches `targetPath`. */ +export function anyImportReaches(parsed: ParsedFile, targetPath: string): boolean { + for (const written of importedModulePaths(parsed)) { + if (modulePathReaches(written, targetPath)) return true; + } + return false; +} + +/** Strip a trailing file extension. `a/b.rs` → `a/b`; `a/b` → `a/b`. */ +export function stripExtension(filePath: string): string { + const slash = filePath.lastIndexOf('/'); + const dot = filePath.lastIndexOf('.'); + return dot > slash ? filePath.slice(0, dot) : filePath; +} diff --git a/gitnexus/src/core/ingestion/scope-resolution/value-ref-edges.ts b/gitnexus/src/core/ingestion/scope-resolution/value-ref-edges.ts new file mode 100644 index 000000000..5507f9d0e --- /dev/null +++ b/gitnexus/src/core/ingestion/scope-resolution/value-ref-edges.ts @@ -0,0 +1,14 @@ +/** + * The `reason` text stamped on the USES edge a `value-ref` site emits. + * + * A leaf module on purpose. Two modules need this string and they sit on + * opposite sides of the product: `passes/property-dispatch.ts` writes it at + * analysis time, and `mcp/local/local-backend.ts` reads it back at query time + * to decide whether an answer is `exact`. Re-typing the literal in the reader + * would make the epistemic signal fail SILENTLY the day the writer's text is + * reworded — the query would simply match nothing and every answer would go + * back to claiming certainty, which is the exact defect (#3399) this constant + * exists to close. Importing `property-dispatch.ts` for it instead would drag + * the whole scope-resolution emit graph into the MCP backend for one string. + */ +export const VALUE_REF_EDGE_REASON = 'scope-resolution: value-ref'; diff --git a/gitnexus/src/core/ingestion/tree-sitter-queries.ts b/gitnexus/src/core/ingestion/tree-sitter-queries.ts index 2a157dea6..a930eb69a 100644 --- a/gitnexus/src/core/ingestion/tree-sitter-queries.ts +++ b/gitnexus/src/core/ingestion/tree-sitter-queries.ts @@ -2610,12 +2610,15 @@ export const ZIG_QUERIES = ` import { SupportedLanguages } from 'gitnexus-shared'; +const OBJECTIVE_C_QUERIES = `((translation_unit) @objc.root)`; + export const LANGUAGE_QUERIES: Record = { [SupportedLanguages.TypeScript]: TYPESCRIPT_QUERIES, [SupportedLanguages.JavaScript]: JAVASCRIPT_QUERIES, [SupportedLanguages.Python]: PYTHON_QUERIES, [SupportedLanguages.Java]: JAVA_QUERIES, [SupportedLanguages.C]: C_QUERIES, + [SupportedLanguages.ObjectiveC]: OBJECTIVE_C_QUERIES, [SupportedLanguages.Go]: GO_QUERIES, [SupportedLanguages.CPlusPlus]: CPP_QUERIES, [SupportedLanguages.CSharp]: CSHARP_QUERIES, diff --git a/gitnexus/src/core/ingestion/ts-js-export-marker.ts b/gitnexus/src/core/ingestion/ts-js-export-marker.ts new file mode 100644 index 000000000..b01440433 --- /dev/null +++ b/gitnexus/src/core/ingestion/ts-js-export-marker.ts @@ -0,0 +1,345 @@ +/** + * Export evidence for ECMAScript declarations — the `@declaration.is-exported` + * marker both the TypeScript and the JavaScript capture emitters synthesize. + * + * `SymbolDefinition.isExported` is tri-state, and this is where the three + * states are decided for TS/JS: + * + * - `true` — the declaration sits under an `export_statement` (`export + * function f`, `export const x`, `export default class`), or the + * file names it in an `export { f }` / `export { f as g }` clause + * or an `export default f` / `export = f` statement. + * - `false` — an ESM-shaped file (no CommonJS export assignment) that does + * neither. The declaration is module-private: `export *` cannot + * republish it and it must not be counted as a wildcard provider. + * - no verdict — the file exports through CommonJS (`module.exports = …`, + * `exports.x = …`, top-level `this.x = …`) or is an ambient + * `.d.ts`, where "not under `export`" says nothing about what the + * module publishes. Nothing is emitted, and the reader keeps its + * prior behavior. One CommonJS shape IS decidable and gets `true`: + * a method or property declared directly in the object literal + * assigned to `module.exports` (`module.exports = { alpha() {} }`) + * is that module's export of `alpha`. + * + * Only a declaration reached from the top level through DECLARATION nodes can + * be a module export. The walk therefore stops with `false` at the first + * nesting boundary — a class body, an interface/enum body, a function body, an + * object literal that is not the `module.exports` value: `export class C { + * m() {} }` exports `C`, not `m`; `function w() { function s() {} }` exports + * nothing even when the file has `export { s }` for a different `s`. + * + * The ancestor walk here is deliberately NOT `tsExportChecker` + * (`export-detection.ts`): that checker's text fallback (`text.startsWith('export ')`) + * fires on the `program` node of any file whose first token is `export`, which + * would mark every declaration in such a file exported. + */ + +import type { SyntaxNode } from './utils/ast-helpers.js'; + +export interface EsmExportEvidence { + /** Local names published by `export { … }`, `export default `, `export = `. */ + readonly namedLocals: ReadonlySet; + /** The file exports through a CommonJS assignment: a plain "not under + * `export`" is no verdict there. */ + readonly commonJs: boolean; +} + +/** Read binding patterns, never initializer expressions or property keys. */ +function bindsReceiver(node: SyntaxNode | null, name: string): boolean { + if (node === null) return false; + if (node.type === 'identifier' || node.type === 'shorthand_property_identifier_pattern') { + return node.text === name; + } + if (node.type === 'variable_declarator') + return bindsReceiver(node.childForFieldName('name'), name); + if (node.type === 'assignment_pattern') + return bindsReceiver(node.childForFieldName('left'), name); + if (node.type === 'pair_pattern') return bindsReceiver(node.childForFieldName('value'), name); + if (node.type === 'import_specifier') { + return bindsReceiver(node.childForFieldName('alias') ?? node.childForFieldName('name'), name); + } + if (node.type === 'required_parameter' || node.type === 'optional_parameter') { + return bindsReceiver(node.childForFieldName('pattern'), name); + } + return ( + [ + 'formal_parameters', + 'object_pattern', + 'array_pattern', + 'rest_pattern', + 'import_clause', + 'named_imports', + 'namespace_import', + ].includes(node.type) && node.namedChildren.some((child) => bindsReceiver(child, name)) + ); +} + +/** A locally bound `module`/`exports` is not Node's export receiver. */ +function isExportReceiverShadowed(node: SyntaxNode, name: string): boolean { + for (let scope = node.parent; scope !== null; scope = scope.parent) { + if ( + ['function_expression', 'generator_function', 'class'].includes(scope.type) && + scope.childForFieldName('name')?.text === name + ) + return true; + if ( + bindsReceiver(scope.childForFieldName('parameters'), name) || + bindsReceiver(scope.childForFieldName('parameter'), name) + ) + return true; + if (scope.type === 'for_in_statement' || scope.type === 'for_of_statement') { + if (bindsReceiver(scope.childForFieldName('left'), name)) return true; + } + if (scope.type === 'for_statement') { + const initializer = scope.childForFieldName('initializer'); + if ( + initializer !== null && + (initializer.type === 'lexical_declaration' || + initializer.type === 'variable_declaration') && + initializer.namedChildren.some((child) => bindsReceiver(child, name)) + ) + return true; + } + if (scope.type !== 'program' && scope.type !== 'statement_block') continue; + for (const statement of scope.namedChildren) { + const declaration = + statement.type === 'export_statement' + ? statement.childForFieldName('declaration') + : statement; + if (declaration === null) continue; + if ( + declaration.type === 'import_statement' && + declaration.namedChildren.some( + (child) => child.type === 'import_clause' && bindsReceiver(child, name), + ) + ) + return true; + if ( + declaration.type === 'lexical_declaration' || + declaration.type === 'variable_declaration' + ) { + if (declaration.namedChildren.some((child) => bindsReceiver(child, name))) return true; + } else if ( + ['function_declaration', 'class_declaration'].includes(declaration.type) && + declaration.childForFieldName('name')?.text === name + ) + return true; + } + } + return false; +} + +/** Static dot and bracket spellings of the same CommonJS export object. */ +function isModuleExportsReference(node: SyntaxNode): boolean { + const object = node.childForFieldName('object'); + if ( + object?.type !== 'identifier' || + object.text !== 'module' || + isExportReceiverShadowed(node, 'module') + ) + return false; + if (node.type === 'member_expression') { + return node.childForFieldName('property')?.text === 'exports'; + } + if (node.type !== 'subscript_expression') return false; + const index = node.childForFieldName('index'); + return index?.type === 'string' && (index.text === "'exports'" || index.text === '"exports"'); +} + +/** + * Does the file touch a CommonJS export object anywhere — `module.exports` or + * `exports.x` / `exports[x]` — as an actual expression? Read from AST nodes, + * not source text, so a comment or string mentioning `module.exports` does not + * disable the file's verdicts. + */ +function hasCommonJsExportSurface(root: SyntaxNode): boolean { + for (const member of root.descendantsOfType('member_expression')) { + const object = member.childForFieldName('object'); + if (object === null) continue; + if ( + object.type === 'identifier' && + object.text === 'exports' && + !isExportReceiverShadowed(member, 'exports') + ) + return true; + if (isModuleExportsReference(member)) return true; + } + for (const sub of root.descendantsOfType('subscript_expression')) { + const object = sub.childForFieldName('object'); + if ( + object?.type === 'identifier' && + object.text === 'exports' && + !isExportReceiverShadowed(sub, 'exports') + ) + return true; + if (isModuleExportsReference(sub)) return true; + } + return false; +} + +/** + * Top-level control-flow wrappers around `this.x =` / `this['x'] =`. + * Recurse through these only — a function or class body is a different `this`. + */ +const TOP_LEVEL_THIS_WALK: ReadonlySet = new Set([ + 'if_statement', + 'else_clause', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'while_statement', + 'do_statement', + 'switch_statement', + 'switch_body', + 'switch_case', + 'switch_default', + 'try_statement', + 'catch_clause', + 'finally_clause', + 'labeled_statement', + 'statement_block', +]); + +/** Module-wrapper `this.x =` / `this['x'] =` as a statement. */ +function isThisExportAssignment(stmt: SyntaxNode): boolean { + if (stmt.type !== 'expression_statement') return false; + const assignment = stmt.namedChildren[0]; + const left = + assignment?.type === 'assignment_expression' ? assignment.childForFieldName('left') : null; + return ( + left !== null && + (left.type === 'member_expression' || left.type === 'subscript_expression') && + left.childForFieldName('object')?.type === 'this' + ); +} + +/** + * A top-level `this` export, including `if (enabled) this.api = api` and the + * braced form. Does not walk into function or class bodies. + */ +function hasTopLevelThisExportAssignment(stmt: SyntaxNode): boolean { + if (isThisExportAssignment(stmt)) return true; + if (!TOP_LEVEL_THIS_WALK.has(stmt.type)) return false; + return stmt.namedChildren.some(hasTopLevelThisExportAssignment); +} + +/** Node types below which a declaration is nested, not module-level. */ +const NESTING_BOUNDARIES: ReadonlySet = new Set([ + 'class_body', + 'interface_body', + 'enum_body', + 'object_type', + 'statement_block', + 'arrow_function', + 'function_expression', + 'function_declaration', + 'generator_function', + 'generator_function_declaration', + 'method_definition', + // `export namespace NS { export function f() {} }` / `declare module 'x' { + // export function q(): void }`: an `export` inside these bodies is an export + // of the namespace/ambient module, not of the file. + 'internal_module', + 'module', + 'ambient_declaration', +]); + +/** + * Scan a file's top level once. `undefined` means the file's export surface + * cannot be read at all (ambient `.d.ts`), so no marker should be emitted. + */ +export function collectEsmExportEvidence( + root: SyntaxNode, + filePath: string, +): EsmExportEvidence | undefined { + if (filePath.endsWith('.d.ts')) return undefined; + const namedLocals = new Set(); + // Any CommonJS export surface anywhere in the file — a direct `module.exports + // = …`, an alias (`const m = module.exports; m.x = …`), an `exports.x` — means + // "not under `export`" says nothing. + let commonJs = hasCommonJsExportSurface(root); + for (const stmt of root.namedChildren) { + if (stmt.type === 'export_statement') { + // `export { a } from './x'` / `export type { T } from './t'` re-export + // ANOTHER module's names: they say nothing about a local `a`, and adding + // them here marked a private local of the same name exported. + if (stmt.childForFieldName('source') !== null) continue; + for (const child of stmt.namedChildren) { + if (child.type === 'export_clause') { + for (const spec of child.namedChildren) { + if (spec.type !== 'export_specifier') continue; + const name = spec.childForFieldName('name')?.text; + if (name !== undefined && name !== '') namedLocals.add(name); + } + } else if (child.type === 'identifier') { + // `export default f;` and TS `export = f;`. + namedLocals.add(child.text); + } + } + } else if (hasTopLevelThisExportAssignment(stmt)) { + commonJs = true; + } + } + return { namedLocals, commonJs }; +} + +/** Is `object` the value of a top-level `module.exports = { … }` assignment? */ +function isModuleExportsObject(object: SyntaxNode): boolean { + const assignment = object.parent; + if (assignment === null || assignment.type !== 'assignment_expression') return false; + if (assignment.childForFieldName('right')?.id !== object.id) return false; + const left = assignment.childForFieldName('left'); + if (left === null || !isModuleExportsReference(left)) return false; + return ( + assignment.parent?.type === 'expression_statement' && + assignment.parent.parent?.type === 'program' + ); +} + +/** + * The export verdict for a declaration whose NAME node is `nameNode`: + * `true` / `false` as documented in the header, `undefined` when this file's + * export surface cannot decide it (CommonJS, other than the `module.exports` + * object literal itself). + */ +export function esmExportVerdict( + nameNode: SyntaxNode, + evidence: EsmExportEvidence, +): boolean | undefined { + // The name node's own declaration node is where the walk starts; the + // declaration itself (a `method_definition`, a `function_declaration`) must + // not count as its own nesting boundary. + let current: SyntaxNode | null = nameNode.parent; + // An `export` keyword is only a FILE-level export when the walk reaches the + // program without crossing a nesting boundary — one inside a namespace or + // ambient-module body is that container's export (see NESTING_BOUNDARIES). + let underExport = false; + let functionScopedVariable = false; + while (current !== null && current.type !== 'program') { + if (current.type === 'variable_declaration') functionScopedVariable = true; + if (current.type === 'export_statement') { + underExport = true; + current = current.parent; + continue; + } + if (current.type === 'object') { + // `module.exports = { alpha() {} }`: the literal's own members are the + // module's exports. Any other object literal is a nesting boundary. + if (isModuleExportsObject(current) && nameNode.parent?.parent?.id === current.id) return true; + return evidence.commonJs ? undefined : false; + } + if (NESTING_BOUNDARIES.has(current.type) && current.id !== nameNode.parent?.id) { + // `var` crosses blocks but never a function/namespace boundary. A + // module-scoped `var` can therefore be exported by a later clause. + if (current.type === 'statement_block' && functionScopedVariable) { + current = current.parent; + continue; + } + return evidence.commonJs ? undefined : false; + } + current = current.parent; + } + if (underExport) return true; + if (evidence.commonJs) return undefined; + return evidence.namedLocals.has(nameNode.text); +} diff --git a/gitnexus/src/core/ingestion/utils/symbol-labels.ts b/gitnexus/src/core/ingestion/utils/symbol-labels.ts index 0751324dc..5f9eb86eb 100644 --- a/gitnexus/src/core/ingestion/utils/symbol-labels.ts +++ b/gitnexus/src/core/ingestion/utils/symbol-labels.ts @@ -23,6 +23,8 @@ export const SYMBOL_NODE_LABELS: ReadonlySet = new Set([ 'Function', 'Method', 'Class', + 'Protocol', + 'Category', 'Interface', 'CodeElement', 'Struct', diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index e6b776228..6c9f8ec10 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -19,7 +19,7 @@ import PHP from 'tree-sitter-php'; import Ruby from 'tree-sitter-ruby'; import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js'; import { SupportedLanguages } from 'gitnexus-shared'; -import { getProvider } from '../languages/index.js'; +import { getLanguageForFileContent, getProvider } from '../languages/index.js'; import { getTreeSitterBufferSize, getTreeSitterContentByteLength, @@ -87,6 +87,11 @@ let Zig: TreeSitterLanguage | null = null; try { Zig = requireVendoredGrammar('tree-sitter-zig') as TreeSitterLanguage; } catch {} + +let ObjectiveC: TreeSitterLanguage | null = null; +try { + ObjectiveC = requireVendoredGrammar('tree-sitter-objc') as TreeSitterLanguage; +} catch {} import { getLanguageFromFilename } from 'gitnexus-shared'; import { buildDefinitionPreScan, @@ -251,7 +256,7 @@ interface ParsedRelationship { id: string; sourceId: string; targetId: string; - type: 'DEFINES' | 'HAS_METHOD' | 'HAS_PROPERTY'; + type: 'DEFINES' | 'DECLARES' | 'HAS_METHOD' | 'HAS_PROPERTY'; confidence: number; reason: string; } @@ -557,6 +562,7 @@ const languageMap: Record = { [SupportedLanguages.Java]: Java, ...(C ? { [SupportedLanguages.C]: C } : {}), [SupportedLanguages.CPlusPlus]: CPP, + ...(ObjectiveC ? { [SupportedLanguages.ObjectiveC]: ObjectiveC } : {}), [SupportedLanguages.CSharp]: CSharp, [SupportedLanguages.Go]: Go, [SupportedLanguages.Rust]: Rust, @@ -1269,7 +1275,7 @@ const processBatch = ( // Group by language to minimize setLanguage calls const byLanguage = new Map(); for (const file of files) { - const lang = getLanguageFromFilename(file.path); + const lang = getLanguageForFileContent(file.path, file.content); if (!lang) continue; let list = byLanguage.get(lang); if (!list) { @@ -1739,6 +1745,52 @@ const processFileGroup = ( result.parsedFiles.push(withChannels); } + const semanticGraph = provider.extractSemanticGraph?.(tree, file.path, parseContent); + if (semanticGraph !== undefined) { + for (const node of semanticGraph.nodes) { + result.nodes.push({ + id: node.id, + label: node.label, + properties: { ...node.properties }, + }); + } + for (const relationship of semanticGraph.relationships) { + if ( + relationship.type === 'DECLARES' || + relationship.type === 'DEFINES' || + relationship.type === 'HAS_METHOD' || + relationship.type === 'HAS_PROPERTY' + ) { + result.relationships.push({ ...relationship, type: relationship.type }); + } + } + for (const symbol of semanticGraph.symbols) { + result.symbols.push({ + filePath: symbol.filePath, + name: symbol.name, + nodeId: symbol.nodeId, + type: symbol.type, + ...(symbol.qualifiedName !== undefined ? { qualifiedName: symbol.qualifiedName } : {}), + ...(symbol.parameterCount !== undefined ? { parameterCount: symbol.parameterCount } : {}), + ...(symbol.requiredParameterCount !== undefined + ? { requiredParameterCount: symbol.requiredParameterCount } + : {}), + ...(symbol.parameterTypes !== undefined + ? { parameterTypes: [...symbol.parameterTypes] } + : {}), + ...(symbol.parameterTypeClasses !== undefined + ? { parameterTypeClasses: [...symbol.parameterTypeClasses] } + : {}), + ...(symbol.returnType !== undefined ? { returnType: symbol.returnType } : {}), + ...(symbol.declaredType !== undefined ? { declaredType: symbol.declaredType } : {}), + ...(symbol.ownerId !== undefined ? { ownerId: symbol.ownerId } : {}), + ...(symbol.visibility !== undefined ? { visibility: symbol.visibility } : {}), + ...(symbol.isStatic !== undefined ? { isStatic: symbol.isStatic } : {}), + ...(symbol.isReadonly !== undefined ? { isReadonly: symbol.isReadonly } : {}), + }); + } + } + // Build per-file type environment + constructor bindings in a single AST walk. // The legacy heritage pre-pass that seeded a file-local parentMap for // buildTypeEnv was removed in RING4-1 (#942) along with the rest of the diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 76d1b56be..1d5c680b6 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -568,6 +568,8 @@ export const streamAllCSVsToDisk = async ( const multiLangHeader = 'id,name,filePath,startLine,endLine,content,description'; const constHeader = `${multiLangHeader},convexEndpointFactory`; const MULTI_LANG_TYPES = [ + 'Protocol', + 'Category', 'Struct', 'Enum', 'Macro', diff --git a/gitnexus/src/core/lbug/graph-emit-sink.ts b/gitnexus/src/core/lbug/graph-emit-sink.ts index 6b263701f..d13ab47c7 100644 --- a/gitnexus/src/core/lbug/graph-emit-sink.ts +++ b/gitnexus/src/core/lbug/graph-emit-sink.ts @@ -70,7 +70,7 @@ * replaced, so the obvious rewrite is not the one that shipped. * 3. The remaining ~90 ms was object allocation itself, irreducible while the * read API returns objects — so the five whole-graph scans moved to - * `forEachRelationshipFields`, which passes the four fields they actually + * `forEachRelationshipFields`, which passes the five fields they actually * read as primitives and allocates nothing. See * {@link GraphEmitSink.forEachRelationshipFields}. * @@ -282,14 +282,19 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { * safe because `buildRelRow` never persists `rel.id` and no consumer keys on * it (audited). * - * The dropped `reason`/`step` are safe too, but for a different reason worth - * stating: the PERSISTED row keeps their true values, because `buildRelRow` is - * handed the original relationship on the way through. Only in-memory reads - * see the `'streamed'` placeholder, and the in-pipeline consumers of streamed - * edges read neither field. So e.g. the `ACCESSES reason: 'read'|'write'` - * distinction that MCP queries rely on survives in the database. A future - * in-pipeline consumer needing `reason` or `step` on a streamed edge must add - * the column, not trust the placeholder. + * `reason` IS now retained, as an interned index — the in-pipeline consumer + * this JSDoc anticipated arrived. Process tracing and large-graph community + * detection must exclude global-name-fallback edges, which are emitted at + * exactly their confidence threshold (0.5) and so cannot be separated by + * confidence alone. Interning keeps the cost at one small integer per edge + * (the reason vocabulary is a fixed set of literals), not one string. + * + * `id` and `step` remain dropped. The PERSISTED row keeps `step`'s true value, + * because `buildRelRow` is handed the original relationship on the way + * through; only in-memory OBJECT reads see the `'streamed'`-era placeholder, + * and no in-pipeline consumer of streamed edges reads `step`. A future + * in-pipeline consumer needing `step` must add the column, not trust the + * placeholder. * * Node ids are interned; the strings are shared by reference with the node * map's, so interning adds bookkeeping, not new text. @@ -300,6 +305,12 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { private readonly tgtIx: number[] = []; private readonly relTypes: RelationshipType[] = []; private readonly confidences: number[] = []; + /** Interned reason strings, and the per-edge index into them. The vocabulary + * is a fixed set of emitter literals, so this is O(vocabulary) text plus one + * small integer per edge. */ + private readonly reasonIds = new Map(); + private readonly reasonByIx: string[] = []; + private readonly reasonIx: number[] = []; private finalized = false; /** * Streaming is OFF until {@link beginStreaming} is called by `parse`. @@ -472,6 +483,16 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { this.tgtIx.push(tgtIx); this.relTypes.push(relationship.type); this.confidences.push(relationship.confidence); + this.reasonIx.push(this.internReason(relationship.reason)); + } + + private internReason(reason: string): number { + const existing = this.reasonIds.get(reason); + if (existing !== undefined) return existing; + const ix = this.reasonByIx.length; + this.reasonByIx.push(reason); + this.reasonIds.set(reason, ix); + return ix; } /** Flush + close every writer and return the COPY manifest. Every fd is @@ -599,7 +620,13 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { * with the object-based graph despite holding relationships columnar. */ forEachRelationshipFields( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ): void { this.real.forEachRelationshipFields(fn); for (let ix = 0; ix < this.srcIx.length; ix++) { @@ -608,6 +635,7 @@ export class GraphEmitSink implements KnowledgeGraph, GraphEmitControl { this.nodeIdByIx[this.tgtIx[ix]], this.relTypes[ix], this.confidences[ix], + this.reasonByIx[this.reasonIx[ix]], ); } } diff --git a/gitnexus/src/core/lbug/pdg-emit-sink.ts b/gitnexus/src/core/lbug/pdg-emit-sink.ts index 12db53ac4..fafcd2c53 100644 --- a/gitnexus/src/core/lbug/pdg-emit-sink.ts +++ b/gitnexus/src/core/lbug/pdg-emit-sink.ts @@ -299,7 +299,13 @@ export class PdgEmitSink implements KnowledgeGraph { this.real.forEachRelationship(fn); } forEachRelationshipFields( - fn: (sourceId: string, targetId: string, type: RelationshipType, confidence: number) => void, + fn: ( + sourceId: string, + targetId: string, + type: RelationshipType, + confidence: number, + reason: string, + ) => void, ): void { this.real.forEachRelationshipFields(fn); } diff --git a/gitnexus/src/core/lbug/schema.ts b/gitnexus/src/core/lbug/schema.ts index d8c66d906..3d3edd8ac 100644 --- a/gitnexus/src/core/lbug/schema.ts +++ b/gitnexus/src/core/lbug/schema.ts @@ -69,6 +69,30 @@ CREATE NODE TABLE Class ( PRIMARY KEY (id) )`; +export const PROTOCOL_SCHEMA = ` +CREATE NODE TABLE Protocol ( + id STRING, + name STRING, + filePath STRING, + startLine INT64, + endLine INT64, + content STRING, + description STRING, + PRIMARY KEY (id) +)`; + +export const CATEGORY_SCHEMA = ` +CREATE NODE TABLE Category ( + id STRING, + name STRING, + filePath STRING, + startLine INT64, + endLine INT64, + content STRING, + description STRING, + PRIMARY KEY (id) +)`; + export const INTERFACE_SCHEMA = ` CREATE NODE TABLE Interface ( id STRING, @@ -494,7 +518,7 @@ const ATTACHMENT_TARGET_LABELS: readonly NodeTableName[] = [ ]; /** - * The 66 pairs NEITHER rule above generates — everything left after the two + * The pairs NEITHER rule above generates — everything left after the two * cross products are subtracted. Carried by CONTAINMENT, inheritance, imports * and DI: a container label crossed with a contained label. No predicate * describes that surface (any container can hold any definition). @@ -507,6 +531,8 @@ const ATTACHMENT_TARGET_LABELS: readonly NodeTableName[] = [ * templates), the two `Route|Process` / `Tool|Process` entry points whose * emitter names both labels as literals, and `BasicBlock|BasicBlock`, the PDG * substrate. + * Spring dynamic lookup in a constructor also emits INJECTS to a synthetic + * @Bean CodeElement (#3238), so Constructor|CodeElement belongs here. * * NOTHING A RULE ALREADY COVERS BELONGS HERE. `generatedRelationPairs` skips * any pair present in this block, so a redundant line does not merely duplicate @@ -581,6 +607,9 @@ export const STRUCTURAL_PAIR_DDL = ` FROM File TO Folder, FROM \`Module\` TO \`Namespace\`, FROM \`Namespace\` TO Function, FROM CodeElement TO CodeElement, + FROM CodeElement TO Class, + FROM CodeElement TO Category, + FROM CodeElement TO Method, FROM CodeElement TO \`Module\`, FROM CodeElement TO \`Property\`, FROM Section TO Section, @@ -597,6 +626,7 @@ export const STRUCTURAL_PAIR_DDL = ` FROM File TO Folder, FROM \`Constructor\` TO \`Impl\`, FROM \`Constructor\` TO \`Namespace\`, FROM \`Constructor\` TO \`Typedef\`, + FROM \`Constructor\` TO CodeElement, FROM Route TO Process, FROM Tool TO Process, FROM Destination TO \`Property\`, @@ -714,6 +744,8 @@ export const NODE_SCHEMA_QUERIES = [ FOLDER_SCHEMA, FUNCTION_SCHEMA, CLASS_SCHEMA, + PROTOCOL_SCHEMA, + CATEGORY_SCHEMA, INTERFACE_SCHEMA, METHOD_SCHEMA, CODE_ELEMENT_SCHEMA, diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 8b70ef10e..d9c07e26f 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -17,6 +17,12 @@ import { constants as fsConstants } from 'node:fs'; import { randomUUID } from 'node:crypto'; import { retryRename } from '../storage/fs-atomic.js'; import { acquireIndexLock } from '../storage/index-lock.js'; +import { invalidateNodeWorkspacePackages } from './ingestion/import-resolvers/node-workspace-packages.js'; +import { + logNameFallbackSummary, + summarizeNameFallback, + countCallsByLanguage, +} from './ingestion/scope-resolution/name-fallback-summary.js'; import { runPipelineFromRepo } from './ingestion/pipeline.js'; import { logUnresolvedReceiverFiles, @@ -1076,6 +1082,9 @@ export async function runFullAnalysis( // Scope the degraded-parse log throttle to this run (module-level counter // would otherwise stay saturated on a reused process). resetDegradedParseCounter(); + // The workspace-package memo is per process: this run must see the tree as it + // is now, not as the previous run in a long-lived watch/server process saw it. + invalidateNodeWorkspacePackages(repoPath); const log = (msg: string) => callbacks.onLog?.(stripControlCharacters(msg)); const acquireOpts = { @@ -3861,6 +3870,14 @@ async function runFullAnalysisInner( const resolutionOutcomes = pipelineResult.resolutionOutcomes ?? []; logUnresolvedReceiverFiles(resolutionOutcomes); + // Census of guessed call sites (before edge coalescing), refused candidates + // and ambiguous `export *` names. The legacy `nameFallbackEdges` metadata + // key stores site counts, not the final population of heuristic edges. + const nameFallbackSummary = summarizeNameFallback( + resolutionOutcomes, + countCallsByLanguage(pipelineResult.resolvedCalleeNamesByCaller, pipelineResult.graph), + ); + logNameFallbackSummary(nameFallbackSummary); // Annotated so the capabilities stamp below is compile-checked against // RepoMeta's status unions (tri-review 4669518496 P1/U3) — an unannotated @@ -3977,6 +3994,7 @@ async function runFullAnalysisInner( // Git-only: non-git repos never take the incremental path. schemaFingerprint: hasGitDir(repoPath) ? SCHEMA_FINGERPRINT : undefined, unresolvedReceiverMembers: summarizeUnresolvedReceivers(resolutionOutcomes), + nameFallbackEdges: nameFallbackSummary, scopeExtractionFailures: summarizeScopeExtractionFailures( pipelineResult.scopeExtractionFailures, ), diff --git a/gitnexus/src/core/search/fts-schema.ts b/gitnexus/src/core/search/fts-schema.ts index eca162448..56acb8385 100644 --- a/gitnexus/src/core/search/fts-schema.ts +++ b/gitnexus/src/core/search/fts-schema.ts @@ -25,6 +25,8 @@ export const FTS_INDEXES: readonly FTSIndexDefinition[] = [ // Original 5 (minus File) gain `description`. { table: 'Function', indexName: 'function_fts', properties: FTS_PROPERTIES }, { table: 'Class', indexName: 'class_fts', properties: FTS_PROPERTIES }, + { table: 'Protocol', indexName: 'protocol_fts', properties: FTS_PROPERTIES }, + { table: 'Category', indexName: 'category_fts', properties: FTS_PROPERTIES }, { table: 'Method', indexName: 'method_fts', properties: FTS_PROPERTIES }, { table: 'Interface', indexName: 'interface_fts', properties: FTS_PROPERTIES }, // Remaining EMBEDDABLE_LABELS symbol tables — all CODE_ELEMENT_BASE-shaped diff --git a/gitnexus/src/core/tree-sitter/parser-loader.ts b/gitnexus/src/core/tree-sitter/parser-loader.ts index 8aed12547..33c49fb4f 100644 --- a/gitnexus/src/core/tree-sitter/parser-loader.ts +++ b/gitnexus/src/core/tree-sitter/parser-loader.ts @@ -96,6 +96,17 @@ const SOURCES: Record = { unavailableNote: 'C++ parsing requires `tree-sitter-cpp`. Check the install and native binding.', }, + [SupportedLanguages.ObjectiveC]: { + load: () => requireVendoredGrammar('tree-sitter-objc'), + optional: true, + severity: 'error', + unavailableNote: + 'Objective-C parsing disabled: vendored `tree-sitter-objc` (under ' + + '`gitnexus/vendor/tree-sitter-objc`) could not be loaded. GitNexus ships ' + + 'prebuilt binaries for supported macOS/Linux runner architectures; this usually ' + + 'indicates a corrupted install or native ABI mismatch with the bundled ' + + 'tree-sitter@0.21.1 runtime.', + }, [SupportedLanguages.Go]: { load: () => _require('tree-sitter-go'), unavailableNote: 'Go parsing requires `tree-sitter-go`. Check the install and native binding.', diff --git a/gitnexus/src/core/tree-sitter/vendored-grammars.ts b/gitnexus/src/core/tree-sitter/vendored-grammars.ts index b920c7474..932ec1a70 100644 --- a/gitnexus/src/core/tree-sitter/vendored-grammars.ts +++ b/gitnexus/src/core/tree-sitter/vendored-grammars.ts @@ -33,6 +33,7 @@ export const VENDORED_GRAMMAR_PACKAGES: ReadonlySet = new Set([ 'tree-sitter-proto', 'tree-sitter-swift', 'tree-sitter-kotlin', + 'tree-sitter-objc', 'tree-sitter-zig', ]); @@ -43,7 +44,7 @@ export const vendoredGrammarDir = (packageName: string): string => /** * Load a vendored tree-sitter grammar by its absolute path under `vendor/`. * - * GitNexus vendors six grammars (c/dart/proto/swift/kotlin/zig) inside its own + * GitNexus vendors seven grammars (c/dart/proto/swift/kotlin/objc/zig) inside its own * package under `vendor/`, each shipping committed per-platform prebuilds. They * are deliberately NOT npm dependencies and must NEVER be copied into * `node_modules`: an undeclared package under `node_modules` is "extraneous" to diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 52eaccea1..bcb2cf1b4 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -36,13 +36,12 @@ import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../../core/lbug/l // git utilities available if needed // import { isGitRepo, getCurrentCommit, getGitRoot } from '../../storage/git.js'; import { - parseDiffHunks, + parseDiffHunksResult, coalesceHunksByPath, hunksOverlapRange, findGitRootByDotGit, getCanonicalRepoRoot, getGitRoot, - type FileDiff, } from '../../storage/git.js'; import { realpathSync } from 'fs'; import { @@ -129,6 +128,7 @@ import type { UnresolvedReceiverSummary } from '../../core/ingestion/scope-resol import type { UndecidedSatisfactionSummary } from '../../core/ingestion/scope-resolution/undecided-satisfaction.js'; import { scopeExtractionFailureTotal } from '../../core/ingestion/scope-resolution/scope-extraction-failures.js'; import { lookupCount } from '../../core/ingestion/scope-resolution/summary-maps.js'; +import { VALUE_REF_EDGE_REASON } from '../../core/ingestion/scope-resolution/value-ref-edges.js'; import { DEFERRED_IMPORT_REASON_SUFFIX, TYPE_ONLY_IMPORT_REASON_SUFFIX, @@ -328,6 +328,8 @@ export const VALID_NODE_LABELS = new Set([ 'Folder', 'Function', 'Class', + 'Protocol', + 'Category', 'Interface', 'Method', 'CodeElement', @@ -709,6 +711,39 @@ export interface EpistemicCauses { * "nothing was undecided", and a re-index is what tells the two apart. */ readonly undecidedSatisfaction: number; + /** + * Symbols that name this callable in VALUE position rather than calling it + * (#3399) — a registration table (`bridge.accessor(Element.getNamespaceUri, + * …)`), a callback argument, a function pointer stored in a field. + * + * Unit: SYMBOLS — distinct referrers, the same unit and the same reason as + * `dispatchBoundary`: the reference edge is per-site but the walk's question + * is "who else might reach this", and a referrer that names the callable + * twice is still one place the value escapes from. + * + * Kept separate from `dispatchBoundary` even though both describe dispatch + * the walk cannot follow. That slot counts implementations and + * interface-level consumers found by the heritage probe; these are neither, + * and folding them in would tell a consumer branching on the numbers that an + * interface boundary exists where there is none. The distinction is also the + * actionable one: a dispatch boundary is irreducible, whereas a callable + * value CAN often be followed once the language models the store/load that + * carries it. + * + * The reference itself IS modelled — that is what makes it countable. What is + * missing is the invocation through the value: it happens later, through a + * struct field, a registry lookup, or comptime reflection, and no CALLS edge + * connects the eventual call site back to this symbol. + * + * Zero when the property-dispatch pass DID synthesize that invocation + * (`x.()` through a registered object-literal key): the walk followed + * the registration, so nothing was missed and the result stays `exact`. + * + * Also zero — WITH a boundary note — when the probe itself could not run. + * The note is the signal there; the count is not, which is why a reader must + * branch on `epistemic` first and read the causes as explanation. + */ + readonly callableValueReferences: number; } function epistemicFrom(dropped: { @@ -718,6 +753,7 @@ function epistemicFrom(dropped: { undecided: number; dispatch: number; scopeExtraction: number; + callableValueReferences: number; }): { epistemic: 'exact' | 'lower-bound'; boundaries?: string[]; @@ -736,6 +772,7 @@ function epistemicFrom(dropped: { dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, undecidedSatisfaction: 0, + callableValueReferences: dropped.callableValueReferences, }, } : { epistemic: 'exact' } @@ -752,6 +789,7 @@ function epistemicFrom(dropped: { dispatchBoundary: dropped.dispatch, externalBoundary: dropped.external, undecidedSatisfaction: dropped.undecided, + callableValueReferences: dropped.callableValueReferences, }, }; } @@ -851,6 +889,153 @@ function undecidedSatisfactionBoundaries( return { notes, undecided }; } +/** + * Boundary evidence for callables named in VALUE position (#3399). + * + * `bridge.accessor(Element.getNamespaceUri, null, .{})`, `{ onClick: handler }`, + * `qsort(xs, n, sz, compareItems)` — each REGISTERS a function somewhere + * instead of calling it. The registration is modelled (`value-ref` → a USES + * edge, Kythe `ref` / Joern `METHOD_REF`); the invocation through the stored + * value is not, because it happens later through a struct field, a registry + * lookup or comptime reflection. + * + * That gap is precisely the first half of `tools.ts`'s definition of + * `lower-bound` — the walk provably missed callers — and it was previously + * reported as `exact`. (The second half, a probe that could not run, is what + * the failure branch below publishes; the contract states both because this + * function can produce either.) A + * public DOM accessor bound into a JS bridge table came back LOW/exact with two + * internal callers, which is worse than no answer: `lower-bound` invites the + * reader to look further, `exact` tells them not to bother. + * + * Counted as DISTINCT REFERRERS rather than sites: the question the count + * serves is "how many places does this value escape from", and a table that + * registers the same callable twice is still one table. + * + * NOT every value reference is a gap. Where the property-dispatch pass + * synthesized the invocation side, the walk followed it and the answer stays + * `exact` — see the second probe below. + * + * Three failure modes, three different answers, none of them silence: + * - the query cannot run → hedge, count 0 (a probe that did not answer + * is not evidence of completeness); + * - the query returns nothing → no hedge (a real, measured zero); + * - the reference was followed → no hedge (nothing was missed). + * + * The probe reads the edge's `reason`, which is why writer and reader share + * {@link VALUE_REF_EDGE_REASON}. Language-neutral by construction — every + * provider that emits a `value-ref` capture participates, and one that emits + * none simply gets no rows. + */ +async function callableValueReferenceBoundaries( + lbugPath: string, + symId: string, +): Promise<{ notes: string[]; referrers: number }> { + // `COUNT(DISTINCT …)`, not a capped row list. A `LIMIT n` here would make the + // published cause silently understate a target with more than n + // registrations — and this number is documented as "how many symbols", so a + // reader comparing its magnitude against `receiverTyping` would be comparing + // a truth to a ceiling. Aggregating in the database keeps the work bounded + // without capping the answer; scalar `sym.id` equality plus an implicit + // group-by is the shape `countByType` below already relies on. + // + // `null`, not `[]`, on failure: see below — an empty result set and an + // unanswerable query must not be the same value. + const rows = await executeParameterized( + lbugPath, + `MATCH (other)-[r:CodeRelation]->(sym) + WHERE sym.id = $symId AND r.type = 'USES' AND r.reason = $reason + RETURN COUNT(DISTINCT other.id) AS cnt`, + { symId, reason: VALUE_REF_EDGE_REASON }, + ).catch(() => null); + + // A probe that could not run must never read as certainty — the same rule the + // `loadMeta` read above states, and the whole reason this function exists. + // Returning zero here would publish `exact` on the strength of a query that + // never answered. + if (rows === null) { + return { + referrers: 0, + notes: [ + 'The callable-value-reference probe could not be run against this index, so whether ' + + 'this symbol is registered somewhere as a value is unknown. Treat the caller list as ' + + 'incomplete until it can be re-checked.', + ], + }; + } + const referrers = rows.length > 0 ? Number((rows[0] as any).cnt ?? (rows[0] as any)[0] ?? 0) : 0; + if (!Number.isFinite(referrers) || referrers <= 0) return { notes: [], referrers: 0 }; + + // Registrations whose invocation side the analyzer ALREADY synthesized are + // not a gap. `emitPropertyDispatchCalls` sweep 2 connects `x.()` member + // calls to every function registered under `` and stamps those edges + // `property-dispatch`; where that happened, the walk did not "provably miss" + // the caller and `lower-bound` would be noise sprayed over an answer the + // analyzer actually computed. Zig — the case this was built for — never sets + // a property key (no object-literal key to dispatch through), so it is never + // excluded here; the exclusion exists to keep TypeScript/JavaScript hook + // tables that ARE followed from being downgraded. + // + // SYMBOL-LEVEL, NOT PER-EDGE, and that is only sound because of an invariant + // that lives nowhere near this line. The graph does not record which + // registration produced which synthesized call, so if one symbol could carry + // both a followed and an unfollowed registration, this would zero the note + // over a gap the analyzer provably did not close — #3399 returning through a + // side door. Today no symbol can: + // + // - sweep 2 synthesizes CALLS only for a registration whose site carried a + // `propertyKey` (sweep 1 skips the index when it is undefined); + // - every JS/TS `@reference.value-ref` rule also captures + // `@reference.property-key` — both are object-literal shapes; + // - no Zig `@reference.value-ref` rule captures one. + // + // So a dispatchable registration is always a JS/TS one, an undispatchable + // registration is always a Zig one, and the two never meet on one symbol. + // `test/unit/scope-resolution/value-ref-dispatchability.test.ts` FAILS the day + // that stops holding — a JS/TS rule for a bare callback argument + // (`register(handler)`), a Zig rule that grows a key. When it does, the + // choice to make here is between (a) splitting the edge `reason` into + // dispatchable / undispatchable so this probe can count them apart, and + // (b) hedging any symbol with an undispatchable registration regardless of + // dispatch. (a) is precise and costs a graph-content change; (b) is cheap and + // over-hedges. What is NOT acceptable is leaving this as-is, because a signal + // that quietly stops firing is the defect this whole feature removes. + // + // Given the invariant, the residual today is only the coarseness of the + // exclusion within JS/TS, and hedging every property-value registration in + // every JS/TS codebase is worse: a signal that fires on everything stops + // carrying information, and the fan-out cap warning still sits behind it. + const dispatched = await executeParameterized( + lbugPath, + `MATCH (other)-[r:CodeRelation]->(sym) + WHERE sym.id = $symId AND r.type = 'CALLS' AND r.reason = 'property-dispatch' + RETURN COUNT(r) AS cnt`, + { symId }, + ).catch(() => null); + // Failure here is NOT a reason to skip the hedge: we already know a value + // reference exists, and being unable to prove it was followed leaves the + // conservative answer standing. + const dispatchedCount = + dispatched === null || dispatched.length === 0 + ? 0 + : Number((dispatched[0] as any).cnt ?? (dispatched[0] as any)[0] ?? 0); + if (Number.isFinite(dispatchedCount) && dispatchedCount > 0) { + return { notes: [], referrers: 0 }; + } + + const one = referrers === 1; + return { + referrers, + notes: [ + `${referrers} ${one ? 'symbol references' : 'symbols reference'} this callable as a VALUE ` + + `rather than calling it (a registration table, a callback argument, a stored function ` + + `pointer). The reference is recorded, but the call made THROUGH that value is not: it is ` + + `dispatched later from wherever the value is stored. Callers reached that way are absent ` + + `from this result — actual impact may be higher.`, + ], + }; +} + interface RepoHandle { id: string; // unique key = repo name (basename) name: string; @@ -992,6 +1177,12 @@ export function buildDetectChangesDiffArgs(scope: string, baseRef?: string): str // not `--default-prefix`, which needs git >= 2.42. `--no-ext-diff` stops a // configured external diff driver from replacing the unified output we parse. const args = [ + // Before the subcommand: default core.quotePath C-quotes non-ASCII so + // `diff --git` / `+++` tokens no longer match the unquoted `a/` `b/` forms + // the parser also accepts. The parser still decodes quoted tokens; this + // pin keeps production git from emitting them. + '-c', + 'core.quotePath=false', 'diff', '--ignore-cr-at-eol', '--no-ext-diff', @@ -4249,7 +4440,8 @@ export class LocalBackend { repo.lbugPath, ` MATCH (caller)-[r:CodeRelation]->(n {id: $symId}) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'MEMBER_OF', 'USES', 'DECLARES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + AND (r.type <> 'MEMBER_OF' OR labels(n) <> 'Community') RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind ORDER BY uid, relType LIMIT 30 @@ -4404,7 +4596,8 @@ export class LocalBackend { repo.lbugPath, ` MATCH (n {id: $symId})-[r:CodeRelation]->(target) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'MEMBER_OF', 'USES', 'DECLARES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + AND (r.type <> 'MEMBER_OF' OR labels(target) <> 'Community') RETURN r.type AS relType, target.id AS uid, target.name AS name, target.filePath AS filePath, labels(target)[0] AS kind ORDER BY uid, relType LIMIT 30 @@ -5418,11 +5611,11 @@ export class LocalBackend { return { error: `Git diff failed: ${err.message}` }; } - const fileDiffs: FileDiff[] = parseDiffHunks(diffOutput); + const { files: fileDiffs, unparsedGitHeaders } = parseDiffHunksResult(diffOutput); if (fileDiffs.length === 0) { - // Git printed a diff but none of it parsed: the `+++ b/` headers were not - // where `parseDiffHunks` looks. That is a PARSE failure, not a clean tree, + // Git printed a diff but none of it parsed: no `diff --git` / `+++` + // file header was recognised. That is a PARSE failure, not a clean tree, // and the clean branch below would report it to the pre-commit gate as // `risk_level:'none'`, no `partial`, exit 0 — a false all-clear (#2915). const parseFailed = diffOutput.trim().length > 0; @@ -5455,7 +5648,9 @@ export class LocalBackend { const changedSymbols = new Map(); // Set if a swallowed graph query fails below — surfaces `partial:true` so a // degraded run cannot report a false-clean `risk_level:'low'` (#2283). - let queryDegraded = false; + // An unparsed `diff --git` is the same class: later hunks must not make + // the gate look complete. + let queryDegraded = unparsedGitHeaders > 0; // Hunks arrive grouped per path and already in the graph's 0-based line // space, so every comparison below is base-neutral (#2377). @@ -6994,6 +7189,19 @@ export class LocalBackend { direction === 'downstream' ? Promise.resolve(undefined) : queryConvexDispatchMetadata(repo.lbugPath, symId, symName, symType); + // #3399 — callables named in value position. Upstream only: the question + // "who can reach this symbol" is the one a registration makes unanswerable. + // A downstream walk asks what THIS symbol reaches, which a reference INTO + // it does not affect. + // + // Issued alongside the heritage probe rather than after it, and read into + // `droppedBoundaries` below, so it hedges even when that probe finds + // nothing AND when it throws — a value reference is an independent reason + // a count is short, exactly as the receiver drops above are. + const valueRefPromise = + direction === 'downstream' + ? Promise.resolve({ notes: [] as string[], referrers: 0 }) + : callableValueReferenceBoundaries(repo.lbugPath, symId); const interfaceRowsPromise = executeParameterized( repo.lbugPath, `MATCH (x)-[r:CodeRelation]->(iface) @@ -7014,12 +7222,14 @@ export class LocalBackend { : []), ]); const convexDispatch = await convexDispatchPromise; + const valueRefDrops = await valueRefPromise; const droppedBoundaries = { ...receiverDrops, notes: [ ...receiverDrops.notes, ...scopeExtractionDrops.notes, ...undecidedDrops.notes, + ...valueRefDrops.notes, ...(convexDispatch === undefined ? [] : [convexDispatch.boundary]), ], undecided: undecidedDrops.undecided, @@ -7028,6 +7238,7 @@ export class LocalBackend { // inventing one from the presence of a note. dispatch: 0, scopeExtraction: scopeExtractionDrops.files, + callableValueReferences: valueRefDrops.referrers, }; try { // Discover the interface / abstract supertypes on the target's boundary. @@ -7119,6 +7330,7 @@ export class LocalBackend { dispatchBoundary: droppedBoundaries.dispatch + dispatchBoundarySymbols, externalBoundary: droppedBoundaries.external, undecidedSatisfaction: droppedBoundaries.undecided, + callableValueReferences: droppedBoundaries.callableValueReferences, }, }; } catch { diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index d68b5b70b..01f2a6a31 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -291,14 +291,15 @@ Handles disambiguation: if multiple symbols share the same name, returns ranked NOTE: ACCESSES edges (field read/write tracking) are included in context results with reason 'read' or 'write'. CALLS edges resolve through field access chains and method-call chains (e.g., user.address.getCity().save() produces CALLS edges at each step). COMPLETENESS OF incoming: alongside symbol/incoming/outgoing the result carries the same epistemic envelope impact() returns: -- epistemic: 'exact' | 'lower-bound' — 'lower-bound' means callers exist that this view provably does not list. +- epistemic: 'exact' | 'lower-bound' — 'lower-bound' means incoming is a FLOOR: either the walk provably missed callers, or a probe that would have established completeness could not run. Do not read it as proof that an omitted caller exists — read boundaries for which of the two it is. - boundaries: string[] — one plain-language sentence per reason. Prose for humans; branch on causes instead. -- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction, callableValueReferences } — machine-readable WHY. Every field counts MISSING THINGS, never sentences: - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — RESOLVER GAP: the analyzer dropped that many call sites on this name because it could not type the receiver, so they are missing from incoming. Do not read an absent caller as proof none exists. - causes.externalBoundary (unit: call sites) > 0 — the calls left the indexed program (System.out.println, fetch(...)). NOT a defect: no in-graph node could have been reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary static analysis cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not decide whether a type satisfies an interface, so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Usually fixable by making the missing dependency available to analysis. + - causes.callableValueReferences (unit: symbols) > 0 — that many symbols name this callable as a VALUE instead of calling it (a Zig registration table or const initialiser, a JS/TS object-literal property value). A bare callback argument in JS/TS is not captured today and is not counted, so a 0 does not rule that shape out; nor does it, on an index built before the language emitted these captures — re-analyze first. The reference is in the graph as a USES edge; the call made THROUGH the value is not, because it is dispatched later from wherever the value was stored. incoming.calls is therefore a floor. Follow the USES edges to find the registration, then the code that reads it. It is 0 when the analyzer DID synthesize the dispatch through a registered property key. That exclusion is per SYMBOL, not per registration: a target with BOTH a followed registration and an unfollowed escape reads 0 here, so a 0 means 'no unfollowed registration was proven', not 'this symbol escapes nowhere'. A 0 alongside epistemic 'lower-bound' can also mean the probe itself could not run — read boundaries for which. REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. @@ -488,15 +489,16 @@ Output includes: - affected_processes: which execution flows break and at which step - affected_modules: which functional areas are hit (direct vs indirect; classification-unavailable when that secondary query fails) - byDepth: affected symbols grouped by traversal depth (paginated by limit/offset; omitted when summaryOnly:true — use byDepthCounts for totals per depth, pagination object when truncated). Each item includes a processes:[{id,label,processType,step}] field listing the execution flows that symbol participates in. Empty when the symbol has no process membership. Can ALSO be empty when partial:true is set — either the process-aggregation pass hit its cap before detecting affected processes, or per-symbol enrichment was capped on a very large page. When partial:true, do NOT treat processes:[] as proof of no participation; cross-check the top-level affected_processes list. An item carries staticGated:true only when the edge that reached it is provably unreachable at compile time from the indexed source (today: Zig calls inside an 'if (CONST_FALSE)' body or the else of 'if (CONST_TRUE)'); the field is absent when the edge is live or the language does not model it. Traversal and risk do NOT filter or rank on it: it is metadata for the caller to weigh. -- epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the walk provably missed callers, so the count is a floor. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). +- epistemic: 'exact' | 'lower-bound' — whether impactedCount is the whole story. 'lower-bound' means the count is a FLOOR: either the walk provably missed callers, or a probe that would have established completeness could not run (a failed callable-value-reference query says so in boundaries). It is not itself proof that an omitted caller exists — branch on causes and read boundaries. Absent only on skipped probes (ambiguous-candidate lists, group fan-out). - boundaries: string[] — one plain-language sentence per reason the count is short. Prose for humans; branch on causes instead. -- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: +- causes: { scopeExtractionFiles, receiverTyping, dispatchBoundary, externalBoundary, undecidedSatisfaction, callableValueReferences } — the machine-readable split of WHY, so an agent gating its own edits can tell a fixable analyzer gap from an irreducible one. Every field counts MISSING THINGS, never sentences: - causes.scopeExtractionFiles (unit: files) > 0 — scope extraction still failed after the fallback pass, so scope-resolution edges from those files are absent. A value of 0 does not prove completeness when epistemic is 'lower-bound' because an older or unverified index has no measured file count. Re-run \`gitnexus analyze --force\`; if the reason persists, inspect the extraction warnings. - causes.receiverTyping (unit: call sites) > 0 — the RESOLVER GAP signal: the analyzer dropped that many call sites because it could not establish the receiver's type (unresolved constructor, factory, chained expression). Those callers are absent from byDepth. Treat the result as incomplete: grep the symbol name before deleting or renaming. - causes.externalBoundary (unit: call sites) > 0 — those calls left the indexed program (System.out.println, fetch(...), os.environ.*). NOT a defect and NOT a reason the count is short: there is no in-graph node any edge could have reached. An epistemic:'exact' result can carry this. - causes.dispatchBoundary (unit: symbols) > 0 — DI or interface dispatch: that many symbols sit on or beyond a boundary a static walk cannot cross. Irreducible. A symbol count, not a site count — per-site multiplicity is not retained for these edges — so compare its magnitude with receiverTyping, not its exact value. A framework runtime-proxy boundary can make epistemic lower-bound while this value remains 0 because endpoint metadata proves the gap but cannot count omitted symbols. - causes.undecidedSatisfaction (unit: unjudged interface/type pairs) > 0 — the analyzer could not DECIDE whether a type satisfies an interface (a type in a required signature named a package it could not resolve), so no IMPLEMENTS edge exists and no dispatch boundary was left for the walk to notice. Distinct from every cause above, which count decided facts that could not be attributed; this one counts questions never answered. It is the only cause that shortens a result WITHOUT leaving a trace in the graph, so an unhedged zero on a symbol reached only through such an interface would otherwise read as 'nobody calls this'. Usually fixable: it most often means a dependency is missing from the analyzed tree. + - causes.callableValueReferences (unit: symbols) > 0 — that many symbols name this callable as a VALUE rather than calling it: 'bridge.accessor(Element.getNamespaceUri, ...)' and 'pub const h = onReset;' in Zig, '{ onClick: handler }' in JS/TS. Those are the shapes actually captured today — a bare callback argument in JS/TS ('qsort'-style, 'setTimeout(tick)') is NOT one of them and is not counted, so a 0 here does not rule that shape out. The registration IS modelled (a USES edge); the invocation through the stored value is NOT, because it happens later via a struct field, a registry lookup or comptime reflection. So impactedCount is a floor and a LOW risk verdict on such a symbol is a floor too. Unlike dispatchBoundary this is often reducible — it usually means the language provider does not yet follow that store/load — but until it is, do NOT read an empty or small caller set as 'safe to change'. It is an exact count, not a capped sample. It is 0 when the analyzer DID synthesize the dispatch through a registered property key, and epistemic stays 'exact' on that account. That exclusion is symbol-level, not edge-level — the graph does not record which registration produced which synthesized call — so a symbol with a mix of followed and unfollowed registrations also reads 0: treat a 0 as 'no unfollowed registration was proven', not as proof the value escapes nowhere. A 0 alongside epistemic 'lower-bound' can instead mean the probe could not run at all, so read boundaries to tell those apart. Read from the graph, so it needs no index-time metadata BEYOND the edges being there: an index built by an analyzer that did not yet emit this language's value-ref captures has none, and reports 0. Re-analyze before reading a 0 as measured. REQUIRES RE-INDEX: causes.scopeExtractionFiles, causes.receiverTyping, causes.externalBoundary, causes.undecidedSatisfaction, and framework runtime-proxy boundary detection depend on index-time metadata that only a current analyzer writes. Against an older index the metadata can be absent, which is indistinguishable from "nothing was dropped" unless the schema probe detects the stale index — re-run \`gitnexus analyze\` before trusting a zero or an apparently exact result. diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 5a731ae0b..7aa8635fd 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -65,6 +65,8 @@ import { GITHUB_TOKEN_HOSTS, } from './git-clone.js'; import { createAnalyzeUploadHandler } from './analyze-upload.js'; +import { checkStalenessAsync } from '../core/git-staleness.js'; +import { projectRepoDetail, projectRepoListEntry, resolveLastCommit } from './repo-projection.js'; // Shared with the CLI's `--branch` (via the analyze-config wrapper) so both // entry points accept the same refs. Imported from core — not cli/ — so // createServer does not close a cycle with cli/serve.ts. @@ -1000,18 +1002,24 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => }); // List all registered repos - app.get('/api/repos', async (_req, res) => { + // Rate-limited (CodeQL js/missing-rate-limiting) because this route now spawns + // one `git rev-list` per registered repo to answer freshness: an unauthenticated + // GET that costs N subprocesses is worth the same 60 rpm/IP ceiling `/api/repo` + // already carries. Web callers hit this on connect/switch, never in a loop. + app.get('/api/repos', createRouteLimiter(), async (_req, res) => { try { const repos = await listRegisteredRepos(); + // Checked in parallel, for the reason `list_repos` already does it that + // way: each check spawns an async `git rev-list`, and the sequential + // variant took ~50s across 200 repos (#1363). Projecting inside the map + // keeps the entry and its own check together — an index-matched second + // array is the shape that silently mispairs them if either is reordered. res.json( - repos.map((r) => ({ - name: r.name, - path: r.path, - repoPath: r.path, - indexedAt: r.indexedAt, - lastCommit: r.lastCommit, - stats: r.stats, - })), + await Promise.all( + repos.map(async (r) => + projectRepoListEntry(r, await checkStalenessAsync(r.path, r.lastCommit)), + ), + ), ); } catch (err: any) { res.status(500).json({ error: err.message || 'Failed to list repos' }); @@ -1038,12 +1046,8 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => return; } const meta = await loadMeta(entry.storagePath); - res.json({ - name: entry.name, - repoPath: entry.path, - indexedAt: meta?.indexedAt ?? entry.indexedAt, - stats: meta?.stats ?? entry.stats ?? {}, - }); + const staleness = await checkStalenessAsync(entry.path, resolveLastCommit(entry, meta)); + res.json(projectRepoDetail(entry, meta, staleness)); } catch (err: any) { res.status(500).json({ error: err.message || 'Failed to get repo info' }); } diff --git a/gitnexus/src/server/repo-projection.ts b/gitnexus/src/server/repo-projection.ts new file mode 100644 index 000000000..9ada19403 --- /dev/null +++ b/gitnexus/src/server/repo-projection.ts @@ -0,0 +1,81 @@ +/** + * Response projections for the `serve` repo routes. + * + * Extracted from the route bodies so the field list is assertable. Inline, the + * projections were only reachable by booting a server and indexing a real + * repository, which is how `branch` came to sit on `RegistryEntry` unexposed + * over HTTP while `gitnexus list` printed it, with no test to notice (#3226). + * + * These are pure: the caller resolves the registry entry, the on-disk metadata + * and the staleness check, and passes the results in. + */ +import type { StalenessInfo } from '../core/git-staleness.js'; +import type { RegistryEntry } from '../storage/repo-manager.js'; +import type { RepoMeta } from '../storage/repo-meta.js'; + +/** + * Staleness in the shape MCP `list_repos` already returns + * (`mcp/local/local-backend.ts`): the key is present only when the index is + * actually behind, so "fresh" stays the absence of a field rather than a second + * thing for a client to interpret. Deliberately identical across the two + * surfaces — the same fact should not have two shapes. + * + * `checkStalenessAsync` self-catches and reports 0 commits behind when the + * commit cannot be resolved, so an unanswerable check degrades to "not stale" + * rather than failing the request that carries it. + * + * That makes the field meaningful mainly for `path`-registered repositories, + * where an operator commits into the working tree the index was built from. + * A `url` repository is cloned `--depth 1` and is re-analyzed by the same run + * that pulls it, so its recorded commit is HEAD; and were the two ever to + * diverge, `git rev-list ..HEAD` cannot walk a shallow history and fails + * closed to "not stale". Reporting fresh there is not a claim that the remote + * has not moved — this measures the index against the local working tree, the + * same thing `gitnexus status` and MCP `list_repos` measure. + */ +export const stalenessField = ( + info: StalenessInfo, +): { staleness?: { commitsBehind: number; hint?: string } } => + info.isStale ? { staleness: { commitsBehind: info.commitsBehind, hint: info.hint } } : {}; + +/** One entry of `GET /api/repos`. */ +export const projectRepoListEntry = (entry: RegistryEntry, staleness: StalenessInfo) => ({ + name: entry.name, + path: entry.path, + repoPath: entry.path, + indexedAt: entry.indexedAt, + lastCommit: entry.lastCommit, + stats: entry.stats, + // The registry has carried these since #2106; #3199 made them load-bearing + // over HTTP, because a branch-pinned analyze now gets its own entry and the + // only other way to tell two entries apart is to parse the clone-directory + // slug — a layout detail, not an API contract. + branch: entry.branch, + branches: entry.branches, + ...stalenessField(staleness), +}); + +/** + * `GET /api/repo`. `meta ?? entry` throughout, for the reason `indexedAt` + * already did it: the on-disk metadata is the fresher record when the two + * disagree, and the entry is the fallback for a repo whose meta cannot be read. + */ +export const projectRepoDetail = ( + entry: RegistryEntry, + meta: RepoMeta | null | undefined, + staleness: StalenessInfo, +) => ({ + name: entry.name, + repoPath: entry.path, + indexedAt: meta?.indexedAt ?? entry.indexedAt, + stats: meta?.stats ?? entry.stats ?? {}, + lastCommit: meta?.lastCommit ?? entry.lastCommit, + branch: meta?.branch ?? entry.branch, + ...stalenessField(staleness), +}); + +/** The commit a `/api/repo` staleness check should be measured against. */ +export const resolveLastCommit = ( + entry: RegistryEntry, + meta: RepoMeta | null | undefined, +): string => meta?.lastCommit ?? entry.lastCommit; diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 792c1010b..1e06b0477 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -750,10 +750,189 @@ export interface FileDiff { hunks: DiffHunk[]; } +/** `parseDiffHunks` plus how many `diff --git` headers could not be decoded. */ +export interface DiffHunkParseResult { + files: FileDiff[]; + unparsedGitHeaders: number; +} + +const DIFF_GIT_PREFIX = 'diff --git '; + +/** + * Decode one Git C-quoted token (`"a/\\344\\270\\255.png"`). Returns + * `undefined` when the quotes are unbalanced or a trailing escape is bare. + */ +function unquoteCStyleGitToken(quoted: string): string | undefined { + if (quoted.length < 2 || quoted[0] !== '"' || quoted[quoted.length - 1] !== '"') { + return undefined; + } + // Git C-quotes are byte-oriented: non-ASCII is `\nnn` octal of the UTF-8 + // code units, not JS UTF-16 characters. + const bytes: number[] = []; + const pushChar = (ch: string): void => { + const code = ch.charCodeAt(0); + if (code < 0x80) bytes.push(code); + else bytes.push(...new TextEncoder().encode(ch)); + }; + for (let i = 1; i < quoted.length - 1; i++) { + const ch = quoted[i]; + if (ch !== '\\') { + pushChar(ch); + continue; + } + const next = quoted[++i]; + if (next === undefined) return undefined; + switch (next) { + case '\\': + case '"': + pushChar(next); + break; + case 'n': + bytes.push(0x0a); + break; + case 't': + bytes.push(0x09); + break; + case 'r': + bytes.push(0x0d); + break; + case 'a': + bytes.push(0x07); + break; + case 'b': + bytes.push(0x08); + break; + case 'v': + bytes.push(0x0b); + break; + case 'f': + bytes.push(0x0c); + break; + default: { + if (next < '0' || next > '7') { + pushChar(next); + break; + } + let oct = next; + while (oct.length < 3 && i + 1 < quoted.length - 1) { + const digit = quoted[i + 1]; + if (digit < '0' || digit > '7') break; + oct += digit; + i++; + } + bytes.push(parseInt(oct, 8)); + } + } + } + return new TextDecoder('utf-8').decode(Uint8Array.from(bytes)); +} + +function takeCQuotedToken( + source: string, + start: number, +): { token: string; end: number } | undefined { + if (source[start] !== '"') return undefined; + for (let i = start + 1; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === '"') return { token: source.slice(start, i + 1), end: i + 1 }; + } + return undefined; +} + +function stripGitDstPrefix(raw: string): string | undefined { + return raw.startsWith('b/') ? raw.slice(2) : undefined; +} + +/** Unified-diff paths end at the first TAB (timestamp / empty terminator). */ +function stripUnifiedDiffTab(pathWithOptionalTab: string): string { + const tab = pathWithOptionalTab.indexOf('\t'); + return tab === -1 ? pathWithOptionalTab : pathWithOptionalTab.slice(0, tab); +} + +function decodeGitPathToken(raw: string): string | undefined { + const trimmed = stripUnifiedDiffTab(raw); + if (trimmed.startsWith('"')) return unquoteCStyleGitToken(trimmed); + return trimmed; +} + +/** + * Destination path from `diff --git a/ b/`. + * + * Same-path headers recover `name` from `a/${name} b/${name}` so a dest that + * itself contains ` b/` is not split at the last occurrence. C-quoted tokens + * (default `core.quotePath`) are decoded. Renames that the greedy split would + * mis-parse stay a best-effort dest; `rename to` / `+++` correct them. + */ +function takeGitHeaderPathToken( + source: string, + start: number, +): { path: string; end: number } | undefined { + if (start >= source.length) return undefined; + if (source[start] === '"') { + const tok = takeCQuotedToken(source, start); + if (!tok) return undefined; + const path = unquoteCStyleGitToken(tok.token); + if (path === undefined) return undefined; + return { path, end: tok.end }; + } + if (source.startsWith('b/', start)) { + return { path: stripUnifiedDiffTab(source.slice(start)), end: source.length }; + } + if (source.startsWith('a/', start)) { + for (let i = start + 2; i < source.length; i++) { + if (source.startsWith(' b/', i) || source.startsWith(' "', i)) { + return { path: source.slice(start, i), end: i }; + } + } + } + return undefined; +} + +function filePathFromGitHeader(line: string): string | undefined { + if (!line.startsWith(DIFF_GIT_PREFIX)) return undefined; + const rest = line.slice(DIFF_GIT_PREFIX.length); + + if (rest.startsWith('a/')) { + for (let i = 2; i < rest.length; i++) { + if (!rest.startsWith(' b/', i)) continue; + const nameA = rest.slice(2, i); + const nameB = rest.slice(i + 3); + if (nameA.length > 0 && nameA === nameB) return nameA; + } + } + + const src = takeGitHeaderPathToken(rest, 0); + if (!src) return undefined; + let i = src.end; + while (rest[i] === ' ') i++; + const dest = takeGitHeaderPathToken(rest, i); + return dest ? stripGitDstPrefix(dest.path) : undefined; +} + +function pathFromPlusPlusPlus(line: string): string | undefined { + if (!line.startsWith('+++ ')) return undefined; + const raw = decodeGitPathToken(line.slice(4)); + if (!raw || raw === '/dev/null') return undefined; + return stripGitDstPrefix(raw); +} + +function pathFromRenameTo(line: string): string | undefined { + if (!line.startsWith('rename to ')) return undefined; + return decodeGitPathToken(line.slice('rename to '.length)); +} + /** * Parse unified diff output (with -U0) into per-file hunk ranges. * Extracts the new-file line ranges from @@ hunk headers. * + * The `diff --git` header is also retained as a file entry. This matters for + * binary, rename-only, and mode-only changes, which have no `+++ b/` header. + * Such entries intentionally have no hunks: callers can count the changed + * path without pretending that a symbol line range was touched. + * * A pure deletion adds no new lines, and unified diff spells that empty range * as the line BEFORE it: `@@ -4,2 +3,0 @@` removed old lines 4–5 from between * new lines 3 and 4 (git emits `+0,0` when the deletion is at the head of the @@ -769,13 +948,52 @@ export interface FileDiff { * gap — the widening {@link coalesceHunks} is careful never to do. */ export function parseDiffHunks(diffOutput: string): FileDiff[] { + return parseDiffHunksResult(diffOutput).files; +} + +/** + * Same as {@link parseDiffHunks}, plus a count of `diff --git` lines that + * could not be decoded. `detect_changes` uses the count to fail closed + * (`partial` + `risk_level:'unknown'`) instead of attaching later hunks to a + * previous file. + */ +export function parseDiffHunksResult(diffOutput: string): DiffHunkParseResult { const files: FileDiff[] = []; let current: FileDiff | null = null; + let unparsedGitHeaders = 0; + // `+++` after the first `@@` of a file is hunk body (`+` plus source text + // that itself starts `++ …`), not another file header. + let inHunk = false; for (const line of diffOutput.split('\n')) { - if (line.startsWith('+++ b/')) { - current = { filePath: line.slice(6), hunks: [] }; - files.push(current); + if (line.startsWith(DIFF_GIT_PREFIX)) { + // Drop the previous file first: an unparsed header must not leave + // `current` live for a later `@@` / quoted `+++` to steal. + current = null; + inHunk = false; + const filePath = filePathFromGitHeader(line); + if (filePath) { + current = { filePath, hunks: [] }; + files.push(current); + } else { + unparsedGitHeaders++; + } + } else if (line.startsWith('rename to ')) { + const filePath = pathFromRenameTo(line); + if (!filePath) continue; + if (current) current.filePath = filePath; + else { + current = { filePath, hunks: [] }; + files.push(current); + } + } else if (!inHunk && line.startsWith('+++ ')) { + const filePath = pathFromPlusPlusPlus(line); + if (!filePath) continue; + if (!current || current.filePath !== filePath) { + current = { filePath, hunks: [] }; + files.push(current); + } } else if (line.startsWith('@@') && current) { + inHunk = true; const match = line.match(/@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/); if (match) { const start = parseInt(match[1], 10); @@ -791,7 +1009,7 @@ export function parseDiffHunks(diffOutput: string): FileDiff[] { } } } - return files; + return { files, unparsedGitHeaders }; } /** diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 743b810b4..0ce2b7dab 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -735,7 +735,35 @@ import { copyV8CacheIfPresent, tryLoadV8Cache, writeV8CacheFile } from './v8-sid // v93: Zig call captures inside a comptime-false branch carry // `@reference.static-gated` (feat/zig-static-gated-edges); the site gains // `staticGated` and the CALLS edge a BOOLEAN column. -const SCHEMA_BUMP = 93; +// v94: Objective-C now elides bare, file-scope macro markers before parsing. +// A warm v93 cache can retain error-recovered trees and provider facts that +// omit Objective-C declarations following markers such as RCT_EXTERN_C_END. +// v95: Objective-C header classification no longer treats framework `#import` +// alone as Objective-C syntax. A warm v94 cache can replay Objective-C worker +// output for a C++ header during `--force`, even though the current classifier +// routes that same header through the C++ provider. +// v96: Objective-C macro-marker preprocessing recognizes all C preprocessing +// whitespace before a directive or bare marker. A warm v95 cache can retain +// error-recovered facts for sources that begin those lines with form feed or +// vertical tab. +// v97: Objective-C macro-marker preprocessing recognizes comment-prefixed +// directives and rejects invalid numeric marker prefixes. A warm v96 cache can +// replay error-recovered facts from the previous normalization behavior. +// v98 (#3219): `ZIG_SCOPE_QUERY` gained three `@reference.value-ref` rules — +// bare call argument, qualified call argument (with `@reference.receiver`), and +// const-binding initialiser — so a Zig callable named in VALUE position now +// produces a `value-ref` entry in `ParsedFile.referenceSites` where it produced +// none before. These captures are PARSE-TIME facts, so a warm pre-v98 cache +// replays unchanged `.zig` files with zero value-ref sites, `--force` included +// (shards are content-addressed): `emitPropertyDispatchCalls` then emits no +// USES edge, `callableValueReferenceBoundaries` measures a real zero, and +// `impact` on a registered accessor republishes `epistemic: "exact"` — the exact +// #3399 defect this change exists to close, silently un-fixed. +// v99: ParsedImport retains declaredAtScope and export evidence changes in +// #3190. Old durable ParsedFiles lack the facts needed for scoped binding; +// invalidate both stores so warm indexing actually applies the correction. +// origin/main took 98 for #3219; 99 is the next free value. +const SCHEMA_BUMP = 99; const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index f65fc8ec1..3dddc911a 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -28,6 +28,7 @@ import fs from 'fs/promises'; import path from 'path'; import type { UnresolvedReceiverSummary } from '../core/ingestion/scope-resolution/unresolved-receivers.js'; +import type { NameFallbackSummary } from '../core/ingestion/scope-resolution/name-fallback-summary.js'; import type { UndecidedSatisfactionSummary } from '../core/ingestion/scope-resolution/undecided-satisfaction.js'; import type { ScopeExtractionFailureSummary } from '../core/ingestion/scope-resolution/scope-extraction-failures.js'; @@ -311,6 +312,15 @@ export interface RepoMeta { * reads as absent, and both correctly mean "no hedge available from here". */ undecidedInterfaceSatisfaction?: UndecidedSatisfactionSummary; + /** + * Census of name-guessed call sites before edge coalescing, the impossible + * candidates the run refused, and the ambiguous + * `export *` names it declined to publish. Absent on indexes built before the + * census existed. The legacy key does not imply final edge counts: a precise + * site may prove a dependency shared with a guessed site. + * See `scope-resolution/name-fallback-summary.ts`. + */ + nameFallbackEdges?: NameFallbackSummary; /** * SHA-256 of every file's content at the time of the last successful * indexing run. The next run computes current hashes and diffs against diff --git a/gitnexus/src/types/pipeline.ts b/gitnexus/src/types/pipeline.ts index d665a7acf..4136cca42 100644 --- a/gitnexus/src/types/pipeline.ts +++ b/gitnexus/src/types/pipeline.ts @@ -33,6 +33,14 @@ export interface PipelineResult { * produced; graph edge semantics are unchanged. */ resolutionOutcomes: readonly ResolutionOutcome[]; + /** + * Caller node id → simple names of every callee it has a CALLS edge to, read + * through the streaming sink when one was active (the raw graph holds no + * streamed edge). Denominator for the name-fallback census + * (`countCallsByLanguage`), so a guess count can be read as a share of the + * call graph. Absent only when scope resolution did not run. + */ + resolvedCalleeNamesByCaller?: ReadonlyMap>; /** * Interfaces whose structural-satisfaction check could not be completed * (#2873). Empty for languages with no structural detection. diff --git a/gitnexus/test/fixtures/go-captures-golden/expected-captures.json b/gitnexus/test/fixtures/go-captures-golden/expected-captures.json index c8c5d6efe..0035a60c4 100644 --- a/gitnexus/test/fixtures/go-captures-golden/expected-captures.json +++ b/gitnexus/test/fixtures/go-captures-golden/expected-captures.json @@ -85,7 +85,7 @@ }, "go-constructor-type-inference/cmd/main.go": { "captureGroups": 19, - "digest": "4f2e8bb6dbca59bcb82fc2efd43899b7a92816af28811cfc1858f0ce7ba9f6fe" + "digest": "8dff73176d5890d7ede1d8b9cd9f8555518a1dcc11fbf02190beb474c0f9e47e" }, "go-constructor-type-inference/models/repo.go": { "captureGroups": 8, @@ -252,8 +252,8 @@ "digest": "ad4cfabcd53397ab664ba4bb05df4e1a803455b5c255a19e2ed283c6523a1660" }, "go-method-enrichment/app.go": { - "captureGroups": 15, - "digest": "35929a31f6921feb074f50385aa91879e93f25b88d7f50774fddcfbb7b4e8539" + "captureGroups": 16, + "digest": "b2ab0936ab28e22d0333119838c929ec6e3d460cc0be8d27aa4d178b9f6452d3" }, "go-mixed-chain/cmd/main.go": { "captureGroups": 22, diff --git a/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go b/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go index 9de68c3b0..c2734574a 100644 --- a/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go +++ b/gitnexus/test/fixtures/lang-resolution/go-method-enrichment/app.go @@ -1,9 +1,10 @@ -package main - -import "animal" +// This fixture exercises method enrichment and a valid same-package free call. +// A different package with a normal import cannot refer to Dog or Classify bare. +package animal func main() { dog := Dog{} sound := dog.Speak() category := Classify("dog") + _, _ = sound, category } diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/AppConfig.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/AppConfig.java new file mode 100644 index 000000000..2f06e41d4 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/AppConfig.java @@ -0,0 +1,12 @@ +package example; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AppConfig { + @Bean + public Handler handler() { + return null; + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Handler.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Handler.java new file mode 100644 index 000000000..2e2c8f480 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Handler.java @@ -0,0 +1,3 @@ +package example; + +public interface Handler {} diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/SpringContextUtil.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/SpringContextUtil.java new file mode 100644 index 000000000..045e01021 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/SpringContextUtil.java @@ -0,0 +1,9 @@ +package example; + +import java.util.Map; + +public class SpringContextUtil { + public static Map getBeans(Class type) { + return Map.of(); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Worker.java b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Worker.java new file mode 100644 index 000000000..eda02aca2 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/spring-constructor-bean-lookup/src/example/Worker.java @@ -0,0 +1,7 @@ +package example; + +public class Worker { + public Worker() { + SpringContextUtil.getBeans(Handler.class); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig index 1cb77e2ee..15c44f34a 100644 --- a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/main.zig @@ -21,6 +21,7 @@ pub usingnamespace @import("mixin.zig"); pub const Interfaces = .{ @import("webapi/AbortController.zig"), @import("webapi/AbortSignal.zig"), + @import("webapi/Element.zig"), }; pub fn main() void { diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Element.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Element.zig new file mode 100644 index 000000000..02bdeed1e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Element.zig @@ -0,0 +1,224 @@ +// The JS-API binding-table idiom, as Lightpanda writes it (#3399). +// +// Every accessor below hands a Zig function to `bridge.accessor` AS A VALUE: +// the function is REGISTERED here, never called here. The eventual invocation +// runs through comptime reflection (`@call(.auto, func, args)` over a +// `func: anytype` field), which no static walk can follow — that terminal hop +// is out of scope. What was NOT acceptable is dropping the reference entirely: +// `impact` then reported the accessor as having only its two in-file callers +// and called that answer `exact`. +// +// A file-as-struct, like every webapi module in the real tree. +const Element = @This(); + +// A NAMESPACE-only module (no `@This()`), imported under a handle. The bridge +// table below registers one of its functions the same way it registers this +// file's own methods. +const dom_utils = @import("dom_utils.zig"); + +// A hub module that re-exports `dom_utils`' members without declaring any. +const hub = @import("hub.zig"); + +// A MODULE-LEVEL binding whose name collides with the container `Gauge.zig` +// declares, bound to something that is NOT a container. This file never imports +// `Gauge.zig`. `findClassBindingInScope` filters the scope chain by +// `isClassLike`, so it walks past this binding, and its workspace-wide +// qualified-name fallback answers with the other file's struct — while the +// shadow guard used to permit exactly this, treating the module scope as a floor +// it need not inspect. +// +// The name is bound by IMPORT rather than by a local `const Gauge: u8 = 3`, +// and that detail is the difference between a live case and a self-defeating +// one: a local declaration would also claim the workspace qualified name +// `Gauge`, leaving two candidates, and the fallback refuses to guess between +// two. An imported alias claims nothing, so the fallback stays unique and fires. +const Gauge = @import("dom_utils.zig").DEFAULT_NS; + +_namespace: u8 = 0, + +// ── Registered accessors ──────────────────────────────────────────────────── + +pub fn getNamespaceUri(self: *Element) u8 { + return self._namespace; +} + +// An ordinary in-file caller. The point of the defect is that the REGISTRATION +// was missing, not that the symbol looked like a leaf: a plausible-but-short +// caller list is exactly what makes `epistemic: "exact"` dangerous. +pub fn lookupNamespaceUri(self: *Element) u8 { + return self.getNamespaceUri(); +} + +// ── The control ───────────────────────────────────────────────────────────── + +// Called normally and registered NOWHERE. Its edges must not move: a change +// that hedges or re-links every method would be indistinguishable from one +// that models value references, and only the second is correct. +pub fn getTagNameLower(self: *Element) u8 { + return self._namespace; +} + +pub fn describe(self: *Element) u8 { + return self.getTagNameLower(); +} + +// ── Owner discrimination ──────────────────────────────────────────────────── + +// Shadowed below by a same-named sibling inside `JsApi`. The registration +// writes `Element.getLocalName`, so THIS is the one it must bind. +pub fn getLocalName(self: *Element) u8 { + return self._namespace; +} + +fn tick(self: *Element) u8 { + return self._namespace; +} + +// ── The binding table ─────────────────────────────────────────────────────── + +pub const JsApi = struct { + pub const bridge = Bridge(Element); + + // QUALIFIED value reference — the accessor names its container explicitly. + // This is the exact line from Element.zig:2296 that #3399 was filed over. + pub const namespaceURI = bridge.accessor(Element.getNamespaceUri, null, .{}); + + // BARE value reference to a sibling declared in this same container. + pub const tagName = bridge.accessor(_tagName, null, .{}); + + fn _tagName(self: *Element) u8 { + return self.getTagNameLower(); + } + + // A sibling with the SAME simple name as the file-struct method above. + // `walkScopeChain` gives a local binding precedence over the enclosing + // scope, so a registration resolved by TAIL NAME alone binds here — the + // wrong function, silently. Resolving `Element.getLocalName` through its + // written owner is what keeps them apart. + fn getLocalName(self: *Element) u8 { + return 0; + } + + pub const localName = bridge.accessor(Element.getLocalName, null, .{}); + + // A receiver this index cannot resolve. There is a file-level `tick`, and + // tail-name resolution would happily bind it even though the source says + // the function belongs to something else entirely. Declining is the only + // safe answer: a missing reference is recoverable, a confident wrong edge + // is not. + pub const ticker = bridge.accessor(unresolvable_ns.tick, null, .{}); + + // QUALIFIED value reference through a MODULE handle rather than a container. + // `dom_utils` is a namespace, not a class, so the class-owner lookup answers + // nothing here — and declining would be silent rather than safe: with no + // USES edge, `impact` on `compare` measures a real zero and reports `exact`, + // which is the claim this whole change exists to stop making. + pub const comparator = bridge.accessor(dom_utils.compare, null, .{}); + + // A namespace member that is NOT callable. Module receivers get the same + // callable gate as container receivers — a registration table full of + // constants must keep emitting nothing. + pub const defaultNs = bridge.accessor(dom_utils.DEFAULT_NS, null, .{}); + + // The receiver IS a known namespace import, but `dom_utils.zig` declares no + // `onlyOnDecoy` — so the namespace channel declines and the container + // channel runs, reaching `decoy.zig`'s same-named struct through the + // workspace-wide qualified-name index. A registration must NOT be minted + // there: the file wrote which module it meant. + pub const decoyed = bridge.accessor(dom_utils.onlyOnDecoy, null, .{}); + + // Through a HUB, whose published names are all imported ones. The CALL form + // resolves — `namespaceExportsIncludeImportedNames` is what makes a Zig hub + // work at all — so the REGISTRATION form has to resolve to the same def, or + // one name means two things depending on whether it is followed by `(`. + pub const scaled = bridge.accessor(hub.scale, null, .{}); + + // …and the callable gate still applies through the hub. + pub const hubNs = bridge.accessor(hub.DEFAULT_NS, null, .{}); + + // `Gauge` names this file's `const Gauge: u8`, not `Gauge.zig`'s container. + pub const level = bridge.accessor(Gauge.read, null, .{}); +}; + +// The CALL form of the same hub member, so the two are pinned side by side. +pub fn callsThroughTheHub(v: u8) u8 { + return hub.scale(v); +} + +// A LOCAL declaration shadowing the module handle. `dom_utils` here is a `u8` +// parameter with no member of its own; resolving `dom_utils.normalize` through +// the file-level import would attach the registration to a module the source +// did not name at this site — a wrong edge, the failure the same guard prevents +// on the member-CALL path. +pub fn shadowsTheModuleHandle(dom_utils: u8) u8 { + register(dom_utils.normalize); + return dom_utils; +} + +// A LOCAL declaration shadowing a CONTAINER name — the class-owner half of the +// same failure. `Ticker` here is a `u8` parameter, and this file neither +// declares nor imports the `Ticker` container that `Ticker.zig` defines. +// `findClassBindingInScope` filters the scope chain by `isClassLike`, so it +// walks straight past the parameter and its qualified-name fallback answers +// with a struct from a file this one never named. Resolving `Ticker.fire` +// through that is a confident edge to a function the source did not write. +pub fn shadowsAContainerName(Ticker: u8) u8 { + register(Ticker.fire); + return Ticker; +} + +// The positive half of the same guard: here the LOCAL declaration IS the +// container the registration names. A shadow test that only asked "is this name +// bound nearer than the module scope" would answer yes and decline — reading the +// declaration as its own shadow — and this whole class of local container would +// stop registering anything. +pub fn registersALocalContainer() u8 { + const Local = struct { + pub fn go() u8 { + return 3; + } + }; + return bridge.accessor(Local.go, null, .{}); +} + +// ── Const binding initialiser ─────────────────────────────────────────────── + +fn onReset(self: *Element) u8 { + return self._namespace; +} + +// A `const` whose initialiser IS a function value. Second value position, same +// class of drop. +pub const defaultHandler = onReset; + +// ── comptime anytype sink ─────────────────────────────────────────────────── + +var registered: ?*const fn (*Element) u8 = null; + +// Takes a callable by value into a `comptime … anytype` parameter and STORES +// it. Nothing in this file calls `f`. +pub fn register(comptime f: anytype) void { + registered = f; +} + +fn onTick(self: *Element) u8 { + return self._namespace; +} + +pub fn boot() void { + // `onTick` is never called in this file — it is handed over as a value and + // invoked later through `registered`. + register(onTick); +} + +fn Bridge(comptime T: type) type { + _ = T; + return struct { + pub fn accessor(comptime getter: anytype, comptime setter: anytype, comptime opts: anytype) u8 { + _ = getter; + _ = setter; + _ = opts; + return 0; + } + }; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Gauge.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Gauge.zig new file mode 100644 index 000000000..5b6071db4 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Gauge.zig @@ -0,0 +1,10 @@ +// The unique workspace definition of the name `Gauge`. `Element.zig` never +// imports it, and declares a module-local `const Gauge` of its own — see +// `readsThroughAShadowedContainerName` there. +const Gauge = @This(); + +_level: u8 = 0, + +pub fn read(self: *Gauge) u8 { + return self._level; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Ticker.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Ticker.zig new file mode 100644 index 000000000..58fb48aa5 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Ticker.zig @@ -0,0 +1,13 @@ +// A file-struct in a module `Element.zig` never imports. +// +// Its only job is to be the unique workspace definition of the name `Ticker`, +// so that `findClassBindingInScope`'s qualified-name fallback can reach it from +// a file that has no binding for that name at all. See the +// `shadowsAContainerName` case in `Element.zig`. +const Ticker = @This(); + +_ticks: u8 = 0, + +pub fn fire(self: *Ticker) u8 { + return self._ticks; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Widget.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Widget.zig new file mode 100644 index 000000000..595c3aab9 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/Widget.zig @@ -0,0 +1,70 @@ +// A file-as-struct whose `@This()` alias is spelled `Self`, NOT `Widget`. +// +// This is the ordinary Zig idiom, and the case #3219 originally declined. The +// container this file mints is named after the FILE STEM (`Widget`), so before +// `bindZigThisAliases` the name `Self` meant nothing class-like here: a +// file-level `@This()` alias mints no Const at all (`isZigFileThisAlias` +// suppresses it so it cannot shadow the type for `w: *Widget`), and a +// container-level one mints a Variable that every `isClassLike` walk steps +// over. Every qualified reference through the alias — a CALL as much as a +// REGISTRATION — resolved to nothing. +// +// Counted on the corpora on hand when this was fixed: 73 of ghostty's 185 +// `@This()` files, 93 of tigerbeetle's 94 and 8 of mach's 42 spell the alias +// differently from the file stem, carrying 302 `Alias.member` references, 96 +// of them calls. +// +// `Element.zig` next door is the OTHER half of the control: it writes +// `const Element = @This();` in `Element.zig`, so its qualified references +// resolved through the stem binding and must keep resolving exactly as before. +const Self = @This(); + +_w: u8 = 0, + +pub fn width(self: *Self) u8 { + return self._w; +} + +// A qualified CALL through the alias — the explicit spelling Zig allows beside +// `self.width()`, and the shape `Alias.member` takes 96 times in the corpora +// above. +pub fn describeWidth(self: *Self) u8 { + return Self.width(self); +} + +pub const WidgetApi = struct { + pub const binder = Binder(Self); + // A qualified REGISTRATION through the alias: the #3399 shape, written the + // way most Zig files spell their own type. + pub const w = binder.accessor(Self.width, null, .{}); +}; + +// A NESTED container with its own differently-spelled alias. The file-level and +// container-level aliases take different code paths — one binds at the module +// scope against the file-struct, the other at the container's own scope — so +// both are exercised. +pub const Metrics = struct { + const Me = @This(); + + _n: u8 = 0, + + pub fn read(self: *Me) u8 { + return self._n; + } + + pub fn readTwice(self: *Me) u8 { + return Me.read(self) +% Me.read(self); + } +}; + +fn Binder(comptime T: type) type { + _ = T; + return struct { + pub fn accessor(comptime g: anytype, comptime s: anytype, comptime o: anytype) u8 { + _ = g; + _ = s; + _ = o; + return 0; + } + }; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/decoy.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/decoy.zig new file mode 100644 index 000000000..1197a40c8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/decoy.zig @@ -0,0 +1,24 @@ +// A container whose NAME collides with `Element.zig`'s `dom_utils` import +// handle, in a file `Element.zig` never imports. +// +// `findClassBindingInScope` does not stop at the scope chain: when its +// `isClassLike` walk misses — and a namespace import binds a Module, not a +// class — it falls back to `scopes.qualifiedNames`, a WORKSPACE-wide index, and +// answers with the unique def of that name. This struct is that unique def. A +// registration written `dom_utils.compare` in `Element.zig` must still bind +// `dom_utils.zig`'s function, not this one: the file said which module it meant. +pub const dom_utils = struct { + pub fn compare(a: u8, b: u8) u8 { + return if (a < b) a else b; + } + + // A callable that exists ONLY here. `dom_utils.zig` has no member of this + // name, so a registration written `dom_utils.onlyOnDecoy` declines in the + // namespace channel and falls through to the container channel — where the + // workspace-wide qualified-name fallback answers with this struct. The + // owner-shadow guard is the only thing standing between that and a + // confident edge into a file `Element.zig` never imported. + pub fn onlyOnDecoy(v: u8) u8 { + return v; + } +}; diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/dom_utils.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/dom_utils.zig new file mode 100644 index 000000000..e8972ee41 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/dom_utils.zig @@ -0,0 +1,24 @@ +// A NAMESPACE-only module: no `const X = @This()`, so this file declares no +// container symbol of its own. Its members are reachable only through an +// `@import` handle — the second kind of owner a qualified name can have, and +// the one `findClassBindingInScope` cannot answer for. +// +// Lightpanda's `libdom.zig` / `parser.zig` helpers are written exactly this +// way, and they are registered into the same bridge tables as the file-struct +// methods next door. + +pub const DEFAULT_NS: u8 = 7; + +pub fn compare(a: u8, b: u8) u8 { + return if (a > b) a else b; +} + +pub fn normalize(v: u8) u8 { + return v; +} + +// Republished by `hub.zig`, and by nothing else — so an assertion about the +// hub cannot be satisfied by an edge some other case emitted. +pub fn scale(v: u8) u8 { + return v +% 1; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/hub.zig b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/hub.zig new file mode 100644 index 000000000..b99f60d9e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-idioms/src/webapi/hub.zig @@ -0,0 +1,10 @@ +// A HUB module: a file made only of re-exports, the shape +// `ScopeResolver.namespaceExportsIncludeImportedNames` exists for (ghostty's +// `src/terminal/`, tigerbeetle's `stdx`). It declares nothing of its own — +// every name it publishes is a name it imported. +// +// A consumer writes `hub.scale(x)` to CALL through it, and +// `bridge.accessor(hub.scale, …)` to REGISTER through it. Those are the same +// name resolved by the same rule, so they must not disagree. +pub const scale = @import("dom_utils.zig").scale; +pub const DEFAULT_NS = @import("dom_utils.zig").DEFAULT_NS; diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/README.md b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/README.md new file mode 100644 index 000000000..4ec85bc38 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/README.md @@ -0,0 +1,19 @@ +A Zig MONOREPO: several build packages, and no `build.zig` at the repo root. + +This is the layout the root-only config loader could not see at all. With no +root `build.zig` / `build.zig.zon`, `loadZigBuildConfig` answered `null`, and +every bare `@import("")` below went unresolved — so cross-file symbol +resolution in a repo like this degraded to relative imports only. + +The three packages are not interchangeable: + +- `core` declares module `core` and is imported by `app`. +- `app` declares module `app` and depends on `core` through a + `build.zig.zon` `.path = "../core"` — a spelling relative to the PACKAGE, + which has to be rebased to repo-relative `packages/core` before it can be + matched against indexed files. +- `tool` is the discriminating control. It binds the SAME alias `core` to its + OWN `src/core.zig`. A workspace that flattened every package's modules into + one repo-wide map would resolve `tool`'s `@import("core")` to `core`'s root + — a confident edge into a package `tool` never depends on. Only per-package + scoping (`zigPackageFor`, the `tsconfigFor` analogue) keeps them apart. diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig new file mode 100644 index 000000000..e237138a8 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig @@ -0,0 +1,7 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const core_dep = b.dependency("core", .{}); + const app = b.addModule("app", .{ .root_source_file = b.path("src/main.zig") }); + app.addImport("core", core_dep.module("core")); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig.zon b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig.zon new file mode 100644 index 000000000..2b666b3d1 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/build.zig.zon @@ -0,0 +1,19 @@ +.{ + .name = "app", + .version = "0.1.0", + .dependencies = .{ + .core = .{ + .path = "../core", + }, + // ABSOLUTE, so it points outside the repository whichever package + // declares it. It sits in a NESTED package on purpose: that is the + // branch where the package prefix used to hide it. `packages/app/` + + // `/src` reads as the relative, in-repo `packages/app/src` — a real + // directory in this fixture — so the absolute dep silently became the + // package's own source tree instead of being rejected. + .escapes = .{ + .path = "/src", + }, + }, + .paths = .{""}, +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/main.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/main.zig new file mode 100644 index 000000000..26798d708 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/main.zig @@ -0,0 +1,18 @@ +// `core` is a PACKAGE dependency, declared by this package's build.zig.zon and +// wired in by its build.zig — not a relative path. Resolving it needs the +// package's own config, which lives two directories below the repo root. +const core = @import("core"); +const util = @import("util.zig"); + +pub fn run(attempt: u8) u8 { + return core.retryBudget(attempt); +} + +pub fn configured() u8 { + var cfg = core.Config{}; + return cfg.load(); +} + +pub fn delegated(attempt: u8) u8 { + return util.clamp(attempt); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/util.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/util.zig new file mode 100644 index 000000000..5387442cd --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/app/src/util.zig @@ -0,0 +1,8 @@ +// A second file of the same package reaching the same dependency: membership in +// a build module is the root plus what it reaches, so a file that is not itself +// a module root must resolve the alias too. +const core = @import("core"); + +pub fn clamp(attempt: u8) u8 { + return core.retryBudget(attempt); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/build.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/build.zig new file mode 100644 index 000000000..53ac256a7 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/build.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + _ = b.addModule("core", .{ .root_source_file = b.path("src/root.zig") }); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/src/root.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/src/root.zig new file mode 100644 index 000000000..b048ced4c --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/core/src/root.zig @@ -0,0 +1,13 @@ +pub const DEFAULT_RETRIES: u8 = 3; + +pub fn retryBudget(attempt: u8) u8 { + return DEFAULT_RETRIES - attempt; +} + +pub const Config = struct { + retries: u8 = DEFAULT_RETRIES, + + pub fn load(self: *Config) u8 { + return self.retries; + } +}; diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/build.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/build.zig new file mode 100644 index 000000000..8a3964bee --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/build.zig @@ -0,0 +1,8 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const tool = b.addModule("tool", .{ .root_source_file = b.path("src/main.zig") }); + // The SAME alias as `app` binds, pointing somewhere else entirely. + const own_core = b.createModule(.{ .root_source_file = b.path("src/core.zig") }); + tool.addImport("core", own_core); +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/core.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/core.zig new file mode 100644 index 000000000..1616db72b --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/core.zig @@ -0,0 +1,5 @@ +// `tool`'s own `core`, unrelated to `packages/core`. Same module name, and that +// is the point: the name is only meaningful inside the package that binds it. +pub fn retryBudget(attempt: u8) u8 { + return attempt; +} diff --git a/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/main.zig b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/main.zig new file mode 100644 index 000000000..979d76d7d --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/zig-monorepo/packages/tool/src/main.zig @@ -0,0 +1,7 @@ +// Resolving this through a repo-wide module map would reach +// `packages/core/src/root.zig` — a package `tool` does not depend on. +const core = @import("core"); + +pub fn measure(attempt: u8) u8 { + return core.retryBudget(attempt); +} diff --git a/gitnexus/test/fixtures/objective-c/SYModuleBridge.mm b/gitnexus/test/fixtures/objective-c/SYModuleBridge.mm new file mode 100644 index 000000000..ffbc923cd --- /dev/null +++ b/gitnexus/test/fixtures/objective-c/SYModuleBridge.mm @@ -0,0 +1,9 @@ +#import "SYModuleCaller.h" + +@interface SYModuleBridge : NSObject +- (void)bridgeValue:(NSInteger)value; +@end + +@implementation SYModuleBridge +- (void)bridgeValue:(NSInteger)value {} +@end diff --git a/gitnexus/test/fixtures/objective-c/SYModuleCaller.h b/gitnexus/test/fixtures/objective-c/SYModuleCaller.h new file mode 100644 index 000000000..c55ed20db --- /dev/null +++ b/gitnexus/test/fixtures/objective-c/SYModuleCaller.h @@ -0,0 +1,32 @@ +#import + +#define RCT_EXTERN_C_BEGIN +#define RCT_EXTERN_C_END + +RCT_EXTERN_C_BEGIN +typedef struct SYModuleMethodInfo { + const char *const name; +} SYModuleMethodInfo; +RCT_EXTERN_C_END + +int SYModuleSupportAdd(int a, int b); + +@protocol SYModuleRunnable +- (void)runTask:(NSString *)name completion:(void (^)(BOOL ok))completion; +@end + +@interface SYBaseCaller : NSObject +- (void)loadData:(NSString *)name completion:(void (^)(BOOL ok))completion; +@end + +@interface SYModuleCaller : SYBaseCaller { + SYBaseCaller *_base; +} +@property (nonatomic, strong) SYBaseCaller *helper; ++ (instancetype)sharedCaller; +- (void)runTask:(NSString *)name completion:(void (^)(BOOL ok))completion; +@end + +@interface SYModuleCaller (Tracing) +- (void)traceEvent:(NSString *)name; +@end diff --git a/gitnexus/test/fixtures/objective-c/SYModuleCaller.m b/gitnexus/test/fixtures/objective-c/SYModuleCaller.m new file mode 100644 index 000000000..9e70c83fc --- /dev/null +++ b/gitnexus/test/fixtures/objective-c/SYModuleCaller.m @@ -0,0 +1,32 @@ +#import "SYModuleCaller.h" +#include "SYModuleSupport.h" +@import Foundation; + +#define SY_OBJC_RECEIVER(x) x + +@interface SYModuleCaller () +@property (nonatomic, strong) SYBaseCaller *privateHelper; +@end + +@implementation SYModuleCaller ++ (instancetype)sharedCaller { return [SYModuleCaller new]; } +- (void)runTask:(NSString *)name completion:(void (^)(BOOL ok))completion { + SYBaseCaller *typed = self.helper; + id dynamic = typed; + [self traceEvent:name]; + [SY_OBJC_RECEIVER(self) traceEvent:name]; + [super loadData:name completion:completion]; + [self loadData:name completion:completion]; + [typed loadData:name completion:completion]; + [dynamic loadData:name completion:completion]; +} +- (void)runProtocol:(id)runner { + [runner runTask:@"x" completion:^(BOOL ok) {}]; +} +@end + +@implementation SYModuleCaller (Tracing) +- (void)traceEvent:(NSString *)name {} +@end + +static int SYModuleCompute(int value) { return value + 1; } diff --git a/gitnexus/test/fixtures/objective-c/SYModuleSupport.h b/gitnexus/test/fixtures/objective-c/SYModuleSupport.h new file mode 100644 index 000000000..08d72ae53 --- /dev/null +++ b/gitnexus/test/fixtures/objective-c/SYModuleSupport.h @@ -0,0 +1,4 @@ +#ifndef SY_MODULE_SUPPORT_H +#define SY_MODULE_SUPPORT_H +int SYModuleSupportAdd(int a, int b); +#endif diff --git a/gitnexus/test/helpers/grammar-introspection.ts b/gitnexus/test/helpers/grammar-introspection.ts index 503290ab5..d61af9b5d 100644 --- a/gitnexus/test/helpers/grammar-introspection.ts +++ b/gitnexus/test/helpers/grammar-introspection.ts @@ -109,7 +109,7 @@ interface NodeTypeEntry { /** Resolve the on-disk directory of an installed package, or null if absent. */ function resolvePackageDir(pkg: string): string | null { - // Vendored grammars (c/dart/proto/swift/kotlin/zig) are NOT in node_modules — they + // Vendored grammars (c/dart/proto/swift/kotlin/objc/zig) are NOT in node_modules — they // load from vendor/ by absolute path (vendored-grammars.ts / #2111), so resolve // their node-types.json from there rather than via _require.resolve. if (VENDORED_GRAMMAR_PACKAGES.has(pkg)) { diff --git a/gitnexus/test/integration/cobol-import-index-reuse.test.ts b/gitnexus/test/integration/cobol-import-index-reuse.test.ts index bcde62b05..d9db7162c 100644 --- a/gitnexus/test/integration/cobol-import-index-reuse.test.ts +++ b/gitnexus/test/integration/cobol-import-index-reuse.test.ts @@ -118,11 +118,9 @@ describe('COBOL COPY resolution — index reuse across imports (#2908)', () => { // Source tier, reached only after every copybook missed. expect(resolveImportTarget('PAYROLL', FROM_FILE, files, undefined)).toBe('src/PAYROLL.cob'); expect(resolveImportTarget('TAXCALC', FROM_FILE, files, undefined)).toBe('src/TAXCALC.cobol'); - // `path.basename(fp, '.cpy')` will not strip `.CPY`, so the stem keeps it. - expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBe( - 'copybooks/Mixed.CPY', - ); - expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBeNull(); + // P1-A: raw extension stripped, so `Mixed.CPY` keys as `MIXED`. + expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBe('copybooks/Mixed.CPY'); + expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBeNull(); // Neither tier: a `.txt`, a file with no extension, and an absent member. expect(resolveImportTarget('NOEXT', FROM_FILE, files, undefined)).toBeNull(); expect(resolveImportTarget('ABSENT', FROM_FILE, files, undefined)).toBeNull(); diff --git a/gitnexus/test/integration/csv-pipeline.test.ts b/gitnexus/test/integration/csv-pipeline.test.ts index b2534d67e..78be85580 100644 --- a/gitnexus/test/integration/csv-pipeline.test.ts +++ b/gitnexus/test/integration/csv-pipeline.test.ts @@ -151,6 +151,50 @@ describe('streamAllCSVsToDisk', () => { expect(await readAllRelRows(result.relsByPair)).toHaveLength(3); }); + it('persists Protocol and Category nodes and their structural relationships', async () => { + const graph = buildTestGraph( + [ + { + id: 'Protocol:objc:protocol:Runnable', + label: 'Protocol', + name: 'Runnable', + filePath: 'src/Runnable.h', + }, + { + id: 'Class:objc:class:Worker', + label: 'Class', + name: 'Worker', + filePath: 'src/Worker.h', + }, + { + id: 'Category:objc:category:Worker:Tracing', + label: 'Category', + name: 'Tracing', + filePath: 'src/Worker+Tracing.m', + }, + ], + [ + { + sourceId: 'Class:objc:class:Worker', + targetId: 'Protocol:objc:protocol:Runnable', + type: 'IMPLEMENTS', + }, + { + sourceId: 'Category:objc:category:Worker:Tracing', + targetId: 'Class:objc:class:Worker', + type: 'MEMBER_OF', + }, + ], + ); + + const result = await streamAllCSVsToDisk(graph, repoDir, csvDir); + + expect(result.nodeFiles.get('Protocol')?.rows).toBe(1); + expect(result.nodeFiles.get('Category')?.rows).toBe(1); + expect(result.relsByPair.get('Class|Protocol')?.rows).toBe(1); + expect(result.relsByPair.get('Category|Class')?.rows).toBe(1); + }); + it('CSV content is properly escaped', async () => { const graph = buildTestGraph([ { diff --git a/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts index 283f68842..4d1d50802 100644 --- a/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts +++ b/gitnexus/test/integration/group/graphql-resolve-symbol.test.ts @@ -7,9 +7,11 @@ import { closeLbug, executeParameterized } from '../../../src/core/lbug/pool-ada import type { GroupConfig, RepoHandle } from '../../../src/core/group/types.js'; import { withTestLbugDB } from '../../helpers/test-indexed-db.js'; +// Graph Method.startLine is 0-based (tree-sitter row). `health()` is source line 5. const SEED = [ - `CREATE (:Method {id:'method:health', name:'health', filePath:'src/health.resolver.ts', startLine:5, endLine:5, content:'', description:''})`, - `CREATE (:Const {id:'const:health-document', name:'HealthDocument', filePath:'src/generated.ts', startLine:1, endLine:1, content:'', description:''})`, + `CREATE (:Method {id:'method:health', name:'health', filePath:'src/health.resolver.ts', startLine:4, endLine:4, content:'', description:''})`, + `CREATE (:Method {id:'method:save', name:'save', filePath:'src/health.resolver.ts', startLine:6, endLine:6, content:'', description:''})`, + `CREATE (:Const {id:'const:health-document', name:'HealthDocument', filePath:'src/generated.ts', startLine:0, endLine:0, content:'', description:''})`, ]; withTestLbugDB( @@ -140,6 +142,42 @@ withTestLbugDB( registrySpy.mockRestore(); } }); + + it('binds a decorated arrow-field provider at the wrapper startLine (#3201)', async () => { + providerRoot = path.join(handle.tmpHandle.dbPath, 'arrow-provider-repo'); + await fs.mkdir(path.join(providerRoot, 'src'), { recursive: true }); + await fs.writeFile( + path.join(providerRoot, 'src/health.resolver.ts'), + `import { Query, Mutation, Resolver } from '@nestjs/graphql'; +@Resolver() +class HealthResolver { + @Query() + health() { return 'ok'; } + + @Mutation() + save = async () => true; +}`, + 'utf8', + ); + const providerRepo: RepoHandle = { + id: handle.repoId, + path: 'api', + repoPath: providerRoot, + storagePath: handle.tmpHandle.dbPath, + }; + const execute = (query: string, params: Record = {}) => + executeParameterized(handle.repoId, query, params); + const contracts = await new GraphqlExtractor().extract(execute, providerRoot, providerRepo); + expect(contracts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contractId: 'graphql::mutation::save', + role: 'provider', + symbolUid: 'method:save', + }), + ]), + ); + }); }, { seed: SEED, poolAdapter: true }, ); diff --git a/gitnexus/test/integration/impact-callable-value-references.test.ts b/gitnexus/test/integration/impact-callable-value-references.test.ts new file mode 100644 index 000000000..6124d76d3 --- /dev/null +++ b/gitnexus/test/integration/impact-callable-value-references.test.ts @@ -0,0 +1,231 @@ +/** + * Integration test: a callable named in VALUE position makes `impact` a lower + * bound (#3399). + * + * The rejected behaviour, in one sentence: `Element.getNamespaceUri` — the DOM + * `Element.namespaceURI` accessor — is bound into a JS bridge table as + * `bridge.accessor(Element.getNamespaceUri, null, .{})`, and `impact` answered + * "2 callers, LOW risk, epistemic: exact". Everything about that answer except + * the number 2 was wrong, and `exact` is the part that made it unusable: it + * tells the reader not to look further. + * + * The seed below is the shape that matters, not the language. A registration + * edge (`USES`, reason `scope-resolution: value-ref`) says a function was + * handed somewhere as a value. Where the value goes next — a struct field, a + * registry lookup, comptime reflection — is not modelled, so no CALLS edge + * connects the eventual invocation back to the target. `tools.ts` defines + * `lower-bound` as "the walk provably missed callers", and that is exactly this + * situation. + * + * WHY the assertions are what they are: + * - `impactedCount` must NOT move. Hedging is not inventing callers; a fix + * that made the number go up would be a different (and unearned) claim. + * - a plain CALLS-only target in the SAME index must stay `exact`, or the + * hedge is noise sprayed over every query and carries no information. + * - the cause has its own slot rather than being folded into + * `dispatchBoundary`: an agent branching on the numbers would otherwise be + * told an interface boundary exists where there is none, and would draw the + * opposite conclusion about whether the gap is reducible. + */ +import { it, expect, beforeAll, vi } from 'vitest'; +import path from 'node:path'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; +import { VALUE_REF_EDGE_REASON } from '../../src/core/ingestion/scope-resolution/value-ref-edges.js'; + +/** + * Fail ONLY the value-reference probe, leaving every other query on the real + * database. Keyed on the bound `$reason` param rather than the query text, so + * it cannot accidentally match a different query that happens to mention USES. + */ +let failValueRefProbe = false; + +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + executeParameterized: (...args: any[]) => { + const params = args[2] as { reason?: string } | undefined; + if (failValueRefProbe && params?.reason === 'scope-resolution: value-ref') { + return Promise.reject(new Error('simulated: index unreadable')); + } + return (actual as any).executeParameterized(...args); + }, + }; +}); + +vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + listRegisteredRepos: vi.fn().mockResolvedValue([]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), + }; +}); +const { listRegisteredRepos, saveMeta } = await import('../../src/storage/repo-manager.js'); + +const SEED = [ + // The registered accessor, with one ordinary in-file caller so the walk has + // something real to report. This mirrors Element.zig: `getNamespaceUri` genuinely + // has internal callers, and it is the REGISTRATION that the answer omits. + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.getNamespaceUri', name: 'getNamespaceUri', filePath: 'webapi/Element.zig', startLine: 439, endLine: 442, isExported: true, content: '', description: ''})`, + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.lookupPrefixForElement', name: 'lookupPrefixForElement', filePath: 'webapi/Element.zig', startLine: 490, endLine: 520, isExported: true, content: '', description: ''})`, + `MATCH (a:Method {id:'Method:webapi/Element.zig:Element.lookupPrefixForElement'}), (b:Method {id:'Method:webapi/Element.zig:Element.getNamespaceUri'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'scope-resolution: local-call', step:0}]->(b)`, + + // The binding table entry: `pub const namespaceURI = bridge.accessor(Element.getNamespaceUri, null, .{});` + // A registration, NOT an invocation — hence USES, not CALLS (Kythe `ref` vs + // `ref/call`; Joern METHOD_REF). + `CREATE (:Struct {id: 'Struct:webapi/Element.zig:Element.JsApi', name: 'JsApi', filePath: 'webapi/Element.zig', startLine: 2281, endLine: 2400, content: '', description: ''})`, + `MATCH (a:Struct {id:'Struct:webapi/Element.zig:Element.JsApi'}), (b:Method {id:'Method:webapi/Element.zig:Element.getNamespaceUri'}) CREATE (a)-[:CodeRelation {type:'USES', confidence:0.85, reason:'${VALUE_REF_EDGE_REASON}', step:0}]->(b)`, + + // Control: same file, same shape of caller, but nothing registers it as a + // value. This is `getTagNameLower` — it must come back `exact`. + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.getTagNameLower', name: 'getTagNameLower', filePath: 'webapi/Element.zig', startLine: 400, endLine: 410, isExported: true, content: '', description: ''})`, + `MATCH (a:Method {id:'Method:webapi/Element.zig:Element.lookupPrefixForElement'}), (b:Method {id:'Method:webapi/Element.zig:Element.getTagNameLower'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.85, reason:'scope-resolution: local-call', step:0}]->(b)`, + + // Second control: an ordinary USES edge that is NOT a value registration (a + // type reference). The probe keys on the reason, so this must not hedge — + // otherwise every type mention in the repo would downgrade its target. + `CREATE (:Method {id: 'Method:webapi/Element.zig:Element.getInnerText', name: 'getInnerText', filePath: 'webapi/Element.zig', startLine: 600, endLine: 610, isExported: true, content: '', description: ''})`, + `MATCH (a:Struct {id:'Struct:webapi/Element.zig:Element.JsApi'}), (b:Method {id:'Method:webapi/Element.zig:Element.getInnerText'}) CREATE (a)-[:CodeRelation {type:'USES', confidence:0.85, reason:'scope-resolution: type-reference', step:0}]->(b)`, + + // Third control: a registration whose invocation the analyzer DID synthesize. + // `emitPropertyDispatchCalls` sweep 2 connects `x.()` member calls to + // every function registered under `` and stamps those edges + // `property-dispatch`. Where that happened the walk followed the + // registration, so hedging would be noise over an answer that was actually + // computed. This is the JS/TS hook-table shape, not the Zig one — Zig has no + // object-literal key to dispatch through and so is never excluded. + `CREATE (:Function {id: 'Function:hooks/registry.js:onClick', name: 'onClick', filePath: 'hooks/registry.js', startLine: 5, endLine: 8, isExported: true, content: '', description: ''})`, + `CREATE (:Function {id: 'Function:hooks/registry.js:registerAll', name: 'registerAll', filePath: 'hooks/registry.js', startLine: 1, endLine: 3, isExported: true, content: '', description: ''})`, + `CREATE (:Function {id: 'Function:hooks/consumer.js:runHandlers', name: 'runHandlers', filePath: 'hooks/consumer.js', startLine: 1, endLine: 6, isExported: true, content: '', description: ''})`, + `MATCH (a:Function {id:'Function:hooks/registry.js:registerAll'}), (b:Function {id:'Function:hooks/registry.js:onClick'}) CREATE (a)-[:CodeRelation {type:'USES', confidence:0.85, reason:'${VALUE_REF_EDGE_REASON}', step:0}]->(b)`, + `MATCH (a:Function {id:'Function:hooks/consumer.js:runHandlers'}), (b:Function {id:'Function:hooks/registry.js:onClick'}) CREATE (a)-[:CodeRelation {type:'CALLS', confidence:0.7, reason:'property-dispatch', step:0}]->(b)`, +]; + +withTestLbugDB( + 'impact-callable-value-references', + (handle) => { + let backend: LocalBackend; + beforeAll(() => { + backend = (handle as any)._backend; + }); + + it('downgrades a registered accessor to lower-bound without inventing callers', async () => { + const result: any = await backend.callTool('impact', { + target: 'getNamespaceUri', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + // Pinned, not implied: hedging must not INVENT callers. The seed gives + // this symbol exactly two inbound edges — one CALLS, one value-ref USES — + // and both are traversed. If a later change made the hedge also widen the + // walk, every other assertion here would still pass. + expect(result.impactedCount).toBe(2); + // The registration is a real inbound edge, so it IS traversed and counted + // — but the call THROUGH the registered value is not, which is why the + // count still cannot be the whole story. + expect(result.epistemic).toBe('lower-bound'); + expect(result.causes.callableValueReferences).toBe(1); + // Its own slot: an agent must not read this as an interface boundary. + expect(result.causes.dispatchBoundary).toBe(0); + expect(result.boundaries.join(' ')).toContain('as a VALUE'); + }); + + it('leaves a symbol with only ordinary calls exact', async () => { + const result: any = await backend.callTool('impact', { + target: 'getTagNameLower', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('exact'); + }); + + it('does not hedge on a USES edge that is not a value registration', async () => { + const result: any = await backend.callTool('impact', { + target: 'getInnerText', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + // A type reference is a use, not an escape: nothing can be invoked + // through it, so the answer stays complete. + expect(result.epistemic).toBe('exact'); + }); + + it('hedges context() for the same reason it hedges impact()', async () => { + const result: any = await backend.callTool('context', { name: 'getNamespaceUri' }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('lower-bound'); + expect(result.causes.callableValueReferences).toBe(1); + }); + + it('stays exact when the analyzer already synthesized the dispatch', async () => { + // `onClick` is registered as a value AND reached by a synthesized + // property-dispatch CALLS edge. The walk did not "provably miss" that + // caller, so `lower-bound` would be wrong — and a signal that fires on + // every hook table in every JS codebase carries no information. + const result: any = await backend.callTool('impact', { + target: 'onClick', + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('exact'); + }); + + it('hedges — never reports exact — when the probe itself cannot run', async () => { + // The failure mode this whole feature exists to remove is silence reading + // as certainty. A probe that threw has not established that the symbol is + // unregistered; swallowing the error into a zero would publish `exact` on + // the strength of a question that was never answered. + failValueRefProbe = true; + try { + const result: any = await backend.callTool('impact', { + target: 'getTagNameLower', // the control: exact when the probe works + direction: 'upstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('lower-bound'); + expect(result.boundaries.join(' ')).toContain('could not be run'); + // No count is claimed — the note is the signal, and inventing a + // magnitude from a failed query would be the same error inverted. + expect(result.causes.callableValueReferences).toBe(0); + } finally { + failValueRefProbe = false; + } + }); + + it('stays exact downstream — a reference INTO a symbol says nothing about what it reaches', async () => { + const result: any = await backend.callTool('impact', { + target: 'getNamespaceUri', + direction: 'downstream', + }); + expect(result).not.toHaveProperty('error'); + expect(result.epistemic).toBe('exact'); + }); + }, + { + seed: SEED, + poolAdapter: true, + afterSetup: async (h) => { + // Without a completeness receipt EVERY answer is hedged ("scope-extraction + // completeness was not recorded"), which would make the controls below + // pass for the wrong reason and prove nothing about this probe. + // `saveMeta` is the only writer production uses. + await saveMeta(path.dirname(h.dbPath), { scopeExtractionReceipt: 1 } as any); + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: h.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc123', + stats: { files: 1, nodes: 5, communities: 0, processes: 0 }, + }, + ] as any); + const backend = new LocalBackend(); + await backend.init(); + (h as any)._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/objective-c-pipeline-benchmark.test.ts b/gitnexus/test/integration/objective-c-pipeline-benchmark.test.ts new file mode 100644 index 000000000..3101ad503 --- /dev/null +++ b/gitnexus/test/integration/objective-c-pipeline-benchmark.test.ts @@ -0,0 +1,247 @@ +/** + * Objective-C ingestion pipeline benchmark. + * + * Generates synthetic .h/.m pairs at increasing scales and measures + * wall-clock time and peak heap through the full pipeline — header + * classification, parsing, provider-owned semantic facts, import + * suffix resolution, and static message-send edges. + * + * Mirrors test/integration/csharp-pipeline-benchmark.test.ts and + * test/integration/php-pipeline-benchmark.test.ts. Two shapes: + * 1. "spread" — each class imports a fixed sibling (constant fan-out; + * see cpp-pipeline-benchmark.test.ts). This is the linear file-count + * gate. + * 2. "protocol" — every class conforms to one shared protocol and + * sends one protocol-typed message. Implementer USES are shared + * per (protocol, selector); the dedicated + * bench/objective-c-resolution protocol arm pins that evidence + * shape at O(implementers). + * + * Run: GITNEXUS_BENCH=1 npx vitest run test/integration/objective-c-pipeline-benchmark.test.ts + */ +import { beforeAll, describe, it, expect, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; + +const BENCH_ENABLED = process.env.GITNEXUS_BENCH === '1'; + +beforeAll(() => vi.stubEnv('GITNEXUS_WORKER_READY_TIMEOUT_MS', '60000')); + +interface BenchResult { + classCount: number; + fileCount: number; + elapsedMs: number; + peakHeapMB: number; + nodeCount: number; + edgeCount: number; +} + +type FixtureShape = 'spread' | 'protocol'; + +function generateObjectiveCFixture( + classCount: number, + shape: FixtureShape, +): { dir: string; fileCount: number } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `objc-bench-${shape}-${classCount}-`)); + + if (shape === 'protocol') { + fs.writeFileSync( + path.join(dir, 'Runnable.h'), + [ + '#import ', + '', + '@protocol Runnable', + '- (void)run;', + '@end', + '', + ].join('\n'), + ); + } + + for (let f = 0; f < classCount; f++) { + const className = `Class${f}`; + const sibling = `Class${(f + 1) % classCount}`; + const header = + shape === 'protocol' + ? [ + '#import ', + '#import "Runnable.h"', + '', + `@interface ${className} : NSObject `, + '- (void)run;', + '- (void)tick:(id)runner;', + '@end', + '', + ].join('\n') + : [ + '#import ', + '', + `@interface ${className} : NSObject`, + '- (void)process;', + '@end', + '', + ].join('\n'); + + const impl = + shape === 'protocol' + ? [ + `#import "${className}.h"`, + '', + `@implementation ${className}`, + '- (void)run {}', + '- (void)tick:(id)runner', + '{', + ' [self run];', + ' [runner run];', + '}', + '@end', + '', + ].join('\n') + : [ + `#import "${className}.h"`, + `#import "${sibling}.h"`, + '', + `@implementation ${className}`, + '- (void)process', + '{', + ' [self process];', + ` ${sibling} *sib = [${sibling} new];`, + ' [sib process];', + '}', + '@end', + '', + ].join('\n'); + + fs.writeFileSync(path.join(dir, `${className}.h`), header); + fs.writeFileSync(path.join(dir, `${className}.m`), impl); + } + + const fileCount = classCount * 2 + (shape === 'protocol' ? 1 : 0); + return { dir, fileCount }; +} + +async function runBenchmark( + classCount: number, + shape: FixtureShape, + budgetMs: number, +): Promise { + const { dir, fileCount } = generateObjectiveCFixture(classCount, shape); + + let peakHeapMB = 0; + const heapSampler = setInterval(() => { + const heap = process.memoryUsage().heapUsed / 1024 / 1024; + if (heap > peakHeapMB) peakHeapMB = heap; + }, 50); + + let budgetTimer: ReturnType | undefined; + try { + const start = Date.now(); + const result = await Promise.race([ + runPipelineFromRepo(dir, () => {}, { skipGraphPhases: true }), + new Promise((_, reject) => { + budgetTimer = setTimeout( + () => + reject( + new Error(`Pipeline exceeded ${budgetMs}ms at ${classCount} classes (${shape})`), + ), + budgetMs, + ); + }), + ]); + const elapsedMs = Date.now() - start; + + return { + classCount, + fileCount, + elapsedMs, + peakHeapMB: Math.round(peakHeapMB), + nodeCount: result.graph.nodeCount, + edgeCount: result.graph.relationshipCount, + }; + } finally { + clearInterval(heapSampler); + clearTimeout(budgetTimer); + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +function printResults(label: string, results: BenchResult[]) { + console.log(`\n${label}`); + console.log('┌──────────┬──────────┬───────────┬──────────┬───────┬───────┐'); + console.log('│ Classes │ Files │ Time (ms) │ Heap MB │ Nodes │ Edges │'); + console.log('├──────────┼──────────┼───────────┼──────────┼───────┼───────┤'); + for (const r of results) { + console.log( + `│ ${String(r.classCount).padStart(8)} │ ${String(r.fileCount).padStart(8)} │ ${String(r.elapsedMs).padStart(9)} │ ${String(r.peakHeapMB).padStart(8)} │ ${String(r.nodeCount).padStart(5)} │ ${String(r.edgeCount).padStart(5)} │`, + ); + } + console.log('└──────────┴──────────┴───────────┴──────────┴───────┴───────┘'); + + if (results.length >= 2) { + console.log('\nScaling ratios (time_ratio / class_ratio):'); + for (let i = 1; i < results.length; i++) { + const classRatio = results[i].classCount / results[i - 1].classCount; + const timeRatio = results[i].elapsedMs / results[i - 1].elapsedMs; + const scaling = timeRatio / classRatio; + console.log( + ` ${results[i - 1].classCount} → ${results[i].classCount}: ${scaling.toFixed(2)}x (${scaling < 1.5 ? 'linear' : scaling < 3 ? 'superlinear' : 'WARNING: quadratic'})`, + ); + } + } +} + +describe.skipIf(!BENCH_ENABLED)('Objective-C pipeline benchmark', () => { + it('scales with file count — typed sibling receivers', async () => { + const scales = [40, 80, 160]; + const results: BenchResult[] = []; + + for (const classCount of scales) { + const result = await runBenchmark(classCount, 'spread', 180_000); + results.push(result); + console.log( + ` ${classCount} classes: ${result.elapsedMs}ms, ${result.peakHeapMB}MB heap, ${result.nodeCount} nodes, ${result.edgeCount} edges`, + ); + } + + printResults('Objective-C Pipeline — Spread', results); + + expect(results[results.length - 1].edgeCount).toBeGreaterThan(0); + + for (let i = 1; i < results.length; i++) { + const classRatio = results[i].classCount / results[i - 1].classCount; + const timeRatio = results[i].elapsedMs / results[i - 1].elapsedMs; + expect(timeRatio / classRatio).toBeLessThan(3); + // Wall-clock at these scales is worker-startup dominated; node/edge + // cardinality is the load-bearing linearity check (cpp-pipeline analog). + expect(results[i].nodeCount / results[i - 1].nodeCount).toBeCloseTo(classRatio, 1); + expect(results[i].edgeCount / results[i - 1].edgeCount).toBeCloseTo(classRatio, 1); + } + }, 600_000); + + it('scales with file count — shared protocol receivers', async () => { + const scales = [40, 80, 160]; + const results: BenchResult[] = []; + + for (const classCount of scales) { + const result = await runBenchmark(classCount, 'protocol', 180_000); + results.push(result); + console.log( + ` ${classCount} classes: ${result.elapsedMs}ms, ${result.peakHeapMB}MB heap, ${result.nodeCount} nodes, ${result.edgeCount} edges`, + ); + } + + printResults('Objective-C Pipeline — Shared Protocol', results); + + expect(results[results.length - 1].edgeCount).toBeGreaterThan(0); + + for (let i = 1; i < results.length; i++) { + const classRatio = results[i].classCount / results[i - 1].classCount; + const timeRatio = results[i].elapsedMs / results[i - 1].elapsedMs; + expect(timeRatio / classRatio).toBeLessThan(3); + expect(results[i].nodeCount / results[i - 1].nodeCount).toBeCloseTo(classRatio, 1); + expect(results[i].edgeCount / results[i - 1].edgeCount).toBeCloseTo(classRatio, 1); + } + }, 600_000); +}); diff --git a/gitnexus/test/integration/objective-c-provider.test.ts b/gitnexus/test/integration/objective-c-provider.test.ts new file mode 100644 index 000000000..949a2f33e --- /dev/null +++ b/gitnexus/test/integration/objective-c-provider.test.ts @@ -0,0 +1,887 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execSync } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import type { GraphNode, RelationshipType } from 'gitnexus-shared'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { classifyObjectiveCFileContent } from '../../src/core/ingestion/languages/objective-c.js'; +import type { PipelineResult } from '../../src/types/pipeline.js'; + +const FIXTURE_DIR = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../fixtures/objective-c', +); + +function readFixture(name: string): string { + return fs.readFileSync(path.join(FIXTURE_DIR, name), 'utf8'); +} + +const HEADER = readFixture('SYModuleCaller.h'); +const IMPL = readFixture('SYModuleCaller.m'); +const MM_IMPL = readFixture('SYModuleBridge.mm'); +const PLAIN_C_HEADER = readFixture('SYModuleSupport.h'); + +const HEADER_V2 = HEADER.replace( + '- (void)runTask:(NSString *)name completion:(void (^)(BOOL ok))completion;\n@end', + '- (void)runTask:(NSString *)name completion:(void (^)(BOOL ok))completion;\n- (void)cancelTask;\n@end', +).replace( + '- (void)traceEvent:(NSString *)name;\n@end', + '- (void)traceEvent:(NSString *)name;\n- (void)traceDetail:(NSString *)name level:(NSInteger)level;\n@end', +); + +const IMPL_V2 = IMPL.replace( + '[self traceEvent:name];', + '[self traceEvent:name];\n [self traceDetail:name level:1];', +).replace( + '@implementation SYModuleCaller (Tracing)\n- (void)traceEvent:(NSString *)name {}\n@end', + '@implementation SYModuleCaller (Tracing)\n- (void)traceEvent:(NSString *)name {}\n- (void)traceDetail:(NSString *)name level:(NSInteger)level {}\n@end', +); + +function git(repoRoot: string, command: string): void { + execSync(command, { cwd: repoRoot, stdio: 'pipe' }); +} + +function gitCommitAll(repoRoot: string, message: string): void { + git(repoRoot, 'git -c user.name=test -c user.email=t@t -c commit.gpgsign=false add -A'); + git( + repoRoot, + `git -c user.name=test -c user.email=t@t -c commit.gpgsign=false commit -q -m "${message}"`, + ); +} + +function writeObjectiveCRepo(repoRoot: string, header = HEADER, impl = IMPL): void { + fs.writeFileSync(path.join(repoRoot, 'SYModuleCaller.h'), header); + fs.writeFileSync(path.join(repoRoot, 'SYModuleCaller.m'), impl); + fs.writeFileSync(path.join(repoRoot, 'SYModuleBridge.mm'), MM_IMPL); + fs.writeFileSync(path.join(repoRoot, 'SYModuleSupport.h'), PLAIN_C_HEADER); +} + +function normalizeRows(rows: unknown): unknown[] { + if (!Array.isArray(rows)) return []; + return rows + .map((row) => { + const record = row as Record; + return { + uid: record.uid ?? record.id, + name: record.name, + filePath: record.filePath, + kind: record.kind, + }; + }) + .sort((left, right) => + `${left.uid ?? ''}:${left.name ?? ''}:${left.filePath ?? ''}`.localeCompare( + `${right.uid ?? ''}:${right.name ?? ''}:${right.filePath ?? ''}`, + ), + ); +} + +function normalizeBuckets(value: unknown): Record { + const record = (value ?? {}) as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, normalizeRows(record[key])]), + ); +} + +function normalizeContext(value: unknown): Record { + const record = value as Record; + const symbol = (record.symbol ?? {}) as Record; + return { + status: record.status, + symbol: { + uid: symbol.uid, + name: symbol.name, + kind: symbol.kind, + filePath: symbol.filePath, + }, + incoming: normalizeBuckets(record.incoming), + outgoing: normalizeBuckets(record.outgoing), + }; +} + +async function readPersistedObjectiveCSurface(repoRoot: string): Promise> { + const backend = new LocalBackend(); + try { + const classContext = await backend.callTool('context', { + name: 'SYModuleCaller', + file_path: 'SYModuleCaller.h', + repo: repoRoot, + }); + const runTaskContext = await backend.callTool('context', { + uid: 'Method:objc:method:objc:class:SYModuleCaller:-:runTask:completion:', + repo: repoRoot, + }); + const runProtocolContext = await backend.callTool('context', { + uid: 'Method:objc:method:objc:class:SYModuleCaller:-:runProtocol:', + repo: repoRoot, + }); + const candidateEvidenceId = ( + ( + (runProtocolContext as Record).outgoing as + | Record + | undefined + )?.uses ?? [] + ) + .map((entry) => (entry as Record).uid) + .find( + (uid): uid is string => + typeof uid === 'string' && uid.startsWith('CodeElement:objc:protocol-candidates:'), + ); + if (candidateEvidenceId === undefined) { + throw new Error('Persisted protocol candidate evidence was not reachable from context'); + } + const candidateEvidenceContext = await backend.callTool('context', { + uid: candidateEvidenceId, + repo: repoRoot, + }); + const categoryContext = await backend.callTool('context', { + uid: 'Category:objc:category:SYModuleCaller:Tracing', + repo: repoRoot, + }); + const queryResult = (await backend.callTool('query', { + search_query: 'SYModuleCaller', + repo: repoRoot, + limit: 5, + include_content: false, + })) as Record; + const protocolAndCategoryResult = await backend.callTool('cypher', { + query: + "MATCH (n) WHERE labels(n) IN ['Protocol', 'Category'] " + + 'RETURN n.id AS id, labels(n)[0] AS kind ORDER BY kind, id', + repo: repoRoot, + }); + const categoryHostResult = await backend.callTool('cypher', { + query: + 'MATCH (category:Category)-[r:CodeRelation]->(host:Class) ' + + "WHERE r.type = 'MEMBER_OF' " + + 'RETURN category.id AS category, host.id AS host', + repo: repoRoot, + }); + const protocolCandidateResult = await backend.callTool('cypher', { + query: + 'MATCH (source:Method)-[sourceRel:CodeRelation]->(e:CodeElement)-[candidateRel:CodeRelation]->(candidate:Method) ' + + "WHERE sourceRel.type = 'USES' AND candidateRel.type = 'USES' " + + "AND e.id STARTS WITH 'CodeElement:objc:protocol-candidates:' " + + 'RETURN source.id AS sourceId, candidate.id AS candidateId, candidateRel.reason AS reason ' + + 'ORDER BY sourceId, candidateId', + repo: repoRoot, + }); + const unresolvedReasonResult = await backend.callTool('cypher', { + query: + 'MATCH (source:Method)-[r:CodeRelation]->(e:CodeElement) ' + + "WHERE r.type = 'USES' AND e.id STARTS WITH 'CodeElement:objc:unresolved:' " + + 'RETURN source.id AS sourceId, e.name AS evidence, r.reason AS reason ' + + 'ORDER BY sourceId, evidence', + repo: repoRoot, + }); + return { + classContext: normalizeContext(classContext), + runTaskContext: normalizeContext(runTaskContext), + runProtocolContext: normalizeContext(runProtocolContext), + candidateEvidenceContext: normalizeContext(candidateEvidenceContext), + categoryContext: normalizeContext(categoryContext), + queryDefinitions: normalizeRows(queryResult.definitions), + protocolAndCategoryResult, + categoryHostResult, + protocolCandidateResult, + unresolvedReasonResult, + }; + } finally { + await backend.disconnect(); + } +} + +async function analyzeObjectiveCRepo( + repoRoot: string, + options: { force?: boolean } = {}, +): Promise { + const logs: string[] = []; + await runFullAnalysis( + repoRoot, + { + force: options.force, + skipAgentsMd: true, + skipSkills: true, + workerPoolSize: 1, + }, + { + onProgress: () => undefined, + onLog: (message) => logs.push(message), + }, + ); + return logs; +} + +describe('Objective-C provider integration', () => { + let repoRoot: string; + let result: PipelineResult; + + beforeAll(async () => { + repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-objc-provider-')); + fs.writeFileSync(path.join(repoRoot, 'SYModuleCaller.h'), HEADER); + fs.writeFileSync(path.join(repoRoot, 'SYModuleCaller.m'), IMPL); + fs.writeFileSync(path.join(repoRoot, 'SYModuleBridge.mm'), MM_IMPL); + fs.writeFileSync(path.join(repoRoot, 'SYModuleSupport.h'), PLAIN_C_HEADER); + result = await runPipelineFromRepo(repoRoot, () => undefined, { + workerPoolSize: 1, + }); + }, 60000); + + afterAll(() => { + fs.rmSync(repoRoot, { recursive: true, force: true }); + }); + + function nodeByQualifiedName(qualifiedName: string): GraphNode | undefined { + return result.graph.nodes.find((node) => node.properties.qualifiedName === qualifiedName); + } + + function expectNode(qualifiedName: string, label: GraphNode['label']): GraphNode { + const node = nodeByQualifiedName(qualifiedName); + expect(node, qualifiedName).toBeDefined(); + expect(node?.label).toBe(label); + if (node === undefined) throw new Error(`Missing expected node ${qualifiedName}`); + return node; + } + + function hasRelationship( + type: RelationshipType, + sourceId: string, + targetId: string, + reason?: string | RegExp, + ): boolean { + return result.graph.relationships.some((rel) => { + if (rel.type !== type || rel.sourceId !== sourceId || rel.targetId !== targetId) return false; + if (reason === undefined) return true; + return typeof reason === 'string' ? rel.reason === reason : reason.test(rel.reason); + }); + } + + it('classifies a macro-wrapped Objective-C header by declarations after the marker', () => { + expect( + classifyObjectiveCFileContent( + 'RCTBridgeModule.h', + [ + 'RCT_EXTERN_C_BEGIN', + 'typedef struct RCTMethodInfo {', + ' const char *const jsName;', + '} RCTMethodInfo;', + 'RCT_EXTERN_C_END', + '@protocol RCTBridgeModule ', + '- (void)run;', + '@end', + '', + ].join('\n'), + ), + ).toBe(true); + }); + + it('indexes Objective-C semantic nodes beyond File nodes', () => { + expectNode('objc:protocol:SYModuleRunnable', 'Protocol'); + expectNode('objc:class:SYBaseCaller', 'Class'); + expectNode('objc:class:SYModuleCaller', 'Class'); + expectNode('objc:class:SYModuleBridge', 'Class'); + expectNode('objc:category:SYModuleCaller:Tracing', 'Category'); + expectNode('objc:method:objc:class:SYModuleCaller:-:runTask:completion:', 'Method'); + expectNode('objc:method:objc:class:SYModuleBridge:-:bridgeValue:', 'Method'); + expectNode('objc:method:objc:class:SYModuleCaller:+:sharedCaller', 'Method'); + expectNode('objc:method:objc:category:SYModuleCaller:Tracing:-:traceEvent:', 'Method'); + expectNode('objc:property:objc:class:SYModuleCaller:helper', 'Property'); + expectNode('objc:ivar:objc:class:SYModuleCaller:_base', 'Variable'); + expectNode('objc:function:SYModuleSupportAdd', 'Function'); + expectNode('objc:function:static:SYModuleCaller.m:SYModuleCompute', 'Function'); + }); + + it('emits imports, inheritance, protocol, and category host relationships', () => { + const caller = expectNode('objc:class:SYModuleCaller', 'Class'); + const base = expectNode('objc:class:SYBaseCaller', 'Class'); + const protocol = expectNode('objc:protocol:SYModuleRunnable', 'Protocol'); + const category = expectNode('objc:category:SYModuleCaller:Tracing', 'Category'); + + expect(hasRelationship('EXTENDS', caller.id, base.id)).toBe(true); + expect(hasRelationship('IMPLEMENTS', caller.id, protocol.id)).toBe(true); + expect(hasRelationship('MEMBER_OF', category.id, caller.id)).toBe(true); + + const importNodes = result.graph.nodes.filter((node) => node.label === 'Import'); + expect(importNodes.map((node) => node.properties.targetRaw)).toEqual( + expect.arrayContaining(['SYModuleCaller.h', 'SYModuleSupport.h', 'Foundation']), + ); + + const mFile = result.graph.nodes.find( + (node) => node.label === 'File' && node.properties.filePath === 'SYModuleCaller.m', + ); + const hFile = result.graph.nodes.find( + (node) => node.label === 'File' && node.properties.filePath === 'SYModuleCaller.h', + ); + const supportFile = result.graph.nodes.find( + (node) => node.label === 'File' && node.properties.filePath === 'SYModuleSupport.h', + ); + expect(mFile).toBeDefined(); + expect(hFile).toBeDefined(); + expect(supportFile).toBeDefined(); + if (mFile === undefined || hFile === undefined || supportFile === undefined) { + throw new Error('Missing Objective-C fixture file nodes'); + } + expect(hasRelationship('IMPORTS', mFile.id, hFile.id)).toBe(true); + expect(hasRelationship('IMPORTS', mFile.id, supportFile.id)).toBe(true); + }); + + it('records implementation evidence for merged declarations', () => { + const caller = expectNode('objc:class:SYModuleCaller', 'Class'); + const runTask = expectNode( + 'objc:method:objc:class:SYModuleCaller:-:runTask:completion:', + 'Method', + ); + + const implementationEvidence = result.graph.nodes.filter( + (node) => + node.label === 'CodeElement' && + node.properties.objectiveCKind === 'implementation-evidence' && + node.properties.filePath === 'SYModuleCaller.m', + ); + expect(implementationEvidence.map((node) => node.properties.targetQualifiedName)).toEqual( + expect.arrayContaining([ + 'objc:class:SYModuleCaller', + 'objc:method:objc:class:SYModuleCaller:-:runTask:completion:', + ]), + ); + expect( + implementationEvidence.some((node) => + hasRelationship('DECLARES', node.id, caller.id, 'objc: implementation of merged symbol'), + ), + ).toBe(true); + expect( + implementationEvidence.some((node) => + hasRelationship('DECLARES', node.id, runTask.id, 'objc: implementation of merged symbol'), + ), + ).toBe(true); + }); + + it('emits conservative Objective-C message-send call edges and unresolved evidence', () => { + const runTask = expectNode( + 'objc:method:objc:class:SYModuleCaller:-:runTask:completion:', + 'Method', + ); + const loadData = expectNode( + 'objc:method:objc:class:SYBaseCaller:-:loadData:completion:', + 'Method', + ); + const traceEvent = expectNode( + 'objc:method:objc:category:SYModuleCaller:Tracing:-:traceEvent:', + 'Method', + ); + const runProtocol = expectNode( + 'objc:method:objc:class:SYModuleCaller:-:runProtocol:', + 'Method', + ); + const protocolRun = expectNode( + 'objc:method:objc:protocol:SYModuleRunnable:-:runTask:completion:', + 'Method', + ); + + expect( + hasRelationship('CALLS', runTask.id, loadData.id, /objc-message: (super|local) receiver/), + ).toBe(true); + expect(hasRelationship('CALLS', runTask.id, loadData.id, 'objc-message: self receiver')).toBe( + true, + ); + expect(hasRelationship('CALLS', runTask.id, traceEvent.id, 'objc-message: self receiver')).toBe( + true, + ); + expect( + hasRelationship('CALLS', runProtocol.id, protocolRun.id, 'objc-message: protocol receiver'), + ).toBe(true); + + const unresolved = result.graph.nodes.find( + (node) => + node.label === 'CodeElement' && + node.properties.objectiveCKind === 'unresolved-message' && + node.properties.receiver === 'dynamic', + ); + expect(unresolved).toBeDefined(); + if (unresolved === undefined) throw new Error('Missing unresolved dynamic message evidence'); + expect( + hasRelationship('CALLS', runTask.id, unresolved.id), + 'dynamic id receiver must not become a certain CALLS edge', + ).toBe(false); + expect( + hasRelationship( + 'USES', + runTask.id, + unresolved.id, + 'objc-message: unresolved: id receiver is dynamic', + ), + ).toBe(true); + expect(unresolved.properties.name).toContain('unresolved: id receiver is dynamic'); + + const macroUnresolved = result.graph.nodes.find( + (node) => + node.label === 'CodeElement' && + node.properties.objectiveCKind === 'unresolved-message' && + node.properties.receiver === 'SY_OBJC_RECEIVER(self)', + ); + expect(macroUnresolved?.properties.name).toContain( + 'unresolved: macro receiver SY_OBJC_RECEIVER is dynamic', + ); + + const candidates = result.graph.nodes.find( + (node) => + node.label === 'CodeElement' && + String(node.properties.qualifiedName).startsWith('objc:protocol-candidates:'), + ); + expect(candidates).toBeDefined(); + if (candidates === undefined) throw new Error('Missing protocol candidate evidence'); + expect( + hasRelationship( + 'USES', + runProtocol.id, + candidates.id, + 'objc-message: protocol receiver candidates: SYModuleRunnable runTask:completion:', + ), + ).toBe(true); + expect( + hasRelationship( + 'USES', + candidates.id, + runTask.id, + 'objc-protocol-candidate: SYModuleRunnable runTask:completion:', + ), + ).toBe(true); + expect( + hasRelationship('CALLS', runProtocol.id, runTask.id), + 'candidate implementations must not become certain CALLS edges', + ).toBe(false); + }); + + it('resolves header member types across files and keeps static C helpers distinct', async () => { + const crossFileRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-objc-cross-file-')); + try { + fs.writeFileSync( + path.join(crossFileRepo, 'Helper.h'), + '@interface Helper\n- (void)ping;\n@end\n', + ); + fs.writeFileSync( + path.join(crossFileRepo, 'Worker.h'), + '#import "Helper.h"\n@interface Worker { Helper *_helper; }\n@property Helper *helper;\n- (void)run;\n@end\n', + ); + fs.writeFileSync( + path.join(crossFileRepo, 'Worker.m'), + '#import "Worker.h"\n@implementation Worker\n- (void)run { [self.helper ping]; [_helper ping]; }\n@end\n', + ); + fs.writeFileSync( + path.join(crossFileRepo, 'First.m'), + 'static int helper(void) { return 1; }\n', + ); + fs.writeFileSync( + path.join(crossFileRepo, 'Second.m'), + 'static int helper(void) { return 2; }\n', + ); + + const crossFileResult = await runPipelineFromRepo(crossFileRepo, () => undefined, { + workerPoolSize: 1, + }); + const run = crossFileResult.graph.nodes.find( + (node) => + node.label === 'Method' && + node.properties.qualifiedName === 'objc:method:objc:class:Worker:-:run', + ); + const ping = crossFileResult.graph.nodes.find( + (node) => + node.label === 'Method' && + node.properties.qualifiedName === 'objc:method:objc:class:Helper:-:ping', + ); + expect(run).toBeDefined(); + expect(ping).toBeDefined(); + if (run === undefined || ping === undefined) { + throw new Error('Missing cross-file Objective-C method nodes'); + } + + const memberCalls = crossFileResult.graph.relationships.filter( + (relationship) => + relationship.type === 'CALLS' && + relationship.sourceId === run.id && + relationship.targetId === ping.id, + ); + expect(memberCalls).toHaveLength(2); + + const staticHelpers = crossFileResult.graph.nodes.filter( + (node) => node.label === 'Function' && node.properties.name === 'helper', + ); + expect(staticHelpers.map((node) => node.properties.filePath).sort()).toEqual([ + 'First.m', + 'Second.m', + ]); + expect(new Set(staticHelpers.map((node) => node.properties.qualifiedName)).size).toBe(2); + } finally { + fs.rmSync(crossFileRepo, { recursive: true, force: true }); + } + }); + + it('resolves inherited protocol members and candidates without looping on protocol cycles', async () => { + const protocolRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-objc-protocols-')); + try { + fs.writeFileSync( + path.join(protocolRepo, 'Protocols.h'), + '@protocol Parent\n' + + '- (void)ping;\n' + + '@end\n' + + '@protocol Child \n' + + '@end\n' + + '@protocol CycleA \n' + + '- (void)cycleA;\n' + + '@end\n' + + '@protocol CycleB \n' + + '- (void)cycleB;\n' + + '@end\n' + + '@interface BaseWorker \n' + + '@end\n' + + '@interface ConcreteWorker : BaseWorker\n' + + '- (void)ping;\n' + + '@end\n' + + '@interface Caller\n' + + '- (void)run:(id)worker;\n' + + '- (void)runCycle:(id)worker;\n' + + '@end\n', + ); + fs.writeFileSync( + path.join(protocolRepo, 'Protocols.m'), + '#import "Protocols.h"\n' + + '@implementation ConcreteWorker\n' + + '- (void)ping {}\n' + + '@end\n' + + '@implementation Caller\n' + + '- (void)run:(id)worker { [worker ping]; }\n' + + '- (void)runCycle:(id)worker { [worker cycleB]; }\n' + + '@end\n', + ); + + const protocolResult = await runPipelineFromRepo(protocolRepo, () => undefined, { + workerPoolSize: 1, + }); + const node = (qualifiedName: string): GraphNode => { + const found = protocolResult.graph.nodes.find( + (item) => item.properties.qualifiedName === qualifiedName, + ); + if (found === undefined) throw new Error(`Missing protocol fixture node ${qualifiedName}`); + return found; + }; + const has = (type: RelationshipType, sourceId: string, targetId: string): boolean => + protocolResult.graph.relationships.some( + (relationship) => + relationship.type === type && + relationship.sourceId === sourceId && + relationship.targetId === targetId, + ); + + const caller = node('objc:method:objc:class:Caller:-:run:'); + const parentMethod = node('objc:method:objc:protocol:Parent:-:ping'); + const workerMethod = node('objc:method:objc:class:ConcreteWorker:-:ping'); + const cycleCaller = node('objc:method:objc:class:Caller:-:runCycle:'); + const cycleMethod = node('objc:method:objc:protocol:CycleB:-:cycleB'); + const candidateEvidence = protocolResult.graph.nodes.find( + (item) => + item.label === 'CodeElement' && + String(item.properties.qualifiedName).startsWith('objc:protocol-candidates:'), + ); + + expect(has('CALLS', caller.id, parentMethod.id)).toBe(true); + expect(has('CALLS', caller.id, workerMethod.id)).toBe(false); + expect(candidateEvidence).toBeDefined(); + if (candidateEvidence === undefined) + throw new Error('Missing inherited protocol candidate evidence'); + expect(has('USES', candidateEvidence.id, workerMethod.id)).toBe(true); + expect(has('CALLS', cycleCaller.id, cycleMethod.id)).toBe(true); + } finally { + fs.rmSync(protocolRepo, { recursive: true, force: true }); + } + }); + + it('emits category membership only when the host class exists locally', async () => { + const categoryRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-objc-category-host-')); + try { + fs.writeFileSync( + path.join(categoryRepo, 'Categories.m'), + '@interface LocalHost\n@end\n' + + '@interface LocalHost (Tracing)\n@end\n' + + '@interface UIView (Tracing)\n@end\n', + ); + + const categoryResult = await runPipelineFromRepo(categoryRepo, () => undefined, { + workerPoolSize: 1, + }); + const localHost = categoryResult.graph.nodes.find( + (node) => node.properties.qualifiedName === 'objc:class:LocalHost', + ); + const localCategory = categoryResult.graph.nodes.find( + (node) => node.properties.qualifiedName === 'objc:category:LocalHost:Tracing', + ); + const sdkCategory = categoryResult.graph.nodes.find( + (node) => node.properties.qualifiedName === 'objc:category:UIView:Tracing', + ); + expect(localHost).toBeDefined(); + expect(localCategory).toBeDefined(); + expect(sdkCategory).toBeDefined(); + if (localHost === undefined || localCategory === undefined || sdkCategory === undefined) { + throw new Error('Missing category host fixture nodes'); + } + + expect( + categoryResult.graph.relationships.some( + (relationship) => + relationship.type === 'MEMBER_OF' && + relationship.sourceId === localCategory.id && + relationship.targetId === localHost.id, + ), + ).toBe(true); + expect( + categoryResult.graph.relationships.some( + (relationship) => + relationship.type === 'MEMBER_OF' && relationship.sourceId === sdkCategory.id, + ), + ).toBe(false); + } finally { + fs.rmSync(categoryRepo, { recursive: true, force: true }); + } + }); + + it('keeps category selector collisions as candidate evidence instead of certain calls', async () => { + const collisionRepo = fs.mkdtempSync( + path.join(os.tmpdir(), 'gitnexus-objc-category-collision-'), + ); + try { + fs.writeFileSync( + path.join(collisionRepo, 'Collision.m'), + `@interface CollisionWorker +- (void)run; +- (void)refresh; +@end + +@interface CollisionWorker (Tracing) +- (void)refresh; +- (void)categoryOnly; +@end + +@interface CollisionWorker (Metrics) +- (void)refresh; +@end + +@implementation CollisionWorker +- (void)run { + [self refresh]; + CollisionWorker *worker = self; + [worker refresh]; + [self categoryOnly]; +} +- (void)refresh {} +@end + +@implementation CollisionWorker (Tracing) +- (void)refresh {} +- (void)categoryOnly {} +@end + +@implementation CollisionWorker (Metrics) +- (void)refresh {} +@end +`, + ); + + const collisionResult = await runPipelineFromRepo(collisionRepo, () => undefined, { + workerPoolSize: 1, + }); + const node = (qualifiedName: string): GraphNode => { + const found = collisionResult.graph.nodes.find( + (item) => item.properties.qualifiedName === qualifiedName, + ); + if (found === undefined) + throw new Error(`Missing category collision node ${qualifiedName}`); + return found; + }; + const caller = node('objc:method:objc:class:CollisionWorker:-:run'); + const collisionTargets = [ + node('objc:method:objc:class:CollisionWorker:-:refresh'), + node('objc:method:objc:category:CollisionWorker:Tracing:-:refresh'), + node('objc:method:objc:category:CollisionWorker:Metrics:-:refresh'), + ]; + const categoryOnly = node('objc:method:objc:category:CollisionWorker:Tracing:-:categoryOnly'); + const collisionTargetIds = new Set(collisionTargets.map((target) => target.id)); + const certainCollisionCalls = collisionResult.graph.relationships.filter( + (relationship) => + relationship.type === 'CALLS' && + relationship.sourceId === caller.id && + collisionTargetIds.has(relationship.targetId), + ); + + expect(certainCollisionCalls).toHaveLength(0); + + const evidenceNodes = collisionResult.graph.nodes.filter( + (item) => + item.label === 'CodeElement' && + String(item.properties.qualifiedName).startsWith('objc:category-dispatch-candidates:'), + ); + expect(evidenceNodes).toHaveLength(2); + + for (const evidence of evidenceNodes) { + expect( + collisionResult.graph.relationships.some( + (relationship) => + relationship.type === 'USES' && + relationship.sourceId === caller.id && + relationship.targetId === evidence.id && + relationship.confidence === 0.7, + ), + ).toBe(true); + for (const target of collisionTargets) { + expect( + collisionResult.graph.relationships.some( + (relationship) => + relationship.type === 'USES' && + relationship.sourceId === evidence.id && + relationship.targetId === target.id && + relationship.confidence === 0.5, + ), + ).toBe(true); + } + } + + const categoryOnlyCalls = collisionResult.graph.relationships.filter( + (relationship) => + relationship.type === 'CALLS' && + relationship.sourceId === caller.id && + relationship.targetId === categoryOnly.id, + ); + expect(categoryOnlyCalls).toHaveLength(1); + expect(categoryOnlyCalls[0]?.confidence).toBe(0.9); + } finally { + fs.rmSync(collisionRepo, { recursive: true, force: true }); + } + }); +}); + +describe('Objective-C provider persisted index behavior', () => { + it('surfaces query/context semantics and keeps incremental results aligned with force rebuild', async () => { + const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-objc-provider-index-')); + try { + writeObjectiveCRepo(repoRoot); + git(repoRoot, 'git init'); + gitCommitAll(repoRoot, 'initial Objective-C fixture'); + + await analyzeObjectiveCRepo(repoRoot); + const initialSurface = await readPersistedObjectiveCSurface(repoRoot); + const reopenedInitialSurface = await readPersistedObjectiveCSurface(repoRoot); + expect(reopenedInitialSurface).toEqual(initialSurface); + expect(initialSurface).toMatchObject({ + classContext: { + status: 'found', + symbol: { + uid: 'Class:objc:class:SYModuleCaller', + kind: 'Class', + filePath: 'SYModuleCaller.h', + }, + incoming: { + declares: expect.arrayContaining([ + expect.objectContaining({ + uid: expect.stringContaining( + 'CodeElement:objc:implementation:objc:class:SYModuleCaller:SYModuleCaller.m:', + ), + filePath: 'SYModuleCaller.m', + }), + ]), + imports: expect.arrayContaining([ + expect.objectContaining({ + uid: 'File:SYModuleCaller.m', + filePath: 'SYModuleCaller.m', + }), + ]), + member_of: expect.arrayContaining([ + expect.objectContaining({ + uid: 'Category:objc:category:SYModuleCaller:Tracing', + }), + ]), + }, + }, + categoryContext: { + outgoing: { + member_of: expect.arrayContaining([ + expect.objectContaining({ uid: 'Class:objc:class:SYModuleCaller' }), + ]), + }, + }, + runProtocolContext: { + outgoing: { + uses: expect.arrayContaining([ + expect.objectContaining({ + uid: expect.stringContaining('CodeElement:objc:protocol-candidates:'), + }), + ]), + }, + }, + candidateEvidenceContext: { + outgoing: { + uses: expect.arrayContaining([ + expect.objectContaining({ + uid: 'Method:objc:method:objc:class:SYModuleCaller:-:runTask:completion:', + }), + ]), + }, + }, + queryDefinitions: expect.arrayContaining([ + expect.objectContaining({ uid: 'Class:objc:class:SYModuleCaller' }), + expect.objectContaining({ + uid: expect.stringMatching(/^Method:objc:method:objc:class:SYModuleCaller:/), + }), + ]), + protocolAndCategoryResult: expect.objectContaining({ + markdown: expect.stringContaining('Protocol:objc:protocol:SYModuleRunnable'), + }), + categoryHostResult: expect.objectContaining({ + markdown: expect.stringContaining('Category:objc:category:SYModuleCaller:Tracing'), + }), + protocolCandidateResult: expect.objectContaining({ + markdown: expect.stringContaining( + 'objc-protocol-candidate: SYModuleRunnable runTask:completion:', + ), + }), + unresolvedReasonResult: expect.objectContaining({ + markdown: expect.stringContaining('objc-message: unresolved: id receiver is dynamic'), + }), + }); + + writeObjectiveCRepo(repoRoot, HEADER_V2, IMPL_V2); + gitCommitAll(repoRoot, 'change Objective-C declarations and implementations'); + const incrementalLogs = await analyzeObjectiveCRepo(repoRoot); + expect(incrementalLogs).toContainEqual(expect.stringContaining('Incremental: changed=')); + const incrementalSurface = await readPersistedObjectiveCSurface(repoRoot); + + await analyzeObjectiveCRepo(repoRoot, { force: true }); + const forceSurface = await readPersistedObjectiveCSurface(repoRoot); + expect(incrementalSurface).toEqual(forceSurface); + expect(forceSurface).toMatchObject({ + classContext: { + outgoing: { + has_method: expect.arrayContaining([ + expect.objectContaining({ + uid: 'Method:objc:method:objc:category:SYModuleCaller:Tracing:-:traceDetail:level:', + }), + ]), + }, + }, + runTaskContext: { + outgoing: { + calls: expect.arrayContaining([ + expect.objectContaining({ + uid: 'Method:objc:method:objc:category:SYModuleCaller:Tracing:-:traceDetail:level:', + }), + ]), + }, + }, + }); + } finally { + fs.rmSync(repoRoot, { recursive: true, force: true }); + } + }, 180000); +}); diff --git a/gitnexus/test/integration/resolvers/barrel-dir-index-wildcard.test.ts b/gitnexus/test/integration/resolvers/barrel-dir-index-wildcard.test.ts new file mode 100644 index 000000000..276015b25 --- /dev/null +++ b/gitnexus/test/integration/resolvers/barrel-dir-index-wildcard.test.ts @@ -0,0 +1,136 @@ +/** + * C2 — the grafana `Button` shape: a workspace barrel `export *`s a components + * index, which re-exports NAMED bindings (with inline `type` modifiers) from a + * DIRECTORY index, which `export *`s the real file, whose `Button` is a + * `React.forwardRef` const. Measured on grafana@871af0720: `Button` resolved 8 + * of 475 ledger entries while siblings through plain hops resolved at scale. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import path from 'path'; +import fs from 'node:fs'; +import os from 'node:os'; +import { + getRelationships, + getResolutionOutcomes, + runPipelineFromRepo, + writeFixtureRepo, + type PipelineResult, +} from './helpers.js'; + +describe('named re-export through a directory index that wildcards (grafana Button shape)', () => { + let result: PipelineResult; + let repoDir: string | undefined; + + beforeAll(async () => { + repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-c2-dir-index-')); + writeFixtureRepo(repoDir, { + 'package.json': '{ "name": "root", "private": true, "workspaces": ["packages/*"] }\n', + 'packages/ui/package.json': + '{ "name": "@x/ui", "version": "1.0.0", "main": "src/index.ts" }\n', + 'packages/ui/src/index.ts': `export * from './components';\nexport * from './themes';\n`, + // Inline `type` modifiers on the same statement as value re-exports. + 'packages/ui/src/components/index.ts': `export { Stack } from './Layout/Stack'; +export { Button, LinkButton, type ButtonVariant, ButtonGroup, type ButtonProps, clearButtonStyles } from './Button'; +`, + // Directory index: wildcard + one named re-export. + 'packages/ui/src/components/Button/index.ts': `export * from './Button';\nexport { ButtonGroup } from './ButtonGroup';\n`, + 'packages/ui/src/components/Button/Button.tsx': `import React from 'react'; +export type ButtonVariant = 'primary' | 'secondary'; +export interface ButtonProps { variant?: ButtonVariant; label: string } +export const Button = React.forwardRef((props, ref) => { + return null; +}); +export const LinkButton = React.forwardRef((props, ref) => { + return null; +}); +export const clearButtonStyles = (theme: string) => { + return theme; +}; +`, + 'packages/ui/src/components/Button/ButtonGroup.tsx': `export function ButtonGroup(children: string) { + return children; +} +`, + 'packages/ui/src/components/Layout/Stack.tsx': `export function Stack(children: string) { + return children; +} +`, + 'packages/ui/src/themes/index.ts': `export * from './hooks';\n`, + 'packages/ui/src/themes/hooks/index.ts': `export * from './useStyles2';\n`, + 'packages/ui/src/themes/hooks/useStyles2.ts': `export function useStyles2(fn: (t: string) => string) { + return fn('theme'); +} +`, + 'packages/app/package.json': + '{ "name": "@x/app", "version": "1.0.0", "main": "src/main.tsx", "dependencies": { "@x/ui": "1.0.0" } }\n', + 'packages/app/tsconfig.json': `{ "compilerOptions": { "jsx": "react-jsx" } }\n`, + 'packages/app/src/main.tsx': `import { Button, LinkButton, ButtonGroup, clearButtonStyles, Stack, useStyles2 } from '@x/ui'; + +export function render() { + const styles = useStyles2((t) => t); + clearButtonStyles(styles); + ButtonGroup('x'); + Stack('y'); + const a =