diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5c59c0ebe..7055efeb2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -76,6 +76,7 @@ updates: tree-sitter-grammars: patterns: - tree-sitter-* + - '@eagleoutice/tree-sitter-*' exclude-patterns: - tree-sitter - tree-sitter-cli diff --git a/.github/scripts/check-tree-sitter-upgrade-readiness.py b/.github/scripts/check-tree-sitter-upgrade-readiness.py index c4cfa0860..12435ff1e 100644 --- a/.github/scripts/check-tree-sitter-upgrade-readiness.py +++ b/.github/scripts/check-tree-sitter-upgrade-readiness.py @@ -69,6 +69,8 @@ GRAMMARS: dict[str, tuple[str, str, str]] = { "tree-sitter-python": ("tree-sitter/tree-sitter-python", "master", "src/parser.c"), "tree-sitter-ruby": ("tree-sitter/tree-sitter-ruby", "master", "src/parser.c"), "tree-sitter-rust": ("tree-sitter/tree-sitter-rust", "master", "src/parser.c"), + # Scoped npm package: the only grammar whose name does not start with "tree-sitter-". + "@eagleoutice/tree-sitter-r": ("r-lib/tree-sitter-r", "main", "src/parser.c"), "tree-sitter-swift": ("alex-pinkus/tree-sitter-swift", "main", "src/parser.c"), "tree-sitter-typescript": ("tree-sitter/tree-sitter-typescript", "master", "typescript/src/parser.c"), # Vendored parsers — kept here so the upstream coords for drift @@ -162,13 +164,15 @@ def read_pinned_grammar_versions() -> dict[str, str]: Looks at both runtime and optional dependencies. Returns the raw range string (e.g. '0.21.4', '^0.23.0', 'file:./vendor/...') so the report can - expose how flexible each pin is. + expose how flexible each pin is. A grammar is recognised by its package + basename, so a scoped package (``@scope/tree-sitter-x``) is included, keyed + by its full name. """ pkg = _load_package_json() pinned: dict[str, str] = {} for section in ("dependencies", "optionalDependencies"): for name, spec in (pkg.get(section) or {}).items(): - if name.startswith("tree-sitter-"): + if name.rsplit("/", 1)[-1].startswith("tree-sitter-"): pinned[name] = spec return pinned @@ -182,7 +186,7 @@ def read_patched_peers() -> list[tuple[str, str, str]]: """ script = GITNEXUS_DIR / "scripts" / "prepare-tree-sitter-bundle.cjs" return re.findall( - r"\['(tree-sitter[\w-]*)@([\d.]+)',\s*'([^']+)'\]", + r"\['((?:@[\w.-]+/)?tree-sitter[\w-]*)@([\d.]+)',\s*'([^']+)'\]", script.read_text(encoding="utf-8"), ) @@ -196,7 +200,8 @@ def npm_view_json(pkg: str) -> dict | None: """ if OFFLINE: return None - url = f"https://registry.npmjs.org/{pkg}/latest" + # A scoped name (`@scope/name`) is one path segment with its slash encoded. + url = f"https://registry.npmjs.org/{urllib.parse.quote(pkg, safe='@')}/latest" try: req = urllib.request.Request(url, headers={"Accept": "application/json"}) with urllib.request.urlopen(req, timeout=8) as resp: diff --git a/.github/scripts/test_check_tree_sitter_upgrade_readiness.py b/.github/scripts/test_check_tree_sitter_upgrade_readiness.py index c75e7caca..8ed421cfa 100644 --- a/.github/scripts/test_check_tree_sitter_upgrade_readiness.py +++ b/.github/scripts/test_check_tree_sitter_upgrade_readiness.py @@ -38,8 +38,10 @@ _spec.loader.exec_module(readiness) # type: ignore[union-attr] # The exact row-diff regex the workflow's change-detection bot uses # (.github/workflows/tree-sitter-upgrade-readiness.yml) — byte-identical so a matrix # format change that would silently break change-detection fails here. Group 2 is -# ONLY the Status cell ([^|]+? before the final `|$`). -_ROW_DIFF_RE = re.compile(r"\| `(tree-sitter-[^`]+)` \|.*\| ([^|]+?) \|$", re.M) +# ONLY the Status cell ([^|]+? before the final `|$`). Group 1 is the grammar +# name, optionally scoped (`@eagleoutice/tree-sitter-r`). +_R_GRAMMAR = "@eagleoutice/tree-sitter-r" +_ROW_DIFF_RE = re.compile(r"\| `((?:@[^/`]+/)?tree-sitter-[^`]+)` \|.*\| ([^|]+?) \|$", re.M) # Mirrors the scheduled issue-update summary extraction in # tree-sitter-upgrade-readiness.yml. If the report prose changes again, the issue # comment should not silently degrade to "?/? ready. ? blocker(s)". @@ -216,6 +218,93 @@ class AssertCurrent(TestCase): self.assertIn("outside current runtime range", buf.getvalue()) +class AssertCurrentCoversR(TestCase): + """R is an npm grammar under a scoped name; --assert-current must check its + installed ABI against the current runtime's range like any other grammar.""" + + def test_out_of_range_r_abi_fails_the_gate_and_names_r(self): + real = readiness.extract_language_version + + def fake(parser_c): + return 99 if "tree-sitter-r" in parser_c.parts else real(parser_c) + + import urllib.request + buf = io.StringIO() + with mock.patch.object(readiness, "extract_language_version", side_effect=fake), \ + mock.patch.object(urllib.request, "urlopen", side_effect=AssertionError("network")), \ + contextlib.redirect_stdout(buf): + code = readiness.assert_current() + self.assertEqual(code, 1) + self.assertIn(f"{_R_GRAMMAR}: installed ABI 99 outside current runtime range", buf.getvalue()) + + +class ScopedGrammarName(TestCase): + """The R grammar's package name is scoped (`@eagleoutice/tree-sitter-r`); every + place that recognises a grammar by a `tree-sitter-` prefix must cope with it.""" + + def test_pinned_versions_include_the_scoped_grammar(self): + pkg = json.loads((_REPO_ROOT / "gitnexus" / "package.json").read_text()) + pinned = readiness.read_pinned_grammar_versions() + self.assertEqual(pinned.get(_R_GRAMMAR), pkg["dependencies"][_R_GRAMMAR]) + # Selection is by grammar basename: the runtime and non-grammar scoped + # dependencies are still excluded. + self.assertNotIn("tree-sitter", pinned) + self.assertNotIn("@ladybugdb/core", pinned) + + def test_r_row_shows_its_pin_and_is_a_blocker_not_a_bump_candidate(self): + # Production-faithful R metadata: npm latest 1.1.2 (what package.json pins) with peer ^0.21.0 — blocks 0.25 and is NOT behind latest. + def fake_npm(pkg: str): + if pkg == _R_GRAMMAR: + return {"version": "1.1.2", "peerDependencies": {"tree-sitter": "^0.21.0"}} + return {"version": "9.9.9", "peerDependencies": {"tree-sitter": "^0.25.0"}} + + def fake_fetch(url: str, timeout: int = 8): + if "parser.c" in url and "alex-pinkus" not in url: + return "#define LANGUAGE_VERSION 14\n" + if "/commits/" in url: + return json.dumps({"sha": "0123456789abcdef"}) + return None + + buf = io.StringIO() + with mock.patch.object(readiness, "npm_view_json", side_effect=fake_npm), \ + mock.patch.object(readiness, "fetch_text", side_effect=fake_fetch), \ + contextlib.redirect_stdout(buf): + code = readiness.main() + report = buf.getvalue() + row = next(l for l in report.splitlines() if l.startswith(f"| `{_R_GRAMMAR}` |")) + cells = [c.strip() for c in row.strip().strip("|").split("|")] + self.assertEqual(cells[1], "1.1.2") # Pinned + self.assertEqual(cells[-1], "Peer patch required") + self.assertEqual(code, 1) + self.assertIn(f"`{_R_GRAMMAR}@1.1.2` — peer `^0.21.0`", report) # Blocked bucket + # On latest already: not a bump candidate (pre-fix the lost pin made it + # one). Other grammars are mocked at 9.9.9 and so do appear in that list; + # only R's own line is asserted absent. + self.assertNotIn(f"- `{_R_GRAMMAR}`: ", report) + self.assertNotIn(f"`{_R_GRAMMAR}` — pinned", report) + + def test_row_diff_regex_captures_scoped_and_unscoped_names(self): + rows = ( + "| `@eagleoutice/tree-sitter-r` | ^1.1.2 | 1.1.2 | ^0.21.0 | **No** | 14 | 14 | Peer patch required |\n" + "| `tree-sitter-go` | ^0.23.0 | 0.25.0 | ^0.25.0 | Yes | 14 | 14 | Ready |\n" + ) + self.assertEqual( + dict(_ROW_DIFF_RE.findall(rows)), + {"@eagleoutice/tree-sitter-r": "Peer patch required", "tree-sitter-go": "Ready"}, + ) + + def test_row_diff_regex_mirror_matches_the_workflow(self): + # The workflow's change-detection regex is the one that matters; the + # mirror above is only worth anything while it is the same expression. + workflow = ( + _REPO_ROOT / ".github" / "workflows" / "tree-sitter-upgrade-readiness.yml" + ).read_text(encoding="utf-8") + match = re.search(r"md\.matchAll\(/(.+)/gm\)", workflow) + self.assertIsNotNone(match, "row-diff regex not found in the workflow") + # JS regex literals escape `/`; Python patterns do not. + self.assertEqual(match.group(1).replace("\\/", "/"), _ROW_DIFF_RE.pattern) + + class FetchHelperReadPhaseErrors(TestCase): """Read-phase transport failures — raised by resp.read() AFTER urlopen has returned (ConnectionResetError, ssl.SSLError, socket.timeout, @@ -272,6 +361,50 @@ class FetchHelperReadPhaseErrors(TestCase): self.assertIsNone(readiness.npm_view_json("tree-sitter-anything")) +class NpmRegistryUrl(TestCase): + """The registry names a scoped package as one path segment with the slash + encoded (``@scope%2Fname``). The unencoded form is served today, but the + encoded one is the documented spelling, so the script does not rely on that.""" + + @staticmethod + def _requested_urls(pkg: str) -> list[str]: + import urllib.request + + seen: list[str] = [] + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self, *a, **k): + return b"{}" + + def _fake_urlopen(req, *a, **k): + seen.append(req.full_url) + return _Resp() + + with mock.patch.object(readiness, "OFFLINE", False), mock.patch.object( + urllib.request, "urlopen", side_effect=_fake_urlopen + ): + readiness.npm_view_json(pkg) + return seen + + def test_scoped_package_slash_is_encoded(self): + self.assertEqual( + self._requested_urls(_R_GRAMMAR), + ["https://registry.npmjs.org/@eagleoutice%2Ftree-sitter-r/latest"], + ) + + def test_unscoped_package_is_unchanged(self): + self.assertEqual( + self._requested_urls("tree-sitter-go"), + ["https://registry.npmjs.org/tree-sitter-go/latest"], + ) + + class ReportRendering(TestCase): @classmethod def setUpClass(cls): @@ -326,6 +459,17 @@ class ReportRendering(TestCase): self.assertNotIn("Could not check", self.report) self.assertNotIn("fetch failed", self.report) + def test_r_grammar_is_listed_npm_queried_and_diffable(self): + # R is an npm dependency, so it takes the npm path (queried, not vendored) + # and its row must be captured by the change-detection regex despite the + # scoped package name. + self.assertEqual( + readiness.GRAMMARS.get(_R_GRAMMAR), + ("r-lib/tree-sitter-r", "main", "src/parser.c"), + ) + self.assertIn(_R_GRAMMAR, _render_report.last_npm_calls) + self.assertIn(_R_GRAMMAR, self.rows) + def test_held_swift_renders_held_and_keeps_exit_nonzero(self): # Status is the last matrix cell (the row-diff regex captures the whole # tail, not just status, so read the cell directly). @@ -361,17 +505,19 @@ class ReportRendering(TestCase): self.assertIsNotNone(ready) self.assertIsNotNone(blockers) # Counts are derived from _render_report()'s mock corpus (all npm peer - # deps mocked permissive): of the 10 npm-installed grammars, 9 render + # deps mocked permissive): of the 11 npm-installed grammars, 10 render # Ready and 1 — tree-sitter-cpp — is the intentional pin (#1242), so it is # not counted ready. The remaining workarounds are that intentional pin # and Swift's declaration-recovery hold. - self.assertEqual(ready.groups(), ("9", "10")) + self.assertEqual(ready.groups(), ("10", "11")) self.assertEqual(blockers.group(1), "2") def test_lists_every_pack_time_peer_patch_with_its_drop_condition(self): patched = readiness.read_patched_peers() - # Guards the AUDITED_PEERS regex: the bundle script patches 8 manifests today. - self.assertGreaterEqual(len(patched), 8) + # Guards the AUDITED_PEERS regex: the bundle script patches 9 manifests + # today, including the scoped R grammar. + self.assertGreaterEqual(len(patched), 9) + self.assertIn((_R_GRAMMAR, "1.1.2", "^0.21.0"), patched) self.assertIn(f"## Pack-time peer patches ({len(patched)})", self.report) for name, version, _ in patched: self.assertIn(f"- `{name}@{version}`", self.report) @@ -382,9 +528,11 @@ class ReportRendering(TestCase): self.assertIn("required while intentionally pinned", line("tree-sitter-cpp@0.23.4")) self.assertIn("transitive", line("tree-sitter-javascript@0.23.1")) self.assertIn("transitive", line("tree-sitter-c@0.23.6")) + # The scoped R grammar is a direct pin, so its patch is listed like any other. + self.assertIn("can drop: bump to `9.9.9`", line(f"{_R_GRAMMAR}@1.1.2")) def test_target_runtime_is_the_bundled_pin(self): - pkg = json.loads((_REPO_ROOT / "gitnexus" / "package.json").read_text()) + pkg = json.loads((_REPO_ROOT / "gitnexus" / "package.json").read_text(encoding="utf-8")) self.assertEqual(readiness.TARGET_RUNTIME, pkg["dependencies"]["tree-sitter"]) self.assertIn(f"Bundled runtime `tree-sitter@{readiness.TARGET_RUNTIME}`", self.report) diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml index 889e6cd76..afdf69882 100644 --- a/.github/workflows/tree-sitter-upgrade-readiness.yml +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -176,7 +176,8 @@ jobs: // `|$`), so change-detection fires on status transitions, not on // unrelated cell drift (e.g. an upstream-ABI bump). Mirror this in // _ROW_DIFF_RE in test_check_tree_sitter_upgrade_readiness.py. - for (const m of md.matchAll(/\| `(tree-sitter-[^`]+)` \|.*\| ([^|]+?) \|$/gm)) { + // The grammar name may be scoped (`@eagleoutice/tree-sitter-r`). + for (const m of md.matchAll(/\| `((?:@[^\/`]+\/)?tree-sitter-[^`]+)` \|.*\| ([^|]+?) \|$/gm)) { map[m[1]] = m[2].trim(); } return map; diff --git a/.gitleaksignore b/.gitleaksignore index aeca3a91f..a2c3c34fe 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -6,3 +6,6 @@ c9fdab17f25ebaf332fba6e6ba55ee328f20fe66:README.md:curl-auth-header:348 bf4fe7666212c1e2a82aea8309095f63e4b23dcc:gitnexus/tools/mutation/runner/protocol.ts:generic-api-key:11 # The previous wording of the comment above quoted that key name and was itself flagged; no credential was present. 02989dd688f863248993122879818ffc4cc95e11:.gitleaksignore:generic-api-key:5 +# Rcpp's public generator-token UUID in an R rename fixture (generated-rcpp-wrapper); no credential was present. +ab0a3acf132b3a97f78d39a97988fb5b325ad3fc:gitnexus/test/fixtures/rename-integration/r/generated-rcpp-wrapper/before/R/RcppExports.R:generic-api-key:2 +ab0a3acf132b3a97f78d39a97988fb5b325ad3fc:gitnexus/test/fixtures/rename-integration/r/generated-rcpp-wrapper/after/R/RcppExports.R:generic-api-key:2 diff --git a/AGENTS.md b/AGENTS.md index 89a34605b..ba869a7ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,5 @@ - - + + Last reviewed: 2026-10-17 @@ -41,6 +41,7 @@ Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING. - **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)** - **Objective-C provider work:** read **[docs/languages/objective-c-provider.md](docs/languages/objective-c-provider.md)** before changing Objective-C parsing or resolution. +- **R provider work:** read **[docs/languages/r-provider.md](docs/languages/r-provider.md)** and **[docs/languages/r-ingestion-gaps.md](docs/languages/r-ingestion-gaps.md)** before changing R parsing or resolution. - **Call & inheritance resolution (RFC #909 Ring 3):** See ARCHITECTURE.md § Scope-Resolution Pipeline. All languages resolve calls and inheritance through the scope-resolution pipeline (`Registry.lookup`, `preEmitInheritanceEdges`, `emitHeritageEdges`, `buildMro` → `MethodDispatchIndex`). **Shared code in `gitnexus/src/core/ingestion/` must not name languages** — plug language behavior in via `LanguageProvider` / `ScopeResolver` hooks. A language plugs in by implementing `ScopeResolver` (`scope-resolution/contract/scope-resolver.ts`) and registering it in `SCOPE_RESOLVERS`. (The legacy call-resolution DAG + `@heritage` capture path were removed in RING4-1 #942.) - **Cursor:** `.cursor/index.mdc` (always-on); `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` deprecated. - **GitNexus:** standard skills in `.claude/skills/gitnexus-*/`; MCP rules in `gitnexus:start` block below. @@ -92,6 +93,8 @@ mirror. `gitnexus/test/unit/shipped-skills-sync.test.ts` guards the copies. Toke | Date | Version | Change | | ---------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-25 | 1.20.0 | Added the R ingestion gaps inventory (`docs/languages/r-ingestion-gaps.md`) next to the R provider guide as a required reference before changing R parsing or resolution. | +| 2026-10-17 | 1.19.0 | Added the R provider guide as the required reference before changing R parsing or resolution. | | 2026-10-17 | 1.18.0 | Require the reviewed preview's single-use `plan_token` when applying semantic rename (#2973). | | 2026-09-24 | 1.17.0 | Clones with the same `origin` URL now share a store automatically; `--no-share` records a lasting opt-out (#3352). | | 2026-09-24 | 1.16.0 | Documented the shared worktree index store (`/stores/`, `analyze --share-with`, `GITNEXUS_SHARED_STORE=off`) in the storage notes (#3352). | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0d9e32b18..5f66a0bde 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -372,6 +372,8 @@ A provider can explicitly restrict its edge mode. COBOL currently uses and scope resolution contributes procedure-pointer calls. Registration alone is not a claim that every language feature uses identical resolution rules. +Shared code does not name languages. When a language needs an extension point that does not exist yet, it is added as an optional `LanguageProvider` hook that no other language has to implement (R's `postParse` and `resolveMemberOwnerNode`; see the hook table below). + ### Pipeline stages ``` @@ -568,7 +570,7 @@ CI auto-discovers the set via `tsx`. No workflow edit required. ## Language-agnostic graph feeding -18 languages → single unified graph. Four abstraction layers: +19 languages → single unified graph. Four abstraction layers: ``` Unified Graph Schema (44 node types, 21 relationship types) @@ -594,8 +596,10 @@ Each language implements `LanguageProvider` (`language-provider.ts`). Key fields | `mroStrategy` | `first-wins` / `c3` / `none` | | `descriptionExtractor` | Optional hook returning a symbol's doc-comment text as its `description`; feeds the embedding metadata header so doc-only terms are semantically searchable (issue #2270). Most languages register `createLeadingDocDescriptionExtractor` (shared, language-neutral; per-language comment/wrapper config passed at the call site) | | `definitionPropertiesExtractor` | Optional language-owned hook for structured, clone-safe definition metadata. Shared ingestion persists these properties opaquely; the owning provider supplies the extraction semantics. | +| `postParse` | Optional. Whole-graph pass a language runs once after every chunk is merged and before scope resolution (deferred owner resolution, manifest-driven export refinement). Called once per analyze, awaited, only when the language has parsed files; a throw aborts the parse phase. Incremental analyse writes back only changed files, so a mutation of an unchanged file's nodes is not persisted; a hook whose result depends on manifests that are not parsed sources needs `--force` after such an edit. R: deferred `setMethod`/R6 owners + NAMESPACE exports. | +| `resolveMemberOwnerNode` | Optional. For languages whose members are owned by a call/assignment rather than an enclosing container node (R: R6/RefClass): returns the node the method and field extractors read members from, or null. Consulted by the enclosing-owner slot after the container walk and `resolveFileTypeOwner` (same slot; Zig file-owner is the precedent). Not an S4 mechanism: R's `setMethod(...)` returns null (its owner is a string-literal class name, not a node); S4 ownership arrives only through the `ownerNameHint` → `postParse` path. | -18 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. +19 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. ### Unified capture tags @@ -794,4 +798,5 @@ Node IDs use arity suffix (`#`): `Method:file:Class.method#1` vs `#2 - [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents - [TESTING.md](TESTING.md) — how to run tests - [docs/languages/objective-c-provider.md](docs/languages/objective-c-provider.md) — Objective-C provider behavior and limits +- [docs/languages/r-provider.md](docs/languages/r-provider.md) — R provider behavior and limits - `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a977ba196..b29d6073e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -204,7 +204,7 @@ The CLI pins and bundles the native `tree-sitter` runtime and its npm grammars. Several compatible grammars still declare older runtime peer ranges. Root npm `overrides` do not propagate to consumers, and older npm versions validate bundled peers against their original ranges even when an override is configured. The -lockfile therefore pins the same eight audited peers to exactly `0.25.1`. +lockfile therefore pins the same nine audited peers to exactly `0.25.1`. `gitnexus/scripts/prepare-tree-sitter-bundle.cjs` runs during `postinstall` and `prepack` to keep installed dependency manifests consistent with that lockfile and the published bundle. It leaves grammar sources unchanged and replaces the diff --git a/README.md b/README.md index 9c0eb862f..ff0a20d9e 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,18 @@ Ruby, Rust, Swift, TypeScript, Vue and Zig have registered providers. Support varies by construct; see the [language inventories](docs/languages/README.md) for measured behavior, exclusions and remaining gaps. +R (experimental) also has a registered provider (`.r` / `.R`, parsed with +`@eagleoutice/tree-sitter-r`): functions, S4, R6 and RefClass classes with their +methods and fields, `library()` / `require()` / `source()` imports to local +packages and files, `obj$method()` and `pkg::fn()` calls, R6 `inherit=` / S4 +`contains=` heritage and NAMESPACE-driven export status. Calls through function +values, `do.call()`, infix operators and S3 dispatch are not linked; `.Rmd`, +`box::use()` and external (CRAN) packages are not supported. With `--pdg` it +builds a control-flow graph (control dependence only, no reaching definitions), +and `rename` handles plain functions while refusing classes, members, S3 methods, +exported package functions and names spelled as strings. See +[docs/languages/r-provider.md](docs/languages/r-provider.md) for coverage and limits. + ## Documentation diff --git a/docs/languages/README.md b/docs/languages/README.md index 23121f260..93b2f6c93 100644 --- a/docs/languages/README.md +++ b/docs/languages/README.md @@ -1,7 +1,7 @@ # Language Ingestion Gap Inventories These inventories are the starting point for language-specific plans. There is one -`-ingestion-gaps.md` file for each of the 18 languages in +`-ingestion-gaps.md` file for each of the 19 languages in [SCOPE_RESOLVERS](../../gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts). Names match provider directories; `jvm` is shared infrastructure, not another language. @@ -29,6 +29,7 @@ receiver examples do not establish whole-language, CFG/PDG or rename completenes | [Objective-C](objective-c-ingestion-gaps.md) | Add receiver measurements without weakening dynamic-dispatch boundaries. | | [PHP](php-ingestion-gaps.md) | Retain PHPDoc container types and establish namespace owners. | | [Python](python-ingestion-gaps.md) | Audit constructor-field and module ownership coverage. | +| [R](r-ingestion-gaps.md) | Type receivers beyond same-function constructors; revisit R-02 and R-12. | | [Ruby](ruby-ingestion-gaps.md) | Isolate YARD parameter bindings before adding recall. | | [Rust](rust-ingestion-gaps.md) | Measure conservative smart-pointer limits before widening them. | | [Swift](swift-ingestion-gaps.md) | Type indexed receivers, then revalidate historical reports. | @@ -55,8 +56,8 @@ The receiver harness has separate states: - **N/A:** the harness declares this spelling inapplicable to its measurement. Read its reason before interpreting language support. - **GRAMMAR-UNAVAILABLE:** the parser did not load. No behavior was measured; this is not a product-gap verdict. -An all-N/A row (C or COBOL) does not mean complete ingestion. Vue now has an executable SFC row. Zig and -Objective-C have dedicated tests but no row in this shape arm. A fully resolving +An all-N/A row (C or COBOL) does not mean complete ingestion. Vue now has an executable SFC row. Zig, +Objective-C and R have dedicated tests but no row in this shape arm (R: the receiver corpus linked from its inventory). A fully resolving row also proves only its fixed examples. The baseline's count arm includes external and unknown receivers, so total drops are not an actionable bug count. diff --git a/docs/languages/local-import-scoping.md b/docs/languages/local-import-scoping.md index e34d8d4a4..7e48fa39c 100644 --- a/docs/languages/local-import-scoping.md +++ b/docs/languages/local-import-scoping.md @@ -39,7 +39,7 @@ pin that distinction. ## Providers with expanded lexical support These seven entries are part of the -[18-provider registry](../../gitnexus/src/core/ingestion/languages/index.ts). +[19-provider registry](../../gitnexus/src/core/ingestion/languages/index.ts). The evidence links cover exact targets and negative cases, not just edge counts. | Provider | Supported behavior | Source and regression evidence | @@ -225,7 +225,7 @@ activate before their declarations. ## Retained provider behavior -The other eleven providers retain their existing import/loading models. This +The other twelve providers retain their existing import/loading models. This work does not reinterpret every dependency construct as a local value import. | Provider | Retained model and boundary | Evidence | @@ -241,6 +241,7 @@ work does not reinterpret every dependency construct as a local value import. | C# | Namespace/file `using` directives remain imports; resource `using` statements are not import directives. | [Directive query](../../gitnexus/src/core/ingestion/languages/csharp/query.ts), [import tests](../../gitnexus/test/unit/scope-resolution/csharp/csharp-imports.test.ts) | | Swift | Imports and target/module visibility retain provider ownership. | [Ownership](../../gitnexus/src/core/ingestion/languages/swift/simple-hooks.ts), [import decomposition](../../gitnexus/test/unit/scope-resolution/swift/import-decomposer.test.ts), [target siblings](../../gitnexus/test/unit/scope-resolution/swift/target-siblings.test.ts) | | Dart | Library imports retain existing package identity and visibility rules. | [Ownership](../../gitnexus/src/core/ingestion/languages/dart/simple-hooks.ts), [library/package regressions](../../gitnexus/test/integration/resolvers/dart.test.ts) | +| R | `library()`/`require()` of a local package attach its exported names (every top-level name when it has no NAMESPACE; wildcard loading, filtered by NAMESPACE, and not told apart inside a function); `source()` binds no name; NAMESPACE `importFrom()` names bind to definitions in local packages; `pkg::fn()` is decided by its qualifier. No function-local loading model. | [Provider guide](r-provider.md), [Import resolver](../../gitnexus/src/core/ingestion/languages/r/import-target.ts), [Ownership](../../gitnexus/src/core/ingestion/languages/r/simple-hooks.ts), [library masking](../../gitnexus/test/integration/resolvers/r-library-masking.test.ts), [importFrom binding](../../gitnexus/test/unit/scope-resolution/r/r-namespace-imports.test.ts) | ## Taint provenance diff --git a/docs/languages/r-ingestion-gaps.md b/docs/languages/r-ingestion-gaps.md new file mode 100644 index 000000000..f43fea6a0 --- /dev/null +++ b/docs/languages/r-ingestion-gaps.md @@ -0,0 +1,180 @@ +# R Ingestion Gaps + +Audited on 2026-10-24 with freshly built production sources at `a9b112ae5`, and +re-checked on 2026-10-25 against the audit fixes (the rename guards landed at `ab0a3acf1`; the +closure commit after them changed only tests, docs and behaviour-neutral code), with the +accompanying receiver corpus and regression tests. See the [measurement audit](receiver-resolution-audit.md) +for the benchmark commands, target checks, exclusions and test limits. R has no row in +the receiver shape arm of that benchmark (no `bench/receiver-resolution` entry was added, as for +Zig and Objective-C), so every state below comes from the dedicated corpus and +regression tests named in each item, not from `baseline.json`. The benchmark's count arm +still records R call drops (21 `receiver-unresolved` drops attributed to `.R` files in its baseline), +but that is a count, not a state per shape. Broader ingestion is not +exhaustively assessed. The supported behaviour is described in the [R provider guide](r-provider.md). + +## Receiver shape snapshot + +States below come from the [receiver corpus test](../../gitnexus/test/integration/resolvers/r-receiver-corpus.test.ts), +a table of call shapes, each with a caller, an exact expected target (or no edge) and the +outcome recorded at the call. Its fixtures are `r-receiver-corpus/` (no `DESCRIPTION`, so +the unique-name fallback is live; R6 `User` and `Address` both define `save`, and free +functions `save`, `greet`, `climb_up` and `nothere` are the decoys) and +`r-receiver-corpus-qualified/` (a package that re-exports another package's `fmt`, and a +caller-local `fmt` as the decoy). A row that resolves asserts the exact node id (for +example `Method:R/models.R:User.save#0`) and that no other edge leaves the caller; a row without +an edge asserts that the caller has no edge at all (so also none to a decoy) and +names the positive control that resolves nearby. Two repository-wide checks add that no edge +reaches a decoy and that the table lists every edge and every dropped site. + +In the harness vocabulary of the [README](README.md#reading-the-evidence): + +- **RESOLVES** (exact target, no decoy edge): a receiver typed by a constructor whose every binding in its frame agrees + (`u <- User$new(); u$save()` in a function, or at module level of the same file for that file's functions), including when `Address` shares the method + name `save` with `User` and with a free function; typed siblings in either order; `self$m()` + inside an R6 method for an own method and for one inherited from a parent (the corpus has the + parent in the same file; `r-self-binding` covers another file), also after a write through + `self` (`self$visits <- 1`), and for a parent's public method when the class has a private + one of that name (`self` is the public interface); `pkg::fmt()` for a package that defines the function, and for a package that re-exports it from a + second package (one hop), each beside a caller-local `fmt`. +- **VISIBLE-GAP, a recorded drop** (no edge, with an outcome that counts toward the `impact` + lower bound): `self$m()` where neither the class nor a parent defines `m` as a public member, + a private method included (`self$secret()`; `member-lookup-ambiguous`, the shared pass's + closest label), `self$m()` in a free function + (`receiver-owned-but-unbound`), a receiver whose name has a disagreeing binding (a second class, an untyped + value, a loop variable, a formal, `rm(x)`, a `<<-` elsewhere in the file: `receiver-unresolved`), a private member + through a typed receiver (`u$secret()`, `member-lookup-ambiguous`), and a chain with a variable at its base: `a$b$save()`, + `User$new()$save()`, `self$repo$save()` (`receiver-unresolved`). These are + deliberately unresolved: a decoy edge would be wrong, and the right answer needs types that + R6 does not declare (R-21). +- **INVISIBLE-GAP** (no edge and no recorded outcome, so drop counts cannot reveal it): an + untyped local (`u <- make_user(); u$save()`), an untyped parameter, an untyped sibling of a + typed function that reuses the parameter name (the typed function does not lend its type, in + either order), a roxygen-typed parameter (`#' @param repo User`, `User` a class of the same + file), the S4 spelling `u <- new("User")`, `super$greet()`, `private$secret()`, a RefClass + method calling a sibling by bare name, `u@save()` (not a member call), calls of an S4 generic, generator object or `new("Pt")`, and + chains whose base is not a variable (`get_user()$save()`, `pkg::User$new()$save()`, + `x[[1]]$f$save()`). The untyped controls are intended to stay unresolved; the rest are + shapes a later slice could resolve (R-18 to R-21, R-23) or that mint no site today (R-06). + A RefClass `.self$m()` call is a recorded drop (`receiver-unresolved`), and a bare call + of a name that is both a RefClass method and a free function is ambiguous: no edge to either. +- Qualified sites that are dropped (`ext::fmt()` for a certainly external package, + `invisiblepkg::fmt()` when package discovery was truncated) get no edge, never the + caller-local `fmt`, and record no outcome (R-04). + +Of 49 corpus cells, 13 resolve, 17 are deliberately unresolved and 19 are `deferred` rows +(absence pinned so that a later feature shows up as a deliberate change); every row +without an edge names a resolving control. Nothing in the corpus reaches a decoy. + +## Incorrect results in the supported slice + +Each item is a wrong, lost or over-broad result inside what the provider claims to +support, with the test that pins today's behaviour. Pinned does not mean accepted: +a change in these results should flip the named test deliberately. + +| ID | Evidence | Gap or boundary | Pinned by | Next planning step and acceptance | +| ---- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| R-01 | Negative control / deliberate limit | An assignment of a function-capable value to a name (`helper <- identity`) makes that name a lexical name of the whole function body. The rule is flow-insensitive: a call written before the assignment, or under an `if` that may skip it, loses its edge to the outer `helper`. Lost result, chosen over a wrong edge. | `r-assignment-locals.test.ts` (d4, d5); `r-lexical-names.test.ts` (flow-insensitive); rename fixtures `assign-local-call-before`, `assign-local-conditional` | Order the binding against the call with the function's control flow (the R CFG exists) before reading it as a lexical name. Keep the rename guard in step: it reads the same assignments through one shared predicate. | +| R-02 | Measured gap | Bindings the provider does not read still resolve to the outer function (wrong target): `assign("helper", ...)`, `delayedAssign()`, `list2env()` and the values of `with(list(helper = ...), helper(x))`. A top-level assignment in the caller's own file (`helper <- identity`, `helper <- 5`) leaves the unique-name guess (0.5) on another file's `helper`. `local({ helper <- ... })` inside a function is read as part of the function body, so the call is unresolved, not wrong; the braces of `local()`, `with()` and `within()` are also block scopes, so a function defined inside them is visible there only (`r-block-scopes.test.ts`). | `r-assignment-locals.test.ts` (d19 to d23, and `r-assignment-locals-module` d10, d14); `r-lexical-names.test.ts` (does not model assign(), delayedAssign(), list2env() or with()) | Model the string-named forms only where the name is a literal; otherwise leave the call unresolved. Module-level: read a top-level assignment in the caller's file as a name claim for calls in that file. | +| R-03 | Recorded gap | `source("../exp/R/a.R")` followed by `hidden()` gets no edge when `hidden` is a private name of a package: R puts a sourced file's names in the global environment, but the name guess is vetoed for callers outside the package tree and `source()` binds no names. | `r-qualified-exports.test.ts` (R-03 source() of a private package file) | Let a caller with a `source()` import whose target is the candidate's file reach it through the guess. Test with a script that sources a package file, and keep the veto for callers that do not. | +| R-04 | Recorded gap | A qualified site dropped for a certainly external package, an unknown-locality qualifier, a duplicate definition or a name the package does not export records no resolution outcome: the drop runs before outcomes exist and no shared reason fits. Impact lower bounds miss these calls. | `r-receiver-corpus.test.ts` (qualified describes: no outcome); `r-qualified-exports.test.ts` (R-04 dropped qualified site records no outcome, in `main` and `decoy`) | Add a shared suppression reason for a qualified site with no precise target, then record it from `populateRQualifiedCalls`. That is a shared-code change. | +| R-05 | Recorded gap | In a replacement chain every call is recorded as its own setter, so the getter half R also evaluates (`bar(x)` in `foo(bar(x)) <- 1`) has no edge (a missing edge, not a wrong one). A setter defined by string (`"foo<-" <- function(...)`), `setReplaceMethod()` and `setGeneric("foo<-")` define no node, so the call has no edge, not even to the getter. | `r-replacement-calls.test.ts` (foo(bar(x)) <- 1; setters with no definition node stay edge-free) | Emit the getter site as well as the setter site once two sites can share an anchor, and give string-defined setters a definition node. Keep the "never the getter for a setter call" control. | +| R-06 | Measured gap | A chain whose base is not a plain variable (`get_user()$save()`, `pkg::Cls$new()$m()`, `x[[1]]$f$m()`, `(x)$f$save()`, or a chain the codec cannot encode) mints no site: no edge and no outcome, INVISIBLE to drop counts. | `r-chained-receivers.test.ts` (no variable base mints no site); `r-receiver-chain.test.ts` (no variable base; longer than the codec accepts; unencodable name); `r-receiver-corpus.test.ts` (`get_user()$save()`, `pkg::User$new()$save()`, `x[[1]]$f$save()`) | Decide what the shared emitter does with a call-node receiver and no chain, then mint an unresolved-receiver site for these bases. Assert the drop outcome and keep the free-function and class-method decoys unreached. | +| R-07 | Measured gap | The `library()` hook has no importer. A `library(ownpkg)` from inside the package (`tests/` included) attaches only the exports, so a private name loses its 0.85 `import-resolved` edge and falls to the 0.5 guess (the guess is allowed for callers inside the package tree). | `r-qualified-exports.test.ts` (exp/tests: library(exp); hidden()) | Give the wildcard-expansion hook the importing file, then exempt callers inside the package. Needs a shared signature change. | +| R-08 | Negative control / deliberate limit | A `pkg::name()` whose qualifier locality is unknown (package discovery was cut short, or no package configuration was recorded) is dropped and loses the old 0.5 guess. A bare call of a name that is not proven private (unreadable NAMESPACE, a cut-short `exportPattern()`) keeps the 0.5 guess. | `r-qualified-calls.test.ts` (discovery truncated); `r-qualified-visibility.test.ts` (drops every qualified site when locality cannot be decided); `r-receiver-corpus.test.ts` (`invisiblepkg::fmt()`); `r-qualified-exports.test.ts` (unr: bare unr_cond() keeps the 0.5 guess) | Raise the depth and directory bounds of package discovery or make the truncation observable; decide whether an unreadable NAMESPACE should veto the guess. | +| R-09 | Recorded boundary | Chained receivers get their unresolved outcome from the shared codec's `truncated` receiver-chain marker, which the shared fold's own comment calls defence in depth that no producer mints. R relies on it, against that producer contract. The marker is not load-bearing end to end today: a mutant that mints `truncated: false` fails only the unit test, and the outcomes and edges of the `r-chained-receivers` fixture do not change. | `r-receiver-chain.test.ts` (the shared fold refuses a truncated chain; returns undefined before reading any index; R mints the marker) | Replace the marker with a supported producer API, or have the shared comment name R's use. Either way the unit test must keep failing if upstream starts folding or refusing a truncated chain, and an end-to-end cell should be added once the marker decides an outcome. | +| R-10 | Recorded boundary | Package-tree membership uses the longest package directory that prefixes the file. For a root-level package (a `DESCRIPTION` at the repository root) every file outside a nested package counts as inside, so the private-name exemption (`pkg::name`, the name guess) is broad there. The same exemption is over-broad for `inst/` and `vignettes/`: they run against the installed package and see its exports only, so a vignette that calls a private name keeps a guessed edge (audit case v1; not measured as a restriction). | `r-qualified-exports.test.ts` (root: a script outside every nested package; a nested package is outside); `r-qualified-visibility.test.ts` (rEnclosingPackageDir claims every file of a root-level package) | Restrict the root package's tree to `R/` and `tests/`; give `inst/` and `vignettes/` exports only. Keep the nested-package control. | +| R-11 | Recorded boundary | A `pkg::name()` whose package defines no `name` resolves through its `importFrom(origin, name)` plus `export(name)` for one hop only, and only for a local origin that defines the name once. A re-export of a re-export, or an external or multiply-defining origin, gives no edge; only `importFrom()` entries are read. | `r-qualified-exports.test.ts` (decoy: l_reexport, l_ext_origin, l_dup_origin, R-11 two hops); `r-receiver-corpus.test.ts` (`pkga::fmt()`) | Follow the chain with a visited set and a depth bound; keep external and duplicate origins unresolved. | +| R-12 | Measured gap | The `library()` export filter works on names. A class reached as a receiver type through the `IMPORTS` edge a `library(pkg)` creates is not filtered: `rs <- ResultSet$new(); rs$count()` still resolves to a private class of `pkg` at 0.85. | `r-qualified-exports.test.ts` (R-12 library(exp); ResultSet$new() then rs$count()) | Apply the visibility rule when a receiver type is looked up through the import edge. Test a private class and an exported class side by side. | +| R-13 | Recorded boundary | In an R6 class whose name contains a dot (`My.Class`), a member call through `self` or a typed receiver gets no receiver lookup: no edge and a `receiver-unresolved` outcome, for an own method, an inherited one and a missing one alike. The shared receiver pass reads a dotted type name as a qualified one and cannot find the owner's members, and the earlier route bound a same-named free function (0.49) or the class's own method (1.0). Unresolved, not wrong. | `r-self-binding.test.ts` (c12 dotted: three rows, no edge, `receiver-unresolved`); `r-typed-receiver-members.test.ts` (c_own, c_inh, c_missing) | Make the shared receiver pass look up a dotted class name's members and parents. Flip the three rows to the own method, the parent's method and unresolved. | +| R-14 | Recorded boundary | A free function in a file that does not bind `self` is treated as owning `self`, so a global `self <- Foo$new()` defined in another file is not followed: `self$m()` has no edge and the outcome is `receiver-owned-but-unbound`. A file that binds `self` itself is exempt. | `r-self-binding.test.ts` (c13 binds self itself; c13b a self bound in another file is not followed) | Resolve a global `self` across files, or stop treating a free function as an owner when any file binds `self`. Keep c9 (no object) unresolved. | +| R-15 | Negative control / deliberate limit | A `self$m()` in a class whose parent is named through a package (`inherit = basepkg::BasePkgClass`) cannot find the member: no edge and the outcome `member-lookup-ambiguous`, the closest label the shared pass offers. The class gets no `EXTENDS` edge either. | `r-call-shapes.test.ts` (creates no EXTENDS edge for inherit = pkg::Base; records member-lookup-ambiguous) | Add a shared "member not found" reason so the outcome names the real cause, then link `pkg::Base` when the package is indexed. | +| R-16 | Recorded gap | `rename` refuses (`incomplete_semantics`) a function while a same-named parameter is read in a body (`function(x, helper) x + helper`) or a `for` variable of that name is called in a loop (`for (helper in xs) helper(1)`): no edge covers the occurrence. Both refuse; neither leaves a call behind. A parameter that is only declared, or called (then `ambiguous_reference`), is as described in the guide. | rename fixtures `parameter-body-read`, `for-variable-call`, `parameter-read-only`, `parameter-no-default-call` | Cover a parameter read and a `for` variable as bindings in the scope graph, then let the rename provider accept them. Keep the call-under-parameter refusal. | + +## Deferred work + +These are not incorrect results: the provider does not model them, and each has a recorded +absence (a corpus row or a named test) so that building one shows up as a deliberate +change. + +| ID | Evidence | Gap or boundary | Pinned by | Next planning step and acceptance | +| ---- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| R-17 | Recorded boundary | S3 dispatch: `UseMethod()` and `NextMethod()` create no edge; S3 methods are indexed as plain functions with dotted names, and a bare call never binds to a dotted definition. | `r-dotted-s3-collision.test.ts`; `r-call-shapes.test.ts` (does not link UseMethod() or NextMethod() to a method) | Resolve `UseMethod("generic")` to the methods registered for the class of the first argument, once that class is known. Needs receiver types first (R-20). | +| R-18 | Recorded boundary | S4 generics and dispatch: a call to a generic (`area(x)` after `setGeneric`), a `setClass` generator (`Person(...)`), `new("Class")` and `Class$new()` get no edge from the class itself. The S4 `new("Class")` constructor binds the per-file type environment only, not the scope, so it types no receiver. The guess can be wrong across load units: with `library(geo)`, a call `area(c)` of geo's generic, or `Circle(r = 1)` of its generator `Circle <- setClass("Circle", ...)`, binds a same-named plain function of a script at 0.5 (audit case g2, pinned). Two cases are right and kept: the bare-`setClass` plus separate constructor-function idiom (`setClass("Circle", ...)` and `Circle <- function(r) new(...)`) keeps its edge to the constructor, and in one package the later-collated `Circle <- function` wins over an earlier `Circle <- setClass(...)`, so its 0.5 edge is correct. That is why no generator-specific veto ships. The query change that would capture generics is deferred: it moves pinned corpus rows. | `r-receiver-corpus.test.ts` (S4 generic, generator and `new("Pt")` calls; `u <- new("User"); u$save()`); `type-env-r.test.ts` (the type-environment binding); `r-s4-generator.test.ts` (generator cells: the wrong 0.5 limits, the constructor idiom, the control cells and the collate-order cell) | Bind `new("Class")` in the scope as `Class$new()` is, then resolve generics through method signatures. Assert signature owners and keep the unbound spelling unresolved until then. | +| R-19 | Recorded boundary | RefClass methods that call a sibling or inherited method by bare name (`log_it(x)`) or through `.self$log_it(x)` get no edge. `self` in a RefClass method is owned but untyped (`receiver-owned-but-unbound`). | `r-self-binding.test.ts` (c10 an unassigned R6Class and a RefClass method); `r-receiver-corpus.test.ts` (RefClass bare-name and `.self$` rows) | Bind `.self` and bare method names to the enclosing RefClass and its `contains=` chain, with the same owner and decoy controls as c1 to c9. | +| R-20 | Coverage gap / audit | Receiver propagation: `x <- Cls$new()` (R6, RefClass) types a receiver only when every binding of `x` in its frame agrees with it (the T1 rule): a second class, an untyped value, a loop variable, a formal, `rm(x)`, `assign("x", ...)` or `class(x) <- v` leaves it untyped, and the call is a recorded `receiver-unresolved` drop. This removes wrong edges and also some right ones: the lost edges are two constructors of different classes with the call after the second, where R calls the last one (`a2` in a function, `n1` for two module-level bindings) and `x <- x$validate()` (`e4`). A local from any other call, a parameter, a roxygen `@param` type (the type extractor feeds the type environment only; no edge reads it) and a typed sibling's parameter name leave the receiver untyped, and an untyped receiver records no outcome (INVISIBLE). | `r-receiver-agreement.test.ts` (the lost-edge cells a2, n1 and e4, the module-level cells, the `with()` and cross-file limits); `r-receiver-corpus.test.ts` (untyped local, untyped parameter, roxygen parameter, both sibling orders, the eleven agreement rows); `r-roxygen-param.test.ts` | Promote restricted roxygen `@param` types through `interpretTypeBinding`, not the type environment, and record a receiver-unresolved outcome for an untyped receiver. Keep the sibling cells: no type may leak between functions. | +| R-21 | Coverage gap / audit | Field and return types: R6 fields and method results carry no declared type, so `Cls$new()$m()`, `a$b$m()`, `self$repo$save()` and typed-base chains such as `p$addr$save()` (a field initialised with `Address$new()`) or `p$me()$save()` get no edge. They record `receiver-unresolved` instead. | `r-chained-receivers.test.ts`; `r-receiver-corpus.test.ts` (`a$b$save()`, `User$new()$save()`, `self$repo$save()`) | Type fields from their initialisers and methods from `invisible(self)` or a constructor return, then walk the chain. Assert the exact target and keep the untyped base unresolved. | +| R-22 | Recorded gap | Reaching definitions and taint: R has no def/use harvest, so the PDG layer reports `no-facts`, emits no `REACHING_DEF` edge and records no taint call site. An empty answer from `pdg_query` or `explain` for R means not analysed. | `unit/cfg/r-visitor.test.ts` and `integration/cfg/r-cfg-join.test.ts` (reaching definitions report `no-facts` for R) | Harvest definitions and uses per function in the CFG visitor, then enable reaching definitions for R with the shared harness. | +| R-23 | Negative control / deliberate limit | `super$m()` and `private$m()` create no edge: nothing binds `super` or `private` to a class. Neither reaches the parent's method or a free function of that name. | `r-self-binding.test.ts` (c6, c7); `r-receiver-corpus.test.ts` (`super$greet()`, `private$secret()`) | Bind `super` to the parent class and `private` to the enclosing class. Test an own and an inherited private method and keep free-function decoys unreached. | +| R-24 | Recorded boundary | Callable values: a function passed as a value (`lapply(x, f)`, `Map`, `pkg::f` as an argument), `do.call("f", ...)`, `match.fun()`, `get()` and registry dispatch (`handlers[["a"]]()`) create no `CALLS` edge. | `r-call-shapes.test.ts` (a function used other than as a callee creates no CALLS edge) | Classify R in the cross-language callable-value analysis and decide which value references count as calls. Keep the string forms unresolved. | +| R-25 | Recorded boundary (outside R) | The shared free-call pass now resolves a qualified site before it applies a lexical name claim. Rust and PHP implement the same hook, publish no such claim for a `let`, a parameter or a `use function`, and measure the same before and after: the reorder is a no-op for them. A Rust `let dispatch = ..; tools::dispatch()` has no edge to `tools.rs` before or after, a gap of another pass. | `free-call-qualified-over-name-claim.test.ts` (fake hook); `rust-qualified-over-local.test.ts`; `php-qualified-over-local.test.ts` | Not an R item. Plan it under the Rust inventory: resolve a module-qualified call beside a local binding of the same name. | + +## Further boundaries from the audit fixes + +These rows were found while fixing the findings of the audit above. Rows R-26 to R-31 and R-33 were +measured in the audit and are not pinned by a repo test; the others are pinned by the named tests. + +| ID | Evidence | Gap or boundary | Pinned by | To close | +| -- | -- | -- | -- | -- | +| R-26 | Measured gap (audit) | A `library(external)` after `library(local)` does not mask the local package's names: the external package is dropped without a trace, so a name both define still binds to the local package at 0.85, where R masks it with the package attached last. | none (measured in the audit; no repo test) | Record external `library()` calls as masking bindings that resolve to nothing; keep the last-attached rule. | +| R-27 | Measured gap (audit) | A name defined in a `source()`d file does not mask a name from `library()`: `source()` binds no name, so the wildcard import wins, where R's global environment masks the attached package. | none (measured in the audit; no repo test) | Bind the sourced file's names as named imports; plan it with R-03. | +| R-28 | Measured gap (audit) | Inheritance across class systems: an R6 `inherit =` naming an S4 class, or an S4 `contains =` naming an R6 class, still gets an `EXTENDS` edge, although the systems do not inherit from each other. The generic heritage query has no veto. | none (measured in the audit; no repo test) | Add an inheritance-base veto hook to the shared heritage pass. | +| R-29 | Measured gap (audit) | A class defined inside a function is reachable from other files at 0.85 (`import-resolved`) through the shared unique-name class fallback, although no other file can see it. | none (measured in the audit; no repo test) | Give the typed-receiver class lookup a visibility filter, or let a language opt out of the unique-name class fallback. | +| R-30 | Measured gap (audit) | `Collate:` in `DESCRIPTION` is never read: where two files of a package define the same name, the same-file preference or the ambiguity rule decides, not the load order R uses. | none (measured in the audit; no repo test) | Read `Collate:` and rank same-name definitions by load order. | +| R-31 | Measured gap (audit) | A RefClass generator assigned under another name (`Acc <- setRefClass("Account")`) names the class from the string: `Account$new()` resolves, `Acc$new()` has no edge. | none (measured in the audit; no repo test) | Capture the assigned name of a generator and alias it to the class. | +| R-32 | Measured gap | A dotted call that spells a full name is looked up by qualified name at 0.35 (`scope-resolution: call`), which also matches a member whose qualified name is `Class.member`: `data.frame(x)` from another file binds the method `frame` of an R6 class `data`, and `Stack.push(x)` a member `Stack.push`. The same tier is the right route for a dotted free function (`read.config()`), so it is documented, not removed. In the same file `data.frame(x)` gets no edge. | `r-typed-receiver-members.test.ts` (u2 `data.frame(x)`, u3 `Stack.push(x)` at 0.35) | Add an opt-in flag so a dotted bare call never matches an owned member; keep the free-function route. | +| R-33 | Measured gap (audit) | Calls inside quoted code (`quote()`, `bquote()`, `expression()`, `substitute()`, `alist()`, `rlang::expr()`, and formulas) are recorded as calls of the declaration the code names, at the same confidence as a direct call. The edge is a may-call to the right declaration, never to an unrelated one, and it keeps `eval()`-ed code and NSE verbs linked and renamed. | none (measured in the audit; no repo test) | Only if wanted: drop free-call matches under the first argument of the quoting functions (about 20 lines in `captures.ts`), keep formulas; `rename` would then refuse those occurrences as uncovered. | +| R-34 | Recorded boundary | Package visibility rules A, B and D (the dependency veto of the name guess; see the guide, Resolution model) apply to package code only. They veto calls that are unsupported for check-clean package code, which is stricter than the shared hook's wording ("impossible"): a user who attaches a package by hand can still make such a call run. Scripts are left alone, since a script may attach anything, so a script call keeps its 0.5 guess across packages. The rules switch off when the package's NAMESPACE exports or imports were unreadable. | `r-package-visibility.test.ts` | Treat a script as a package of its own: its `library()` and `source()` calls declare its dependencies. | +| R-35 | Recorded boundary | `with()` and `within()` are not modelled as bindings of a receiver: `with(data, u$save())` after `u <- User$new()` keeps its typed edge even when `data` holds a `u` (b6: a wrong edge kept), and a module-level `u` is left unresolved only through another function's `<<-` poison (m7), not because of `with()`. | `r-receiver-agreement.test.ts` (b6, m7) | Model `with()`/`within()` as frames whose bindings come from the data argument. | +| R-36 | Recorded boundary | The receiver-typing frames are per file, but a package namespace spans files: a module binding in `a.R` rebound by `<<-` in `b.R` stays typed (`otherfile` q1 keeps its edge to `Db.query`). A test file that reuses one name for different classes in several `test_that` blocks leaves the name untyped in all of them, because blocks are not frames. | `r-receiver-agreement.test.ts` (q1, o1, reset) | Merge the frames of the files of one load unit; make block scopes frames. | +| R-37 | Recorded boundary | `rename` does not see code that reads the function name off the call (`match.call()`, `deparse(substitute(f))`, snapshot tests of printed calls) or builds code from a value (`{name}` interpolation of a variable, `paste0`-built code). A string that spells a call of the name is refused; these are not. | none (documented limit) | Not detectable from the sources; keep the documentation. | +| R-38 | Measured gap | The rename guard for R files the index does not cover (extension scan of skipped directories, `.gitignore`d files, over-cap files, `.Rprofile`, `R/*.S`) retains the identifier tokens of all of them with the plan. On a very large unindexed R tree this exceeds the retained-plan budget and the preview fails closed with `plan_too_large`: this repository's own 6,378 R files (fixtures and scratch copies included, all counted as unindexed) charge 117% of the 64 MiB budget, while a 511-file R tree charges 4.06%. Callers hidden by `.gitnexusignore` or in `.git`, `node_modules`, `*.Rcheck`, `renv/library` and `packrat/lib*` are not scanned, nor are extensionless Rscript files. | `r-rename-unindexed-sources.test.ts`; `rename-provider-r.test.ts` (unindexed-bin-script, rprofile-mention, unindexed-r-dir-s-file, unindexed-no-mention) | Keep a digest plus a bit per file once `refreshResolutionConfig` receives the old name. | +| R-39 | Recorded boundary | The local-definition guard (a call before a function-local or block-local definition of the name) skips a body that contains no call of the name, and ignores a call inside the qualifying literal itself (recursion). The plan text would have refused a non-direct local literal even with zero calls; the guard refuses only where a call can run before the definition. | R rename fixtures, `rename-provider-r*.test.ts` (local-def-call-before, local-def-if, local-def-switch, local-def-loop, local-def-closure-before, local-def-first, block-def-call-before, block-def-conditional, block-def-nested-first) | None needed; recorded because the guard is narrower than first planned. | +| R-40 | Recorded gap | A block-local or nested definition still counts toward the uniqueness of its name, so a call of that name in another block, where R calls the one top-level definition in another file, gets no edge and no outcome (`stubonly()`, `outerh()`, `mockme()`): a lost edge, not a wrong one. Functions nested in functions behave the same. | `r-block-scopes.test.ts` (limit rows) | Exclude definitions not visible from the caller from the uniqueness count, with caller-scope plumbing in the visibility hook. | +| R-41 | Recorded boundary (outside R) | A file over the shared size cap (512 KB by default) is skipped for every language. With the cap raised, a 50,000-line (2.4 MB) file took about 138 s and ended with its definitions dropped, for R and TypeScript alike. Not R-specific and left unchanged. | none (shared behaviour) | Shared work: bound the per-file parse time for all languages. | +| R-42 | Recorded boundary | Work bounds. (1) A matcher visit costs about 50 to 70 ns, and no fixed per-test overhead is charged on top of the counted visits: charging one was tried and dropped because it moved a pinned visit count and a visibility verdict. (2) At call-nesting depth 1,500 the R query returns 1,028 matches for 1,500 calls (tree-sitter behaviour), so a match count is not a call count; the pathological-input tests assert the sentinel node, not a number of matches. | `r-replacement-scaling.test.ts`; `r-pathological-inputs.test.ts`; `r-export-pattern-shared-budget.test.ts`; `r-qualified-visibility.test.ts` | Retake the per-test overhead with a raised visit budget in `r-qualified-visibility`. | +| R-43 | Measured gap | A call of a name written before a function literal assigned to that name in the same function body or block scope (`test_that("x", { expect_equal(helper(), 1); helper <- function() "stub" })`, or a literal assigned under `if`) resolves to the local literal (0.85 `local-call`), where R calls the outer definition: the local binds only once the assignment has run. Rename refuses such a name (`ambiguous_reference`), since the call would be left behind. The edge was measured in the C15 probes and is not pinned. | R rename fixtures, `rename-provider-r*.test.ts` (local-def-call-before, block-def-call-before, block-def-conditional) | Bind a call to a local literal only when the literal is assigned, unconditionally, before it. | +| R-44 | Measured gap | Methods added to a RefClass generator by `Gen$methods(name = function() ...)` are not modelled: they create no Method node, and a bare call inside one resolves like a top-level call (`Foo$methods(run = function() helper())` reaches the top-level `helper` at 0.85 `local-call`, where R calls the class method `helper`). Rename refuses a name that any `x$methods(...)` call mentions (`ambiguous_reference`); the receiver is not traced to a `setRefClass()` call. | R rename fixtures, `rename-provider-r*.test.ts` (refclass-methods-call) | Read `$methods()` calls on a generator as members of its class, then bind bare calls in RefClass methods (R-19). | + +## Code and regression entry points + +- [Language provider directory](../../gitnexus/src/core/ingestion/languages/r/) and its + [scope resolver](../../gitnexus/src/core/ingestion/languages/r/scope-resolver.ts). +- [Receiver corpus](../../gitnexus/test/integration/resolvers/r-receiver-corpus.test.ts) + with fixtures [`r-receiver-corpus/`](../../gitnexus/test/fixtures/lang-resolution/r-receiver-corpus/) + and [`r-receiver-corpus-qualified/`](../../gitnexus/test/fixtures/lang-resolution/r-receiver-corpus-qualified/). +- Receivers and `self`: + [r-self-binding.test.ts](../../gitnexus/test/integration/resolvers/r-self-binding.test.ts), + [r-chained-receivers.test.ts](../../gitnexus/test/integration/resolvers/r-chained-receivers.test.ts), + [r-receiver-chain.test.ts](../../gitnexus/test/unit/scope-resolution/r/r-receiver-chain.test.ts), + [r-call-shapes.test.ts](../../gitnexus/test/integration/resolvers/r-call-shapes.test.ts), + [r-receiver-agreement.test.ts](../../gitnexus/test/integration/resolvers/r-receiver-agreement.test.ts), + [r-typed-receiver-members.test.ts](../../gitnexus/test/integration/resolvers/r-typed-receiver-members.test.ts). +- Locals and replacement calls: + [r-assignment-locals.test.ts](../../gitnexus/test/integration/resolvers/r-assignment-locals.test.ts), + [r-formal-shadows.test.ts](../../gitnexus/test/integration/resolvers/r-formal-shadows.test.ts), + [r-replacement-calls.test.ts](../../gitnexus/test/integration/resolvers/r-replacement-calls.test.ts), + [r-lexical-names.test.ts](../../gitnexus/test/unit/scope-resolution/r/r-lexical-names.test.ts), + [r-block-scopes.test.ts](../../gitnexus/test/integration/resolvers/r-block-scopes.test.ts). +- Qualified calls and exports: + [r-qualified-exports.test.ts](../../gitnexus/test/integration/resolvers/r-qualified-exports.test.ts), + [r-qualified-calls.test.ts](../../gitnexus/test/integration/resolvers/r-qualified-calls.test.ts), + [r-qualified-visibility.test.ts](../../gitnexus/test/unit/scope-resolution/r/r-qualified-visibility.test.ts), + [r-package-visibility.test.ts](../../gitnexus/test/integration/resolvers/r-package-visibility.test.ts), + [r-s3method-registration.test.ts](../../gitnexus/test/integration/resolvers/r-s3method-registration.test.ts), + [r-s4-generator.test.ts](../../gitnexus/test/integration/resolvers/r-s4-generator.test.ts). +- Rename: [rename-provider-r.test.ts](../../gitnexus/test/integration/rename-provider-r.test.ts) and + `rename-provider-r-part2.test.ts` to `rename-provider-r-part5.test.ts` (one fixture list dealt + into five parts for the CI time budget; only part 1 is on the Windows and macOS list, since + cmd.exe's 8191-char command limit leaves no room for the other four), and + [r-rename-unindexed-sources.test.ts](../../gitnexus/test/unit/r-rename-unindexed-sources.test.ts), + with fixtures under `gitnexus/test/fixtures/rename-integration/r/`. +- Work bounds: + [r-replacement-scaling.test.ts](../../gitnexus/test/unit/scope-resolution/r/r-replacement-scaling.test.ts), + [r-pathological-inputs.test.ts](../../gitnexus/test/integration/resolvers/r-pathological-inputs.test.ts). +- Shared free-call ordering: + [free-call-qualified-over-name-claim.test.ts](../../gitnexus/test/unit/scope-resolution/free-call-qualified-over-name-claim.test.ts), + [rust-qualified-over-local.test.ts](../../gitnexus/test/integration/resolvers/rust-qualified-over-local.test.ts), + [php-qualified-over-local.test.ts](../../gitnexus/test/integration/resolvers/php-qualified-over-local.test.ts). + +Evidence details: [receiver baseline notes](../../gitnexus/bench/receiver-resolution/BASELINE.md) (no R row) and +[current roadmap](../../type-resolution-roadmap.md#current-backlog). diff --git a/docs/languages/r-provider.md b/docs/languages/r-provider.md new file mode 100644 index 000000000..f0b6bb6fe --- /dev/null +++ b/docs/languages/r-provider.md @@ -0,0 +1,209 @@ +# R Language Provider + +Status: implemented (experimental) + +The provider is covered by unit and integration tests, listed under [Tests](#tests). It is classified as experimental in the language support table. This status describes what is implemented today; it does not promise R's dynamic dispatch. + +Wrong or lost results inside the supported slice, and the work that is deliberately deferred, are tracked with IDs (`R-NN`), evidence labels and pinning tests in [r-ingestion-gaps.md](r-ingestion-gaps.md). This guide describes the behaviour; that inventory is the starting point for a plan. + +## Scope + +GitNexus indexes `.r` and `.R` files with the `@eagleoutice/tree-sitter-r` grammar and resolves them through the scope-resolution pipeline (see ARCHITECTURE.md, Scope-Resolution Pipeline). R-specific logic lives in `gitnexus/src/core/ingestion/languages/r/`, with the per-language extractor files beside those of the other languages (the import resolver is `languages/r/import-target.ts`). Apart from the registrations every language has (the language enum, the file-extension and classification tables, the grammar loader and parse-worker grammar map, and the query, provider and scope-resolver registries), shared ingestion code does not name R; the provider plugs in through `LanguageProvider` and `ScopeResolver` hooks. The grammar is loaded through the shared grammar loader on first use, and the parse worker loads it behind a guard like the vendored grammars: if the native binding cannot load, R files are skipped with one logged warning per run and no other language is affected (the warning names the file count and either "parsing disabled via GITNEXUS_SKIP_OPTIONAL_GRAMMARS" when the grammar was opted out, or "parser not available (native binding may not have built)" with an `npm rebuild @eagleoutice/tree-sitter-r` hint). R's own warnings are the package-configuration ones in `package-config.ts`. + +R has no class syntax: classes are function calls (`R6Class(...)`, `setClass(...)`, `setRefClass(...)`) and package exports live in a NAMESPACE file rather than in the source. That is why the provider uses three optional hooks: `postParse` (attach each member to its owner and refine export status once every file is parsed), `resolveMemberOwnerNode` (find the owner of a member from a call rather than an enclosing node) and `resolveDetachedOwnerName` (qualify a member's id with its class, so two classes in one file that share a member name stay distinct nodes: `Method:R/a.R:Stack.push#0`, `Property:R/a.R:Stack.items`). + +## Supported constructs + +Definitions: + +- Functions defined by assignment (`<-`, `=`, `<<-`). S3 methods appear as ordinary functions with dotted names such as `print.foo`. +- S4 classes (`setClass`), generics (`setGeneric`), methods (`setMethod`) and slots (`representation`, `slots`). +- A `setMethod()` is owned by the class its signature names: the string, or the first argument of `signature(...)` or `c(...)`. A method on several classes (multiple dispatch) belongs to its first class, so `signature("A", "B")` and `signature("A", "C")` give one method of `A` (the first written wins) and a signature whose first argument is not a string names no owner. +- R6 classes with their methods, fields and active bindings. +- RefClass (`setRefClass`) classes with their methods and fields. +- Slots and fields given as `list(...)`, `representation(...)` or `c(...)`. A named entry (`c(a = "numeric")`) records its type; an unnamed string in `slots = c("a", "b")` or `fields = c("a", "b")` names an untyped member, as in R. An unnamed string in `representation(...)` or `list(...)` is not a member. +- The argument that names an S4 or RefClass class, a generic or a method is the argument spelled with the first formal (`Class=`, `name=`, `f=`) wherever it sits, else the first unnamed argument (so a later `Class=` takes the formal from an earlier unnamed argument, as in R). Other strings (`contains=`, `valueClass=`, a `setMethod` signature) create no node. + +Types: + +- `x <- Type$new()` (R6, RefClass) and `x <- new("Class")` (S4) type `x`. Resolution of `x$method()` is asserted by tests for R6 only (`r-receiver-corpus`). The S4 `new()` binding is made in the per-file type environment, not in the scope, so it types no receiver (`u <- new("User"); u$save()` has no edge); it is unit-tested only. A constructor types a name only when every binding of that name in its frame agrees with it (`receiver-agreement.ts`): a second class, an untyped value, a loop variable, a formal, `rm(x)`, `assign("x", ...)` or `class(x) <- v` in the same frame leaves the receiver untyped, and `obj$m()` then ends as `receiver-unresolved` with no edge, never as an edge to another class's method. `NULL`, a constant and a write through the object (`x$a <- v`, `x[[i]] <- v`) do not count against it. A frame is the file's top level or one `function_definition`; the braces of `local({...})` and of other calls belong to the enclosing frame; `<<-` and `->>` bind in no frame of their own but are recorded in every enclosing frame, up to the top level. A module-level `u <- User$new()` also types `u` in the file's functions, unless a function binds `u` itself or the file `<<-`s it. Frames are per file: a package namespace spans files, so a rebinding in another file is not seen (R-36). `with()` and `within()` are not modelled (R-35). A test file that reuses one name for different classes in several `test_that` blocks leaves that name untyped in all of them, because blocks are not frames. A local from any other call, a parameter and a roxygen type leave it untyped (see [r-ingestion-gaps.md](r-ingestion-gaps.md), R-20). +- roxygen2 `@param` types in a restricted form: the description is the type alone (`#' @param name Type`) or starts with it in braces (`#' @param name {Type} prose`), the type is an R6, S4 or RefClass class defined in the same file, and the name is a formal of the function the block documents. The type is bound inside that function only. Prose that begins with a capitalised word binds nothing. Other roxygen2 tags (`@return`, `@field`, `@inheritParams`, `@rdname`, `@noRd`, `@export`) are not read. + +Imports: + +- `library()` and `require()` resolve to local packages, and `source()` to local files. The argument is `package=` / `file=` wherever it sits, else the first unnamed argument after any named one; named-first arguments such as `lib.loc=` and `local=` are not imports. +- NAMESPACE `importFrom()` names bind to definitions in local packages. A name that can never bind (not exported, not defined at top level, or dotted) creates no import, so its calls keep the `global-name-fallback` guess, unless the name is provably private to its package: then the guess is vetoed too and the call gets no edge (see Export status). + +Calls: + +- Free calls, `obj$method()` calls (receiver typed as above), R6 `self$method()` calls, and calls inside native pipe (`|>`) chains. +- A chained member call, `a$b$m()`, `Cls$new()$m()` or `self$f$m()` (the receiver is itself a `$`/`@` access or a call), is never typed: R6 fields and method results carry no declared type, so the chain is not followed. The site is minted with the shared codec's `truncated` receiver chain, which the shared fold refuses, and the call ends with a `receiver-unresolved` outcome and no edge, instead of reaching a free function or a method of that name elsewhere (it counts toward the `impact` lower bound). The first link must be a plain variable: a chain based on a call (`get_user()$save()`, `pkg::Cls$new()$m()`), on `x[[1]]`, `x[1]` or `(x)`, or one the codec cannot encode (more steps than its depth limit, a name with a space) mints no site at all: no edge and no outcome. That shape is invisible, not unresolved. The shared fold's comment calls the `truncated` marker defence in depth that no producer mints; R relies on it: `r-receiver-chain.test.ts` fails if R stops minting the marker or if upstream starts folding a truncated chain. End to end the marker is not load-bearing today: with it removed (`truncated: false`), the outcomes and edges of the `r-chained-receivers` fixture are unchanged and only that unit test fails (R-09). Renaming a function whose name is the last step of such a call (a free `save` beside `a$b$save()`) is refused as `ambiguous_reference`. +- A replacement-function call, `foo(x) <- value`, is recorded as a call of `` `foo<-` ``, as R runs it (`=`, `<<-`, `->` and `->>` too, and `pkg::foo(x) <- v` as a call of ``pkg::`foo<-` ``, which passes the export check of the setter name: a NAMESPACE entry may be written ``export(`foo<-`)`` or `export("foo<-")`, and an unexported setter is dropped from outside the package, never replaced by the getter). A call is in a replacement position when it lies in the target chain, reached only through first arguments and the bases of `$`, `@`, `[` and `[[`; every call of the chain is recorded as its own setter (`foo(bar(x)) <- 1` calls `` `foo<-` `` and `` `bar<-` ``), so the getter half that R also evaluates (`bar(x)`) has no edge: a missing edge, not a wrong one. Other arguments (`foo(x, bar(1)) <- v`, the index of `foo(x)[bar(1)] <- v`) and the assigned value stay ordinary calls, and `x$a <- 1` or `x[i] <- 1` (a builtin setter) has no call site at all. A function defined as `` `foo<-` <- function(x, value) ... `` is now found by the call; a string-spelled definition (`"foo<-" <- function(...)`) and `setReplaceMethod()` / `setGeneric("foo<-")` define no node, so such a call has no edge (not even to the getter). An explicit `` `foo<-`(x, v) `` is unchanged. `rename` refuses both halves of a replacement pair (see Rename). +- `self` in an R6 method is typed as the class: `self$m()` resolves through the class and its parents (`inherit =`) at the receiver-call confidence (0.85), never to a free function of that name. The class is the name the `R6Class` call is assigned to (`Child <- R6::R6Class(...)`, bare or `R6::`), and the method is a function in its `public`, `private` or `active` list (a private method has a typed `self` too). `self` is the public interface, so `self$m()` reaches only public and active members: a method in a `private` list, of the class or a parent, is skipped, and a parent's public method of that name is the target even when the class has a private one (`private$m()` itself is not resolved). A closure inside a method sees the method's `self`. Writing through `self` (`self$count <- 1`, `self[["x"]] <- v`, `self$items[[i]] <- v`, `names(self$items) <- v`) leaves it the instance. Where `self` is no R6 instance (a free function, a parameter called `self`, a plain `self <- other()` in the method or function, an `R6Class` call that is assigned to nothing, a `setRefClass` method) a `self$m()` call gets no edge and the resolution outcome is `receiver-owned-but-unbound`, unless that binding is itself typed (`self <- Foo$new()` makes `self` a `Foo`); a method that neither the class nor a parent defines as a public member (or a parent that is not indexed, as in `inherit = pkg::Base`) gets no edge and the outcome `member-lookup-ambiguous` (the closest label the shared pass offers for "member not found"). A file that binds `self` itself (`self <- Foo$new()` at top level) is exempt: its free functions read that object. The same lookup rules hold for every typed receiver, not only `self`: `u <- User$new(); u$secret()` does not reach a private method (a typed miss records `member-lookup-ambiguous`), `u@save()` is not a member call and makes no site, and a call through a generator object (`g <- setRefClass(...)`) is not a receiver. A class whose name contains a dot (`My.Class`) gets no receiver lookup at all, for `self$m()` and for typed receivers alike: the call ends `receiver-unresolved` with no edge, where it used to reach a same-named free function (R-13). The edge of a `self$m()` call is labelled `global` at 0.85 when the class is in the caller's file and `import-resolved` at 0.85 when it is in another file. +- A call through a name that a function's formal or one of its `for` variables binds (`function(data, transform) transform(data)`) gets no edge, not even to a top-level function of that name. The scope publishes those names as lexical names: they stop the lookup and the unique-name fallback. Every named formal is a lexical name, whatever its default: a default (`n = 2`, `helper = NULL`) proves nothing about the argument a caller passes, which may be a function, so a call through the formal is never bound to a function of that name outside. A `for` loop outside every function binds nothing. +- An ordinary assignment in a function's own body is a lexical name too, when its value may be a function (`helper <- identity`, `helper = some_fn()`, `identity -> helper`): a call of the name in that function, or in a closure nested in it, gets no edge to a top-level function of the same name. A function literal (`helper <- function(y) 2`) is a real definition and the call resolves to it; a constant (`helper <- 2`, a string, `TRUE`) is skipped by R at a call, so the call still reaches the outer function; `<<-` and `->>` assign outward and bind nothing in the frame. A complex target binds its base (`names(x) <- v`, `x$a <- v` bind `x`, never the called name). The rule is flow-insensitive, a deliberate trade for a conservative result: a call written before the assignment, or under an `if` that may skip it, is left unresolved too. Not modelled, so the call still reaches the outer function: `assign()`, `delayedAssign()`, `list2env()` and `with(list(...), ...)` values (R-02; pinned by `r-assignment-locals`), and a string target (`"helper" <- identity`). The braces of `local({ ... })` are read as part of the function body, so an assignment inside them makes the name lexical (the call gets no edge). Braces that R evaluates in an environment of their own (`local({...})`, `with(data, {...})`, `within()`, testthat's `test_that(desc, {...})`, `describe()` and `it()`, bare or `pkg::`-qualified) are block scopes: a function literal defined inside is visible in those braces only, so a call from another block or file does not reach it (`fallback-refused`, never the 0.5 guess). `withr::with_*()` and `suppressWarnings({...})` evaluate their braces in the caller's frame and get no scope. A block-local definition still counts toward the uniqueness of its name, so a call elsewhere of a name that is also defined at top level in another file gets no edge where R would call the top-level definition: a lost edge, not a wrong one (R-40). Module-level assignments are not modelled either: a top-level `helper <- identity` (or `helper <- 5`) in the caller's own file still lets the unique-name fallback reach another file's `helper` at 0.5. A qualified call is decided by its qualifier alone: `function(f) mypkg::f()` and `helper <- identity; pkga::helper(x)` still bind `mypkg`'s `f` and `pkga`'s `helper` (the shared free-call pass resolves the qualifier before it applies a lexical name claim). +- Named nested functions (closures) are indexed, and calls inside attribute to the nested function, which can shadow a top-level name; a nested function reads the lexical names of the function around it. Top-level script calls outside any function are attributed to the File node. +- `pkg::fn()` and `pkg:::fn()`, decided by the qualifier (see [Resolution model](#resolution-model)). + +Heritage: + +- R6 `inherit=` and S4 `contains=` (a single class or `c(...)`). `contains = "VIRTUAL"` links to one placeholder class named `VIRTUAL`. + +Exports: + +- Read from each local package's NAMESPACE: `export()`, `exportClasses()`, `exportMethods()`, `S3method()` and `exportPattern()`. `S3method(generic, class, method)` registers `method`, the function R registers, else `generic.class` (a generic of another package, `S3method(pkg::generic, class)`, registers `generic.class`). A registration is not an export: `pkg::generic.class`, `library(pkg)` and `importFrom(pkg, generic.class)` do not reach a method that only `S3method()` names, though a function also named by `export()` or `exportPattern()` is still exported. The graph node of a registered method stays public (`isExported`), and `rename` refuses it. Conditional (`if`/`else`) exports, raw-string and multi-line arguments and the text after a malformed directive are not read (see Limits and known gaps). A `DESCRIPTION` file with a `Package:` line marks a package root. + +## Resolution model + +The scope query (`languages/r/query.ts`) captures four call shapes only: `f(x)`, `pkg::f(x)`, `obj$f(x)` and a chained `a$b$f(x)` (an `extract_operator` or a call as the receiver; see the chained-call bullet under Calls). A free or qualified call in the target of an assignment (`f(x) <- v`) is a call of the setter `` `f<-` `` (see the replacement-function bullet under Calls). + +- A bare call binds lexically first. `library()` and `require()` of a local package attach that package's exports, as R does for an installed package: every top-level name when the package has no NAMESPACE file, otherwise the names the NAMESPACE provably exports (`export()`, `exportClasses()`, `exportMethods()`, a matching `exportPattern()`; `S3method()` registers a method and does not export it). A name the NAMESPACE does not export is not attached, and neither is a name it may export but cannot be proven to (an unreadable NAMESPACE, or an `exportPattern()` that was dropped or cut short by its work budget): the 0.85 edge a `library()` yields needs proof. The hook has no importer, so a `library(ownpkg)` issued from inside the package (its `tests/` included) attaches only the exports too. When two attached packages define the same name, the package attached last wins, as R masks it: of the `library()` calls whose package defines the name, the one that comes last in the file decides (calls inside a function are not told apart from top-level ones, and a package named by several calls counts at the first of them, because R leaves an attached package where it is: `library(a); library(b); library(a)` still lets `b` mask `a`). Two files of that one package that both define the name stay ambiguous, with no edge. This differs from `devtools::load_all()`, which attaches every top-level name; the earlier behaviour of this provider followed `load_all()`. `source()` binds no name; a call into a sourced file, or into another file of the same package, binds through a workspace-wide unique-name fallback at confidence 0.5 (`global-name-fallback`). That guess skips a candidate its package's readable, complete NAMESPACE does not export, unless the caller sits inside that package's directory tree (see Export status); a name that is not provably private (unreadable NAMESPACE, a cut-short `exportPattern()`) keeps the guess. An ambiguous name yields no edge, and a function defined inside another function, or inside a block scope (see Calls), is never a candidate (no other function can reach it by name). In package code the guess is also bounded by the package's declared dependencies. When the caller is in `P/R/`, P has a NAMESPACE and neither its exports nor its imports were unreadable: Rule A vetoes a candidate in another repository package Q unless P has an `importFrom` entry for the name, imports Q (`import(Q)`, `importClassesFrom(Q, ...)` or `importMethodsFrom(Q, ...)`), lists Q in `Depends:` (transitively, since `library(A)` also attaches A's `Depends:`), or the caller file has a `library(Q)` / `require(Q)`; Rule B vetoes a candidate that is in no package's `R/` (a loose script, another tree's `tests/` or `inst/`) unless an `importFrom` entry exists; Rule D vetoes a call from `P/tests/` to a definition under `P/data-raw/` or `P/inst/`, which tests reach only by sourcing them, unless the caller file calls `source()`, `system.file()` or `test_path()`. Rules A and B veto calls that are unsupported for check-clean package code: the package's declared dependencies cannot justify them. That is stricter than the shared hook's wording ("impossible"), since a user can attach Q by hand and the call then runs; the provider prefers an unresolved call to an edge into a package the code does not declare. A vetoed guess records `fallback-refused`. Scripts are not constrained by these rules (R-34). +- A bare call never binds to a dotted definition, so `foo(x)` cannot reach `print.foo`. Dotted (S3-style) names are never import-resolved and skip the 0.5 fallback; a call that spells the full dotted name (`read.config(x)`, `print.foo(x)`) binds by qualified-name lookup at confidence 0.35 (`scope-resolution: call`) to a unique definition, in the same package or another file, whether or not a `library()` attached it. That lookup also matches a member whose qualified name is `Class.member`, so `data.frame(x)` can bind the method `frame` of an R6 class `data`, and `Stack.push(x)` a member `Stack.push`: a wrong edge when the call means a base-R function or a function from another package (R-32). Dotted names are not bound through `importFrom()`. +- `pkg::name()` is decided by its qualifier: + - `pkg` is a package of the repository and defines `name` exactly once: the call binds to that definition at confidence 0.85 (`import-resolved`), subject to the export rule below. + - `pkg` is certainly not a repository package (discovery completed and found no such package, including a repository with no `DESCRIPTION`): no edge. + - `pkg` defines `name` more than once: no edge. + - `pkg` is a repository package that defines no `name`: the call binds only when `pkg` re-exports it, i.e. its NAMESPACE has `export(name)` and `importFrom(origin, name)` for a repository package `origin` that defines `name` exactly once (one hop; the last `importFrom` of the name wins). The edge goes to `origin`'s definition at 0.85. Anything else (not exported, no `importFrom`, an external or multiply-defining origin, `:::`) gives no edge: a same-file or other-package definition of that name is never reached through a qualifier. + - Discovery may have missed a package (its limits were reached): the call gets no edge. No qualified call is left to the lexical or name-guess tiers. +- `::` and `:::` differ. `pkg:::name` binds to the definition whether or not the package exports it. `pkg::name` binds only when the package has no NAMESPACE file or exports the name, or when the caller sits inside the package's directory tree (see Export status); a `::` to a private name (or one not provably exported) from outside gets no edge, as R errors there. +- Package discovery is bounded to depth 3 and 200 directories (when the bound may have hidden a package this is flagged internally and nothing is logged), reads directories in code-unit order, ranks duplicate package names by shallowest then by whole path, and never searches the directories the indexer itself ignores (dot-directories, `vendor`, `build`, `dist`, `lib64`, `fixtures` and the rest of its built-in list) or those named exactly `renv`, `packrat`, `revdep` or `.Rcheck`. A `DESCRIPTION` with a `Built:` field is an installed library package (`lib/`, `library/`) and registers nothing. `DESCRIPTION` and `NAMESPACE` are read only as regular files (a symlink is not followed) and only up to 4 MiB; a larger `DESCRIPTION` leaves its package undiscovered (discovery counts as incomplete) and a larger `NAMESPACE`, or one that is a symlink, is treated as absent. The shared file scanner has no `renv`, `packrat`, `revdep` or `.Rcheck` entries, so `.R` files inside those trees are still parsed even though discovery skips them. +- Export status: the language-level export check treats every symbol as public. The NAMESPACE pass then marks the remaining `R/` symbols of a NAMESPACE-bearing package non-exported. Symbols outside `R/` (`tests/`, `vignettes/`, `inst/`) are not package members and stay public; files in `R/` subdirectories are treated as `R/` files although R itself does not load them. A package without a NAMESPACE stays fully public, and a nested package without one does not inherit its enclosing package's. roxygen2 `@export` tags are not consulted by the index (`rename` does read them, see Rename). The refinement changes the graph nodes' `isExported` only; the definitions scope resolution works from carry no export verdict (`isExported` stays undefined there, since a `false` would drop the name from wildcard providers). Resolution instead computes a visibility per top-level name from the same NAMESPACE reading, once per pass: `public` (the package has no NAMESPACE file, or exports the name), `private` (a readable, complete NAMESPACE does not export it) or `unknown` (not exported, but the NAMESPACE is unreadable or an `exportPattern()` was dropped or cut short). `library()` and `pkg::name` need `public` (strict); the global-name fallback vetoes `private`, and, for package code, the dependency rules A, B and D (see Resolution model), but not `unknown`. The inputs are tested in sorted order and completeness is read after the whole batch, so a work-budget cutoff gives the same verdicts whatever the order of files or call sites. A caller is inside package P when the longest package directory (a package with a `DESCRIPTION` or a NAMESPACE) that is a path prefix of its file is P's: `tests/`, `inst/` and `vignettes/` of P count as inside, a nested package claims its own files, and for a root-level package every file outside a nested package counts as inside (so the private-name exemption is broad there). Callers inside P reach P's private names through `pkg::name` and the name guess. +- `exportPattern()` text comes from the analysed repository, so patterns are compiled to a linear-time automaton (`linear-regex.ts`) with a work budget. Constructs with no linear-time algorithm (back-references, look-around), constructs TRE reads differently from JavaScript (`\<` `\>`, `\a`, `\e`, `\Q`, `\E`, `\x{`, `\u`, any backslash inside `[...]` (TRE reads it as a literal member, so `[a\-z]` is a range from `\` to `z` and `[\n]` holds `n`; this drops the common `exportPattern("^[^\\.]")`, whose names then read as not provably exported; write `"^[^.]"`), `[.x.]` and `[=x=]`, `{,n}`, a leading `]` in a bracket expression that has no POSIX class) and oversized patterns are dropped with one logged warning per pattern per package, and `rename` then reports `incomplete_semantics`. Character classes use ASCII semantics. A directive argument the NAMESPACE reader cannot read at all is dropped without a warning: an R 4.0 raw string (`exportPattern(r"(^[^.])")`, `export(r"(foo)")`) and any string that spans lines. It exports nothing, so the names it was meant to cover read as non-exported in the index. The reader records that it dropped something that may hide an export (this, a conditional `export()` below, or a malformed directive that swallowed one), and `rename` refuses on that record (see Rename). + +## Limits and known gaps + +Not edges or nodes: + +- A function passed as a value (`lapply(x, f)`, `Map`, `lapply(x, pkg::f)`), dynamic calls (`do.call("f", ...)`, `match.fun()`, `get()`) and dispatch through a registry (`handlers[["a"]]()`) create no CALLS edge, because only the four call shapes above are captured; the cross-language callable-value analysis classifies R as not applicable. +- R6 `super$method()` and `private$method()` calls (nothing binds `super` or `private` to a class), S4 slot access (`obj@slot`), and `UseMethod()` / `NextMethod()` create no edge. +- Anonymous functions (`function(x)`, `\(x)`) passed as arguments are not indexed as definitions; calls inside are attributed to the enclosing function, or to the File at top level. +- Infix operators (`a %op% b`) are not linked to their definitions. +- Constants and other top-level non-function assignments, and environments (`new.env()`, `local()`, `assign()` / `get()`), are not nodes; the only value-like nodes are R6, RefClass and S4 fields and slots. `setValidity()`, `setClassUnion()`, `setReplaceMethod()` and `Class$methods(...)` create no definition or edge; a bare call inside a `Class$methods(...)` method resolves as if it were outside the class (R-44). +- No framework or entry-point detection (Shiny, plumber, targets, testthat). Plumber `#*` annotations are not read and no route nodes are created, and no route or fetch edges link R to other languages. + +Known gaps in call resolution and class handling. Those that decide which target a call reaches are tracked with IDs, evidence and pinning tests in [r-ingestion-gaps.md](r-ingestion-gaps.md): the dotted R6 class name (R-13), a global `self` bound in another file (R-14), RefClass calls by bare name and through `.self$` (R-19), and the S4 generic, generator, `new()` and `Class$new()` calls that get no edge (R-18). Rows R-26 to R-44 record the boundaries found while fixing the audit. The rest were reproduced and are left as is: + +- `source()` paths are matched by suffix: leading directories that do not exist are dropped until a file matches, so `source("secrets/config.R")` binds `R/config.R`, and the first of several matches wins. Backslash paths and absolute paths resolve to nothing. +- `library(localpkg)` creates one `IMPORTS` edge per file of the package from every importing file, so 100 scripts and a package of 200 files give 20,000 edges. +- When two repository packages each define a helper of the same name, a bare call to it from another file of either package gets no edge: the name is ambiguous in the repository, although the caller's package decides it in R. +- `METHOD_OVERRIDES` compares parameter counts, so an R6 or RefClass method that overrides one with a different formals list (`speak = function(loud = TRUE)` over `speak = function()`) gets no edge, although R has no overloading. +- Calls inside the function of a `setMethod()`, `setGeneric()` or `setValidity()` call are credited to the File, not to the `Method` node the same call creates. Calls inside `validity =` of `setClass()` are credited to the Class. +- R6 `inherit = pkg::Base` and legacy S4 `representation("Base")` create no `EXTENDS` edge. +- R6 members named by a string (`"c d" = function() ...`, `'e' = 3`) are dropped; backticked names are kept with their backticks. A class or owner name read from a string keeps its escapes undecoded, so `setClass("Hello\x41")` is a different class from `setClass("HelloA")`, which R treats as one. +- A function nested in an R6, RefClass or `setMethod()` body has an id such as `Counter.function.bump@7:21`: the enclosing function's name is a placeholder and the line and column keep ids apart, so ids change when lines shift. +- A base class named by `inherit =` or `contains =` that is defined more than once in the repository (in two files, or twice in one) gets no `EXTENDS` edge from a class in another file; a base defined in the deriving class's own file is found. +- Export patterns use JavaScript regular-expression syntax where R uses TRE. The constructs where the two differ are refused, not guessed (see the NAMESPACE bullet above), including every backslash inside `[...]`: `exportPattern("^[^\\.]")` is dropped although TRE and JavaScript disagree on it only for names that contain a backslash, because `[a\-z]`, `[\n]` and `[\]]` differ for ordinary names and no narrower rule has been checked against R. +- NAMESPACE `if (...)` and `else` bodies are skipped, so a function exported only conditionally (`if (getRversion() >= "4.1.0") export(new)`) reads as not exported in the index. So does anything after a malformed directive up to the next line that starts with `name(` in column 0. Skipping a body or a malformed directive that names an export-family directive (`export`, `exportClasses`, `exportMethods`, `S3method`, `exportPattern`) is recorded, and `rename` refuses a function of such a package; the persisted `isExported` is not changed by it. + +A class name that is not unique: a member attaches to the class of its own file when several classes share the name (the same helper class in `R/` and `tests/`, a class defined twice in one file counts once). When the member's file defines none of them, or several, the member stays unowned with its `ownerNameHint`, and one warning for methods and one for fields name the class names concerned (at most five each). + +Not supported: + +- The magrittr pipe `%>%` has no specific support and none is claimed; the native pipe `|>` is supported. tidyverse non-standard evaluation (bare column names, `.data$x`) is not modelled. +- `.Rmd` and `.qmd` files are not indexed (the provider handles `.r` and `.R` only; `.Rprofile` and `.Renviron` are not indexed either), and `box::use()`, `import::from()`, `loadNamespace()`, `attachNamespace()` and `sys.source()` imports are not read. +- S3 dispatch is not resolved: S3 methods are indexed as functions only. S4 and RefClass dispatch is not resolved either; there are definitions and heritage, not dispatch edges. +- NAMESPACE `import()`, `importClassesFrom()` and `importMethodsFrom()` create no import edge and bind no name. They are read only as a declared link to a repository package (the dependency rules A and B in Resolution model), and `DESCRIPTION` `Depends:` likewise. Imports of external (CRAN) packages produce no edges. +- NAMESPACE `exportClassPattern()` is not read. An S4 class exported only through such a pattern reads as not exported (`isExported` false), and an `importFrom()` of it does not bind; classes listed in `export()` or `exportClasses()` are read. +- S4 `contains=` links only to classes defined in the repository or the `VIRTUAL` marker; basic-type bases such as `"numeric"` add no edge. +- Package discovery is bounded (depth 3, 200 directories), so a package beyond the bound is treated like an unknown one. A package found only under `renv`, `packrat`, `revdep`, `.Rcheck`, a dot-directory or another directory the indexer ignores, or whose `DESCRIPTION` has a `Built:` field, is not a repository package. +- A `pkg::name` whose package defines no `name` resolves only through a one-hop re-export (`export` plus `importFrom` of a repository package); a re-export through a second hop, through `exportPattern()` only, or through `import()` gets no edge. `pkg::name` used as a value is not qualifier-resolved. Call sites dropped for certainly-external, unknown-locality, duplicate-definition or not-exported qualifiers leave no recorded outcome, because the drop runs before outcomes exist (R-04, R-11). +- When discovery may have missed a package, a `pkg::name` call gets no edge (it used to keep the name-based guess). A `library(pkg)` attaches no private or not-provably-exported name, and a bare call of a name whose NAMESPACE is unreadable or whose `exportPattern()` was cut short (`unknown`) still gets the 0.5 guess, because it is not proven private. A `library(ownpkg)` from inside the package loses the private names at the 0.85 tier too (they fall to the 0.5 guess, which is allowed for callers inside the package). A `source()` of a private package file does not reach its names through the guess: the guess is vetoed for callers outside the package tree, although R puts sourced names in the global environment. A class reached as a receiver type through a `library()` IMPORTS edge (`ResultSet$new()` then `rs$count()`) is not filtered by this rule (R-03, R-07, R-08, R-12). +- Files in `R/` subdirectories are treated as package-owned, although R itself does not load them. +- NAMESPACE is read from disk during analysis, but neither it nor `DESCRIPTION` is a parsed source file. After an edit to either alone, an incremental analyse (no `--force`) keeps stale manifest-derived data: the persisted `isExported` of symbols in unchanged `R/` files, a `CALLS` edge bound through an `importFrom()` whose target changed, a `pkg::fn` edge after a `Package:` rename, and a `library()`, `pkg::fn` or name-guess edge whose target's export status changed. A full analyse is correct and `analyze --force` recomputes it. This is a limitation of the shared incremental write path (only changed files are re-processed), not of R resolution; other language manifests that feed resolution can be affected in the same way. A stale `isExported` does not reach `rename`: the rename guard does not take the persisted flag as proof that a function is private, and reads the package's NAMESPACE again at plan time (see Rename). +- A name written in backticks (`` `%+%` ``, `` `my fn` ``) keeps its backticks on the node, and a definition and its call resolve only when both spell the name alike: `` `helper`(x) `` against `helper <- function(...)` (or the reverse) gets no `CALLS` edge. NAMESPACE `export()`, `exportPattern()`, `importFrom()`, `pkg::name()` and `rename` match the symbol whichever way it is written. `importFrom(pkg, `%||%`)` binds a call spelled like the definition (`` `%||%`(x, y) ``), and a call spelled the other way gets the name-based guess instead. +- Roxygen `@param` types live in the per-file type environment; no graph edge consumes them. +- Argument matching follows R for named and reordered arguments but does not model partial argument names: `setClass(Cla = "D")`, `new(Cla = "E")`, `setRefClass(Cla = "A")` and `setMethod("show", sig = "Foo")` are not recognised, so no definition or owner is attached. +- Members are read only from the named forms. The positional second argument of `setClass("A", representation(x = "numeric"))`, `R6Class("X", list(...))` and `setRefClass("A", list(...))` creates no field or slot nodes; use `slots=` / `representation=`, `public =` / `private =` / `active =` and `fields =`. +- S4 and RefClass constructors, and the import calls, are recognised only as a bare callee: `setClass(...)`, `setRefClass(...)`, `setGeneric(...)`, `setMethod(...)`, `library(...)`, `require(...)` and `source(...)`. A namespace-qualified call such as `methods::setClass("A", slots = list(x = "numeric"))` creates no class, slot, generic or method, and `base::library("pkg")` creates no import. `R6::R6Class(...)` is recognised. +- For `S3method(generic, class, method)` the positional third argument names the registered method. A named third argument such as `S3method(print, foo, method = impl)` is not read: R registers `impl`, but the registration is recorded for `print.foo`, and `rename` treats the NAMESPACE as unreadable. + +### Bounds on NAMESPACE exportPattern matching + +- Matching work and NFA states are counted in deterministic work units, never wall-clock time, so the outcome is the same on every machine. A NAMESPACE file is still read in full. +- Caps, in `linear-regex.ts`: `MAX_TOTAL_WORK` (40,000,000 state visits per matcher), `MAX_SHARED_WORK` (100,000,000 matching visits per load), `MAX_PACKAGE_WORK` (30,000,000 per package), `MAX_SHARED_STATES` (500,000 NFA states per load) and `MAX_PACKAGE_STATES` (100,000 per package); a single pattern is also capped by `MAX_PATTERN_LENGTH` (16,384 characters) and `MAX_STATES` (16,384). +- A pattern refused after its NFA was partly built is charged for the build (eight visits per state or build step), and once the budget is spent later patterns are refused without being parsed (dropped, with a reason naming the work budget). A NAMESPACE of thousands of oversized patterns therefore spends the same budget as one of hostile matches. +- Groups nested deeper than 256 (`MAX_NESTING`) are refused with the reason "groups are nested too deeply". The limit sits well inside the call stack (a quarter of the default stack still compiles 256 levels), so the verdict is the same cold or warm and on every machine. +- Once the work budget is spent the matchers are no longer asked. A name an `exportPattern()` matched while the budget lasted keeps that answer (a second definition of the same name stays exported; only the first 100,000 distinct matched names per package are remembered, so past that a name reads as not exported once the budget is spent); a name not decided by then reads as not exported (fail closed), and the cut-short patterns are named in a warning. +- A verdict answered from a matcher's memo (a name asked about again) costs one visit of the work budget, so many matchers asked about the same names over and over are bounded by the budget too. +- A counted visit costs about 50 to 70 ns of real time on realistic patterns (13 ns for a hostile one, 13 ns for a memo hit; measured), so the package cap is about 2 s and the load cap about 7 s in the worst case: bounded, never a hang. No fixed per-test overhead is charged on top of the visits (it was tried and dropped, because it moved a pinned visit count and a visibility verdict; R-42). +- A file over the shared size cap (512 KB by default, `GITNEXUS_MAX_FILE_SIZE`) is skipped for every language. With the cap raised, a 50,000-line (2.4 MB) R file took about 138 s and ended with its definitions dropped, as an equivalent TypeScript file does: shared behaviour, not R-specific, left unchanged (R-41). Separately, the R query returns 1,028 matches for 1,500 nested calls (tree-sitter behaviour), so a match count is not a call count. +- A pattern that does not fit is dropped, exports nothing and is warned about: at most 50 patterns are named per package (`MAX_LISTED_DROPPED_PATTERNS`), followed by a summary line counting the rest. +- The first pattern refused for lack of states closes that package's allowance, so later patterns in the package are refused without being parsed. +- One hostile package cannot exhaust the load budget alone (it is limited to its own share), but several can (three, for matching work); packages processed after that degrade, with warnings. Which package degrades follows graph node order. +- Budget exhaustion while resolving NAMESPACE `importFrom()` names is not reported: the scope resolver loads its own configuration, consults the same matchers, and a name a cut-short pattern would have exported then reads as not exported, with no warning. The `postParse` warnings above are the only report. +- Explicit `export()` names are unaffected by these bounds. + +## Control flow + +R provides a control-flow-only CFG visitor (`cfg/visitors/r.ts`), used under `analyze --pdg`. It gives R `BasicBlock` nodes, `CFG` edges and control-dependence (`CDG`) edges. + +Modelled, in statement position inside a function body: `if` / `else if` / `else`, `for`, `while`, `repeat`, `break`, `next`, `return(...)` and bare `{ }` blocks. Every loop, including `repeat` and `while (TRUE)`, has a structural exit edge so that EXIT stays reachable for control dependence. Each `function_definition` (`function(x)` and `\(x)`, assigned or anonymous) gets its own CFG; a nested function is opaque in its parent. An `if` arm that is missing (no `else`) or empty (`{}`) produces no block: it is the header's `seq` edge to the join, and control dependence labels it as the complement of the other arm's explicit `cond-true` / `cond-false` edge (`if (c) {} else work()` makes `work()` control-dependent on `c` with label `F`). + +Not modelled: + +- No def/use harvest and no binding table, so reaching definitions report `no-facts` and R emits no `REACHING_DEF` edges; no taint call sites are recorded. This is a coverage gap, not a result of "no flows found". The PDG tools do not say so for R: `pdg_query` with `mode: "flows"` returns `{"results": [], "total": 0}` with no note, `explain` returns `findings: []` with its generic absence note, and the `impact` note for `mode: "pdg"` still reads "over CDG + REACHING_DEF". Read an empty answer for an R function as "not analysed". The shared tools have no per-language capability signal to report this from, so it is documented here rather than patched into them. +- Control flow nested inside an expression stays one straight-line block: `y <- if (c) a else b`, `switch()`, `&&` / `||`, `ifelse()`, and `return()` / `break` inside `tryCatch({ ... })`, `local({ ... })` or similar wrappers. +- `stop()`, `on.exit()`, `tryCatch` handlers and other non-local exits are ordinary calls: no throw edges and no finaliser routing. A `break` / `next` with no enclosing loop falls back to EXIT. +- Function join: a CFG is anchored at the `function_definition` token, while the `Function` node starts at the whole assignment. They share a start line for `f <- function`, `f = function` and `f <- \(x)`, but not for `f <-` followed by `function` on the next line; there the CFG is still emitted and its blocks lie inside the `Function` node's span, but the exact start-line join used by taint summaries does not resolve (inert for R today, since reaching definitions have no facts). Anonymous functions have a CFG but no `Function` node. + +## Rename + +`languages/r/rename.ts` gives the `rename` tool a deliberately narrow, fail-closed R provider (fixtures under `test/fixtures/rename-integration/r/`). R binds names dynamically, so only what the scope graph proves is renamed. + +- **Succeeds:** a plain function (`name <- function(...)`) and the calls the graph resolves to it, in one file or across files; a package function that `NAMESPACE` leaves private is included. Every parameter is claimed as a binding, whatever its default, so a parameter of the old name that the body never uses is left alone (`parameter-read-only`); a call of the name under a same-named parameter is refused (see below), and so is a body that reads it (`incomplete_semantics`, below). +- **Refused (`unsupported_occurrence`, planning status `unsupported`):** + - any target that is not a plain `Function` (S4 classes, R6 classes, members); + - a function whose old or new name is the whole value of a string literal anywhere in the indexed R sources (`UseMethod("area")`, `do.call("f", ...)`, `match.fun`, `get`, `setGeneric`, ...), because string lookup is not resolved. The literal is decoded first, so escapes (`"f\x6fo"`, `"\u{66}oo"`, octal) and raw strings (`r"(foo)"`) count. A string that contains a call of the name (`"foo(1)"`, glue or cli interpolation such as `"{foo(x)}"`, `parse(text = "foo(1)")`, a formula string) is refused too (`unsupported_occurrence`); a string that only mentions the name as a word, `{name}` value interpolation, a name built at run time (`paste0("fo", "o")`), and strings in files that are not indexed R sources (R Markdown, `NAMESPACE`) are not detected; + - a new name `generic.class` where any dot-separated prefix is a function defined in the indexed R sources or a known base generic, which would register an S3 method (`to.json.foo` for a defined `to.json`, `print.foo`). The known generics are the S3 group generics (`Ops`, `Math`, `Summary`, `Complex`, `matrixOps`) with every function each group dispatches (`sum.items`, `abs.diff`), the `is.*` and `as.*` generics (`is.numeric.foo`, `as.complex.foo`), and the common generics of base, stats, graphics and utils (`print`, `format`, `summary`, `plot`, `unlist`, `pretty`, `weighted.mean`, ...). Generics of other packages (`tidy`, `filter`, ...) are not listed; + - an infix function or operator, as the old or the new name (`` `%+%` ``, `` `%||%` ``, `` `+` ``, `` `[[` ``): `a %+% b` is an operator token, not an identifier, so a rename would edit only the definition; + - a replacement function and its getter (`unsupported_occurrence`): a call of the name anywhere in an assignment target (`helper(z) <- 5`, `helper(z)$a <- 1`, `names(helper(z)) <- v`, the `->` forms), because R then calls `` `helper<-` ``; an old or new name ending in `<-`; and a string-defined setter (`"helper<-" <- function(...)`) of either name; + - a new name that is already a global (`name_collision`): a top-level variable or function of that name in any indexed file (`assist <- 3`, a `for` variable, `3 -> assist`), a `<<-` or `->>` to it anywhere, or a function of that name in another file. Same-file function collisions were already refused by the shared engine; local variables, parameters and members of that name are allowed; + - a new name that the sources already read as a value or call (`name_collision`): `sapply(x, paste)`, `Reduce(max, x)`, `x <- letters`, a default `f = assist`, a formula term, `assist[1]`, `assist(1)`. After the rename the function is the global of that name and would capture the read. A parameter of an enclosing function of that name covers a plain read but not a call: a default proves nothing about the argument, so a call of the new name under such a parameter may reach the global the renamed function becomes. Not reads: bound names, argument names (`f(assist = 1)`), members (`x$assist`, `x@assist`) and qualified names (`pkg::assist`); + - a call of the name under a same-named parameter (`ambiguous_reference`), whatever its default (none, a constant, `NULL`, a function or a computed value) and whether or not the body assigns it, because the call may reach the parameter or the function being renamed; + - a function while a same-named parameter is read in a body (`function(x, helper) x + helper`), or a `for` variable of that name is called in a loop (`incomplete_semantics`; fixtures `parameter-body-read`, `for-variable-call`): no edge covers the occurrence, so the planner stops; it never renames and leaves the call behind (R-16); + - a function body that assigns the name a value that may be a function (`ambiguous_reference`; `helper <- identity`, `helper = f()`, `identity -> helper`, `helper$a <- v`), whether or not the name is called there and wherever the assignment sits relative to a call, because the local may shadow the function being renamed (the call-resolution side treats the same assignments as lexical names, see Calls). A function literal (`helper <- function(y) 2`) is a separate definition and does not refuse when it is defined first (see the local-definition guard below); a constant (`helper <- 2`) is not read as a binding, so the shared engine refuses it as an uncovered occurrence (`incomplete_semantics`); + - a function that is also assigned with `<<-` or `->>` anywhere in the indexed R sources (`ambiguous_reference`, as the top-level form already was): the global re-assignment is a second definition that a rename would leave behind; + - a package function under `R/` that is not proven private, because `NAMESPACE` is not an indexed source and its entry would not be renamed. Two checks must both pass. The persisted `isExported` flag must be false, which it is only for a function that a `NAMESPACE` file beside the package leaves out of `export()`, `exportPattern()` and `S3method()` (the third argument of `S3method(generic, class, method)` counts): a package with no `NAMESPACE` file, or an `R/` folder that belongs to no package, keeps the default public flag and is refused. That includes projects that keep their functions in `R/` without a `DESCRIPTION` (targets, drake and analysis repositories): every rename of a function under `R/` is refused there, and a function outside `R/` is not. The flag is not enough alone, because an edit of `NAMESPACE` with no `R/` file touched leaves it stale until a full analyse. So every rename plan also reads the package's `NAMESPACE` from disk again, through the parsing the indexer uses, and refuses (`unsupported_occurrence`) when it now exports the name by `export()`, `exportPattern()` or `S3method()`. It fails closed with `incomplete_semantics` when that `NAMESPACE` was deleted or cannot be read, when an `exportPattern()` of it was dropped or ran out of work budget (a name such a pattern was meant to cover is then not provably private), or when the reader skipped or could not read something that may hide an export: a conditional `if (...) export(x)`, a raw-string or multi-line argument, an argument that is an expression, an `S3method()` it cannot read, or a malformed directive that swallowed an export. It also refuses (`unsupported_occurrence`) a function whose roxygen2 block (the `#'` lines directly above the definition) carries `@export`, `@exportMethod`, `@exportS3Method` or another `@export*` tag, because the next `document()` would export the new name; the tag is found by text, other roxygen2 tags (`@rdname`, `@inheritParams`, `@seealso`) are not read. A private function in a package whose `NAMESPACE` is unchanged is renamed; a script outside `R/` is not affected. The check does not need `--force`; re-run `analyze` to bring the flag itself up to date. The persisted flag stays a second check on purpose, although the plan-time read decides the stale-private case on its own: it costs nothing, it needs no NAMESPACE read to answer, and when it is stale the other way round (the index says exported, NAMESPACE no longer does) it refuses until the next analyse, which is the safe direction. + - a bare call of the name inside a method of a portable R6 class that defines a member of that name (`ambiguous_reference`): a bare call in a portable method does not see class members, so it reaches the function being renamed but has no proven edge. A class with `portable = FALSE` is not refused; + - a name defined at top level in more than one file of the same load unit (`ambiguous_reference`; the same package, or both files outside every package): load order decides which definition callers reach. Definitions inside functions and inside block scopes (`local`, `test_that`, `with`, ...) do not count; + - an R file the index does not cover that uses the name, or that cannot be read in full (`incomplete_semantics`). The rename preview walks the repository for `.R`/`.r` files in directories the indexer skips (`bin/`, `build/`, `tmp/`), `.gitignore`d files, files over the indexer's size cap, `.Rprofile` and the `.S`/`.q`/`.s` code files under `R/`, and records their identifier tokens; a file over 8 MB or one that cannot be read fails closed. Not scanned: `.git`, the index store, `node_modules`, `*.Rcheck`, `renv/library`, `packrat/lib*`, whatever `.gitnexusignore` excludes, extensionless Rscript files and R Markdown code. Editing a scanned file between preview and apply makes the apply fail as `resolution_changed`. The tokens of every unindexed file are retained with the plan, so a very large unindexed R tree can push the retained plan past its budget, and the preview then fails closed with `plan_too_large` (measured as an upper bound, treating every file as unindexed: 511 R files, 59,468 tokens, use 4.06% of the 64 MiB budget; the 6,378 R files of this repository, fixtures included, use 117%; R-38); + - a function body, or the braces of a block scope (`local`, `test_that`, `describe`, `it`, `with`, `within`, bare or `pkg::`), that defines the name as a function literal, unless the definition is a direct statement of those braces (not under `if`, `switch`, a loop, `tryCatch` or any call) and every call of the name in them, nested functions and blocks included, starts after it (`ambiguous_reference`: a call may run before the local definition). So `test_that("x", { expect_equal(helper(), 1); helper <- function() "stub" })` and `test_that("x", { if (cond) helper <- function() 2; helper() })` refuse: R calls the outer `helper` there, but the index binds every call in the block to the block-local stub, so a rename would leave that call behind (R-43). A definition in a nested function or block is judged by that body alone. A body with no call of the name is not refused, and a call inside the qualifying literal itself (recursion) is ignored (R-39). A top-level call of the name that precedes the top-level definition, in the file or in a script that attaches it, is refused too, because it reaches another function of that name; + - a new name that `NAMESPACE` already exports through `export()` or `exportPattern()` (`unsupported_occurrence`): the rename would change the package API; + - a generated file (`unsupported_occurrence`): the target file, or any indexed R file in which the name occurs as an identifier, has \"Generated by\" or \"do not edit by hand\" in its first five lines (`RcppExports.R`, `cpp11.R`), because the generator would restore the old name; + - a call of the name inside a `setRefClass(..., methods = list(...))` method of a class with `contains =` when a class of the chain defines a method or field of that name, or when a parent is not indexed (other than `envRefClass` and `VIRTUAL`) (`ambiguous_reference`): the call may reach the inherited member; + - any `x$methods(...)` call (methods added to a RefClass generator after `setRefClass()`) whose arguments mention the name (`ambiguous_reference`): such methods are not modelled (Limits and known gaps; R-44), so a call inside one is bound to the function being renamed although R reaches a class method of that name first, and a method of the name added there would shadow the function in every method of the class. The receiver is not traced to a `setRefClass()` call; + - a roxygen2 `@rawNamespace` line, or an `@export ` line, of the package that names the function (`unsupported_occurrence`): the next `document()` would write the old name into `NAMESPACE`. +- **Not updated by a successful rename:** files that are not indexed R sources. R Markdown and Quarto documents (`.Rmd`, `.qmd`, vignettes, `README.Rmd`), `man/*.Rd` (`\alias`, `\usage`), roxygen2 tags that name the function (`@rdname`, `@inheritParams`, `@seealso`), `_pkgdown.yml`, `pkg:::name` uses in `inst/`, and C++ or other code that looks it up by string keep the old name. So does code that reads the name off the call: `match.call()`, `deparse(substitute(f))` and snapshot tests of printed calls (R-37). The result does not list them; search for the old name in those files after a rename. +- **Refused by the shared engine (fail-closed, not provider code):** S3 methods and any function name with a dot after its first character (`area.circle` reads as an owner-qualified member, so `stale_graph`). A name whose only dot is the leading one (`.helper`, the usual internal-function convention) is renamed like any other function, with one exception: R's hook functions `.onLoad`, `.onAttach`, `.onUnload`, `.onDetach`, `.First`, `.Last`, `.First.sys`, `.First.lib` and `.Last.lib` are refused (`unsupported_occurrence`), because R calls them by name; S4 class names (`unsupported_occurrence`, no exact source-name span). +- **Invalid names (`invalid_name`):** anything that is not a nonreserved R identifier: the reserved words, `...` and `..1`, `..2`, ..., and any name with a non-ASCII character, because new names are limited to ASCII letters, digits, `.` and `_` (conservative by design; the message says so). An old name made only of non-ASCII characters (`λ`) is refused as not a supported identifier. Backtick-quoting a new name is never offered: a name that needs it is refused rather than quoted at every use. +- **Refused when any indexed R file does not parse (`parse_error`):** the rename plan parses every indexed R file, and a syntax error in one of them, related to the target or not, refuses the rename and names that file. Fix the file (or exclude it) and plan again. +- **Not proven:** unqualified calls to a function defined in another file (including the package `R/` layout) fail with `ambiguous_reference`, because R's global-name fallback is not a provable binding. Renaming across files works only where the call is qualified or otherwise resolved. + +## Type annotations + +R has no inline type annotations, so the "Type Annotations" column of the language support table is not ticked for R. The roxygen2 `@param` extraction above is a comment-based fallback, in the same class as the JSDoc, PHPDoc and YARD fallbacks listed in type-resolution-system.md (Ruby's YARD support is likewise not ticked). + +## Extending the provider + +- Definition captures are in `R_QUERIES` (`tree-sitter-queries.ts`); scope, import, type-binding, heritage and reference captures are in `languages/r/query.ts`. Keep each declaration pattern and its matching `@scope.function` pattern in lockstep, as the comments in `query.ts` explain. +- The captures for `setClass`, `setRefClass`, `setGeneric`, `setMethod` and for `library` / `require` / `source` are deliberately loose. The argument that names the definition or the import is chosen afterwards in `languages/r/naming-argument.ts`, because a query-level prefix that skips leading comments and named arguments is quadratic in the argument count. +- Resolution wiring is in `languages/r/scope-resolver.ts`, registered in `SCOPE_RESOLVERS`. Import resolution is in `import-target.ts`; package discovery and NAMESPACE parsing are in `package-config.ts` and `namespace-imports.ts`; qualified calls in `qualified-call.ts`; owner and export post-processing in `post-parse.ts` and `owner-hooks.ts`. +- Changing what the scope query captures changes what cached parsed files contain. There is no counter to raise: the parse-cache identity is derived from the bytes of the owned source roots and the grammar packages, so the change invalidates stale caches by itself. Rebuild `dist` and verify a cold parse followed by warm reuse, as CONTRIBUTING's "Parser changes and cache compatibility" section describes. +- Shared code under `gitnexus/src/core/ingestion/` must not name languages; add behaviour through the provider hooks. + +## Tests + +Unit tests are under `gitnexus/test/unit/` (`r-*.test.ts`, `type-env-r.test.ts` and `scope-resolution/r/`) and integration tests under `gitnexus/test/integration/resolvers/` (`r*.test.ts`), with `r-*` fixture projects in `gitnexus/test/fixtures/lang-resolution/`. They cover: + +- definitions, owners and the naming-argument rules (`r-definition-anchors`, `r-owner-hooks`, `r-post-parse-owners`, `r-scope-naming-argument`, `r-s4-anchors`, `r-s4-formals`, `r-scope-args`); +- NAMESPACE parsing, exports and `exportPattern()` safety and scale (`r-namespace-parser`, `r-namespace-exports`, `r-export-pattern*`, `r-export-status-*`); +- package discovery and root-package imports (`r-package-discovery-*`, `r-import-root-package`, `r-namespace-imports`); +- qualified calls, exports and visibility, dotted S3 names and native pipes (`r-qualified-*`, `r-qualified-exports`, `r-qualified-visibility`, `r-dotted-s3-collision`, `r-native-pipes` fixture in `r.test.ts`); +- receivers, `self` and chains (`r-self-binding`, `r-chained-receivers`, `r-receiver-chain`, `r-call-shapes`, `r-receiver-agreement`, `r-typed-receiver-members`) and the receiver corpus (`r-receiver-corpus`: one table of call shapes with exact expected targets, decoys and recorded outcomes, summarised in [r-ingestion-gaps.md](r-ingestion-gaps.md)); +- lexical names, block scopes, assignments and replacement calls (`r-lexical-names`, `r-formal-shadows`, `r-assignment-locals`, `r-rename-assigned-locals`, `r-replacement-calls`, `r-block-scopes`); +- package visibility, S3 registration and S4 generators (`r-package-visibility`, `r-s3method-registration`, `r-s4-generator`); +- work bounds and pathological inputs (`r-replacement-scaling`, `r-pathological-inputs`); +- the shared free-call ordering that lets a qualified hit outrank a lexical name claim (`free-call-qualified-over-name-claim`, `rust-qualified-over-local`, `php-qualified-over-local`); +- rename (`rename-provider-r` and `rename-provider-r-part2` to `-part5`, which deal one fixture list into five parts so each shard stays inside the CI time budget; only part 1 runs on the Windows and macOS shards, because the Windows command line has no room for the other four; `r-rename-*`, including `r-rename-unindexed-sources`); +- roxygen2 `@param` types (`type-env-r`, `r-roxygen-param`); +- control flow (`test/unit/cfg/r-visitor.test.ts`, `test/integration/cfg/r-cfg-join.test.ts`). diff --git a/docs/languages/receiver-resolution-audit.md b/docs/languages/receiver-resolution-audit.md index c64bb83b9..77dfc10cf 100644 --- a/docs/languages/receiver-resolution-audit.md +++ b/docs/languages/receiver-resolution-audit.md @@ -37,7 +37,8 @@ tests. No active shape corpus reported GRAMMAR-UNAVAILABLE. The visible-gap counts include the three deliberate controls: JavaScript and Ruby `untypedChain`, and PHP `arrowCallChain`. They must remain unresolved. Zig and Objective-C still lack shape-arm rows; their dedicated suites were run -separately. C and COBOL's all-N/A rows do not measure their ordinary call paths; +separately. R has no shape-arm row either; its receiver corpus +(`r-receiver-corpus.test.ts`) is described in [r-ingestion-gaps.md](r-ingestion-gaps.md). C and COBOL's all-N/A rows do not measure their ordinary call paths; their focused integration suites were also run. The count arm is unchanged: **138 call drops, 168 total drops, 30 reads, no writes**. diff --git a/gitnexus-shared/src/language-detection.ts b/gitnexus-shared/src/language-detection.ts index 583a0fd5f..eb465959d 100644 --- a/gitnexus-shared/src/language-detection.ts +++ b/gitnexus-shared/src/language-detection.ts @@ -53,6 +53,7 @@ const EXTENSION_MAP: Record = { [SupportedLanguages.Swift]: ['.swift'], [SupportedLanguages.Dart]: ['.dart'], [SupportedLanguages.Vue]: ['.vue'], + [SupportedLanguages.R]: ['.r'], [SupportedLanguages.Cobol]: ['.cbl', '.cob', '.cpy', '.cobol'], [SupportedLanguages.Zig]: ['.zig'], } satisfies Record; // Ensure exhaustiveness @@ -127,6 +128,7 @@ const SYNTAX_MAP: Record = { [SupportedLanguages.Swift]: 'swift', [SupportedLanguages.Dart]: 'dart', [SupportedLanguages.Vue]: 'typescript', + [SupportedLanguages.R]: 'r', [SupportedLanguages.Cobol]: 'cobol', [SupportedLanguages.Zig]: 'zig', } satisfies Record; // Ensure exhaustiveness diff --git a/gitnexus-shared/src/languages.ts b/gitnexus-shared/src/languages.ts index eec45e612..85a67e9ad 100644 --- a/gitnexus-shared/src/languages.ts +++ b/gitnexus-shared/src/languages.ts @@ -21,6 +21,7 @@ export enum SupportedLanguages { Swift = 'swift', Dart = 'dart', Vue = 'vue', + R = 'r', /** Standalone regex processor — no tree-sitter, no LanguageProvider. */ Cobol = 'cobol', Zig = 'zig', diff --git a/gitnexus-shared/src/scope-resolution/language-classification.ts b/gitnexus-shared/src/scope-resolution/language-classification.ts index 2e635c8e2..1d9a1fff0 100644 --- a/gitnexus-shared/src/scope-resolution/language-classification.ts +++ b/gitnexus-shared/src/scope-resolution/language-classification.ts @@ -16,6 +16,7 @@ * * Added after Ring 1: zig enters as `experimental` (new language * integration; promotion to `production` is a separate governance PR). + * Same for r, added later. */ import { SupportedLanguages } from '../languages.js'; @@ -47,6 +48,7 @@ export const LanguageClassifications: Readonly` returns `{ group, query, results, per_repo }` and does not include `process_symbols`; `results[].symbol_count` is that member's post-slice count, and a `service` prefix counts only attaches under the prefix. Query `@/` for that member's attach rows (#3351) diff --git a/gitnexus/README.md b/gitnexus/README.md index 4cbc9d66b..f80d9e8df 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -515,7 +515,7 @@ GitNexus supports indexing multiple repositories. Each `gitnexus analyze` regist ## Supported Languages -TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust, PHP, Kotlin, Swift, Ruby, Dart, Zig +TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust, PHP, Kotlin, Swift, Ruby, Dart, Zig, R ### Language Feature Matrix @@ -536,9 +536,12 @@ TypeScript, JavaScript, Python, Java, C, C++, C#, Go, Rust, PHP, Kotlin, Swift, | C++ | — | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | | Dart | ✓ | — | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | | Zig | ✓ | — | ✓ | — | ✓ | ✓ | ✓ | — | ✓ | +| R (experimental) | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | — | — | **Imports** — cross-file import resolution · **Named Bindings** — `import { X as Y }` / re-export tracking · **Exports** — public/exported symbol detection · **Heritage** — class inheritance, interfaces, mixins · **Type Annotations** — explicit type extraction for receiver resolution · **Constructor Inference** — infer receiver type from constructor calls (`self`/`this` resolution included for all languages) · **Config** — language toolchain config parsing (tsconfig, go.mod, etc.) · **Frameworks** — AST-based framework pattern detection · **Entry Points** — entry point scoring heuristics +**R (experimental)** — `.r` / `.R` files, parsed with `@eagleoutice/tree-sitter-r`. Extracts functions, S4 classes, R6 classes and RefClass (`setRefClass`) classes with their methods and fields, and resolves `library()` / `require()` / `source()` imports to local packages and files, `obj$method()` and `pkg::fn()` calls, R6 `inherit=` / S4 `contains=` heritage and NAMESPACE-driven export status. Calls through function values, `do.call()`, infix operators and S3 dispatch are not linked, and `.Rmd` files, `box::use()` and external (CRAN) packages are not supported. With `--pdg` it also builds a control-flow graph (control dependence only; R has no reaching definitions), and `rename` handles plain functions while refusing classes, members, S3 methods, exported package functions and names spelled as strings. See [docs/languages/r-provider.md](../docs/languages/r-provider.md) for the full coverage and limits. + ## Agent Skills GitNexus ships with skill files that teach AI agents how to use the tools effectively: diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index ab34328ac..6701144f6 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -2,6 +2,7 @@ "_what": "Baselines for bench/import-target/measure.mjs cover every import-target resolver registered in SCOPE_RESOLVERS on a shared corpus, plus a configured C# arm for the branch the default call cannot reach. measure.mjs derives its arm inventory from LANG_REGISTRY and --check reconciles the registered languages in both directions. The single PHP arm supplies its production PSR-4 Composer mapping; csharp_csproj supplies csproj configuration. C and C++ also receive their production resolutionConfig header corpus. The timing, shape, fingerprint, context, and retained-heap gates therefore cover each production resolver path without splitting PHP into configured and unconfigured identities.", "_rebaselined_2960_kotlin_declared_packages": "Kotlin now resolves only from parsed package facts and local module bindings. This deliberately changes its five fingerprints, removes path-depth sensitivity, adds the context probe, and reduces the 32000-file retained index from 40.82 MiB to 4.31 MiB. External same-name path decoys now remain unresolved.", "_zig_arm_1432": "zig was registered in SCOPE_RESOLVERS by PR #1432 with no arm here, which is the exact hole the inventory arm exists for, and it reported it. The arm passes NO build config: resolveZigImportInternal walks an importer-relative @import path component by component and probes allFiles.has() twice (as spelled, then + '.zig'), so its fingerprints pin that walk alone and do not move when build.zig / build.zig.zon parsing (the bare-name legs, gated by test/unit/zig-import-resolver.test.ts) changes. Same corpus proportions as rust — 979 of 3200 resolve at 400 files — and rust's collide design, a deep tree whose spellings carry thirteen components against the unique arm's three, because component count is the only axis the cost has. Measured on one box: small 1.036 ms, collide 1.700 ms, scaling 1.050, collide scaling 1.011, depth 1.573; budgets take the file's usual ~1.5x on ratios (2.4 depth) and ~4x on absolute ms (4 / 7). Heap reads 16 B at both 8000 and 32000 files — it builds nothing, exactly rust's reading — so it sits in heap_bound_bytes at rust's absolute 1048576 B.", + "_r_arm": "R was registered in SCOPE_RESOLVERS without an arm here, which is the hole the inventory arm exists for, and it reported it twice: no LANG_REGISTRY row, and a five-parameter resolveImportTarget with no context arm. The arm drives the registered hook with the three import forms R code has, at the shared corpus proportions (1153 of 3200 resolve at 400 files, as for go, java and objc): library(pkg) as a wildcard, which resolves to every file of the package's R/ directory; source(\"R/file.R\") as a side-effect import, which misses the exact join and is answered by the suffix index; and a NAMESPACE importFrom(pkg, name) as a named import, spelled pkg::name in the corpus and split back in resolveOne, which reads the parsed workspace. The config is the DESCRIPTION-derived package map rebuilt from the corpus paths, with no NAMESPACE, so exports are not filtered. The collide arm repeats mod{k}.R in every package and spells source() at the project root (R/mod{k}.R), which is ambiguous across every package. The context probe is importFrom(pkgA, f): the parsed workspace answers the one file defining f, the context-free call answers every file of pkgA/R. Heap reads 57.26 MiB at 32000 files through the shared workspace file index, ratio 0.970, so R is budgeted. Measured over three runs: small 11.9-12.1 ms, collide 10.8-11.2 ms, scaling 1.38-1.43, collide scaling 1.39-1.45, depth 2.77-2.89. Budgets take the file's usual ~1.5x on ratios and ~4x on absolute ms. R sits above every other linear-budget language on scaling and collide scaling because two of its legs are genuinely O(files) per import rather than keyed: library() of a local package scans the whole file list for the R/ prefix, and every source() path calls normalizedFileList.indexOf before it reaches the index. Measured per leg at 400/1600/6400 files: library of a local package 2.5/8.9/32.4 us, source hit 1.2/6.3/35.9 us, against a flat 0.04 us for an external library() and about 13 us for a source miss. The mixed corpus is dominated by the flat legs, which is why the aggregate stays under the shared 1.8 scaling budget; the collide and depth budgets are set with that in mind rather than hiding it. Fixing the two scans is a resolver change, outside this arm's scope.", "_objc_arm_3179": "Objective-C resolves local header and source imports through a cached suffix map. The map preserves the previous first-path tie-break, which is covered directly in objective-c-provider.test.ts. This arm covers the registered resolver's five dispatch paths, including the depth-sensitive index construction and a collision layout. Measured on this branch: small 1.844 ms, collide 1.861 ms, scaling 1.111, collide scaling 1.067, depth 2.779, and 44101944 retained bytes. The 3.6 depth budget allows normal timing variation while remaining below a full per-import file scan; collision scaling stays at the shared linear 1.8 budget because suffix lookup is keyed.", "_cobol_copybook_dir_preference_2967": "COBOL now prefers well-known copybook directories (copybooks/, cpy/, copy/, plus importer dir) over vendor paths when resolving COPY statements, so COPY EXTERNAL no longer hits same-named decoys outside the project structure. This intentional behavior change moves the resolver from depth-free (prior measured ~0.885) to depth-sensitive (measured 1.751 on CI), because the new preferredCopybookDirs check walks path components. The 2.4 depth budget is ~1.37x the measured ratio, consistent with the ~1.5x convention for depth arms. The unique-arm fingerprints and resolved counts change to reflect the new target set: small/deep drop from 1153 to 442 resolved because files outside preferred directories are now correctly filtered. The collide arm (svc${d}/copybooks layout) keeps 1153 resolved because ALL its files are in preferred copybook directories, so the filter does not change its resolution outcomes — the collide arm now measures collision behavior within the preferred class rather than across mixed layouts. Fixes #2967.", "_php_composer_gate_2962": "The canonical PHP arm supplies an authoritative App PSR-4 mapping. A deterministic Vendor0 suffix decoy makes deletion of the external gate change every timing fingerprint, while the heap arm separately pins a mapped miss, the rendered mapping, and the external null result. Three serial samples measured depth ratios 1.058, 1.114, and 1.158; the 1.8 budget is 1.55x the observed maximum. The mapped-miss heap reading peaked at 39607216 retained bytes at 32000 files.", @@ -31,7 +32,8 @@ "c": 3.8, "cpp": 4, "objc": 1.8, - "zig": 1.8 + "zig": 1.8, + "r": 2.2 }, "depth_budget": { "go": 1.4, @@ -52,7 +54,8 @@ "c": 3, "cpp": 3, "objc": 3.6, - "zig": 2.4 + "zig": 2.4, + "r": 4.2 }, "small_ms_ceiling": { "go": 7, @@ -73,7 +76,8 @@ "c": 7, "cpp": 7, "objc": 8, - "zig": 4 + "zig": 4, + "r": 50 }, "collide_ms_ceiling": { "go": 28, @@ -94,7 +98,8 @@ "c": 11, "cpp": 12, "objc": 8, - "zig": 7 + "zig": 7, + "r": 45 }, "heap_ceiling_bytes": { "rust": 31300000, @@ -109,7 +114,8 @@ "php": 59410824, "java": 5600000, "python": 9541404, - "c": 15000000 + "c": 15000000, + "r": 90100000 }, "_heap_reading_note": "heap_reading_bytes records each measured large-corpus reading so the 0.5x floor is independent of its ceiling. Ceilings use the standard 1.5x allowance for cross-platform and Node heap-accounting differences; re-baseline the reading and ceiling together.", "_kotlin_declared_package_gate": "#2960 replaces Kotlin\u0027s path-suffix cascade with one declared-package/module-binding index. The 32000-file arm measured 4516944 retained bytes (4.31 MiB), with a 6800000-byte ceiling. Correctness and reuse are pinned separately by kotlin-module-resolution.test.ts, external-import-conformance.test.ts, the shared index-reuse contract, and bench/kotlin-import-target.", @@ -126,7 +132,8 @@ "php": 39607216, "java": 3676984, "python": 6360936, - "c": 10018816 + "c": 10018816, + "r": 60040752 }, "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 6200000 (1.508x of 4112464 B measured after copybook-dir preference; previously 3500000 based on 2320456 B before the behavior change), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 3.9x below the smallest real index measured here (cobol's 3.92 MB at the same file count after copybook-dir preference), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", "heap_bound_bytes": { @@ -1162,6 +1169,62 @@ "scaling_ratio": 1.05, "small_ms": 1.036 } + }, + "r": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 3080, + "fingerprint": "b77a56af9b59034404801d79b3ed7760bbc63069b50caab996123051e97bf755" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 12297, + "fingerprint": "159bb6eacfff674cc2c7fec1ffe3b814104fb500d6cb8416a38942ec11ec97fd" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 3080, + "fingerprint": "5ea8bb989760f40b4e47986efeb8b6e67d8363f38caac1c341d24c000af25a05" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 3057, + "fingerprint": "b53fe6dce1eaa9bd6c1c5083aa36bec45ea5afe485cc9ec28db97b78cc41fce5" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 12212, + "fingerprint": "225bda6d01439bc48eda483a76e7b4c1c3f8328ab243d52adcf2d4c018b0cf0b" + }, + "fingerprint": "159bb6eacfff674cc2c7fec1ffe3b814104fb500d6cb8416a38942ec11ec97fd", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/R/missing.R" + }, + "context": { + "target": "pkgA::f", + "with_context": "pkgA/R/a.R", + "without_context": "pkgA/R/a.R,pkgA/R/b.R" + }, + "_measured": { + "collide_ms": 11.022, + "collide_scaling_ratio": 1.434, + "depth_ratio": 2.82, + "scaling_ratio": 1.39, + "small_ms": 12.145 + } } }, "_blind_spot": "MEASURED, so nobody has to rediscover it: a full workspace scan reintroduced on 1-in-32 imports passes EVERY arm here \u2014 dart scored 1.458 scaling and 1.736 ms against the 1.8 budget and 4 ms ceiling of an earlier revision. At 1-in-8 the scaling arm catches it (2.414). The gate that NARROWS this is not a timing gate at all: test/unit/scope-resolution/import-target-index-parity.test.ts counts iterations of the file-set Set and reads 14 instead of 1 for that same 1-in-32 mutation, deterministically and for all five languages. It does NOT close it. The counter watches the Set, and the resolvers no longer read the Set \u2014 they read materialized copies of the same file list: WorkspaceFileIndex.normalized and .all (C#, Ruby), Dart's byBasename buckets, and PackageDirIndex.filesByDir (Go, C#). A 1-in-32 scan over any of those three touches the Set zero extra times, so it passes the parity test AND passes --check. Closing it would take an iteration counter on the materialized arrays themselves. Read the two gates together; tightening these ceilings toward the noise floor to chase that case would only buy flaky CI. CONFIRMED THE HARD WAY by PR #2911: JavaScript resolution was scanning ImportPassCache.normalizedFileList on every import \u2014 a materialized array, not the Set \u2014 at 25972 us per import at 8000 files, and no instrument on the #2901-#2909 branch could see it. It took a differential parity test over 211200 old-vs-new pairs to find. The arms added here would have caught THAT one on absolute ms (85 ms budget against a 20 ms arm; the unindexed resolver costs ~83000 ms on the same corpus), which is the argument for gating every registered language rather than only the ones a PR happens to touch. THE SECOND BLIND SPOT IS CLOSED, and this records what closing it changed. This harness used to call the inner resolvers with the NO-CONTEXT shape: run.ts calls provider.resolveImportTarget with five arguments, the fifth being { parsedFiles, parsedImport }, and resolveOne supplied three. resolveOne now makes the production call, newPass mints the ParsedFile[] FIRST and derives the path set from it exactly as run.ts does, and both legs behind the argument run on every import of their arms \u2014 PHP's named/alias function-or-const leg over filesByDirectory(context.parsedFiles), whose memo defeated measures 197.0 us -> 9976.2 us per import (50.6x), and Python's from-import submodule-precedence branch, the only spelling that reads context.parsedFiles at all. Fifteen of the seventeen arms cannot observe a context (their hooks declare three or four parameters) and are handed none, so their numbers did not move; which two CAN is now reconciled against SCOPE_RESOLVERS' hook arity rather than asserted in prose. NOTHING ELSE IN THIS FILE COULD HAVE GATED IT, which is why the context arm exists: fingerprints and shape can remain unchanged while dropping context only makes timing faster. The deterministic context arm is therefore the guard for this wiring. The arm is one import per language resolved through resolveOne twice, with and without the pass's parsedFiles, whose two answers must DIFFER and must both match what is recorded. WHAT REMAINS UNMEASURED, narrowed rather than deleted: Python's parsedFileByPath memo is exercised by the five timing arms and cannot be reached by the heap arm at all, because retainedPassBytes requires a probe that MISSES while every path that builds that memo returns a non-null packageTarget \u2014 so no ceiling bounds that Map (one pointer per parsed file, O(files), no depth term) and the contract test's count gate is what holds it to one build per pass. PHP's sole arm carries a representative Composer PSR-4 map, so mapped hits and authoritative misses exercise that production branch directly. And the const tail of PHP's leg is a different ANSWER at the same cost \u2014 it runs the identical candidate gather and localDefs filter and diverges in the last two lines \u2014 so it is gated by count in test/unit/scope-resolution/import-target-index-reuse.contract.test.ts, which stays the gate to read alongside this file.", diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index a4d87ba38..f3f180a6e 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -5,8 +5,8 @@ * — over ONE shared corpus so the arms are directly comparable. One arm per * registered language, plus a second `csharp` arm carrying csproj configs * (#2902), so there is one more arm than there are languages. The newest row - * is `zig` (PR #1432), added the day its resolver registered — the inventory - * arm below is what noticed it missing, which is the arm doing its job. + * is `r`, which registered without a row and was reported by the inventory arm + * below — the arm doing its job — after `zig` (PR #1432), which did the same. * * NO LANGUAGE IS OMITTED, and that is the point of the list rather than an * accident of it. Nine of these arms (go, csharp, csharp_csproj, dart, ruby, @@ -127,6 +127,19 @@ * path-walking resolver alone and does not move when that config parsing * changes. * + * - r: one resolver behind three import forms, and the arm builds all three + * (see `uniqueTarget`): `library(pkg)` (a wildcard that resolves to every + * file of the package's `R/`), `source("R/file.R")` (a side-effect import + * answered by the suffix index after an exact-join miss) and a NAMESPACE + * `importFrom(pkg, name)` (a named import that reads the parsed workspace, + * spelled `pkg::name` in the corpus). It passes the package map a + * `DESCRIPTION` scan yields and is a `CONTEXT_LANGS` member. Two of its + * legs are O(files) per import rather than keyed (a local `library()` scans + * the file list for the `R/` prefix; every `source()` path runs + * `normalizedFileList.indexOf` before the index), which is why R reads the + * highest scaling of the linear-budget languages; `_r_arm` in + * baselines.json has the per-leg numbers. + * * Two properties of the corpus are load-bearing and must not be "simplified": * * 1. **Most imports are unresolvable.** In real source the majority of imports @@ -494,6 +507,7 @@ import { typescriptScopeResolver } from '../../src/core/ingestion/languages/type import { cScopeResolver } from '../../src/core/ingestion/languages/c/scope-resolver.ts'; import { cppScopeResolver } from '../../src/core/ingestion/languages/cpp/scope-resolver.ts'; import { objectiveCScopeResolver } from '../../src/core/ingestion/languages/objective-c/scope-resolver.ts'; +import { rScopeResolver } from '../../src/core/ingestion/languages/r/scope-resolver.ts'; import { csharpScopeResolver } from '../../src/core/ingestion/languages/csharp/scope-resolver.ts'; // `SCOPE_RESOLVERS` is NOT imported here — see the inventory arm at the bottom, // which loads it dynamically. Statically it costs 6-10 s of module load @@ -606,6 +620,9 @@ const HEAP_BUDGETED = [ 'objc', // Lexical module lookup retains a workspace index over parsed Rust files. 'rust', + // R retains the shared workspace file index (57.26 MiB at 32000 files, ratio + // 0.970 against the 1.25 budget), so it takes the full ceiling + floor + ratio. + 'r', ]; // javascript, typescript and vue were budgeted here until #2953 and are now // BOUNDED, which is a demotion in gate strength and a promotion in what the @@ -645,6 +662,7 @@ const CONTEXT_LANGS = [ 'cpp', 'csharp', 'csharp_csproj', + 'r', ]; /** @@ -788,6 +806,7 @@ const EXTENSION = { cpp: '.cpp', objc: '.m', zig: '.zig', + r: '.R', }; /** C and C++ resolve `#include` against HEADERS, which reach the resolver * through `resolutionConfig` rather than through `allFilePaths` — see @@ -920,6 +939,12 @@ function uniqueDir(lang, d, i) { // share the other unique arms take from a nested directory comes from the // target instead (see `uniqueTarget`). if (lang === 'zig') return `src/mod${d}`; + // An R package keeps its sources in `/R/` and nothing else in the + // namespace, so one package per index with its files flat under `R/` is the + // layout `library()` (every file of `/R/`) and `importFrom()` (the file + // defining one name) both address. Flat on purpose: R has no nested source + // directories inside `R/`, so there is no `d % 7` nested slice to mint. + if (lang === 'r') return `pkg${d}/R`; throw unwiredLanguage('uniqueDir', lang); } @@ -1015,6 +1040,11 @@ function collideDir(lang, d, i) { // axis that CAN grow — `collideTarget` spells its imports up through the // tree and back down, ~4x the components of the unique arm. if (lang === 'zig') return `src/l0/l1/l2/l3/l4/mod${d}`; + // `svc{d}/R/mod{k}.R` in every package: the conventional flat `R/` directory + // with a repeated basename per package, so the `source("R/mod{k}.R")` spelling + // below is ambiguous across every package and its suffix lookup answers from + // a bucket that grows with the corpus. + if (lang === 'r') return `svc${d}/R`; throw unwiredLanguage('collideDir', lang); } @@ -1069,7 +1099,8 @@ function buildFiles(lang, fileCount, pad, shape) { layout === 'vue' || layout === 'python' || layout === 'c' || - layout === 'cpp'; + layout === 'cpp' || + layout === 'r'; const [fileStem, modStem] = PASCAL_CASE_FILES.has(layout) ? ['File', 'Mod'] : ['file', 'mod']; let stem = shape === 'collide' && collideStem ? `${modStem}${Math.floor(i / dirs)}` : `${fileStem}${i}`; @@ -1190,6 +1221,58 @@ const kotlinProbeFile = (filePath, packageName, exportName) => { }; }; +/** + * An R file whose Module scope binds each of `names` to a top-level function, + * which is exactly what `rFileTopLevel` reads — and ONLY that, because + * `localDefs` also lists nested functions and R6 members and cannot answer "is + * this defined at top level". The resolver reads the bindings, never the def's + * other fields, so the rest of the shape is the inert `probeFile` one. + */ +const rProbeFile = (filePath, names) => { + const base = probeFile( + filePath, + names.map((name) => ['Function', name]), + ); + const moduleScope = `module:${filePath}`; + return { + ...base, + moduleScope, + scopes: [ + { + id: moduleScope, + parent: null, + kind: 'Module', + range: { startLine: 1, startCol: 0, endLine: 1, endCol: 1 }, + filePath, + bindings: new Map( + base.localDefs.map((def) => [def.qualifiedName, [{ def, origin: 'local' }]]), + ), + ownedDefs: base.localDefs, + imports: [], + typeBindings: new Map(), + }, + ], + }; +}; + +/** + * What `loadRPackageConfig` reads from disk (one `DESCRIPTION` per package), + * rebuilt from the corpus paths: every `/R/` is the `R/` directory of the + * package named after its last component. `namespaceInfoByPackageDir` is empty + * — no NAMESPACE file — so `importFrom()` exports are not filtered, which is + * the leg that reads `parsedFiles` and is the one this arm exists to measure. + */ +const rPackageConfigFor = (files) => { + const packages = new Map(); + for (const file of files) { + const at = file.indexOf('/R/'); + if (at < 0) continue; + const dir = file.slice(0, at); + packages.set(dir.slice(dir.lastIndexOf('/') + 1), dir); + } + return { packages, namespaceInfoByPackageDir: new Map() }; +}; + function javaBenchmarkPackage(filePath) { const uniquePackage = /\/com\/example\/(pkg\d+)(?:\/|$)/.exec(`/${filePath}`)?.[1]; if (uniquePackage !== undefined) return `com.example.${uniquePackage}`; @@ -1238,6 +1321,17 @@ function buildParsedFiles(lang, files) { parsedFiles.push(kotlinProbeFile(filePath, kotlinBenchmarkPackage(filePath), stem)); continue; } + if (lang === 'r') { + // One top-level function named for the file, plus an S3 method on it + // (`file7.print`), which is a dotted top-level name: the resolver's + // `dottedTails` guard reads it, and its tail (`print`) never equals a + // name an import asks for, so the guard runs on every lookup and refuses + // none. + const slash = filePath.lastIndexOf('/'); + const stem = filePath.slice(slash + 1, filePath.lastIndexOf('.')); + parsedFiles.push(rProbeFile(filePath, [stem, `${stem}.print`])); + continue; + } const slash = filePath.lastIndexOf('/'); const stem = filePath.slice(slash + 1, filePath.lastIndexOf('.')); const parent = slash < 0 ? '' : filePath.slice(0, slash); @@ -1467,6 +1561,32 @@ function uniqueTarget(lang, { local, r, d, j, dirs }) { if (miss === 1) return `ghost${(r >>> 4) % 97}`; return `../vendor${(r >>> 4) % 97}/missing.zig`; } + if (lang === 'r') { + // Three import forms, one per leg of the resolver, in the shares R code + // actually has them: + // - `library(pkg)` — `pkg{d}` is a LOCAL package and resolves to every + // file of its `R/` directory (a list); an external one (`dplyr`) is + // refused by the wildcard guard before any lookup; + // - `source("R/file{j}.R")` — the project-root spelling. The importer's + // own directory does not hold `R/R/…`, so the exact-join probe misses + // and the suffix index answers; an external one is a vendored path in + // no package, which runs the whole cascade to null; + // - `importFrom(pkg, name)` — spelled `pkg::name` in this corpus because + // the import list is `[from, target]` pairs and `resolveOne` splits it + // back (`::` is not legal in a package name, so the split is exact). + // Resolves through the parsed workspace's top-level definitions; an + // external package is not in the config and answers null. + if (local) { + const leg = (r >>> 3) % 3; + if (leg === 0) return `pkg${d}`; + if (leg === 1) return `R/file${j}.R`; + return `pkg${j % dirs}::file${j}`; + } + const miss = (r >>> 3) % 3; + if (miss === 0) return ['dplyr', 'ggplot2', 'data.table', 'stringr'][(r >>> 4) % 4]; + if (miss === 1) return `vendor${(r >>> 4) % 97}/R/missing.R`; + return `ext${(r >>> 4) % 97}::tidy`; + } throw unwiredLanguage('uniqueTarget', lang); } @@ -1700,6 +1820,27 @@ function collideTarget(lang, { local, r, d, j, dirs }) { if (miss === 1) return `ghost${(r >>> 4) % 97}`; return `${up}/vendor${(r >>> 4) % 97}/missing.zig`; } + if (lang === 'r') { + // The same three forms in the same proportions as the unique arm, so the + // resolved count is identical by construction (asserted). The source + // spelling is the one that collides: `R/mod{k}.R` matches the file of that + // name in EVERY package, so the suffix lookup is ambiguous across `dirs` + // packages — the project-root `source("R/utils.R")` shape a multi-package + // repo really has. The external spellings are unchanged and deliberately + // share no stem with a real file, or they would suffix-match and stop + // being misses. + const k = Math.floor(j / dirs); + if (local) { + const leg = (r >>> 3) % 3; + if (leg === 0) return `svc${d}`; + if (leg === 1) return `R/mod${k}.R`; + return `svc${j % dirs}::mod${k}`; + } + const miss = (r >>> 3) % 3; + if (miss === 0) return ['dplyr', 'ggplot2', 'data.table', 'stringr'][(r >>> 4) % 4]; + if (miss === 1) return `vendor${(r >>> 4) % 97}/R/missing.R`; + return `ext${(r >>> 4) % 97}::tidy`; + } throw unwiredLanguage('collideTarget', lang); } @@ -1806,7 +1947,9 @@ function newPass(lang, files, pad = 0) { ? phpComposerConfigFor(pad) : lang === 'csharp_csproj' ? { csharpConfigs: CSPROJ_CONFIGS } - : undefined, + : lang === 'r' + ? rPackageConfigFor(files) + : undefined, parsedFiles, }; } @@ -1992,6 +2135,33 @@ function resolveOne(lang, from, target, pass) { if (lang === 'objc') { return objectiveCScopeResolver.resolveImportTarget(target, from, allFilePaths, pass.config); } + // The registered hook, because R's adapter is where its per-pass file index, + // the wildcard guard and the `importFrom` branch all live. `parsedImport` is + // rebuilt from the target's spelling exactly the way `interpretRImport` and + // `populateRNamespaceImports` mint it: `library()` is a `wildcard`, `source()` + // a `side-effect`, and a NAMESPACE `importFrom()` a `named` import whose + // `targetRaw` is the package. Only the named form reads `parsedFiles`. + if (lang === 'r') { + const sep = target.indexOf('::'); + const parsedImport = + sep >= 0 + ? { + kind: 'named', + localName: target.slice(sep + 2), + importedName: target.slice(sep + 2), + targetRaw: target.slice(0, sep), + } + : target.endsWith('.R') + ? { kind: 'side-effect', targetRaw: target } + : { kind: 'wildcard', targetRaw: target }; + return rScopeResolver.resolveImportTarget( + parsedImport.targetRaw, + from, + allFilePaths, + pass.config, + contextFor(pass, parsedImport), + ); + } if (lang === 'csharp' || lang === 'csharp_csproj') { return csharpScopeResolver.resolveImportTarget( target, @@ -2246,6 +2416,9 @@ const HEAP_PROBE_TARGET = { typescript: 'vendor0/lib/missing', vue: 'vendor0/lib/Missing.vue', cpp: 'vendor0/missing.hpp', + // A vendored `source()` path in no package: the exact-join probe misses and + // the suffix cascade runs to the end, forcing the workspace file index. + r: 'vendor0/R/missing.R', }; /** @@ -2488,6 +2661,24 @@ const CONTEXT_PROBE = { probeFile('include/util.hpp', []), ], }, + /** + * `importFrom(pkgA, f)` where `pkgA` holds `a.R` (defining `f`) and `b.R` + * (defining `g`). With the parsed workspace the named import resolves to the + * one file that defines `f`; without it the call is the context-free one, + * which cannot tell a named import from `library(pkgA)` and answers every + * file of the package's `R/`. Two distinct non-null answers, so a dropped + * `parsedFiles` cannot look like a miss. `f` is spelled with the `pkg::name` + * convention `resolveOne` splits. + */ + r: { + from: 'app/R/main.R', + target: 'pkgA::f', + parsedFiles: [ + rProbeFile('app/R/main.R', ['main']), + rProbeFile('pkgA/R/a.R', ['f']), + rProbeFile('pkgA/R/b.R', ['g']), + ], + }, }; /** Resolve the probe twice through `resolveOne` — once with the pass's parsed @@ -2501,7 +2692,9 @@ function measureContext(lang) { ? phpComposerConfigFor(0) : lang === 'csharp_csproj' ? { csharpConfigs: CSPROJ_CONFIGS } - : undefined; + : lang === 'r' + ? rPackageConfigFor(parsedFiles.map((f) => f.filePath)) + : undefined; const answer = (files) => { restoreBenchmarkSideChannels(lang, files ?? []); const pass = { allFilePaths, config, parsedFiles: files }; @@ -2662,6 +2855,7 @@ const LANG_REGISTRY = { cpp: SupportedLanguages.CPlusPlus, objc: SupportedLanguages.ObjectiveC, zig: SupportedLanguages.Zig, + r: SupportedLanguages.R, }; const LANGS = Object.keys(LANG_REGISTRY); /** diff --git a/gitnexus/bench/receiver-resolution/BASELINE.md b/gitnexus/bench/receiver-resolution/BASELINE.md index 10391e1e3..448edfb84 100644 --- a/gitnexus/bench/receiver-resolution/BASELINE.md +++ b/gitnexus/bench/receiver-resolution/BASELINE.md @@ -1518,3 +1518,7 @@ Main `c30449394` is merged normally. The regenerated baseline is byte-identical ### Typed shape bodies on the #3667 stack parent The validated #3667 head `fb6316b90` is merged normally. Outside the three receiver-resolution bench files the source is byte-identical to that parent. The regenerated baseline is byte-identical to the textual merge. Against the parent, only this branch's cells move: JavaScript `plainChain`, `plainDeepChain`, `optionalChain` and `awaitParen`, Ruby `plainChain` and `optionalChain`, and PHP `plainDeepChain` and `optionalChain` go to RESOLVES; the JavaScript and Ruby `untypedChain` controls read VISIBLE-GAP; the three C dispatch cells go to N/A. Ruby `plainDeepChain`, PHP `arrowCallChain` and every `indexElement` cell keep the parent's states. Ruby `indexElement` is VISIBLE-GAP from the parent's walker change. Its `idx_repos` now carries a YARD element type, but Ruby has no `elementTypeOf`. The count arm equals the parent: 138 call drops, 168 across all kinds, 30 reads and zero writes, with 57 in-program, 49 unknown, 30 external and 2 unclassified origins. Fresh package build, both pinned TypeScript configurations and the receiver benchmark update and check pass on the combined source. + +### R receiver typing and rename-guard fixtures (#3513) + +The R audit fixes add R fixtures whose untyped receivers end as `receiver-unresolved` drops: a constructor that types a name only when every binding of it agrees, typed private members, dotted class names and `with()` limits. The count arm moves from 141 to 162 call drops and from 175 to 196 across all kinds; reads (34) and writes (0) are unchanged. All 21 added drops are `.R` files (`callDropsByExtension..R` goes from absent to 21), and no other extension moves. By shape: chain-field 57 to 64, no-chain 39 to 42, chain-call 31 to 34 and unclassified 13 to 21. By origin: in-program 53 to 60, unknown 64 to 70 and unclassified 2 to 10; external stays 22. The bench counts only `suppressed` outcomes with the reason `receiver-unresolved`, so the movement is that reason alone. No R arm is added to the shape corpora; only the count-arm numbers change. The baseline was refreshed with `--update-baseline` and `--check` passes. diff --git a/gitnexus/bench/receiver-resolution/baseline.json b/gitnexus/bench/receiver-resolution/baseline.json index 1d417ecaf..5114c8913 100644 --- a/gitnexus/bench/receiver-resolution/baseline.json +++ b/gitnexus/bench/receiver-resolution/baseline.json @@ -202,15 +202,16 @@ } }, "countArm": { - "callDrops": 141, - "totalDropsAllKinds": 175, + "callDrops": 162, + "totalDropsAllKinds": 196, "bySiteKind": { - "call": 141, + "call": 162, "read": 34 }, "callDropsByExtension": { ".java": 50, ".py": 24, + ".R": 21, ".ts": 18, ".zig": 10, ".kt": 8, @@ -224,17 +225,17 @@ ".swift": 1 }, "callDropsByShape": { - "chain-field": 57, - "no-chain": 39, - "chain-call": 31, - "<>": 13, + "chain-field": 64, + "no-chain": 42, + "chain-call": 34, + "<>": 21, "chain-unwrap": 1 }, "callDropsByOrigin": { - "unknown": 64, - "in-program": 53, + "unknown": 70, + "in-program": 60, "external": 22, - "<>": 2 + "<>": 10 } } } diff --git a/gitnexus/bench/scope-resolution/README.md b/gitnexus/bench/scope-resolution/README.md index 86a1615f9..d0c404b56 100644 --- a/gitnexus/bench/scope-resolution/README.md +++ b/gitnexus/bench/scope-resolution/README.md @@ -13,7 +13,9 @@ plus callers and helpers. Rust also needs a root module file; Objective-C has a header and implementation per entity. Java adds an isolated producer file and exercises transitive factory returns and repeated unreachable return-name lookups. Swift uses same-module visibility; COBOL exercises -program references and procedure pointers. Other arms import the preceding file and call through its +program references and procedure pointers. R sources the preceding file and calls +through an R6 instance; the second call of its chain stays unresolved, because R6 +method results carry no type. Other arms import the preceding file and call through its return value. These generated sources exercise the analyzer; they are not builds or representative application benchmarks. diff --git a/gitnexus/bench/scope-resolution/baselines.json b/gitnexus/bench/scope-resolution/baselines.json index 5d13bfa00..8eedc48ad 100644 --- a/gitnexus/bench/scope-resolution/baselines.json +++ b/gitnexus/bench/scope-resolution/baselines.json @@ -191,6 +191,22 @@ "moduleTypeBindings": 0, "typeBindings": 40999 }, + "r/100/20": { + "fingerprint": "862f164acf5e0ddcd5fc1bf8a2f9f3ca29479e97f6bfc8f163abae61197fb334", + "nodes": 2299, + "edges": 2397, + "crossFileCalls": 99, + "moduleTypeBindings": 0, + "typeBindings": 2198 + }, + "r/1000/20": { + "fingerprint": "a2fd8b23407c2ee55a99d4754d87eb5297d98d71d94c15431ab0599a1d871883", + "nodes": 22999, + "edges": 23997, + "crossFileCalls": 999, + "moduleTypeBindings": 0, + "typeBindings": 21998 + }, "ruby/100/20": { "fingerprint": "a2472fd7ca5b2493d24a354b65cd605946b19a00d65df9d5d9788d44da50865c", "nodes": 2299, diff --git a/gitnexus/bench/scope-resolution/corpus.mts b/gitnexus/bench/scope-resolution/corpus.mts index 5f38eb3ec..b1a6818d9 100644 --- a/gitnexus/bench/scope-resolution/corpus.mts +++ b/gitnexus/bench/scope-resolution/corpus.mts @@ -175,6 +175,15 @@ ${previous(i, (p) => ` SET TARGET-PTR TO ENTRY 'C${p}'\n CAL PERFORM MAKE-${i}-0 GOBACK. ${members(n, (j) => ` MAKE-${i}-${j}.\n CONTINUE.`)} +`, + }), + r: (i, n) => ({ + path: `src/c${i}.R`, + content: `${previous(i, (p) => `source("c${p}.R")`)} +C${i} <- R6::R6Class("C${i}", public = list( +${members(n, (j) => ` make${i}_${j} = function() self${i > 0 || j < n - 1 ? ',' : ''}`)} +${previous(i, (p) => ` use_previous = function() {\n value <- C${p}$new()\n value$make${p}_0()$make${p}_1()\n }`)} +)) `, }), zig: (i, n) => ({ diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 31f95ea8a..2b1764d4b 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -8,6 +8,7 @@ "name": "gitnexus", "version": "1.6.12", "bundleDependencies": [ + "@eagleoutice/tree-sitter-r", "tree-sitter", "tree-sitter-c-sharp", "tree-sitter-cpp", @@ -23,6 +24,7 @@ "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { + "@eagleoutice/tree-sitter-r": "1.1.2", "@ladybugdb/core": "0.22.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", @@ -891,6 +893,26 @@ "node": ">=20.19.0" } }, + "node_modules/@eagleoutice/tree-sitter-r": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@eagleoutice/tree-sitter-r/-/tree-sitter-r-1.1.2.tgz", + "integrity": "sha512-HR0RyoI5uxcfsdZvTMqSM8GJyGo6DQTkfdgqygQ6N+D0KQObRH4RxYgBZ6ePsGq/36RBAqv8y0NeQvAFL2N4dQ==", + "hasInstallScript": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0" + }, + "peerDependencies": { + "tree-sitter": "0.25.1" + }, + "peerDependenciesMeta": { + "tree_sitter": { + "optional": true + } + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index e2af706a5..1c82bf520 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -81,6 +81,7 @@ "version": "node scripts/sync-plugin-manifests.mjs" }, "dependencies": { + "@eagleoutice/tree-sitter-r": "1.1.2", "@ladybugdb/core": "0.22.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", @@ -164,6 +165,7 @@ "node": "^22.18.0 || >=24.11.0" }, "bundleDependencies": [ + "@eagleoutice/tree-sitter-r", "tree-sitter", "tree-sitter-c-sharp", "tree-sitter-cpp", diff --git a/gitnexus/scripts/assert-native-parsers.mjs b/gitnexus/scripts/assert-native-parsers.mjs index ef73148c3..765ab9e6b 100644 --- a/gitnexus/scripts/assert-native-parsers.mjs +++ b/gitnexus/scripts/assert-native-parsers.mjs @@ -31,6 +31,7 @@ const SAMPLES = { dart: 'void main() { print(1); }\n', kotlin: 'class A {\n fun m() {}\n}\n', zig: 'pub fn main() void {}\n', + r: 'f <- function(x) x\n', }; async function check(installedDir) { diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index bcae19f36..69f2668ba 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -187,6 +187,9 @@ const LBUG_NATIVE = [ 'test/integration/rename-provider-objective-c.test.ts', 'test/integration/rename-provider-php.test.ts', 'test/integration/rename-provider-python.test.ts', + // R parts 2-5 run on the Linux coverage shards only: cmd.exe's 8191-char limit leaves no room + // (the Windows shard command for this list is already 7827 chars without R). + 'test/integration/rename-provider-r.test.ts', 'test/integration/rename-provider-ruby.test.ts', 'test/integration/rename-provider-rust.test.ts', 'test/integration/rename-provider-swift.test.ts', diff --git a/gitnexus/scripts/prepare-tree-sitter-bundle.cjs b/gitnexus/scripts/prepare-tree-sitter-bundle.cjs index 4052e12f6..24919ab95 100644 --- a/gitnexus/scripts/prepare-tree-sitter-bundle.cjs +++ b/gitnexus/scripts/prepare-tree-sitter-bundle.cjs @@ -16,6 +16,7 @@ const { isArm64NapiSharedObject } = require('./arm64-prebuild.cjs'); const RUNTIME = '0.25.1'; // Upstream metadata is an input to validate, never the peer range we ship. const AUDITED_PEERS = new Map([ + ['@eagleoutice/tree-sitter-r@1.1.2', '^0.21.0'], ['tree-sitter-c@0.23.6', '^0.22.1'], ['tree-sitter-cpp@0.23.4', '^0.21.1'], ['tree-sitter-java@0.23.5', '^0.21.1'], @@ -75,8 +76,10 @@ function prepareTreeSitterBundle( if (manifest.dependencies['tree-sitter'] !== RUNTIME) { throw new Error(`Tree-sitter bundle is only validated for runtime ${RUNTIME}`); } + // A grammar is recognised by its package basename, so a scoped package such as + // @eagleoutice/tree-sitter-r is held to the same bundling and pinning rules. const grammars = Object.keys(manifest.dependencies).filter( - (name) => name === 'tree-sitter' || name.startsWith('tree-sitter-'), + (name) => name === 'tree-sitter' || name.split('/').pop().startsWith('tree-sitter-'), ); for (const name of grammars) { if ( diff --git a/gitnexus/src/core/ingestion/cfg/visitors/r.ts b/gitnexus/src/core/ingestion/cfg/visitors/r.ts new file mode 100644 index 000000000..9f7d62d5f --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/visitors/r.ts @@ -0,0 +1,286 @@ +/** + * R CfgVisitor — control flow only (no def/use harvest). + * + * Walks an R `function_definition` (covers both `function(x)` and the `\(x)` + * lambda form; the grammar uses one node type) and drives the language-agnostic + * {@link CfgBuilder}. Shapes were checked against the @eagleoutice/tree-sitter-r + * grammar: + * - `function_definition` fields `parameters` / `body`. The body is either a + * `braced_expression` (children carry the field `body`) or a bare expression + * (`function(x) x + 1`, `function(x) if (x) 1 else 2`). + * - `if_statement` fields `condition` / `consequence` / optional `alternative` + * (an `else if` chain is a nested `if_statement` in `alternative`). + * - `for_statement` fields `variable` / `sequence` / `body`; `while_statement` + * fields `condition` / `body`; `repeat_statement` field `body`. + * - `break` and `next` are their own node types; `return(...)` is a `call` + * whose `function` field is a `return` node. + * + * Edge-kind contract (same as the other visitors; CDG consumes it): + * - if / else if / else -> `cond-true` / `cond-false` + * - for / while / repeat -> `cond-true` / `loop-back` / `cond-false` + * - break / next / return(…) -> `break` / `continue` / `return` + * - straight-line -> `seq` + * + * A fall-through arm (no `else`, or an empty `{}` arm) produces no block and is the + * header's `seq` edge to the join; control dependence labels it as the complement of + * the sibling arm's explicit `cond-true` / `cond-false`, so no branch sense is lost. + * + * Every loop (including `repeat` and `while (TRUE)`) gets the structural + * `header -> loopExit` `cond-false` edge, so EXIT stays reverse-reachable and + * the post-dominator / control-dependence pass is never skipped for a function + * with a non-terminating loop. For `repeat` that edge is an analysis device, not + * a real condition. + * + * What is modelled: statement-position `if` / `for` / `while` / `repeat` / + * `break` / `next` / `return(...)` / bare `{ }` blocks, in function bodies and in + * the bodies of those constructs. R functions are values, so a nested + * `function_definition` (assigned, anonymous, or an argument) is opaque in its + * parent's CFG and gets its own CFG from the pipeline's function collector. + * + * Known limitations (coverage gaps, deliberately not guessed at): + * - NO def/use harvest and no binding table. `finish()` is called without + * bindings, so the reaching-defs solver reports `no-facts` for R and emits no + * REACHING_DEF edges; taint `sites[]` are likewise absent. + * - Control flow nested inside an expression is not lowered: `y <- if (c) a + * else b`, a `return()` inside `tryCatch({ ... })` / `local({ ... })` / + * `withCallingHandlers`, `switch(...)`, `&&` / `||` short-circuits, and + * `ifelse()` all stay a single straight-line block. + * - `stop()`, `on.exit()`, `tryCatch` handlers, `Recall`, `quit()` and + * non-local exits (`return` from a promise, `break` from `sapply` closures) + * are ordinary calls: no `throw` edges, no finalizer routing. + * - `return` / `break` / `next` written without parentheses or as arguments + * (`invisible(return(x))`) are not recognised as terminators. + * - A `break` / `next` with no enclosing loop falls back to EXIT. + * + * Unlike the harvesting visitors this one does not catch internally: the + * pipeline's `collectFunctionCfgs` isolates each function's build, and letting + * `CfgNestingDepthError` through keeps it counted as `tooDeeplyNested`. + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { CfgBuilder } from '../cfg-builder.js'; +import { ControlFlowContext } from '../control-flow-context.js'; +import type { TraversalResult } from '../traversal-result.js'; +import type { CfgVisitor, FunctionCfg } from '../types.js'; + +/** R node types that own a CFG-bearing function body (`function(x)` and `\(x)`). */ +const R_FUNCTION_TYPES = new Set(['function_definition']); + +/** Statement node types that always break a basic block. */ +const CONTROL_FLOW_TYPES = new Set([ + 'if_statement', + 'for_statement', + 'while_statement', + 'repeat_statement', + 'braced_expression', + 'break', + 'next', +]); + +const startLineOf = (n: SyntaxNode): number => n.startPosition.row + 1; +const endLineOf = (n: SyntaxNode): number => n.endPosition.row + 1; + +/** `return(...)` is a call whose callee is the `return` keyword node. */ +const isReturnCall = (n: SyntaxNode): boolean => + n.type === 'call' && n.childForFieldName('function')?.type === 'return'; + +const isControlStatement = (n: SyntaxNode): boolean => + CONTROL_FLOW_TYPES.has(n.type) || isReturnCall(n); + +/** A statement sequence that produced no blocks (empty body) is "transparent". */ +type SeqResult = TraversalResult | null; + +/** Per-function walk state: one instance per function so loop frames never leak. */ +class RCfgWalk { + private readonly cfc = new ControlFlowContext(); + + constructor(private readonly builder: CfgBuilder) {} + + private statementsOf(block: SyntaxNode): SyntaxNode[] { + return block.namedChildren.filter((c) => c.type !== 'comment'); + } + + /** Visit a body that may be a `braced_expression` or a single expression. */ + private visitBody(node: SyntaxNode | undefined | null): SeqResult { + return this.builder.withNesting(() => { + if (!node) return null; + return this.visitStmt(node); + }); + } + + /** Wire a sequence of statements, coalescing straight-line runs into blocks. */ + visitSeq(stmts: SyntaxNode[]): SeqResult { + return this.builder.withNesting(() => { + let entry: number | undefined; + let dangling: number[] = []; + let openSimple: number | undefined; + + for (const stmt of stmts) { + if (isControlStatement(stmt)) { + openSimple = undefined; // close any open straight-line block + const res = this.visitStmt(stmt); + if (res === null) continue; // transparent (empty `{}`) + if (entry === undefined) entry = res.entry; + else this.builder.connect(dangling, res.entry, 'seq'); + dangling = [...res.exits]; + } else if (openSimple === undefined) { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + if (entry === undefined) entry = idx; + else this.builder.connect(dangling, idx, 'seq'); + openSimple = idx; + dangling = [idx]; + } else { + this.builder.extendBlock(openSimple, endLineOf(stmt), stmt.text); + } + } + + if (entry === undefined) return null; + return { entry, exits: dangling }; + }); + } + + /** Dispatch one statement to its handler. Non-null except for an empty `{}`. */ + visitStmt(stmt: SyntaxNode): SeqResult { + switch (stmt.type) { + case 'if_statement': + return this.visitIf(stmt); + case 'for_statement': + return this.visitFor(stmt); + case 'while_statement': + return this.visitWhile(stmt); + case 'repeat_statement': + return this.visitRepeat(stmt); + case 'braced_expression': + return this.visitSeq(this.statementsOf(stmt)); + case 'break': + return this.visitJump(stmt, 'break'); + case 'next': + return this.visitJump(stmt, 'continue'); + default: + if (isReturnCall(stmt)) return this.visitReturn(stmt); + return this.visitSimple(stmt); + } + } + + private visitSimple(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + return { entry: idx, exits: [idx] }; + } + + /** `return(expr)` -> EXIT. */ + private visitReturn(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + this.builder.edge(idx, this.builder.exitIndex, 'return'); + return { entry: idx, exits: [] }; + } + + /** `break` -> loop exit, `next` -> loop header; EXIT when no loop encloses it. */ + private visitJump(stmt: SyntaxNode, kind: 'break' | 'continue'): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const res = kind === 'break' ? this.cfc.resolveBreak() : this.cfc.resolveContinue(); + this.builder.edge(idx, res?.target ?? this.builder.exitIndex, kind); + return { entry: idx, exits: [] }; + } + + /** `if (cond) a [else b]`; an `else if` chain recurses through `alternative`. */ + private visitIf(stmt: SyntaxNode): TraversalResult { + const cond = stmt.childForFieldName('condition') ?? stmt; + const header = this.builder.newBlock(startLineOf(stmt), endLineOf(cond), cond.text); + + const exits: number[] = []; + const thenRes = this.visitBody(stmt.childForFieldName('consequence')); + if (thenRes) { + this.builder.edge(header, thenRes.entry, 'cond-true'); + exits.push(...thenRes.exits); + } else { + exits.push(header); // empty then — true path falls through + } + + const alternative = stmt.childForFieldName('alternative'); + const elseRes = alternative ? this.visitBody(alternative) : null; + if (elseRes) { + this.builder.edge(header, elseRes.entry, 'cond-false'); + exits.push(...elseRes.exits); + } else { + exits.push(header); // no (or empty) else — false path falls through + } + + return { entry: header, exits: [...new Set(exits)] }; + } + + /** Shared loop wiring: header tests, body loops back, structural exit edge. */ + private visitLoop(stmt: SyntaxNode, headerText: string, headerEndLine: number): TraversalResult { + const header = this.builder.newBlock(startLineOf(stmt), headerEndLine, headerText); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, []); + const body = this.visitBody(stmt.childForFieldName('body')); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); // empty body re-tests + } + // Structural escape: keeps EXIT reverse-reachable even for `repeat`/`while (TRUE)`. + this.builder.edge(header, loopExit, 'cond-false'); + return { entry: header, exits: [loopExit] }; + } + + private visitFor(stmt: SyntaxNode): TraversalResult { + const variable = stmt.childForFieldName('variable'); + const sequence = stmt.childForFieldName('sequence'); + const text = `${variable?.text ?? ''} in ${sequence?.text ?? ''}`; + return this.visitLoop(stmt, text, sequence ? endLineOf(sequence) : startLineOf(stmt)); + } + + private visitWhile(stmt: SyntaxNode): TraversalResult { + const cond = stmt.childForFieldName('condition') ?? stmt; + return this.visitLoop(stmt, cond.text, endLineOf(cond)); + } + + private visitRepeat(stmt: SyntaxNode): TraversalResult { + return this.visitLoop(stmt, 'repeat', startLineOf(stmt)); + } +} + +/** + * Build the CFG for one R `function_definition`, or `undefined` if not modelable. + * + * Join key: `functionStartLine` / `functionStartColumn` are those of the + * `function_definition` node itself (the `function` / `\` token), NOT of an + * enclosing `f <- ` assignment. See docs/languages/r-provider.md § Control flow. + */ +function buildFunctionCfg(fnNode: SyntaxNode, filePath: string): FunctionCfg | undefined { + if (!R_FUNCTION_TYPES.has(fnNode.type)) return undefined; + const body = fnNode.childForFieldName('body'); + if (!body) return undefined; + + const builder = new CfgBuilder( + filePath, + startLineOf(fnNode), + endLineOf(fnNode), + fnNode.startPosition.column, + ); + const walk = new RCfgWalk(builder); + const res = walk.visitStmt(body); + if (!res) { + builder.edge(builder.entryIndex, builder.exitIndex, 'seq'); // empty body + return builder.finish(); + } + builder.edge(builder.entryIndex, res.entry, 'seq'); + builder.connect(res.exits, builder.exitIndex, 'seq'); // normal fall-off -> EXIT + return builder.finish(); +} + +/** Whether a node is an R function this visitor builds a CFG for. */ +function isFunction(node: SyntaxNode): boolean { + return R_FUNCTION_TYPES.has(node.type); +} + +/** The R CFG visitor. */ +export function createRCfgVisitor(): CfgVisitor { + return { buildFunctionCfg, isFunction }; +} + +export { R_FUNCTION_TYPES }; diff --git a/gitnexus/src/core/ingestion/field-extractors/r.ts b/gitnexus/src/core/ingestion/field-extractors/r.ts new file mode 100644 index 000000000..69f8bbc58 --- /dev/null +++ b/gitnexus/src/core/ingestion/field-extractors/r.ts @@ -0,0 +1,295 @@ +// gitnexus/src/core/ingestion/field-extractors/r.ts + +/** + * R field extractor — hand-written because R expresses classes as function + * calls (R6Class(), setClass(), setRefClass()) rather than dedicated class syntax, so the + * generic config-driven factory cannot navigate the AST. + * + * Handles: + * - R6 fields: non-function entries in public/private list() + * - R5 fields: setRefClass("Foo", fields = list(name = "character")) + * - S4 representation/slots: setClass("Foo", representation(name = "character")) + */ + +import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { BaseFieldExtractor } from '../field-extractor.js'; +import { isRLeftAssignment } from '../languages/r/assignment.js'; +import { isRNamingArgument } from '../languages/r/naming-argument.js'; +import type { + FieldExtractorContext, + ExtractedFields, + FieldInfo, + FieldVisibility, +} from '../field-types.js'; + +export class RFieldExtractor extends BaseFieldExtractor { + language = SupportedLanguages.R; + + isTypeDeclaration(node: SyntaxNode): boolean { + return isR6Class(node) || isS4Class(node); + } + + protected extractVisibility(_node: SyntaxNode): FieldVisibility { + return 'public'; + } + + extract(node: SyntaxNode, context: FieldExtractorContext): ExtractedFields | null { + if (isR6Class(node)) return this.extractR6Fields(node, context); + if (isS4Class(node)) return this.extractS4Fields(node, context); + return null; + } + + // -------------------------------------------------------------------------- + // R6 field extraction + // -------------------------------------------------------------------------- + + private extractR6Fields( + node: SyntaxNode, + context: FieldExtractorContext, + ): ExtractedFields | null { + const ownerFqn = node.childForFieldName('lhs')?.text; + if (!ownerFqn) return null; + + const call = node.childForFieldName('rhs'); + if (!call) return null; + + const args = call.childForFieldName('arguments'); + if (!args) return null; + + const fields: FieldInfo[] = []; + + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (!arg || arg.type !== 'argument') continue; + + const section = arg.childForFieldName('name')?.text; + if (section !== 'public' && section !== 'private') continue; + + const visibility: FieldVisibility = section === 'private' ? 'private' : 'public'; + const listCall = arg.childForFieldName('value'); + if (!listCall || listCall.type !== 'call') continue; + + const listArgs = listCall.childForFieldName('arguments'); + if (!listArgs) continue; + + for (let j = 0; j < listArgs.namedChildCount; j++) { + const entry = listArgs.namedChild(j); + if (!entry || entry.type !== 'argument') continue; + + const nameNode = entry.childForFieldName('name'); + const valueNode = entry.childForFieldName('value'); + if (!nameNode) continue; + + // Skip function entries (those are methods, not fields) + if (valueNode?.type === 'function_definition') continue; + + fields.push({ + name: nameNode.text, + type: inferR6FieldType(valueNode), + visibility, + isStatic: false, + isReadonly: false, + sourceFile: context.filePath, + line: entry.startPosition.row + 1, + }); + } + } + + return fields.length > 0 ? { ownerFqn, fields, nestedTypes: [] } : null; + } + + // -------------------------------------------------------------------------- + // S4 field extraction (representation / slots / fields) + // -------------------------------------------------------------------------- + + private extractS4Fields( + node: SyntaxNode, + context: FieldExtractorContext, + ): ExtractedFields | null { + const args = node.childForFieldName('arguments'); + if (!args) return null; + + // The class name is the `Class` argument (by name, else the first unnamed one). + const ownerFqn = getRClassNameArgument(args); + if (!ownerFqn) return null; + + const fields: FieldInfo[] = []; + + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (!arg || arg.type !== 'argument') continue; + + const paramName = arg.childForFieldName('name')?.text; + if (paramName !== 'representation' && paramName !== 'slots' && paramName !== 'fields') + continue; + + const slotCall = arg.childForFieldName('value'); + if (!slotCall || slotCall.type !== 'call') continue; + + const slotArgs = slotCall.childForFieldName('arguments'); + if (!slotArgs) continue; + + // Only an unnamed character vector (`slots = c("a", "b")`) names its slots by + // their strings; elsewhere an unnamed string is not a slot (`representation("Base")`). + const unnamedNamesSlots = + paramName !== 'representation' && slotCall.childForFieldName('function')?.text === 'c'; + + for (let j = 0; j < slotArgs.namedChildCount; j++) { + const slot = slotArgs.namedChild(j); + if (!slot || slot.type !== 'argument') continue; + + const slotName = slot.childForFieldName('name'); + const slotType = slot.childForFieldName('value'); + if (!slotName) { + const unnamed = unnamedNamesSlots ? extractQuotedTypeName(slotType) : null; + if (unnamed) { + fields.push({ + name: unnamed, + type: null, + visibility: 'public', + isStatic: false, + isReadonly: false, + sourceFile: context.filePath, + line: slot.startPosition.row + 1, + }); + } + continue; + } + + const typeName = extractQuotedTypeName(slotType); + + fields.push({ + name: slotName.text, + type: typeName, + visibility: 'public', + isStatic: false, + isReadonly: false, + sourceFile: context.filePath, + line: slot.startPosition.row + 1, + }); + } + } + + return fields.length > 0 ? { ownerFqn, fields, nestedTypes: [] } : null; + } +} + +// --------------------------------------------------------------------------- +// AST helpers +// --------------------------------------------------------------------------- + +/** + * The class name of a `setClass` / `setRefClass` argument list: the string value of + * the argument that takes the `Class` formal. Uses the same selection as the scope + * query and definition hook (`isRNamingArgument`), so `contains = "Base"` or + * `representation(...)` ahead of `Class = "A"` never names the owner. + */ +export function getRClassNameArgument(args: SyntaxNode): string | null { + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (!arg || arg.type !== 'argument') continue; + if (!isRNamingArgument(arg, 'Class')) continue; + const value = arg.childForFieldName('value'); + if (value?.type !== 'string') continue; + const name = extractQuotedTypeName(value); + if (name) return name; + } + return null; +} + +export function findRFieldOwnerNode(node: SyntaxNode): SyntaxNode | null { + let current: SyntaxNode | null = node; + while (current) { + if (isR6Class(current) || isS4Class(current)) return current; + current = current.parent; + } + return null; +} + +export function getRTopLevelPropertyOwnerName(node: SyntaxNode): string | null { + const ownerNode = findRFieldOwnerNode(node); + if (!ownerNode) return null; + return getRClassNameOfOwnerNode(ownerNode); +} + +/** + * The class name an R6 / S4 / RefClass owner node declares: the binding name of + * `Foo <- R6Class(...)`, or the `Class` argument of `setClass(...)` / + * `setRefClass(...)`. Null when `ownerNode` is not such a node. Looks at this node + * only (no ancestor walk), so a caller visiting every ancestor stays linear. + */ +export function getRClassNameOfOwnerNode(ownerNode: SyntaxNode): string | null { + if (!isR6Class(ownerNode) && !isS4Class(ownerNode)) return null; + + if (ownerNode.type === 'binary_operator') { + return ownerNode.childForFieldName('lhs')?.text ?? null; + } + + const args = ownerNode.childForFieldName('arguments'); + if (!args) return null; + return getRClassNameArgument(args); +} + +/** Check if a node is an R6 class assignment (`X <- R6Class(...)`; `X + R6Class(...)` is not). */ +function isR6Class(node: SyntaxNode): boolean { + if (!isRLeftAssignment(node)) return false; + const rhs = node.childForFieldName('rhs'); + if (!rhs || rhs.type !== 'call') return false; + const fn = rhs.childForFieldName('function'); + if (!fn) return false; + // R6Class(...) + if (fn.type === 'identifier' && fn.text === 'R6Class') return true; + // R6::R6Class(...) + if (fn.type === 'namespace_operator') { + const rhsFn = fn.childForFieldName('rhs'); + return rhsFn?.text === 'R6Class'; + } + return false; +} + +/** Check if a call node is setClass() or setRefClass(). */ +function isS4Class(node: SyntaxNode): boolean { + if (node.type !== 'call') return false; + const fn = node.childForFieldName('function'); + return fn?.type === 'identifier' && (fn.text === 'setClass' || fn.text === 'setRefClass'); +} + +/** Infer a basic type from an R6 field default value. */ +function inferR6FieldType(valueNode: SyntaxNode | null): string | null { + if (!valueNode) return null; + switch (valueNode.type) { + case 'null': + return null; + case 'true': + case 'false': + return 'logical'; + case 'float': + return 'numeric'; + case 'integer': + return 'integer'; + case 'string': + return 'character'; + case 'na': + // The grammar parses NA, NA_integer_, NA_real_, NA_character_ and NA_complex_ as `na` + // nodes, not identifiers. Plain NA is logical. + return NA_FIELD_TYPES[valueNode.text] ?? null; + default: + return null; + } +} + +/** The type of each NA constant, by its spelling. */ +const NA_FIELD_TYPES: Readonly> = Object.assign(Object.create(null), { + NA: 'logical', + NA_integer_: 'integer', + NA_real_: 'numeric', + NA_character_: 'character', + NA_complex_: 'complex', +}); + +function extractQuotedTypeName(node: SyntaxNode | null): string | null { + if (!node || node.type !== 'string') return null; + const content = node.namedChildren.find((c) => c.type === 'string_content'); + return content?.text ?? node.text.replace(/^["']|["']$/g, ''); +} diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index 4d9e55634..fff495bd4 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -40,6 +40,8 @@ import type { CfgVisitor } from './cfg/types.js'; import type { CollectedCfgs } from './cfg/collect.js'; import type { RenameProvider } from './rename-provider.js'; import type { GraphNode, NodeLabel, ParameterTypeClass, RelationshipType } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../graph/types.js'; +import type { MutableSemanticModel } from './model/semantic-model.js'; import type { ExtractedRoute } from './route-extractors/laravel.js'; import type { SharedSpringType } from './route-extractors/spring-shared.js'; import type { @@ -260,6 +262,16 @@ export function shouldHarvestModuleConstants( return !provider.moduleConstantHeuristic || provider.moduleConstantHeuristic(content); } +/** + * Everything a {@link LanguageProviderConfig.postParse} hook may read or mutate: the merged graph, the + * populated semantic model, and the repository root. Hooks mutate `graph`/`model` in place. + */ +export interface PostParseContext { + readonly graph: KnowledgeGraph; + readonly model: MutableSemanticModel; + readonly repoPath: string; +} + interface LanguageProviderConfig { // ── Identity ────────────────────────────────────────────────────── readonly id: SupportedLanguages; @@ -444,6 +456,16 @@ interface LanguageProviderConfig { filePath: string, ) => { readonly name: string; readonly label: NodeLabel } | null; + /** + * For languages whose member owner is not a syntactic ancestor container + * (R: `R6Class(...)` / `setRefClass(...)` calls): return the node the + * method/field extractors read members from, or `null`. + * + * Consulted by the enclosing-owner slot after the container walk and + * `resolveFileTypeOwner` (same slot as the Zig file-owner). + * Default: undefined. */ + readonly resolveMemberOwnerNode?: (definitionNode: SyntaxNode) => SyntaxNode | null; + /** * The type a CONTAINER node declares, when the language names it from its * context rather than from a name child of the node — a binding wrapper, @@ -844,6 +866,24 @@ interface LanguageProviderConfig { */ readonly prepareRouteConstants?: (repo: RepoConstants) => void; + /** + * Whole-graph work that can only run once every chunk is merged and before scope resolution + * (deferred owner resolution, package-manifest-driven export refinement). "Post-parse" means after + * all chunks are merged but still inside the parse phase — not the later `crossFile` phase. + * + * Called once per analyze, only if this language has parsed files, awaited, on the main thread. + * Mutates `ctx.graph` / `ctx.model` in place. A throw aborts the parse phase (it is not isolated, + * because the hook mutates shared state and a swallowed mid-mutation throw could leave the graph + * half-updated). + * + * Incremental analyse writes back only changed files, so a mutation of an unchanged file's nodes is + * not persisted; a hook whose result depends on manifests that are not parsed sources needs + * `--force` after such an edit. + * + * Default: undefined (nothing to do after the merge). + */ + readonly postParse?: (ctx: PostParseContext) => void | Promise; + /** * Spring async messaging facts captured for one file — the listener * annotations that subscribe to a broker destination and the template calls @@ -1398,6 +1438,22 @@ export function prepareRouteConstantsByProvider( } } +/** + * Run each present language's {@link LanguageProviderConfig.postParse} hook, sequentially and in + * language-id order (independent of scan order), awaiting each before the next: the hooks share one + * mutable graph/model. The provider lookup is a callback so this file needs no registry import. + * A rejection propagates and later hooks do not run. + */ +export async function runPostParseHooks( + presentLanguages: ReadonlySet, + providerFor: (language: SupportedLanguages) => Pick | undefined, + ctx: PostParseContext, +): Promise { + for (const language of [...presentLanguages].sort()) { + await providerFor(language)?.postParse?.(ctx); + } +} + const DEFAULTS: Pick = { mroStrategy: 'first-wins', }; diff --git a/gitnexus/src/core/ingestion/languages/index.ts b/gitnexus/src/core/ingestion/languages/index.ts index b10822a6f..0ade266c9 100644 --- a/gitnexus/src/core/ingestion/languages/index.ts +++ b/gitnexus/src/core/ingestion/languages/index.ts @@ -29,6 +29,7 @@ import { rubyProvider } from './ruby.js'; import { swiftProvider } from './swift.js'; import { dartProvider } from './dart.js'; import { vueProvider } from './vue.js'; +import { rProvider } from './r.js'; import { cobolProvider } from './cobol.js'; import { zigProvider } from './zig.js'; @@ -49,6 +50,7 @@ export const providers = { [SupportedLanguages.Swift]: swiftProvider, [SupportedLanguages.Dart]: dartProvider, [SupportedLanguages.Vue]: vueProvider, + [SupportedLanguages.R]: rProvider, [SupportedLanguages.Cobol]: cobolProvider, [SupportedLanguages.Zig]: zigProvider, } satisfies Record; diff --git a/gitnexus/src/core/ingestion/languages/r.ts b/gitnexus/src/core/ingestion/languages/r.ts new file mode 100644 index 000000000..d0df960ab --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r.ts @@ -0,0 +1,212 @@ +/** + * R language provider. + * + * `library()` / `require()` of a local package are wildcard imports: the package's + * exported names become visible (every top-level name when it has no NAMESPACE file). + * `source()` binds no name; it only records a file + * dependency. Named imports come from NAMESPACE `importFrom()` entries, which + * `populateRNamespaceImports` turns into `named` imports before resolution. No special + * call routing is needed (unlike Ruby); `emitRScopeCaptures` (`r/captures.ts`) produces + * the import captures from the R scope query rather than from standard import nodes. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import { defineLanguage } from '../language-provider.js'; +import { typeConfig as rTypeConfig } from '../type-extractors/r.js'; +import { R_QUERIES } from '../tree-sitter-queries.js'; +import { RFieldExtractor } from '../field-extractors/r.js'; +import { rMethodExtractor } from '../method-extractors/r.js'; +import { emitRScopeCaptures, rScopeOwnsReceivers } from './r/captures.js'; +import { collectRCaptureSideChannel } from './r/r6-self.js'; +import { interpretRImport, interpretRTypeBinding } from './r/interpret.js'; +import { rPostParse } from './r/post-parse.js'; +import { + rDefinitionProperties, + rResolveDetachedOwnerName, + rResolveMemberOwnerNode, +} from './r/owner-hooks.js'; +import { isRNonNamingArgumentCapture } from './r/naming-argument.js'; +import { createRCfgVisitor } from '../cfg/visitors/r.js'; +import { rRenameProvider } from './r/rename.js'; + +const R_BUILT_INS: ReadonlySet = new Set([ + // Base R + 'c', + 'list', + 'vector', + 'matrix', + 'array', + 'data.frame', + 'length', + 'nrow', + 'ncol', + 'dim', + 'names', + 'colnames', + 'rownames', + 'print', + 'cat', + 'paste', + 'paste0', + 'sprintf', + 'message', + 'warning', + 'stop', + 'tryCatch', + 'withCallingHandlers', + 'try', + 'on.exit', + 'sys.call', + 'match.arg', + // Type checks / coercion + 'is.null', + 'is.na', + 'is.numeric', + 'is.character', + 'is.logical', + 'is.list', + 'as.character', + 'as.numeric', + 'as.integer', + 'as.logical', + 'as.data.frame', + // Apply family + 'lapply', + 'sapply', + 'vapply', + 'mapply', + 'tapply', + 'apply', + 'Map', + 'Reduce', + 'Filter', + // Functional + 'do.call', + 'Recall', + 'match.fun', + // I/O + 'readRDS', + 'saveRDS', + 'readLines', + 'writeLines', + 'read.csv', + 'write.csv', + // Environment + 'new.env', + 'environment', + 'parent.env', + 'globalenv', + 'baseenv', + 'exists', + 'get', + 'assign', + 'rm', + 'ls', + // String + 'nchar', + 'substr', + 'substring', + 'gsub', + 'sub', + 'grep', + 'grepl', + 'regmatches', + 'trimws', + 'toupper', + 'tolower', + 'startsWith', + 'endsWith', + 'strsplit', + // Math + 'sum', + 'mean', + 'median', + 'min', + 'max', + 'abs', + 'sqrt', + 'log', + 'exp', + 'round', + 'ceiling', + 'floor', + 'seq', + 'seq_len', + 'seq_along', + 'rep', + 'which', + 'range', + 'cumsum', + 'diff', + // Logical + 'any', + 'all', + 'xor', + 'identical', + // Sort / order + 'sort', + 'order', + 'rev', + 'unique', + 'duplicated', + 'table', + // S4 / R6 internals + 'setClass', + 'setGeneric', + 'setMethod', + 'setRefClass', + 'new', + 'validObject', + 'is', + 'extends', + 'isVirtualClass', + 'R6Class', + // Package / namespace + 'library', + 'require', + 'requireNamespace', + 'loadNamespace', + 'getNamespace', + 'source', + 'sys.source', + 'attachNamespace', + // Control flow (builtins that appear as calls) + 'return', + 'invisible', + 'missing', + 'nargs', + 'Sys.time', + 'Sys.sleep', + 'proc.time', +]); + +export const rProvider = defineLanguage({ + id: SupportedLanguages.R, + extensions: ['.R', '.r'], + entryPointPatterns: [], + astFrameworkPatterns: [], + treeSitterQueries: R_QUERIES, + typeConfig: rTypeConfig, + // Every definition is exported by default. NAMESPACE-driven export refinement lives in + // `rPostParse`: an `ExportChecker` has no file-path context, and NAMESPACE detection needs it. + exportChecker: () => true, + fieldExtractor: new RFieldExtractor(), + methodExtractor: rMethodExtractor, + builtInNames: R_BUILT_INS, + emitScopeCaptures: emitRScopeCaptures, + // Where the R6 private members are (`self$m()` never reaches one), carried to the main thread. + collectCaptureSideChannel: collectRCaptureSideChannel, + scopeOwnsReceivers: rScopeOwnsReceivers, + interpretImport: interpretRImport, + interpretTypeBinding: interpretRTypeBinding, + postParse: rPostParse, + resolveMemberOwnerNode: rResolveMemberOwnerNode, + resolveDetachedOwnerName: rResolveDetachedOwnerName, + definitionPropertiesExtractor: rDefinitionProperties, + // R_QUERIES captures every string argument of setClass/setGeneric/setMethod; + // only the one that names the definition is kept. + shouldSkipDefinitionCapture: isRNonNamingArgumentCapture, + // Control flow only (no def/use harvest): CFG + CDG, REACHING_DEF reports no-facts. + cfgVisitor: createRCfgVisitor(), + rename: rRenameProvider, +}); diff --git a/gitnexus/src/core/ingestion/languages/r/assignment.ts b/gitnexus/src/core/ingestion/languages/r/assignment.ts new file mode 100644 index 000000000..23a54b528 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/assignment.ts @@ -0,0 +1,299 @@ +/** + * Assignment predicates shared by the scope captures and the rename provider. + * + * What makes an R binary expression a definition: the assignment operators `<-`, `<<-` + * and `=`. The grammar gives every infix operator the same `binary_operator` node, so + * `x <- R6Class(...)` and `x + R6Class(...)` differ only in the `operator` field. The scope + * and definition queries carry the same constraint as `operator: ["<-" "<<-" "="]`. + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { rBareName } from './bare-name.js'; + +const LEFT_ASSIGN_OPERATORS: ReadonlySet = new Set(['<-', '<<-', '=']); + +/** Whether `node` is `target <- value`, `target <<- value` or `target = value`. */ +export const isRLeftAssignment = (node: SyntaxNode): boolean => + node.type === 'binary_operator' && + LEFT_ASSIGN_OPERATORS.has(node.childForFieldName('operator')?.text ?? ''); + +const CONSTANT_VALUE_TYPES: ReadonlySet = new Set([ + 'float', + 'integer', + 'complex', + 'string', + 'true', + 'false', + 'na', + 'inf', + 'nan', +]); + +/** + * Whether `node` is a constant that can never be a function: a number, a string, `TRUE`/`FALSE`, + * `NA`, `Inf`, `NaN` or a signed number (`-1`, `+2`). `NULL` is not one: it is the usual + * placeholder of an optional callback (`helper = NULL`), so a name holding it may hold a function. + */ +export function isRConstantValue(node: SyntaxNode): boolean { + if (CONSTANT_VALUE_TYPES.has(node.type)) return true; + return ( + node.type === 'unary_operator' && + ['-', '+'].includes(node.childForFieldName('operator')?.text ?? '') && + ['float', 'integer', 'complex', 'inf', 'nan'].includes( + node.childForFieldName('rhs')?.type ?? '', + ) + ); +} + +/** The name a plain assignment (`<-`, `=`, `<<-`, `->`, `->>`) or a `for` loop binds, or null. */ +export function rBoundName(node: SyntaxNode): string | null { + if (node.type === 'for_statement') { + const variable = node.childForFieldName('variable'); + return variable?.type === 'identifier' ? rBareName(variable.text) : null; + } + if (node.type !== 'binary_operator') return null; + const operator = node.childForFieldName('operator')?.text; + if (operator === undefined) return null; + let target: SyntaxNode | null = null; + if (operator === '<-' || operator === '=' || operator === '<<-') { + target = node.childForFieldName('lhs'); + } else if (operator === '->' || operator === '->>') { + target = node.childForFieldName('rhs'); + } + return target?.type === 'identifier' ? rBareName(target.text) : null; +} + +/** + * Whether assigning `value` to a name binds something a call of that name may reach. + * + * R looks a callee up as a function and skips any binding that is not one, so what matters is + * whether the value can be a function: + * - a `function_definition` is a real definition (the scope extractor already records it, and a + * call resolves to it), so it is not a lexical name here; + * - a constant (`isRConstantValue`) is skipped by R at a call, which then reaches the outer + * function, so it is not either; + * - everything else (an identifier, a call, anything computed, `NULL`) may hold a function. + */ +export function isLexicalAssignment(value: SyntaxNode): boolean { + return value.type !== 'function_definition' && !isRConstantValue(value); +} + +/** The first argument of a call node (`x` of `names(x)`), or null when there is none. */ +function firstArgumentValue(call: SyntaxNode): SyntaxNode | null { + const argument = call.childForFieldName('arguments')?.namedChildren.find((child) => { + return child.type === 'argument'; + }); + return argument?.childForFieldName('value') ?? null; +} + +/** + * The identifier an assignment target writes through: `x` for `x`, `x$a`, `x@a`, `x[i]`, `x[[i]]` + * and `names(x)`, `names(x)[2]`, `pkg::foo(x)` (a replacement call assigns its first argument, never + * the called name). Null for a target with no identifier at its base (`f()$a`, a string). + */ +function assignmentBase(target: SyntaxNode): SyntaxNode | null { + for (let node: SyntaxNode | null = target; node !== null; ) { + switch (node.type) { + case 'identifier': + return node; + case 'extract_operator': + node = node.childForFieldName('lhs'); + break; + case 'subset': + case 'subset2': + node = node.childForFieldName('function'); + break; + case 'call': + node = firstArgumentValue(node); + break; + default: + return null; + } + } + return null; +} + +/** The name (as written) an assignment node binds to a value that may be a function, or null. */ +function lexicalAssignmentOf(node: SyntaxNode): string | null { + if (node.type !== 'binary_operator') return null; + const operator = node.childForFieldName('operator')?.text; + let target: SyntaxNode | null; + let value: SyntaxNode | null; + if (operator === '<-' || operator === '=') { + target = node.childForFieldName('lhs'); + value = node.childForFieldName('rhs'); + } else if (operator === '->') { + target = node.childForFieldName('rhs'); + value = node.childForFieldName('lhs'); + } else { + // `<<-` and `->>` assign outward and bind nothing in this frame. + return null; + } + if (target === null || value === null) return null; + if (target.type === 'identifier') { + return isLexicalAssignment(value) && target.text.length > 0 ? target.text : null; + } + // `x$a <- v`, `names(x) <- v`: R evaluates `x`, modifies it and binds the result to `x` here, + // whatever the value. Not for `self`: writing a field of an R6 instance (`self$count <- 1`, + // `self$items[[i]] <- v`) leaves `self` the same instance, so it stays the method's typed + // receiver. Only a plain `self <- v` (above) rebinds it. + const base = assignmentBase(target); + if (base === null || base.text.length === 0 || base.text === 'self') return null; + return base.text; +} + +const lexicalAssigned = new WeakMap>>(); + +/** + * The names a function's own body binds by ordinary assignment to a value that may be a function + * (`isLexicalAssignment`), as written (backticks included, like a formal): `x <- v`, `x = v`, + * `v -> x`, and the base identifier of a complex target (`x$a <- v`, `names(x) <- v`, never the + * called name, so `foo(x) <- 1` binds `x`; a complex target based on `self` binds nothing, see + * `lexicalAssignmentOf`). Flow-insensitive: a name counts when ANY assignment + * of it in the body does. A nested function binds its own variables and is not read; `<<-` and + * `->>` assign outward; `assign()`, `delayedAssign()`, `list2env()`, `local()` and `with()` are not + * modelled (`local({ x <- f })` is read as an assignment in this frame, which only costs an edge). + * + * The one implementation behind both the call side (`rFunctionLexicalNames`) and the rename guard + * (`checkSafety`), so the two cannot drift apart. + */ +export function rLexicalAssignedNames(fn: SyntaxNode): ReadonlySet { + let byFunction = lexicalAssigned.get(fn.tree); + if (byFunction === undefined) lexicalAssigned.set(fn.tree, (byFunction = new Map())); + const known = byFunction.get(fn.id); + if (known !== undefined) return known; + const names = new Set(); + const body = fn.childForFieldName('body'); + const pending = body === null ? [] : [body]; + for (let node = pending.pop(); node !== undefined; node = pending.pop()) { + if (node.type === 'function_definition') continue; + const bound = lexicalAssignmentOf(node); + if (bound !== null) names.add(bound); + for (const child of node.namedChildren) pending.push(child); + } + byFunction.set(fn.id, names); + return names; +} + +/** Whether `node` is the callee of a call: `f(x)`, `x |> f()`. */ +export function isCallee(node: SyntaxNode): boolean { + const call = node.parent; + return call?.type === 'call' && call.childForFieldName('function')?.id === node.id; +} + +/** The left-hand side of `<-`, `=`, `<<-`, or the right-hand side of `->`, `->>`; null for any other node. */ +export function assignmentTarget(node: SyntaxNode): SyntaxNode | null { + if (node.type !== 'binary_operator') return null; + const operator = node.childForFieldName('operator')?.text; + if (operator === '<-' || operator === '=' || operator === '<<-') + return node.childForFieldName('lhs'); + if (operator === '->' || operator === '->>') return node.childForFieldName('rhs'); + return null; +} + +/** + * Whether the call node `call` lies in the target chain of an assignment, which makes it a call of + * the replacement function: `foo(x) <- v` runs `` `foo<-` ``, not `foo`. The chain is reached from the + * assignment target (`assignmentTarget`) only through a call's first argument and the base of `$`, + * `@`, `[` and `[[`, the same walk `assignmentBase` makes to find the variable written. Any other + * argument (`foo(x, bar(1)) <- v`, the index of `x[bar(1)]`) and the value being assigned are + * ordinary reads. + * + * For one call only: it walks up, and tree-sitter's `.parent` re-descends from the root, so asking + * it about every call of a file is cubic in the nesting depth. The capture layer asks + * {@link rReplacementCallTest} instead, which gives the same answer in one linear pass; this + * function is the reference that pass is tested against. + */ +export function isRReplacementCall(call: SyntaxNode): boolean { + for (let node = call; ; ) { + const parent: SyntaxNode | null = node.parent; + if (parent === null) return false; + switch (parent.type) { + case 'binary_operator': + return assignmentTarget(parent)?.id === node.id; + case 'extract_operator': + if (parent.childForFieldName('lhs')?.id !== node.id) return false; + node = parent; + break; + case 'subset': + case 'subset2': + if (parent.childForFieldName('function')?.id !== node.id) return false; + node = parent; + break; + case 'argument': { + const list = parent.parent; + const owner = list?.parent; + if (parent.childForFieldName('value')?.id !== node.id) return false; + if (list?.type !== 'arguments' || owner?.type !== 'call') return false; + if (owner.childForFieldName('arguments')?.id !== list.id) return false; + if (list.namedChildren.find((child) => child.type === 'argument')?.id !== parent.id) { + return false; + } + node = owner; + break; + } + default: + return false; + } + } +} + +/** + * The ids of every call under `root` that is in a replacement position: the calls for which + * {@link isRReplacementCall} is true, found top-down in one pass. From the target of each + * assignment, the chain is followed down through a call's first argument and the base of `$`, + * `@`, `[` and `[[`, the reverse of the upward walk, so no `.parent` is read. Each node is visited + * once and chains of different assignments never share a node, so the pass is linear in the + * size of the tree; it uses an explicit stack, never recursion, however deep the nesting. + */ +export function rReplacementCallIds(root: SyntaxNode): Set { + const ids = new Set(); + const pending = [root]; + for (let node = pending.pop(); node !== undefined; node = pending.pop()) { + const target = assignmentTarget(node); + if (target !== null) markReplacementChain(target, ids); + for (const child of node.namedChildren) pending.push(child); + } + return ids; +} + +/** Add every call of the target chain that starts at `target` to `ids`. */ +function markReplacementChain(target: SyntaxNode, ids: Set): void { + for (let node: SyntaxNode | null = target; node !== null; ) { + switch (node.type) { + case 'call': { + ids.add(node.id); + const list = node.childForFieldName('arguments'); + const first = + list?.type === 'arguments' + ? list.namedChildren.find((child) => child.type === 'argument') + : undefined; + node = first?.childForFieldName('value') ?? null; + break; + } + case 'extract_operator': + node = node.childForFieldName('lhs'); + break; + case 'subset': + case 'subset2': + node = node.childForFieldName('function'); + break; + default: + return; + } + } +} + +/** + * Whether a call of the tree under `root` is in a replacement position, for asking about many + * calls of one file: the set of {@link rReplacementCallIds} is built on the first question and + * reused, so the answers cost one linear pass in all. + */ +export function rReplacementCallTest(root: SyntaxNode): (call: SyntaxNode) => boolean { + let ids: Set | undefined; + return (call) => (ids ??= rReplacementCallIds(root)).has(call.id); +} + +/** The name of the replacement function a getter name stands for: `foo` -> `` `foo<-` ``. */ +export function rSetterName(name: string): string { + return '`' + rBareName(name) + '<-`'; +} diff --git a/gitnexus/src/core/ingestion/languages/r/bare-name.ts b/gitnexus/src/core/ingestion/languages/r/bare-name.ts new file mode 100644 index 000000000..d9bb76ca2 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/bare-name.ts @@ -0,0 +1,22 @@ +/** + * R lets any name be written in backticks (`` `%+%` ``, `` `my fn` ``, + * `` `helper` ``); the symbol is the text between them. The graph and the scope + * model keep a definition's name exactly as written, while NAMESPACE entries, + * roxygen tags and rename requests carry the symbol. The scope, qualified-call, + * post-parse, rename and type code compare the two spellings through + * {@link rBareName}, so there is one definition of "the same name" there. The one + * other normaliser is `stripBackticks` in `languages/r/import-target.ts`, which removes + * every leading and trailing backtick from a NAMESPACE entry (a quoted + * `` "`x`" `` entry keeps its backticks after `normaliseArg`) before comparing it + * with a symbol. + * + * One pair of surrounding backticks is removed, in constant time. An escaped + * backtick inside the name stays as written, as `normaliseArg` in + * `namespace-imports.ts` leaves it when it unquotes a NAMESPACE entry. + */ +export function rBareName(name: string): string { + const last = name.length - 1; + return last >= 1 && name.charCodeAt(0) === 96 && name.charCodeAt(last) === 96 + ? name.slice(1, last) + : name; +} diff --git a/gitnexus/src/core/ingestion/languages/r/captures.ts b/gitnexus/src/core/ingestion/languages/r/captures.ts new file mode 100644 index 000000000..314286276 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/captures.ts @@ -0,0 +1,269 @@ +import type { Capture, CaptureMatch, MixedChainStep } from 'gitnexus-shared'; +import { nodeToCapture, syntheticCapture } from '../../utils/ast-helpers.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { getRParser, getRScopeQuery } from './query.js'; +import { isRNonNamingScopeMatch } from './naming-argument.js'; +import { MAX_CHAIN_DEPTH } from '../../utils/call-analysis.js'; +import { encodeReceiverChain } from '../../utils/receiver-chain-codec.js'; +import { getTreeSitterBufferSize } from '../../constants.js'; +import { parseSourceSafe } from '../../../tree-sitter/safe-parse.js'; +import { rLexicalAssignedNames, rReplacementCallTest, rSetterName } from './assignment.js'; +import { rConstructorAnchorTest } from './receiver-agreement.js'; +import { + createSelfOwnerClassifier, + fileBindsSelf, + rDefinitionPositionKey, + recordRR6PrivateMembers, + rR6Method, +} from './r6-self.js'; + +/** + * Emit R scope captures — a single tree-sitter query run against the R + * grammar (see `query.ts` for the capture shapes and the rationale for each + * one). No programmatic enrichment is needed: unlike C/Zig/Rust, R's + * declarations carry no synthesized arity/parameter metadata, and heritage + * sites resolve directly by name (no marker encoding, unlike Ruby's + * `include`/`extend`/`prepend`). + */ +export function emitRScopeCaptures( + sourceText: string, + filePath: string, + cachedTree?: unknown, +): readonly CaptureMatch[] { + let tree = cachedTree as ReturnType['parse']> | undefined; + if (tree === undefined) { + tree = parseSourceSafe(getRParser(), sourceText, undefined, { + bufferSize: getTreeSitterBufferSize(sourceText), + }); + } + + const rawMatches = getRScopeQuery().matches(tree.rootNode); + const out: CaptureMatch[] = []; + // A file that binds `self` itself (`self <- Foo$new()` at top level) has a variable of that name + // for its free functions to read, so they do not own `self`. + const bindsSelf = fileBindsSelf(tree.rootNode); + const functionOwnsSelf = createSelfOwnerClassifier(); + const typedMethods = new Set(); + // Where the R6 private members are, for `rResolveSelfMember` (`self$m()` never reaches one). + const privateMembers = new Set(); + // One linear pass over the tree on the first free call, not a walk up from every call. + const isReplacementCall = rReplacementCallTest(tree.rootNode); + // A constructor binding types its name only when every binding of the name in its frame agrees + // (`receiver-agreement.ts`); the tree is read once, on the first constructor binding. + const isAgreedConstructor = rConstructorAnchorTest(tree.rootNode); + + for (const m of rawMatches) { + // The scope query captures every candidate argument of setClass/library/require/ + // source; keep only the one that names the class or import. + if (isRNonNamingScopeMatch(m.captures)) continue; + const grouped: Record = {}; + let hasRealCapture = false; + let functionNode: SyntaxNode | undefined; + let methodNode: SyntaxNode | undefined; + let freeCallNode: SyntaxNode | undefined; + let chainedReceiver: SyntaxNode | undefined; + let constructorAnchor: SyntaxNode | undefined; + for (const c of m.captures) { + const tag = '@' + c.name; + if (tag.startsWith('@_')) continue; + grouped[tag] = nodeToCapture(tag, c.node); + hasRealCapture = true; + if (tag === '@scope.function' && c.node.type === 'function_definition') functionNode = c.node; + if (tag === '@scope.function' && c.node.type === 'argument') methodNode = c.node; + if (tag === '@reference.call.free') freeCallNode = c.node; + if (tag === '@reference.receiver' && c.node.type !== 'identifier') chainedReceiver = c.node; + if (tag === '@type-binding.constructor') constructorAnchor = c.node; + if (tag === '@scope.module' && c.node.type === 'program' && !bindsSelf) { + grouped['@receiver-owner.self'] = syntheticCapture('@receiver-owner.self', c.node, 'self'); + } + } + if (!hasRealCapture) continue; + // Dropping the type leaves the receiver untyped, and an untyped receiver gets no edge: the + // call is unresolved, never bound to another class's method. + if (constructorAnchor !== undefined && !isAgreedConstructor(constructorAnchor)) continue; + if (freeCallNode !== undefined && isReplacementCall(freeCallNode)) { + recordAsSetterCall(grouped, freeCallNode); + } + // A member call whose receiver is itself a `$` access or a call (`a$b$m()`): mint the + // truncated chain, or drop the match when the chain has no variable base. + if (chainedReceiver !== undefined) { + const chain = rReceiverChain(chainedReceiver); + if (chain === undefined) continue; + grouped['@reference.receiver-chain'] = syntheticCapture( + '@reference.receiver-chain', + chainedReceiver, + chain, + ); + } + // The function's own scope names what its formals and loop variables bind, so that a call + // through one of them stops the lexical walk there instead of reaching a top-level function + // of the same name. + if (functionNode !== undefined) { + const names = rFunctionLexicalNames(functionNode); + if (names.length > 0) { + grouped['@scope.lexical-names'] = syntheticCapture( + '@scope.lexical-names', + functionNode, + JSON.stringify(names), + ); + } + // A function that binds `self` itself (`self <- other()`, a formal or loop variable called + // `self`) owns it, R6 method or not: the method's typed `self` is no longer the receiver + // there, so `self$m()` is unresolved unless that binding is typed (`self <- Foo$new()`). + if (names.includes('self') || (!bindsSelf && functionOwnsSelf(functionNode))) { + grouped['@receiver-owner.self'] = syntheticCapture( + '@receiver-owner.self', + functionNode, + 'self', + ); + } + } + out.push(grouped); + // An R6 method types `self` as its class. The binding is anchored on the method's name, so it + // lands in the method's own scope (an anchor that spans the whole scope would hoist it to the + // class scope, which every method of the class shares). + if (methodNode !== undefined && !typedMethods.has(methodNode.id)) { + typedMethods.add(methodNode.id); + const method = rR6Method(methodNode); + const methodName = methodNode.childForFieldName('name'); + if (method !== null && methodName !== null) { + out.push({ + '@type-binding.self': syntheticCapture('@type-binding.self', methodName, 'self'), + '@type-binding.name': syntheticCapture('@type-binding.name', methodName, 'self'), + '@type-binding.type': syntheticCapture( + '@type-binding.type', + methodName, + method.className, + ), + }); + // The method's definition is anchored on this same `argument` node. + if (method.section === 'private') privateMembers.add(rDefinitionPositionKey(methodNode)); + } + } + } + + recordRR6PrivateMembers(filePath, privateMembers); + return out; +} + +/** + * The receiver chain of a chained member call (`a$b$m()`, `Cls$new()$m()`), encoded with the + * shared codec's `truncated` marker, or `undefined` when the receiver has no variable base. + * + * The walk goes down `extract_operator` (`x$f`, `x@f`: a field step) and `call(function: + * extract_operator)` (`x$f()`: a call step) to an identifier, which is the base. R6 fields and + * method results carry no declared type, so a chain is never typed; the marker makes the shared + * fold refuse it, and the site ends as an unresolved receiver (an outcome, no edge) instead of + * falling to a free function or a method of the same name elsewhere. + * + * `undefined` (the caller drops the match, so the call is invisible: no edge, no outcome) when + * the base is not a variable (a call of a function or of `pkg::f`, `x[[1]]`, `x[1]`, `(x)`), a + * step is not an identifier (`a$"b"$m()`), or the codec cannot + * encode the chain (more steps than it allows, a name with a space). No honest outcome exists + * for a receiver that is not a variable, and the walk is bounded so a pathological chain costs + * a constant amount of work. + */ +export function rReceiverChain(receiverNode: SyntaxNode): string | undefined { + const outerToInner: MixedChainStep[] = []; + let node: SyntaxNode = receiverNode; + while (outerToInner.length <= MAX_CHAIN_DEPTH) { + if (node.type === 'identifier') { + const steps = outerToInner.reverse(); + return encodeReceiverChain(node.text, steps, { truncated: true }); + } + const isCall = node.type === 'call'; + const access = isCall ? node.childForFieldName('function') : node; + if (access?.type !== 'extract_operator') return undefined; + const lhs = access.childForFieldName('lhs'); + const rhs = access.childForFieldName('rhs'); + if (lhs === null || rhs?.type !== 'identifier') return undefined; + outerToInner.push({ kind: isCall ? 'call' : 'field', name: rhs.text }); + node = lhs; + } + return undefined; +} + +/** + * Rewrites the captures of a call in a replacement position (`foo(x) <- v`) so that the site names + * the setter: `@reference.name` becomes `` `foo<-` `` and a qualified `pkg::foo` becomes + * ``pkg::`foo<-` ``. The identifier node keeps its range, so the site stays at one anchor: the getter + * half of a nested form (`foo(bar(x)) <- 1` also reads `bar(x)`) is not recorded as a second site. + * Explicit `` `foo<-`(x, 1) `` is a plain call of that name and never reaches here. + */ +function recordAsSetterCall(grouped: Record, call: SyntaxNode): void { + const callee = call.childForFieldName('function'); + if (callee === null) return; + if (callee.type === 'identifier') { + grouped['@reference.name'] = syntheticCapture( + '@reference.name', + callee, + rSetterName(callee.text), + ); + return; + } + const name = callee.type === 'namespace_operator' ? callee.childForFieldName('rhs') : null; + if (name?.type !== 'identifier') return; + grouped['@reference.name'] = syntheticCapture('@reference.name', name, rSetterName(name.text)); + grouped['@reference.qualified-name'] = syntheticCapture( + '@reference.qualified-name', + callee, + callee.text.slice(0, name.startIndex - callee.startIndex) + rSetterName(name.text), + ); +} + +/** + * The names a `function_definition` binds in its own frame without a definition node: all its + * named formals (`...` is not a name; a default proves nothing about the argument, which may be a + * function, so a formal with a constant default is a lexical name too), the variable of every + * `for` loop in its body and every name an ordinary assignment binds to a value that may be a + * function (`rLexicalAssignedNames`: `helper <- identity`, not `helper <- function(y) 1` nor + * `helper <- 2`, and not `<<-`), outside any function nested in it (that one binds its own). The + * assignment names are flow-insensitive: a call before the assignment, or under an `if` that may + * skip it, stops at the local too and is left unresolved rather than bound to the outer function. + * A name is + * kept as written, backticks included, which is how a call site spells it. Read from the tree once + * per function, so the work over a file is linear in its size; the walk is iterative because a + * body can nest deeply. + */ +export function rFunctionLexicalNames(fn: SyntaxNode): string[] { + const names = new Set(); + const parameters = fn.childForFieldName('parameters'); + if (parameters !== null) { + for (let i = 0; i < parameters.namedChildCount; i++) { + const parameter = parameters.namedChild(i); + if (parameter?.type !== 'parameter') continue; + const name = parameter.childForFieldName('name'); + if (name?.type !== 'identifier' || name.text.length === 0) continue; + names.add(name.text); + } + } + for (const assigned of rLexicalAssignedNames(fn)) names.add(assigned); + const stack: SyntaxNode[] = []; + const body = fn.childForFieldName('body'); + if (body !== null) stack.push(body); + for (let node = stack.pop(); node !== undefined; node = stack.pop()) { + if (node.type === 'function_definition') continue; + if (node.type === 'for_statement') { + const variable = node.childForFieldName('variable'); + if (variable?.type === 'identifier' && variable.text.length > 0) names.add(variable.text); + } + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child !== null) stack.push(child); + } + } + return [...names]; +} + +/** + * `LanguageProvider.scopeOwnsReceivers` for R: the module scope, each function frame that is + * not an R6 method (and not nested in a function), and any function that binds `self` itself + * (`self <- other()`, a formal called `self`) bind `self` with no type of their own, which makes + * a `self$m()` call there a definitive negative unless that binding is typed. The tag is written by + * `emitRScopeCaptures`, where the tree is at hand. + */ +const R_SELF_OWNER: ReadonlySet = Object.freeze(new Set(['self'])); + +export function rScopeOwnsReceivers(match: CaptureMatch): ReadonlySet | undefined { + return match['@receiver-owner.self'] !== undefined ? R_SELF_OWNER : undefined; +} diff --git a/gitnexus/src/core/ingestion/languages/r/export-pattern.ts b/gitnexus/src/core/ingestion/languages/r/export-pattern.ts new file mode 100644 index 000000000..b9de4ff38 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/export-pattern.ts @@ -0,0 +1,299 @@ +import { compileLinearRegexDetailed, type SharedWorkBudget } from './linear-regex.js'; + +/** + * `exportPattern("...")` compilation for R NAMESPACE files. + * + * R evaluates `exportPattern()` arguments as POSIX (TRE) extended regular + * expressions, whose bracket expressions support named classes such as + * `[[:alpha:]]`. JavaScript's `RegExp` has no such classes: it reads + * `[[:alpha:]]` as the set `[:alph` followed by a literal `]`. The RStudio + * new-package template writes `exportPattern("^[[:alpha:]]+")`, so without a + * translation step that pattern never matches anything. + * + * {@link compileRExportPattern} rewrites POSIX classes inside bracket + * expressions to ASCII JS equivalents (the C-locale meaning of each class) and + * compiles the result once. Patterns without `[:` are compiled verbatim, so + * their behaviour is unchanged. It never throws. + * + * The compiled form is a linear-time matcher, not a `RegExp`: the pattern text comes + * from the repository being analysed, and a backtracking engine lets a hostile + * pattern such as `^(a+)+$` hang the analyzer (R itself uses the non-backtracking + * TRE). See `linear-regex.ts` for the supported subset; a pattern outside it + * (including one that uses a construct TRE reads differently from JavaScript, such as + * `\<`, `{,n}`, a leading `]` or any backslash inside `[...]`) compiles to null and therefore + * matches nothing. A leading `]` is a literal member when the pattern also has a POSIX class, + * because the translation below escapes it. Callers that care why use + * {@link compileRExportPatternDetailed}, so that a dropped pattern is reported + * rather than silently treated as an empty export list. + * + * The argument reaches {@link compileRExportPattern} as an R string *value*: + * {@link unescapeRString} first turns the NAMESPACE source text `"\\."` (the + * R string value `\.`) into the regex `\.`. Callers that read a pattern out + * of NAMESPACE text must unescape before compiling. + */ + +const POSIX_CLASSES: Readonly> = { + alpha: 'A-Za-z', + alnum: 'A-Za-z0-9', + digit: '0-9', + upper: 'A-Z', + lower: 'a-z', + punct: '!-\\/:-@\\[-`{-~', + space: ' \\t\\n\\r\\f\\v', + blank: ' \\t', + cntrl: '\\x00-\\x1f\\x7f', + print: '\\x20-\\x7e', + graph: '\\x21-\\x7e', + xdigit: '0-9A-Fa-f', +}; + +/** + * Translate one bracket expression starting at `src[start] === '['`. + * Returns the JS equivalent and the index just past its closing `]`, or null + * when the expression is unterminated or names an unknown class. + */ +function translateBracket(src: string, start: number): { text: string; end: number } | null { + let i = start + 1; + let out = '['; + if (src[i] === '^') { + out += '^'; + i++; + } + // A `]` in first position is a literal member, not the terminator. + if (src[i] === ']') { + out += '\\]'; + i++; + } + while (i < src.length) { + const ch = src[i]; + if (ch === ']') return { text: out + ']', end: i + 1 }; + if (ch === '[' && src[i + 1] === ':') { + const close = src.indexOf(':]', i + 2); + if (close === -1) return null; + // An own-property lookup: `constructor`, `toString` or `__proto__` would otherwise + // resolve on the prototype chain and splice that function's source text into the pattern. + const name = src.slice(i + 2, close); + if (!Object.hasOwn(POSIX_CLASSES, name)) return null; + out += POSIX_CLASSES[name]; + i = close + 2; + } else if (ch === '\\' && i + 1 < src.length) { + out += ch + src[i + 1]; + i += 2; + } else if (ch === '[' && (src[i + 1] === '.' || src[i + 1] === '=')) { + return null; // a collating or equivalence element: not modelled, so the pattern is refused + } else if (ch === '[' || ch === '^') { + out += '\\' + ch; + i++; + } else { + out += ch; + i++; + } + } + return null; +} + +/** + * Whether `src` has a backslash inside a bracket expression. TRE (POSIX) reads a backslash there + * as a literal member; JavaScript, the translation above and `linear-regex.ts` read it as an + * escape. The readings differ for more than names that contain a backslash (`[a\-z]` is a range + * from `\` to `z` in TRE, `[\n]` holds `n`, `[\]]` ends at the first `]`), so such a pattern is + * refused. Brackets are found as TRE finds them: a `[` opens one unless it is escaped, a `]` + * straight after `[` or `[^` is a member, and the `]` that ends a `[:name:]`, `[.x.]` or `[=x=]` + * element does not close it. This checks the pattern as written, before the translation adds + * escapes of its own. + */ +function hasBackslashInBracket(src: string): boolean { + for (let i = 0; i < src.length; i++) { + if (src[i] === '\\') { + i++; + continue; + } + if (src[i] !== '[') continue; + let j = i + 1; + if (src[j] === '^') j++; + if (src[j] === ']') j++; + for (; j < src.length && src[j] !== ']'; j++) { + if (src[j] === '\\') return true; + const kind = src[j + 1]; + if (src[j] === '[' && (kind === ':' || kind === '.' || kind === '=')) { + const close = src.indexOf(kind + ']', j + 2); + if (close === -1) return false; // unterminated: refused by the parser instead + j = close + 1; + } + } + i = j; + } + return false; +} + +function translatePosixClasses(src: string): string | null { + let out = ''; + let i = 0; + while (i < src.length) { + const ch = src[i]; + if (ch === '\\' && i + 1 < src.length) { + out += ch + src[i + 1]; + i += 2; + } else if (ch === '[') { + const bracket = translateBracket(src, i); + if (bracket === null) return null; + out += bracket.text; + i = bracket.end; + } else { + out += ch; + i++; + } + } + return out; +} + +/** A compiled `exportPattern()`; `test` answers whether the pattern matches anywhere in `name`. */ +export interface RExportMatcher { + /** The pattern text that was compiled (after POSIX-class translation). */ + readonly source: string; + test(name: string): boolean; + /** + * True once the matcher stopped answering because it spent its work budget (its own, or + * the one shared by a package-config load); names it had not yet decided were then + * treated as not matching. Absent on matchers that + * cannot run out (a plain `RegExp`). + */ + readonly exhausted?: boolean; +} + +/** The outcome of {@link compileRExportPatternDetailed}: a matcher, or why there is none. */ +export type RExportPatternResult = + | { readonly matcher: RExportMatcher } + | { readonly matcher: null; readonly reason: string }; + +/** + * Compile an `exportPattern()` argument, saying why when it cannot be compiled safely. + * Matchers compiled with the same `shared` budget stop together once it is spent (see + * `linear-regex.ts`); a package-config load passes one so that the number of patterns a + * repository carries cannot multiply the per-matcher cost. + */ +export function compileRExportPatternDetailed( + source: string, + shared?: SharedWorkBudget, +): RExportPatternResult { + try { + if (hasBackslashInBracket(source)) { + return { + matcher: null, + reason: 'backslash inside a bracket expression (TRE reads it as a literal member)', + }; + } + let translated = source; + if (source.includes('[:')) { + const posix = translatePosixClasses(source); + if (posix === null) { + return { + matcher: null, + reason: 'unterminated bracket expression or unknown POSIX class', + }; + } + translated = posix; + } + const result = compileLinearRegexDetailed(translated, shared); + return 'reason' in result + ? { matcher: null, reason: result.reason } + : { matcher: result.regex }; + } catch { + return { matcher: null, reason: 'pattern could not be compiled' }; + } +} + +/** Compile an `exportPattern()` argument; null when it cannot be compiled safely. */ +export function compileRExportPattern( + source: string, + shared?: SharedWorkBudget, +): RExportMatcher | null { + return compileRExportPatternDetailed(source, shared).matcher; +} + +const SIMPLE_R_ESCAPES: Readonly> = { + n: '\n', + t: '\t', + r: '\r', + a: '\x07', + b: '\b', + f: '\f', + v: '\v', + '\\': '\\', + '"': '"', + "'": "'", + '`': '`', + ' ': ' ', +}; + +/** + * Turn the body of an R string literal (the text between the quotes) into the + * string value R would hold: `\\` -> `\`, `\"` -> `"`, `\n` -> newline, and + * likewise `\t \r \a \b \f \v \' \``, backslash-space, `\nnn` (1-3 octal digits, the low + * byte of the value; `\0` is a NUL, which R rejects, so it is kept verbatim), `\x` + * (1-2 hex digits), `\u` (1-4 hex digits, bare or in braces) and `\U` (1-8 hex digits, + * bare or in braces), the last two as code points. The digit limits are R's own: a + * longer run is not part of the escape (tree-sitter-r's `escape_sequence` has the same limits). R + * rejects any other backslash sequence; this leniently keeps it verbatim, so a + * mistaken `"\."` still reaches the regex compiler as `\.`. Never throws. + */ +export function unescapeRString(body: string): string { + if (!body.includes('\\')) return body; + let out = ''; + for (let i = 0; i < body.length; i++) { + const ch = body[i]; + if (ch !== '\\' || i + 1 >= body.length) { + out += ch; + continue; + } + const next = body[i + 1]; + const simple = SIMPLE_R_ESCAPES[next]; + if (simple !== undefined) { + out += simple; + i++; + continue; + } + const octal = readOctalEscape(body, i + 1); + if (octal !== null) { + out += octal.text; + i = octal.end - 1; + continue; + } + const hex = readHexEscape(body, i + 1); + if (hex !== null) { + out += hex.text; + i = hex.end - 1; + continue; + } + out += ch; // Unknown escape: keep the backslash; the next char is emitted normally. + } + return out; +} + +/** `\nnn` escape whose first digit is at `body[at]`; null when it is not an octal digit or the value is NUL. */ +function readOctalEscape(body: string, at: number): { text: string; end: number } | null { + let i = at; + while (i < body.length && i - at < 3 && body[i] >= '0' && body[i] <= '7') i++; + if (i === at) return null; + const code = parseInt(body.slice(at, i), 8) & 0xff; + return code === 0 ? null : { text: String.fromCharCode(code), end: i }; +} + +/** `x`/`u`/`U` escape whose introducer is at `body[at]`; null when malformed. */ +function readHexEscape(body: string, at: number): { text: string; end: number } | null { + const kind = body[at]; + const maxDigits = kind === 'x' ? 2 : kind === 'u' ? 4 : kind === 'U' ? 8 : 0; + if (maxDigits === 0) return null; + let i = at + 1; + const braced = kind !== 'x' && body[i] === '{'; + if (braced) i++; + const start = i; + while (i < body.length && i - start < maxDigits && /[0-9A-Fa-f]/.test(body[i])) i++; + if (i === start) return null; + if (braced) { + if (body[i] !== '}') return null; + } + const code = parseInt(body.slice(start, i), 16); + if (code > 0x10ffff) return null; + return { text: String.fromCodePoint(code), end: braced ? i + 1 : i }; +} diff --git a/gitnexus/src/core/ingestion/languages/r/import-target.ts b/gitnexus/src/core/ingestion/languages/r/import-target.ts new file mode 100644 index 000000000..e8b5d82e9 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/import-target.ts @@ -0,0 +1,265 @@ +/** + * R import resolution. + * Handles library(), require(), source(), and pkg::func() resolution. + */ + +import path from 'path'; +import type { SuffixIndex } from '../../utils/suffix-index.js'; +import { suffixResolve } from '../../utils/suffix-index.js'; +import { EXTENSIONS } from '../import-candidates.js'; +import type { ParsedFile } from 'gitnexus-shared'; +import { rExportPatternMatches, type RPackageConfig } from './package-config.js'; +import { rBareName } from './bare-name.js'; +import { rFileTopLevel, rPackageDirForFile, type RFileTopLevel } from './namespace-imports.js'; +import type { ImportResolutionContext } from '../../scope-resolution/contract/scope-resolver.js'; +import { getWorkspaceFileIndex } from '../../utils/workspace-file-index.js'; + +/** + * Low-level R import resolver (internal helper). + * Resolves library/require/source paths to matching .R files. + * + * `normToRaw` (normalized path → first raw path with that spelling, built once per pass by + * `getWorkspaceFileIndex`) answers the exact-path lookups of `source()` in O(1). Without it + * they fall back to `indexOf` over `normalizedFileList`, which picks the same file (the first + * in list order) but costs O(files) per `source()` import. + */ +export function resolveRImportInternal( + filePath: string, + rawImportPath: string, + normalizedFileList: readonly string[], + allFileList: readonly string[], + rConfig: RPackageConfig | null, + index?: SuffixIndex, + normToRaw?: ReadonlyMap, +): string | string[] | null { + const exactFile = (normalizedPath: string): string | undefined => { + if (normToRaw !== undefined) return normToRaw.get(normalizedPath); + const idx = normalizedFileList.indexOf(normalizedPath); + return idx >= 0 ? allFileList[idx] : undefined; + }; + const cleaned = rawImportPath.replace(/^["']|["']$/g, ''); + + // source() with file path + if (cleaned.endsWith('.R') || cleaned.endsWith('.r')) { + if (path.isAbsolute(cleaned)) { + const normalized = cleaned.replace(/\\/g, '/'); + return exactFile(normalized) ?? null; + } + const dir = path.dirname(filePath); + const resolved = path.join(dir, cleaned).replace(/\\/g, '/'); + const direct = exactFile(resolved); + if (direct !== undefined) return direct; + + const pathParts = cleaned.split('/').filter(Boolean); + return suffixResolve(pathParts, normalizedFileList, allFileList, EXTENSIONS, index); + } + + // library("pkg") / require("pkg") — resolve to ALL files in the local package + if (rConfig) { + const pkgDir = rConfig.packages.get(cleaned); + // A root-level package has the directory '' (falsy), so test for absence, not truthiness. + if (pkgDir !== undefined) { + const rDirPrefix = (pkgDir ? pkgDir + '/' : '') + 'R/'; + const files: string[] = []; + for (let i = 0; i < normalizedFileList.length; i++) { + if ( + normalizedFileList[i].startsWith(rDirPrefix) && + (normalizedFileList[i].endsWith('.R') || normalizedFileList[i].endsWith('.r')) + ) { + files.push(allFileList[i]); + } + } + if (files.length > 0) return files; + } + } + + // Fallback: suffix-based resolution + const pathParts = cleaned.split('/').filter(Boolean); + return suffixResolve(pathParts, normalizedFileList, allFileList, EXTENSIONS, index); +} + +/** Per-package index of top-level definitions, built once per `parsedFiles` array. */ +interface RPackageDefIndex { + /** Files of the package's `R/` directory, with what each defines at top level. */ + readonly files: ReadonlyMap; + /** Top-level symbol (backticks removed) → files (in `parsedFiles` order) that define it. */ + readonly definingFiles: ReadonlyMap; +} + +const packageDefIndexes = new WeakMap< + readonly ParsedFile[], + WeakMap> +>(); + +function packageDefIndex( + parsedFiles: readonly ParsedFile[], + rConfig: RPackageConfig, + pkg: string, +): RPackageDefIndex | undefined { + let byConfig = packageDefIndexes.get(parsedFiles); + if (byConfig === undefined) packageDefIndexes.set(parsedFiles, (byConfig = new WeakMap())); + let byPackage = byConfig.get(rConfig); + if (byPackage === undefined) { + const files = new Map>(); + const defining = new Map>(); + for (const parsed of parsedFiles) { + const location = rPackageDirForFile(parsed.filePath, rConfig); + if (location === undefined) continue; + const topLevel = rFileTopLevel(parsed); + let pkgFiles = files.get(location.name); + let pkgDefining = defining.get(location.name); + if (pkgFiles === undefined || pkgDefining === undefined) { + files.set(location.name, (pkgFiles = new Map())); + defining.set(location.name, (pkgDefining = new Map())); + } + pkgFiles.set(parsed.filePath, topLevel); + for (const name of topLevel.names) { + const list = pkgDefining.get(name); + if (list === undefined) pkgDefining.set(name, [parsed.filePath]); + else list.push(parsed.filePath); + } + } + const built = new Map(); + for (const [name, pkgFiles] of files) { + built.set(name, { files: pkgFiles, definingFiles: defining.get(name) ?? new Map() }); + } + byConfig.set(rConfig, (byPackage = built)); + } + return byPackage.get(pkg); +} + +/** + * `s` without its leading and trailing backticks. Two scans, not `/^`+|`+$/g`: the pattern is + * quadratic on `x` + n backticks + `y`, and a NAMESPACE entry is repository-controlled text. + */ +const stripBackticks = (s: string): string => { + let start = 0; + let end = s.length; + while (start < end && s.charCodeAt(start) === 96) start++; + while (end > start && s.charCodeAt(end - 1) === 96) end--; + return s.slice(start, end); +}; + +/** The backticked entries of one export set, stripped, and the set size they were read at. */ +const backtickedExportSymbols = new WeakMap< + ReadonlySet, + { readonly size: number; readonly symbols: ReadonlySet } +>(); + +/** + * The symbols of the entries of `exports` that start or end with a backtick (a quoted + * `` "`%+%`" `` entry keeps its backticks), stripped. Built in one pass per export set, so a + * lookup is a set membership test instead of a walk over every export; a set that has grown or + * shrunk since is read again. + */ +function backtickedExports(exports: ReadonlySet): ReadonlySet { + const cached = backtickedExportSymbols.get(exports); + if (cached !== undefined && cached.size === exports.size) return cached.symbols; + const symbols = new Set(); + for (const entry of exports) { + if (entry.charCodeAt(0) === 96 || entry.charCodeAt(entry.length - 1) === 96) { + symbols.add(stripBackticks(entry)); + } + } + backtickedExportSymbols.set(exports, { size: exports.size, symbols }); + return symbols; +} + +/** + * Resolve a NAMESPACE `importFrom(pkg, importedName)` to the `R/` file(s) of the + * local package `pkg` that define `importedName` at top level, or `null`. + * `importedName` is the symbol (`%||%`) or a definition's own spelling (`` `%||%` ``): files + * are found by the symbol, while the dotted-tail guard below compares the spelling, because + * finalize indexes a definition by the text of its name as written. + * + * `null` (no binding, no IMPORTS edge) when: `pkg` is not a local package + * (external packages never suffix-match anything); `importedName` is dotted + * (finalize keys defs by the text after the last `.`, so it can never bind); + * the provider has a NAMESPACE that does not export the name; no file defines + * it at top level; or a defining file also holds a dotted top-level def whose + * tail equals `importedName` (finalize would index `print.tidy` under `tidy` + * and bind the WRONG def — the guard is order-blind, so it also refuses the + * layout where the bare def comes first). `null` degrades the call site to the + * baseline `global-name-fallback` guess. Never builds a workspace file index. + */ +export function resolveRNamedImport( + rConfig: RPackageConfig | null, + targetRaw: string, + importedName: string, + parsedFiles: readonly ParsedFile[], +): readonly string[] | null { + const pkg = targetRaw.replace(/^["']|["']$/g, ''); + const pkgDir = rConfig?.packages.get(pkg); + if (rConfig === null || pkgDir === undefined) return null; + + if (importedName.includes('.')) return null; + + const symbol = rBareName(importedName); + const info = rConfig.namespaceInfoByPackageDir.get(pkgDir); + if (info !== undefined && info.hasNamespaceFile) { + // An `S3method()` registration alone is not an export (`RNamespaceInfo.s3RegisteredOnly`). + const exported = + (info.s3RegisteredOnly?.has(symbol) !== true && + (info.namedExports.has(symbol) || backtickedExports(info.namedExports).has(symbol))) || + rExportPatternMatches(info, symbol); + if (!exported) return null; + } + + const index = packageDefIndex(parsedFiles, rConfig, pkg); + const defining = index?.definingFiles.get(symbol); + if (index === undefined || defining === undefined || defining.length === 0) return null; + if (defining.some((file) => index.files.get(file)?.dottedTails.has(importedName) === true)) { + return null; + } + return defining; +} + +/** + * `ScopeResolver.resolveImportTarget`-shaped adapter over + * `resolveRImportInternal`. Two responsibilities live only here, not in the + * internal resolver: + * + * - Memoization: `getWorkspaceFileIndex` derives `{ normalized, all, index }` + * once per `allFilePaths` Set identity (see its own doc comment) and this + * adapter calls it on every invocation instead of materializing fresh + * arrays per import — the O(imports × files) regression PR #1918/#2911 + * exist to catch (`import-target-index-reuse.contract.test.ts`). + * - Cross-language false-positive guard: a `library()`/`require()` package + * name not found in `rConfig.packages` would otherwise fall through to + * `resolveRImportInternal`'s generic suffix fallback — and `EXTENSIONS` + * (`languages/import-candidates.ts`) has no `.R`/`.r` entry, so a bare R package reference can + * suffix-match an unrelated same-named file in another language. + * `context.parsedImport.kind === 'wildcard'` is exactly the + * `library()`/`require()` shape (see `interpretRImport`), so the guard is + * scoped to that kind only — `source()`'s own suffix-based file-path + * fallback is unaffected. + * - NAMESPACE `importFrom()` names (`kind: 'named'`, synthesised by + * `populateRNamespaceImports`): resolved by {@link resolveRNamedImport} + * before `getWorkspaceFileIndex`, so they build no file index. + */ +export function resolveRImportTarget( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, + resolutionConfig?: unknown, + context?: ImportResolutionContext, +): string | readonly string[] | null { + const rConfig = (resolutionConfig as RPackageConfig | null | undefined) ?? null; + + if (context?.parsedImport?.kind === 'wildcard') { + const cleaned = targetRaw.replace(/^["']|["']$/g, ''); + if (rConfig === null || !rConfig.packages.has(cleaned)) return null; + } + + if (context?.parsedImport?.kind === 'named') { + return resolveRNamedImport( + rConfig, + targetRaw, + context.parsedImport.importedName, + context.parsedFiles, + ); + } + + const { normalized, all, index, normToRaw } = getWorkspaceFileIndex(allFilePaths); + return resolveRImportInternal(fromFile, targetRaw, normalized, all, rConfig, index, normToRaw); +} diff --git a/gitnexus/src/core/ingestion/languages/r/interpret.ts b/gitnexus/src/core/ingestion/languages/r/interpret.ts new file mode 100644 index 000000000..4538b423d --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/interpret.ts @@ -0,0 +1,47 @@ +import type { CaptureMatch, ParsedImport, ParsedTypeBinding } from 'gitnexus-shared'; + +/** + * `source()` is a side-effect import — it establishes a file dependency but + * binds no name (cross-file free calls into a `source()`d file resolve via + * `rScopeResolver.allowGlobalFreeCallFallback` instead). + * + * `library()`/`require()` are wildcard imports — the exported names of the named + * local package (every top-level name when it has no NAMESPACE) become visible + * (`rScopeResolver.expandsWildcardTo`). + * + * `resolveRImportTarget` (`languages/r/import-target.ts`) guards on + * `context.parsedImport.kind === 'wildcard'` specifically to refuse a + * `library()`/`require()` package name that isn't a known local package + * (rather than falling through to a cross-language suffix-match false + * positive) — the two kinds set here are what that guard keys on. + */ +export function interpretRImport(captures: CaptureMatch): ParsedImport | null { + const source = captures['@import.source']?.text; + // The capture keeps its quotes, so `source("")` is two characters long: test the bare text. + if (source === undefined || source.replace(/^["']|["']$/g, '').length === 0) return null; + + if (captures['@import.wildcard'] !== undefined) { + return { kind: 'wildcard', targetRaw: source }; + } + return { kind: 'side-effect', targetRaw: source }; +} + +/** + * `x <- Type$new(...)` — R6's constructor spelling. `@type-binding.type` is + * the `$` receiver (the class name), captured only when the extract + * operator's rhs is literally `new` (see `query.ts`), so this always + * resolves via the shared `constructor-inferred` sub-kind — the same one + * Zig's `interpretZigTypeBinding` uses for `Counter.init()`. + */ +export function interpretRTypeBinding(captures: CaptureMatch): ParsedTypeBinding | null { + const name = captures['@type-binding.name']?.text; + const type = captures['@type-binding.type']?.text; + if (name === undefined || type === undefined) return null; + + // `self` in an R6 method, typed as its class by `emitRScopeCaptures` (the shared `self` source, + // as Ruby's interpreter reads it). + if (captures['@type-binding.self'] !== undefined) { + return { boundName: name, rawTypeName: type, source: 'self' }; + } + return { boundName: name, rawTypeName: type, source: 'constructor-inferred' }; +} diff --git a/gitnexus/src/core/ingestion/languages/r/linear-regex.ts b/gitnexus/src/core/ingestion/languages/r/linear-regex.ts new file mode 100644 index 000000000..a5e996e47 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/linear-regex.ts @@ -0,0 +1,1141 @@ +/** + * A linear-time matcher for the regular-expression subset that `exportPattern()` + * arguments use. + * + * `exportPattern()` text comes from the repository being analysed and is run against + * every node name. Compiling it to a JavaScript `RegExp` lets a pattern such as + * `^(a+)+$` take exponential time on a short name and hang the analyzer, whereas R + * evaluates the same text with TRE, which does not backtrack. {@link compileLinearRegex} + * parses the pattern (JavaScript syntax, non-Unicode mode, the dialect `RegExp` + * accepted before) into a Thompson NFA and simulates it with a set of active states. + * A match attempt costs O(name length x NFA size) whatever the pattern says, and the + * total work one matcher may do is capped (see the cost model below). + * + * Supported: literals, `.`, bracket expressions, `\d \D \w \W \s \S \b \B`, the + * single-character escapes, `^ $`, groups (capturing and `(?:)`), alternation and the + * quantifiers `* + ? {n} {n,} {n,m}` (lazy forms behave identically for a yes/no + * test). Not supported, and compiled to `null`: constructs TRE reads differently from + * JavaScript (`\<` `\>` word boundaries, `\a` `\e` `\Q` `\E`, `\x{`, `\u` in any form, a + * letter class escape such as `\d` inside `[...]`, `[.x.]` and `[=x=]` elements, `{,n}`, a + * leading `]` in a bracket expression), which fail closed rather than match differently + * from R; back-references (`\1` to `\9`, whether or not + * the group exists, as in TRE) and look-around (no + * linear-time algorithm), named groups, legacy octal escapes, patterns longer than + * {@link MAX_PATTERN_LENGTH} and patterns whose expansion exceeds {@link MAX_STATES}. + * A syntax error also yields `null`. Matching is over UTF-16 code units, as + * `RegExp` without the `u` flag does. {@link compileLinearRegexDetailed} reports why a + * pattern was refused. + * + * Cost model. Compiling is O(pattern length + states). Matching one name is + * O(name length x active states), and a state count alone does not make a pattern + * slow: `[[:alpha:]]{1,2000}` has 4,000 states but one of them is active at a time, + * and an alternation of literals keeps only the ones whose next character matches. + * What is slow is many active states on a long name, and a repository controls both. + * So the caps are sized for legitimate patterns and the hostile case is bounded + * separately, by {@link MAX_TOTAL_WORK}: each matcher counts every state it visits + * and, once the budget is spent, answers false for names it has not already decided + * and reports `exhausted`. A state visit inside a long search costs 10-20 ns, but a test + * costs more than its state visits (the memo lookup and insert, setting up the search), and + * that part is not charged: many short tests cost up to about 70 ns a counted visit (measured + * on the development machine: 50-56 ns for 2,000 trivial patterns against 5,000 names, about + * 70 ns for 50,000 names against two patterns), so the 30M package cap is about 2 s and the + * 100M load cap about 7 s at worst: bounded, never a hang. A hostile pattern spends its + * visits inside long searches: `(a?){100}a{100}` against 5,000 names spends the package cap + * in about 0.4 s. + * Realistic patterns stay far below it: 500 alternated identifiers, anchored or not, + * cost about 20 visits per name, because alternatives that share a prefix share + * states. Verdicts are memoised per distinct name, so a repeated name costs one lookup, which is + * charged as one visit. + * + * The per-matcher cap does not bound how many matchers there are, and a NAMESPACE can + * carry as many `exportPattern()` lines as it likes (200 hostile ones cost ~77 s). So + * matchers may also draw on a {@link SharedWorkBudget}, which the caller creates once + * for a whole package-config load and hands to every matcher it compiles, across all + * the packages it discovers; a matcher stops when its own cap or the shared budget is + * spent, whichever comes first. A count cap would not do: a per-package cap still + * allows many packages, and a per-run cap would penalise a monorepo whose packages + * each carry a few benign patterns, which use next to no work. + * + * Matching is not the only cost that grows with the number of patterns. Every compiled + * pattern keeps an NFA (about 100 bytes a state), and a NAMESPACE can carry hundreds of + * thousands of them, so the budget also holds an allowance of NFA states: a pattern that + * does not fit in what is left is refused (the build stops at the allowance, so its NFA is + * never allocated in full) and, once nothing is left, later patterns are refused without + * being parsed (the first pattern that does not fit ends the allowance; a pattern is at most + * {@link MAX_STATES} states, so that only happens after the allowance is nearly used up). + * A pattern refused after its NFA was partly built is charged for the build (see + * {@link REFUSED_BUILD_WORK_PER_UNIT}), and once the work budget is spent later patterns are + * refused without being parsed, so refused patterns cost no more than matched ones. + * Budgets nest: a package's budget is a {@link SharedWorkBudget.child} of the + * load's, so one package cannot take the whole load's allowance and leave every other + * package's patterns unable to run. + */ + +/** Longest pattern accepted: room for an alternation of ~1,000 identifiers. */ +export const MAX_PATTERN_LENGTH = 16384; + +/** + * Most NFA states a pattern may expand to. Bounded repetition such as `a{1000}` is + * expanded copy by copy, so this caps memory and compile time; the matching cost is + * bounded by {@link MAX_TOTAL_WORK}. Room for ~1,000 alternated identifiers of 15 + * characters, or `[[:alpha:]]{1,2000}`. + */ +export const MAX_STATES = 16384; + +/** + * Deepest group nesting accepted; bounds the parser's and the builder's recursion. It is set + * well inside what the call stack allows, so that the verdict does not depend on the machine: + * a cold process (no JIT warm-up, so the biggest frames) compiled 1,500 levels in the + * measurements and ran out of stack somewhere before 2,048, and 256 levels still compile on a + * quarter of the default stack (`node --stack-size=250`). No NAMESPACE nests groups this deep. + */ +export const MAX_NESTING = 256; + +/** + * Most `build` calls for one pattern. States are capped separately, but a body that + * creates no states (`(){1000}`) can still be repeated exponentially when nested. + */ +const MAX_BUILD_STEPS = 4 * MAX_STATES; + +/** + * State visits one matcher may spend across all the names it is asked about (see the + * cost model above). + */ +export const MAX_TOTAL_WORK = 40_000_000; + +/** + * State visits all the matchers of one package-config load may spend together (the + * default limit of a {@link SharedWorkBudget}), in addition to the per-matcher + * {@link MAX_TOTAL_WORK}. About one second for hostile searches (10 ns a visit), up to + * about 7 s for many short tests (see the cost model above): enough for two + * and a half pathological patterns to run to their own cap, after which the rest of + * the load's patterns stop and are reported, so a hostile NAMESPACE cannot cost more + * than that however many patterns it carries. Benign work is a tiny fraction of it: a + * 50-package monorepo with three realistic patterns per package and 2,500 functions + * spends about 15,000 visits, and one realistic pattern run against 50,000 names + * spends 0.2-7 million. Counted in visits, not wall-clock time, so the outcome is the + * same on every machine. + */ +export const MAX_SHARED_WORK = 100_000_000; + +/** + * NFA states all the patterns of one load may keep together. A state costs about 700 + * bytes once compiled (parallel arrays plus a character set per character state; + * measured: 2,000,000 states held 1.4 GB), so this bounds what a NAMESPACE can make the + * analyzer hold at about 350 MB. A realistic pattern is tens to a few thousand states + * (an alternation of 500 identifiers is about 7,500; `[[:alpha:]]{1,2000}` is 4,000), so + * the 150 benign patterns of the 50-package monorepo above use under 5,000 states in + * total, 1 % of this, and the allowance holds some 65 such alternations. + */ +export const MAX_SHARED_STATES = 500_000; + +/** + * State visits one package may spend on matching, out of {@link MAX_SHARED_WORK} (30 %, + * about 0.4 s for a hostile pattern, up to about 2 s for many short tests): one hostile + * package then cannot take the load's whole budget, so the packages after it keep their + * patterns. Three hostile packages spend the load's budget; the packages after those + * degrade, with a warning per pattern. + */ +export const MAX_PACKAGE_WORK = 30_000_000; + +/** + * NFA states one package may keep, out of {@link MAX_SHARED_STATES} (20 %, about 70 MB): + * six patterns at the {@link MAX_STATES} cap, or some 13 alternations of 500 identifiers, + * far beyond what a NAMESPACE writes by hand, while five such packages still cannot + * exceed the load's allowance on their own. + */ +export const MAX_PACKAGE_STATES = 100_000; + +/** Why a pattern was refused because the work budget was spent before it was compiled. */ +const WORK_BUDGET_REASON = + 'the work budget for this package or repository is spent (too many or too costly patterns)'; + +/** Why a pattern was refused because the allowance of states was spent. */ +const STATE_ALLOWANCE_REASON = + 'the NFA state allowance for this package or repository is spent (too many or too large patterns)'; + +/** + * A work allowance shared by several matchers: state visits spent matching, and NFA + * states kept by compiled patterns. Create one per package-config load and give each + * package a {@link child}; every matcher compiled with a budget adds the state visits + * it spends to {@link spent} (and to every ancestor's), and all of them stop once any + * budget up the chain is spent. Counted in work units, never time. + */ +export class SharedWorkBudget { + /** State visits spent so far by every matcher drawing on this budget. */ + spent = 0; + /** NFA states kept so far by every pattern compiled against this budget. */ + states = 0; + /** + * Set by the caller that skipped matching because the budget was spent, so that the + * patterns it did not get to run can be reported. + */ + cutShort = false; + + constructor( + /** State visits the matchers may spend together. */ + readonly limit: number = MAX_SHARED_WORK, + /** NFA states the compiled patterns may keep together. */ + readonly stateLimit: number = MAX_SHARED_STATES, + private readonly parent?: SharedWorkBudget, + ) {} + + /** A sub-budget for one package: it draws on this budget too, and cannot exceed it. */ + child( + limit: number = MAX_PACKAGE_WORK, + stateLimit: number = MAX_PACKAGE_STATES, + ): SharedWorkBudget { + return new SharedWorkBudget(limit, stateLimit, this); + } + + /** True once this budget or any ancestor is spent; a matcher then answers false for undecided names. */ + get isSpent(): boolean { + return this.spent >= this.limit || this.parent?.isSpent === true; + } + + /** + * NFA states that may still be kept, here and in every ancestor (never negative). Zero + * once a pattern was refused for not fitting (see {@link closeStates}). + */ + get statesLeft(): number { + const own = this.statesClosed ? 0 : Math.max(0, this.stateLimit - this.states); + return this.parent === undefined ? own : Math.min(own, this.parent.statesLeft); + } + + /** + * A pattern was refused because it did not fit in `room` states: close every budget up + * the chain whose own allowance is that tight, so later patterns are refused at once, + * without being parsed. Otherwise a small remainder that no pattern fits would make + * every one of hundreds of thousands of later patterns cost a parse and a partial build. + */ + closeStates(room: number): void { + if (Math.max(0, this.stateLimit - this.states) <= room) this.statesClosed = true; + this.parent?.closeStates(room); + } + + private statesClosed = false; + + /** Add `visits` state visits to this budget and every ancestor. */ + charge(visits: number): void { + this.spent += visits; + this.parent?.charge(visits); + } + + /** Record `count` NFA states kept by a compiled pattern, here and in every ancestor. */ + reserveStates(count: number): void { + this.states += count; + this.parent?.reserveStates(count); + } +} + +/** Most distinct names whose verdict a matcher remembers. */ +const MAX_MEMO_ENTRIES = 100_000; + +/** + * Work charged for answering from a remembered verdict: one state visit, about the cost of a + * map lookup. Keeps a repeated name from being free, so that matchers x lookups is bounded by + * the budget like every other cost. + */ +const MEMO_HIT_WORK = 1; + +/** + * Work charged per state or build step of a pattern's NFA that was built and then refused. Building + * a state costs about eight state visits (85 ns against 10 ns, measured), and a refused pattern + * keeps no states to be counted against the allowance, so the budget would otherwise never see + * the time it took. + */ +const REFUSED_BUILD_WORK_PER_UNIT = 8; + +/** Thrown internally for unsupported or invalid input; never escapes this module. */ +class Reject extends Error {} + +const reject = (why: string): never => { + throw new Reject(why); +}; + +type Range = readonly [number, number]; + +interface CharSet { + readonly ranges: readonly Range[]; + readonly negated: boolean; +} + +type Node = + | { readonly kind: 'empty' } + | { readonly kind: 'set'; readonly set: CharSet } + | { readonly kind: 'assert'; readonly which: 'bol' | 'eol' | 'wordb' | 'nwordb' } + | { readonly kind: 'seq'; readonly items: readonly Node[] } + | { readonly kind: 'alt'; readonly items: readonly Node[] } + | { readonly kind: 'repeat'; readonly min: number; readonly max: number; readonly node: Node }; + +/** + * The code units of `node` when it matches exactly one fixed string (a literal, or a + * sequence of literals), else null. + */ +function literalWord(node: Node): number[] | null { + switch (node.kind) { + case 'empty': + return []; + case 'set': { + const { ranges, negated } = node.set; + return !negated && ranges.length === 1 && ranges[0][0] === ranges[0][1] + ? [ranges[0][0]] + : null; + } + case 'seq': { + const word: number[] = []; + for (const item of node.items) { + const part = literalWord(item); + if (part === null) return null; + for (const code of part) word.push(code); + } + return word; + } + default: + return null; + } +} + +/** The strings of an alternation whose every alternative is a fixed string, else null. */ +function literalWords(items: readonly Node[]): number[][] | null { + const words: number[][] = []; + for (const item of items) { + const word = literalWord(item); + if (word === null) return null; + words.push(word); + } + return words; +} + +const MAX_CODE_UNIT = 0xffff; + +const DIGIT: readonly Range[] = [[0x30, 0x39]]; +const WORD: readonly Range[] = [ + [0x30, 0x39], + [0x41, 0x5a], + [0x5f, 0x5f], + [0x61, 0x7a], +]; +const SPACE: readonly Range[] = [ + [0x09, 0x0d], + [0x20, 0x20], + [0xa0, 0xa0], + [0x1680, 0x1680], + [0x2000, 0x200a], + [0x2028, 0x2029], + [0x202f, 0x202f], + [0x205f, 0x205f], + [0x3000, 0x3000], + [0xfeff, 0xfeff], +]; +const LINE_TERMINATORS: readonly Range[] = [ + [0x0a, 0x0a], + [0x0d, 0x0d], + [0x2028, 0x2029], +]; + +/** Complement of a sorted, non-overlapping range list over the UTF-16 code units. */ +function complement(ranges: readonly Range[]): Range[] { + const out: Range[] = []; + let next = 0; + for (const [lo, hi] of ranges) { + if (lo > next) out.push([next, lo - 1]); + next = hi + 1; + } + if (next <= MAX_CODE_UNIT) out.push([next, MAX_CODE_UNIT]); + return out; +} + +const isWordUnit = (c: number): boolean => + (c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || c === 0x5f || (c >= 0x61 && c <= 0x7a); + +const isDecimal = (c: string | undefined): boolean => c !== undefined && c >= '0' && c <= '9'; +const isHexDigit = (c: string | undefined): boolean => c !== undefined && /^[0-9A-Fa-f]$/.test(c); +const isAsciiLetter = (c: string | undefined): boolean => c !== undefined && /^[A-Za-z]$/.test(c); + +/** What `\x`, `\u` and friends resolve to: a single code unit, or a class. */ +type EscapeResult = + | { readonly code: number } + | { readonly ranges: readonly Range[]; readonly classEscape: true } + | { readonly assertion: 'wordb' | 'nwordb' }; + +/** `{n}`, `{n,}`, `{n,m}` at the sticky cursor. */ +const BRACES = /\{(\d+)(?:(,)(\d*))?\}/y; +const LOWER_BOUND_OMITTED = /\{,\d*\}/y; + +class Parser { + private pos = 0; + private depth = 0; + + constructor(private readonly src: string) {} + + parse(): Node { + const node = this.parseDisjunction(); + if (this.pos < this.src.length) reject('unmatched )'); + return node; + } + + private peek(offset = 0): string | undefined { + return this.src[this.pos + offset]; + } + + private parseDisjunction(): Node { + const alternatives: Node[] = [this.parseAlternative()]; + while (this.peek() === '|') { + this.pos++; + alternatives.push(this.parseAlternative()); + } + return alternatives.length === 1 ? alternatives[0] : { kind: 'alt', items: alternatives }; + } + + private parseAlternative(): Node { + const items: Node[] = []; + while (this.pos < this.src.length) { + const ch = this.peek(); + if (ch === '|' || ch === ')') break; + items.push(this.parseTerm()); + } + if (items.length === 0) return { kind: 'empty' }; + return items.length === 1 ? items[0] : { kind: 'seq', items }; + } + + private parseTerm(): Node { + const ch = this.peek() as string; + if (ch === '^' || ch === '$') { + this.pos++; + this.rejectQuantifierAfterAssertion(); + return { kind: 'assert', which: ch === '^' ? 'bol' : 'eol' }; + } + const atom = this.parseAtom(); + if (atom.kind === 'assert') { + this.rejectQuantifierAfterAssertion(); + return atom; + } + return this.parseQuantifier(atom); + } + + private rejectQuantifierAfterAssertion(): void { + const ch = this.peek(); + if (ch === '*' || ch === '+' || ch === '?') reject('nothing to repeat'); + if (ch === '{' && this.tryReadBraces() !== null) reject('nothing to repeat'); + } + + private parseAtom(): Node { + const ch = this.peek() as string; + switch (ch) { + case '.': + this.pos++; + return { kind: 'set', set: { ranges: LINE_TERMINATORS, negated: true } }; + case '(': + return this.parseGroup(); + case '[': + return this.parseClass(); + case '\\': + return this.parseAtomEscape(); + case '*': + case '+': + case '?': + return reject('nothing to repeat'); + case '{': + if (this.tryReadBraces() !== null) reject('nothing to repeat'); + this.pos++; + return literal(ch.charCodeAt(0)); + default: + this.pos++; + return literal(ch.charCodeAt(0)); + } + } + + private parseGroup(): Node { + this.pos++; // ( + if (this.peek() === '?') { + // `(?:` is a plain group; `(?=`, `(?!`, `(?<` and everything else is unsupported or invalid. + // Only a `(` the parser reaches here opens a group: one inside a character class + // (`[(?<]`) or after a backslash (`\(?<`) is a literal and never gets this far. + if (this.peek(1) === '<') reject('named groups and look-behind are unsupported'); + if (this.peek(1) !== ':') reject('look-around and group modifiers are unsupported'); + this.pos += 2; + } + if (++this.depth > MAX_NESTING) reject('groups are nested too deeply'); + const inner = this.parseDisjunction(); + this.depth--; + if (this.peek() !== ')') reject('unterminated group'); + this.pos++; + // Wrapped so that `(?:$)*` and `(\b)+` stay quantifiable, unlike a bare assertion. + return { kind: 'seq', items: [inner] }; + } + + /** Read `{n}`, `{n,}` or `{n,m}` at the cursor without consuming; null when not a quantifier. */ + private tryReadBraces(): { min: number; max: number; length: number } | null { + BRACES.lastIndex = this.pos; + const m = BRACES.exec(this.src); + if (m === null) { + // TRE reads `{,n}` as `{0,n}`; JavaScript as literal text. Refuse rather than guess. + LOWER_BOUND_OMITTED.lastIndex = this.pos; + if (LOWER_BOUND_OMITTED.test(this.src)) reject('{,n} quantifier is read differently by TRE'); + return null; + } + const min = Number(m[1]); + const max = m[2] === undefined ? min : m[3] === '' ? Infinity : Number(m[3]); + return { min, max, length: m[0].length }; + } + + private parseQuantifier(atom: Node): Node { + const ch = this.peek(); + let min: number; + let max: number; + if (ch === '*') { + min = 0; + max = Infinity; + this.pos++; + } else if (ch === '+') { + min = 1; + max = Infinity; + this.pos++; + } else if (ch === '?') { + min = 0; + max = 1; + this.pos++; + } else if (ch === '{') { + const braces = this.tryReadBraces(); + if (braces === null) return atom; // a literal `{`, parsed as the next atom + if (braces.max < braces.min) reject('numbers out of order in quantifier'); + min = braces.min; + max = braces.max; + this.pos += braces.length; + } else { + return atom; + } + if (this.peek() === '?') this.pos++; // lazy: irrelevant for a yes/no match + return { kind: 'repeat', min, max, node: atom }; + } + + /** `\` outside a bracket expression. */ + private parseAtomEscape(): Node { + const result = this.readEscape(false); + if ('assertion' in result) return { kind: 'assert', which: result.assertion }; + if ('classEscape' in result) { + return { kind: 'set', set: { ranges: result.ranges, negated: false } }; + } + return literal(result.code); + } + + /** + * Decode the escape at the cursor (`src[pos] === '\\'`) and advance past it. + * `inClass` selects the bracket-expression meanings (`\b` is backspace there). + */ + private readEscape(inClass: boolean): EscapeResult { + this.pos++; // backslash + const c = this.peek(); + if (c === undefined) return reject('\\ at end of pattern'); + this.pos++; + // TRE reads a backslash inside a bracket expression as a literal member, not as an + // escape, so a letter class escape there means different things to R and to this parser. + if (inClass && 'dDwWsSb'.includes(c)) { + reject('letter class escape inside a bracket expression (TRE reads the backslash literally)'); + } + switch (c) { + case 'd': + return { ranges: DIGIT, classEscape: true }; + case 'D': + return { ranges: complement(DIGIT), classEscape: true }; + case 'w': + return { ranges: WORD, classEscape: true }; + case 'W': + return { ranges: complement(WORD), classEscape: true }; + case 's': + return { ranges: SPACE, classEscape: true }; + case 'S': + return { ranges: complement(SPACE), classEscape: true }; + case 'b': + return inClass ? { code: 0x08 } : { assertion: 'wordb' }; + case 'B': + return inClass ? { code: 0x42 } : { assertion: 'nwordb' }; + case 't': + return { code: 0x09 }; + case 'n': + return { code: 0x0a }; + case 'v': + return { code: 0x0b }; + case 'f': + return { code: 0x0c }; + case 'r': + return { code: 0x0d }; + case '0': + if (isDecimal(this.peek())) reject('legacy octal escape'); + return { code: 0 }; + case 'c': { + const letter = this.peek(); + // Inside a bracket expression Annex B also accepts a digit or `_` after `\c`. + if (isAsciiLetter(letter) || (inClass && (isDecimal(letter) || letter === '_'))) { + this.pos++; + return { code: (letter as string).charCodeAt(0) % 32 }; + } + // `\c` without a letter is a literal backslash; the `c` is read next. + this.pos--; + return { code: 0x5c }; + } + case '<': + case '>': + return reject('\\< and \\> are TRE word boundaries, not modelled'); + case 'a': + case 'e': + case 'Q': + case 'E': + return reject(`\\${c} has a TRE meaning that is not modelled`); + case 'x': { + if (this.peek() === '{') return reject('\\x{...} is not modelled'); + if (isHexDigit(this.peek()) && isHexDigit(this.peek(1))) { + const code = parseInt(this.src.slice(this.pos, this.pos + 2), 16); + this.pos += 2; + return { code }; + } + return { code: 0x78 }; + } + case 'u': + return reject('\\u is not modelled (TRE reading unverified)'); + case 'k': + // Identity escape (no named groups exist; `parseGroup` rejects those). + return { code: 0x6b }; + default: + // `\1`-`\9` are back-references to R's TRE, whether or not that group exists (an + // absent one is an invalid regex there), so `\8` and `\9` are refused too although + // JavaScript's Annex B reads them as a literal digit when the pattern has fewer groups. + if (c >= '1' && c <= '9') reject('back-reference or legacy octal escape'); + return { code: c.charCodeAt(0) }; + } + } + + private parseClass(): Node { + this.pos++; // [ + let negated = false; + if (this.peek() === '^') { + negated = true; + this.pos++; + } + // POSIX makes a leading `]` a literal member; JavaScript reads `[]` as an empty class. + if (this.peek() === ']') reject('leading ] in a bracket expression is read differently by TRE'); + const ranges: Range[] = []; + for (;;) { + const ch = this.peek(); + if (ch === undefined) reject('unterminated character class'); + if (ch === '[' && (this.peek(1) === '.' || this.peek(1) === '=')) { + reject('collating and equivalence elements are not modelled'); + } + if (ch === ']') { + this.pos++; + break; + } + const lo = this.readClassAtom(); + if (this.peek() === '-' && this.peek(1) !== ']' && this.peek(1) !== undefined) { + // Possible range: a class escape on either side makes the `-` a literal (Annex B). + this.pos++; // - + const hi = this.readClassAtom(); + if ('code' in lo && 'code' in hi) { + if (lo.code > hi.code) reject('range out of order in character class'); + ranges.push([lo.code, hi.code]); + } else { + this.addClassAtom(ranges, lo); + ranges.push([0x2d, 0x2d]); + this.addClassAtom(ranges, hi); + } + } else { + this.addClassAtom(ranges, lo); + } + } + return { kind: 'set', set: { ranges: normalise(ranges), negated } }; + } + + private addClassAtom( + ranges: Range[], + atom: { readonly code: number } | { readonly ranges: readonly Range[] }, + ): void { + if ('code' in atom) ranges.push([atom.code, atom.code]); + else ranges.push(...atom.ranges); + } + + private readClassAtom(): { readonly code: number } | { readonly ranges: readonly Range[] } { + const ch = this.peek() as string; + if (ch !== '\\') { + this.pos++; + return { code: ch.charCodeAt(0) }; + } + const result = this.readEscape(true); + if ('assertion' in result) return reject('unreachable: no assertions inside a class'); + if ('classEscape' in result) return { ranges: result.ranges }; + return { code: result.code }; + } +} + +const literal = (code: number): Node => ({ + kind: 'set', + set: { ranges: [[code, code]], negated: false }, +}); + +/** Sort and merge overlapping or adjacent ranges. */ +function normalise(ranges: readonly Range[]): Range[] { + const sorted = [...ranges].sort((a, b) => a[0] - b[0]); + const out: Array<[number, number]> = []; + for (const [lo, hi] of sorted) { + const last = out[out.length - 1]; + if (last !== undefined && lo <= last[1] + 1) last[1] = Math.max(last[1], hi); + else out.push([lo, hi]); + } + return out; +} + +// --- NFA --------------------------------------------------------------------------- + +const K_CHAR = 0; +const K_SPLIT = 1; +const K_ASSERT = 2; +const K_MATCH = 3; + +const A_BOL = 0; +const A_EOL = 1; +const A_WORDB = 2; +const A_NWORDB = 3; + +const ASSERT_CODES = { bol: A_BOL, eol: A_EOL, wordb: A_WORDB, nwordb: A_NWORDB } as const; + +/** Thompson NFA over parallel arrays; state `i` is described by index `i` of each. */ +class Nfa { + constructor( + /** Most states this NFA may have: {@link MAX_STATES}, or less when an allowance is low. */ + private readonly limit: number = MAX_STATES, + private readonly limitReason: string = 'pattern expands beyond the state cap', + ) {} + + readonly kind: number[] = []; + readonly out: number[] = []; + /** Second branch of a split; unused otherwise. */ + readonly out1: number[] = []; + /** Character set (K_CHAR) or assertion code (K_ASSERT). */ + readonly arg: Array = []; + private steps = 0; + + /** States built plus build steps taken so far: what the build has cost, win or lose. */ + get buildWork(): number { + return this.kind.length + this.steps; + } + + add(kind: number, out: number, out1: number, arg: CharSet | number | null): number { + if (this.kind.length >= this.limit) reject(this.limitReason); + this.kind.push(kind); + this.out.push(out); + this.out1.push(out1); + this.arg.push(arg); + return this.kind.length - 1; + } + + /** Build `node` so that it continues into `next`; returns its entry state. */ + build(node: Node, next: number): number { + if (++this.steps > MAX_BUILD_STEPS) reject('pattern expands beyond the work cap'); + switch (node.kind) { + case 'empty': + return next; + case 'set': + return this.add(K_CHAR, next, -1, node.set); + case 'assert': + return this.add(K_ASSERT, next, -1, ASSERT_CODES[node.which]); + case 'seq': { + let cont = next; + for (let i = node.items.length - 1; i >= 0; i--) cont = this.build(node.items[i], cont); + return cont; + } + case 'alt': { + const words = literalWords(node.items); + if (words !== null) return this.buildWords(words, next); + let entry = this.build(node.items[node.items.length - 1], next); + for (let i = node.items.length - 2; i >= 0; i--) { + entry = this.add(K_SPLIT, this.build(node.items[i], next), entry, null); + } + return entry; + } + case 'repeat': + return this.buildRepeat(node, next); + } + } + + /** + * Build an alternation of literal strings as a trie, so that alternatives sharing a + * prefix share states (`my_function_0|my_function_1|...` keeps one state active on + * the common prefix instead of one per alternative). Matches the same strings as the + * plain alternation. Iterative: a literal may be thousands of characters long. + */ + private buildWords(words: readonly (readonly number[])[], next: number): number { + interface TrieNode { + readonly children: Map; + terminal: boolean; + } + const root: TrieNode = { children: new Map(), terminal: false }; + for (const word of words) { + let at = root; + for (const code of word) { + let child = at.children.get(code); + if (child === undefined) { + child = { children: new Map(), terminal: false }; + at.children.set(code, child); + } + at = child; + } + at.terminal = true; + } + // Each entry: a trie node, and the character state whose `out` is that node's entry + // (-1 for the root, whose entry is the result). + let rootEntry = next; + const pending: Array<{ node: TrieNode; via: number }> = [{ node: root, via: -1 }]; + for (let task = pending.pop(); task !== undefined; task = pending.pop()) { + if (++this.steps > MAX_BUILD_STEPS) reject('pattern expands beyond the work cap'); + const branches: number[] = []; + if (task.node.terminal) branches.push(next); + for (const [code, child] of task.node.children) { + const state = this.add(K_CHAR, -1, -1, { + ranges: [[code, code]], + negated: false, + }); + branches.push(state); + pending.push({ node: child, via: state }); + } + let entry = branches[branches.length - 1]; + for (let i = branches.length - 2; i >= 0; i--) { + entry = this.add(K_SPLIT, branches[i], entry, null); + } + if (branches.length === 0) entry = next; + if (task.via === -1) rootEntry = entry; + else this.out[task.via] = entry; + } + return rootEntry; + } + + private buildRepeat(node: Extract, next: number): number { + // Reject before expanding: a count beyond the cap cannot fit whatever the body is. + if (node.min > MAX_STATES) reject('repetition count beyond the state cap'); + let cont: number; + if (node.max === Infinity) { + const loop = this.add(K_SPLIT, -1, next, null); + this.out[loop] = this.build(node.node, loop); + cont = loop; + } else { + if (node.max - node.min > MAX_STATES) reject('repetition count beyond the state cap'); + cont = next; + for (let i = node.min; i < node.max; i++) { + cont = this.add(K_SPLIT, this.build(node.node, cont), next, null); + } + } + for (let i = 0; i < node.min; i++) cont = this.build(node.node, cont); + return cont; + } +} + +/** Shared by {@link LinearRegex.startStates} entries that reach the match state. */ +const NO_STATES: readonly number[] = []; + +/** Empty-transition closure of the start state under one assertion context. */ +interface StartClosure { + /** True when the match state is reachable without consuming a character. */ + readonly matches: boolean; + /** Character states reachable without consuming a character. */ + readonly chars: readonly number[]; +} + +/** A compiled pattern. `test` has the same yes/no contract as `RegExp.prototype.test`. */ +export class LinearRegex { + /** Number of NFA states, exposed so tests can assert the structural bound. */ + readonly stateCount: number; + /** + * True once the work budget (this matcher's own, or the shared one) ran out; later + * unseen names are answered false. + */ + exhausted = false; + /** State visits spent so far, against {@link MAX_TOTAL_WORK}. */ + work = 0; + private readonly kind: readonly number[]; + private readonly out: readonly number[]; + private readonly out1: readonly number[]; + private readonly arg: ReadonlyArray; + private readonly start: number; + private readonly mark: Int32Array; + private readonly stack: number[] = []; + private stamp = 0; + /** Start closures by assertion context (see {@link contextAt}). */ + private readonly startClosures: Array = new Array(16); + /** Character states of a start closure that accept one code unit, by context and unit. */ + private readonly startStates = new Map(); + private readonly verdicts = new Map(); + + constructor( + /** The pattern text this matcher was compiled from (after any POSIX-class translation). */ + readonly source: string, + nfa: Nfa, + start: number, + /** Allowance shared with other matchers; they all stop when it is spent. */ + private readonly shared?: SharedWorkBudget, + ) { + this.kind = nfa.kind; + this.out = nfa.out; + this.out1 = nfa.out1; + this.arg = nfa.arg; + this.start = start; + this.stateCount = nfa.kind.length; + this.mark = new Int32Array(this.stateCount); + } + + /** True when the pattern matches anywhere in `name` (an unanchored search). */ + test(name: string): boolean { + const known = this.verdicts.get(name); + if (known !== undefined) { + // A remembered verdict is cheap but not free: a caller that asks about the same names of + // thousands of matchers over and over would otherwise cost time no budget counts. + this.charge(MEMO_HIT_WORK); + return known; + } + if (this.work >= MAX_TOTAL_WORK || this.shared?.isSpent === true) { + this.exhausted = true; + return false; + } + const verdict = this.search(name); + // A search the budget cut short has no verdict; do not remember it. + if (!this.exhausted && this.verdicts.size < MAX_MEMO_ENTRIES) this.verdicts.set(name, verdict); + return verdict; + } + + /** Count `visits` state visits against this matcher's cap and the shared budget. */ + private charge(visits: number): void { + this.work += visits; + this.shared?.charge(visits); + } + + private search(name: string): boolean { + const length = name.length; + let current: number[] = []; + let following: number[] = []; + this.stamp++; + if (this.seed(current, name, 0)) return true; + for (let p = 0; p < length; p++) { + if (this.work > MAX_TOTAL_WORK || this.shared?.isSpent === true) { + this.exhausted = true; + return false; + } + const c = name.charCodeAt(p); + this.stamp++; + following.length = 0; + this.charge(current.length); + for (let i = 0; i < current.length; i++) { + const s = current[i]; + if (inSet(this.arg[s] as CharSet, c) && this.add(following, this.out[s], name, p + 1)) { + return true; + } + } + // Unanchored search: a match may also begin at the next position. + if (this.seed(following, name, p + 1)) return true; + const swap = current; + current = following; + following = swap; + } + return false; + } + + /** + * Start a match attempt at position `p`: true when the pattern matches there without + * consuming anything, otherwise the states that can consume `name[p]` are added to + * `list`. Those are the only start states worth keeping (the rest die on the next + * step), and looking them up by code unit keeps a long alternation from costing + * one visit per alternative at every position. + */ + private seed(list: number[], name: string, p: number): boolean { + const context = contextAt(name, p); + const closure = (this.startClosures[context] ??= this.closeStart(context)); + if (closure.matches) return true; + if (p === name.length) return false; + const c = name.charCodeAt(p); + const key = context * 0x10000 + c; + let states = this.startStates.get(key); + if (states === undefined) { + // A miss tests every character state of the closure against `c`, and each test may + // scan every range of its class; charge that, or a large class costs nothing. + const accepting: number[] = []; + for (const s of closure.chars) { + const set = this.arg[s] as CharSet; + this.charge(1 + set.ranges.length); + if (inSet(set, c)) accepting.push(s); + } + states = accepting; + if (this.startStates.size < 4096) this.startStates.set(key, states); + } + this.charge(1 + states.length); + for (let i = 0; i < states.length; i++) { + const s = states[i]; + if (this.mark[s] !== this.stamp) { + this.mark[s] = this.stamp; + list.push(s); + } + } + return false; + } + + /** Closure of the start state where the assertion outcomes are those of `context`. */ + private closeStart(context: number): StartClosure { + const seen = new Uint8Array(this.stateCount); + const chars: number[] = []; + const stack = [this.start]; + while (stack.length > 0) { + const s = stack.pop() as number; + if (seen[s] === 1) continue; + seen[s] = 1; + switch (this.kind[s]) { + case K_MATCH: + return { matches: true, chars: NO_STATES }; + case K_CHAR: + chars.push(s); + break; + case K_SPLIT: + stack.push(this.out1[s], this.out[s]); + break; + default: + if (holdsIn(this.arg[s] as number, context)) stack.push(this.out[s]); + } + } + return { matches: false, chars }; + } + + /** + * Follow empty transitions from `state` at position `p`, pushing character states + * onto `list`. Returns true on reaching the match state. A state is entered at most + * once per list (the stamp), which also stops empty-loop cycles such as `(a*)*`. + * Iterative: a long alternation is a long chain of splits. + */ + private add(list: number[], state: number, name: string, p: number): boolean { + const stack = this.stack; + stack.length = 0; + stack.push(state); + while (stack.length > 0) { + const s = stack.pop() as number; + if (this.mark[s] === this.stamp) continue; + this.mark[s] = this.stamp; + this.charge(1); + switch (this.kind[s]) { + case K_MATCH: + return true; + case K_CHAR: + list.push(s); + break; + case K_SPLIT: + stack.push(this.out1[s], this.out[s]); + break; + default: + if (holdsIn(this.arg[s] as number, contextAt(name, p))) stack.push(this.out[s]); + } + } + return false; + } +} + +const CTX_BOL = 1; +const CTX_EOL = 2; +const CTX_WORD_BEFORE = 4; +const CTX_WORD_AFTER = 8; + +/** What the assertions can observe at position `p`: the two ends and word-ness either side. */ +function contextAt(name: string, p: number): number { + let context = 0; + if (p === 0) context |= CTX_BOL; + else if (isWordUnit(name.charCodeAt(p - 1))) context |= CTX_WORD_BEFORE; + if (p === name.length) context |= CTX_EOL; + else if (isWordUnit(name.charCodeAt(p))) context |= CTX_WORD_AFTER; + return context; +} + +function holdsIn(assertion: number, context: number): boolean { + switch (assertion) { + case A_BOL: + return (context & CTX_BOL) !== 0; + case A_EOL: + return (context & CTX_EOL) !== 0; + default: { + const before = (context & CTX_WORD_BEFORE) !== 0; + const after = (context & CTX_WORD_AFTER) !== 0; + return assertion === A_WORDB ? before !== after : before === after; + } + } +} + +/** + * Whether `c` is in `set`. Every CharSet's ranges are sorted, merged and disjoint (a + * class goes through {@link normalise}; the escape classes are constants or their + * {@link complement}), so a binary search finds the one range that could hold `c`. A + * linear scan here would make each state visit cost a whole class, work the budget + * does not count. + */ +function inSet(set: CharSet, c: number): boolean { + const ranges = set.ranges; + let lo = 0; + let hi = ranges.length - 1; + let hit = false; + while (lo <= hi) { + const mid = (lo + hi) >>> 1; + const r = ranges[mid]; + if (c < r[0]) hi = mid - 1; + else if (c > r[1]) lo = mid + 1; + else { + hit = true; + break; + } + } + return hit !== set.negated; +} + +/** The outcome of {@link compileLinearRegexDetailed}: a matcher, or why there is none. */ +export type LinearRegexResult = + | { readonly regex: LinearRegex } + | { readonly regex: null; readonly reason: string }; + +/** + * Compile a JavaScript-syntax pattern to a linear-time matcher. When the pattern is + * invalid, unsupported or too large the result carries the reason instead. Never throws. + * A `shared` budget makes the matcher draw on it as well as on its own cap, and the + * pattern's NFA states count against the budget's allowance of states. + */ +export function compileLinearRegexDetailed( + source: string, + shared?: SharedWorkBudget, +): LinearRegexResult { + // With nothing left of the state allowance, or of the work budget (which refused patterns + // draw on too, see below), refuse without even parsing the pattern. A matcher compiled from + // a spent budget would answer false for every name anyway. + const room = shared?.statesLeft ?? MAX_STATES; + if (room <= 0) return { regex: null, reason: STATE_ALLOWANCE_REASON }; + if (shared?.isSpent === true) return { regex: null, reason: WORK_BUDGET_REASON }; + if (source.length > MAX_PATTERN_LENGTH) { + return { regex: null, reason: `pattern is longer than ${MAX_PATTERN_LENGTH} characters` }; + } + let nfa: Nfa | undefined; + try { + const ast = new Parser(source).parse(); + // The build stops at the allowance, so a pattern that does not fit never has its NFA + // allocated in full. + nfa = room < MAX_STATES ? new Nfa(room, STATE_ALLOWANCE_REASON) : new Nfa(); + const match = nfa.add(K_MATCH, -1, -1, null); + const start = nfa.build(ast, match); + shared?.reserveStates(nfa.kind.length); + return { regex: new LinearRegex(source, nfa, start, shared) }; + } catch (err) { + if (err instanceof Reject && err.message === STATE_ALLOWANCE_REASON) shared?.closeStates(room); + // A refused pattern may already have built up to the whole state cap (or the build step + // cap) before it was refused, and a NAMESPACE can carry any number of them. Charge what it + // built, so that hostile refusals use up the work budget as hostile matches do, and the + // spent budget then refuses later patterns before they are parsed. (`Reject` is the + // expected path; a stack overflow or any other failure also means "cannot match safely", + // which for an exportPattern is the same as never matching.) + if (nfa !== undefined) shared?.charge(nfa.buildWork * REFUSED_BUILD_WORK_PER_UNIT); + return { + regex: null, + reason: err instanceof Reject ? err.message : 'pattern could not be compiled', + }; + } +} + +/** {@link compileLinearRegexDetailed} without the reason: null when there is no matcher. */ +export function compileLinearRegex(source: string, shared?: SharedWorkBudget): LinearRegex | null { + return compileLinearRegexDetailed(source, shared).regex; +} diff --git a/gitnexus/src/core/ingestion/languages/r/namespace-imports.ts b/gitnexus/src/core/ingestion/languages/r/namespace-imports.ts new file mode 100644 index 000000000..9db91bb1f --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/namespace-imports.ts @@ -0,0 +1,1440 @@ +/** + * NAMESPACE `importFrom()` parsing for R packages. + * + * `parseRNamespaceImportFrom` is a pure, package-blind tokenizer: it reads the + * text of one NAMESPACE file and returns every `importFrom(pkg, name, ...)` + * pair in file order. It deliberately returns *all* entries — including a + * package importing from itself and duplicate names from different packages — + * because those policy decisions (own-package drop, last-wins, external-package + * skip) belong to the consumer that knows the workspace, not to the parser. + * + * The tokenizer ({@link scanRNamespaceDirectives}) is generic (directive head + + * balanced argument list across newlines, honouring `"…"`, `'…'` and backtick + * quoting, `#` comments stripped outside quotes). Two consumers sit on it: + * {@link parseRNamespaceImportFrom} (`importFrom` only; `import`, + * `importClassesFrom` and `importMethodsFrom` are deferred) and + * {@link parseRNamespaceExports} (`export`, `exportClasses`, `exportMethods`, + * `S3method`, `exportPattern`). `useDynLib` and anything under an + * `if (…)` / `else` conditional are ignored. Unbalanced or garbled input never + * throws: a malformed directive is discarded and scanning RESYNCHRONISES at the + * next line that begins, at column 0, with `identifier(` (see + * {@link resyncAfterMalformed}); everything parsed before and after is kept. + * Quoted strings never span lines, and a column-0 `identifier(` line ends any + * argument list still open, so a broken directive cannot swallow those after it. + * + * Runtime import direction is `language-config.ts` → this file; any type from + * `language-config.ts` must be taken with `import type` only, so no runtime + * import cycle forms. + * + * The second half of the file binds those entries to workspace definitions: + * {@link populateRNamespaceImports} synthesises `named` imports for + * `importFrom()` names that a *local* package provides, and + * {@link resolveRImportTarget}'s `named` branch (`languages/r/import-target.ts`) + * resolves them using {@link rFileTopLevel} / {@link rPackageDirForFile}. + * + * The file also hosts the pieces of `pkg::name()` qualifier handling that the + * global-name-fallback veto ({@link isRGlobalNameFallbackPlausible}) shares + * with `qualified-call.ts`: {@link parseRQualifier} and the per-package + * definition-count registry. The qualified-call decisions themselves live in + * `qualified-call.ts`. + */ + +import type { ParsedFile, ParsedImport } from 'gitnexus-shared'; +import type { RNamespaceInfo, RPackageConfig } from './package-config.js'; +import { unescapeRString } from './export-pattern.js'; +import { rBareName } from './bare-name.js'; +import { resolveRNamedImport } from './import-target.js'; + +/** One `importFrom(pkg, name)` pair, in NAMESPACE file order. */ +export interface RNamespaceImportFromEntry { + readonly pkg: string; + readonly name: string; +} + +const IMPORT_FROM_DIRECTIVES: ReadonlySet = new Set(['importFrom']); + +/** Directives that import from a package: whole (`import`), by name, or S4 classes and methods. */ +const IMPORT_DIRECTIVES: ReadonlySet = new Set([ + 'import', + 'importFrom', + 'importClassesFrom', + 'importMethodsFrom', +]); + +/** An import-family directive head followed by `(`, anywhere in a piece of NAMESPACE text. */ +const IMPORT_HEAD_IN_TEXT = new RegExp( + `(? = new Set([ + 'export', + 'exportClasses', + 'exportMethods', +]); + +const EXPORT_DIRECTIVES: ReadonlySet = new Set([ + ...EXPORT_NAME_DIRECTIVES, + 'S3method', + 'exportPattern', +]); + +/** An export-family directive head followed by `(`, anywhere in a piece of NAMESPACE text. */ +const EXPORT_HEAD_IN_TEXT = new RegExp( + `(? = new Set(['if', 'else']); + +const isIdentStart = (ch: string): boolean => /[A-Za-z_.]/.test(ch); +const isIdentPart = (ch: string): boolean => /[A-Za-z0-9_.]/.test(ch); +const isQuote = (ch: string): boolean => ch === '"' || ch === "'" || ch === '`'; + +/** + * Advance past a quoted region starting at `start` (which holds the opening + * quote). Returns the index just after the closing quote, or `-1` when the + * quote does not close on the same line. Backslash escapes the next character + * (R string rules; harmless inside backticks). Quotes never span lines: a + * NAMESPACE directive argument is a one-line token, and letting a stray quote + * run on would pair with a later one and swallow the directives between them. + */ +function skipQuoted(text: string, start: number): number { + const quote = text[start]; + for (let i = start + 1; i < text.length; i++) { + const ch = text[i]; + if (ch === '\n') return -1; + if (ch === '\\') { + i++; + if (text[i] === '\n') return -1; + continue; + } + if (ch === quote) return i + 1; + } + return -1; +} + +function skipLineComment(text: string, start: number): number { + const nl = text.indexOf('\n', start); + return nl === -1 ? text.length : nl + 1; +} + +/** A directive-looking line start: `identifier(` (no leading whitespace, no space before `(`). */ +const DIRECTIVE_LINE_START = /[A-Za-z_.][A-Za-z0-9_.]*\(/y; + +/** True when `i` is the start of a line (column 0) that begins with `identifier(`. */ +function isDirectiveLineStart(text: string, i: number): boolean { + if (i > 0 && text[i - 1] !== '\n') return false; + DIRECTIVE_LINE_START.lastIndex = i; + return DIRECTIVE_LINE_START.test(text); +} + +/** + * Resynchronisation point after a malformed directive that began at `from`: the + * start of the first line AFTER the line containing `from` that begins, at + * column 0, with `identifier(`; `text.length` when there is none. The result + * is always greater than `from` (or the end of text), so callers make forward + * progress and each failure costs at most one pass over the text it skips. + */ +function resyncAfterMalformed(text: string, from: number): number { + let nl = text.indexOf('\n', from); + while (nl !== -1) { + const lineStart = nl + 1; + if (isDirectiveLineStart(text, lineStart)) return lineStart; + nl = text.indexOf('\n', lineStart); + } + return text.length; +} + +/** + * Parse a balanced `( … )` group whose opening paren is at `openIdx`. Returns + * the comma-separated top-level arguments (raw, comment-stripped, unquoted only + * at the outer layer) and the index just after the closing paren, or `null` + * when the group is unbalanced, holds an unterminated quote, or is still open + * when a line starting `identifier(` at column 0 begins (which bounds the scan, + * so an unbalanced group cannot run to the end of the file). + */ +function parseArgs(text: string, openIdx: number): { args: string[]; end: number } | null { + const args: string[] = []; + let current = ''; + let depth = 0; + for (let i = openIdx; i < text.length; ) { + if (i > openIdx && isDirectiveLineStart(text, i)) return null; + const ch = text[i]; + if (ch === '#') { + i = skipLineComment(text, i); + continue; + } + if (isQuote(ch)) { + const end = skipQuoted(text, i); + if (end === -1) return null; + current += text.slice(i, end); + i = end; + continue; + } + if (ch === '(') { + depth++; + if (depth > 1) current += ch; + i++; + continue; + } + if (ch === ')') { + depth--; + if (depth === 0) { + args.push(current); + return { args, end: i + 1 }; + } + current += ch; + i++; + continue; + } + if (ch === ',' && depth === 1) { + args.push(current); + current = ''; + i++; + continue; + } + current += ch; + i++; + } + return null; +} + +/** A bare directive argument R reads as a symbol: a letter or `.`, then letters, digits, `.` and `_`. */ +const BARE_NAME = /^[\p{L}.][\p{L}\p{N}._]*$/u; + +/** + * Strip the whitespace around a raw argument and one layer of surrounding quotes/backticks; the + * text inside the quotes is kept as written, spaces included. Returns `''` for empty or + * non-simple arguments (unterminated quote, or anything that is not a single quoted token or bare + * symbol) so callers can skip them. A bare operator or indexing expression (`a+b`, `x[1]`) is not a symbol. + */ +function normaliseArg(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) return ''; + const first = trimmed[0]; + if (isQuote(first)) { + if (trimmed.length < 2 || trimmed[trimmed.length - 1] !== first) return ''; + const end = skipQuoted(trimmed, 0); + if (end !== trimmed.length) return ''; + const inner = trimmed.slice(1, -1); + // A string is R-unescaped; a backticked name keeps an escaped backtick as written (see `rBareName`). + // Not trimmed: R reads the quoted text as it stands, so `" a "` names the symbol ` a `. + return first === '`' ? inner : unescapeRString(inner); + } + // Bare token: a syntactic name. Anything else (`x = y`, a nested call, an expression) is rejected. + return BARE_NAME.test(trimmed) ? trimmed : ''; +} + +/** + * `text` without its trailing colons: what `text.replace(/:+$/, '')` gives, in linear time. The + * regex retries from every colon of a long run, which is quadratic in its length, and a + * NAMESPACE is repository input. + */ +export function trimTrailingColons(text: string): string { + let end = text.length; + while (end > 0 && text.charCodeAt(end - 1) === 58) end--; + return text.slice(0, end); +} + +/** + * The generic named by the first argument of `S3method()`. R >= 3.6 also accepts a generic of + * another package, `S3method(pkg::generic, class)` (roxygen2 writes it for + * `@exportS3Method pkg::generic`), whose method is still `generic.class`: the text after the + * last `::` or `:::` outside quotes and backticks is the generic. `''` when the argument is not + * a plain name, a quoted name, or a `pkg::name` pair of those. + */ +function s3GenericArg(raw: string): string { + const trimmed = raw.trim(); + let operatorEnd = -1; + for (let i = 0; i < trimmed.length; i++) { + const ch = trimmed[i]; + if (isQuote(ch)) { + const end = skipQuoted(trimmed, i); + if (end === -1) return ''; + i = end - 1; + } else if (ch === ':' && trimmed[i + 1] === ':') { + operatorEnd = trimmed[i + 2] === ':' ? i + 3 : i + 2; + i = operatorEnd - 1; + } + } + if (operatorEnd === -1) { + // A quoted generic spelled with its package: `"pkg::generic"`. + const name = normaliseArg(trimmed); + const colons = name.lastIndexOf('::'); + return colons === -1 ? name : name.slice(colons + 2).replace(/^:/, ''); + } + // Both sides must be names; the package side is only checked, never used. + const before = trimTrailingColons(trimmed.slice(0, operatorEnd)); + return normaliseArg(before) ? normaliseArg(trimmed.slice(operatorEnd)) : ''; +} + +/** + * When `text[i]` starts an `if (cond)`, the index just after its condition group; `-1` + * when the condition is malformed; `null` when no `if` starts here. + */ +function nestedIfCondition(text: string, i: number): number | null { + if (i >= text.length || !isIdentStart(text[i])) return null; + let j = i + 1; + while (j < text.length && isIdentPart(text[j])) j++; + if (text.slice(i, j) !== 'if') return null; + while (j < text.length && /\s/.test(text[j])) j++; + if (text[j] !== '(') return null; + const group = parseArgs(text, j); + return group ? group.end : -1; +} + +/** + * Where each `{ … }` block of NAMESPACE text ends, answered in constant time after one linear + * pass over the text. Scanning a block from its own `{` instead costs a pass over the rest of + * the file whenever the block never closes, and a file of `if (a) {` lines with no `}` made + * that quadratic (a 300 KB input took 8.8 s). + * + * The pass tokenises like the block scan it replaces: `#` comments run to the end of the line, + * a quote that does not close on its line aborts every block still open at that point, and a + * `}` closes the innermost open one. A comment or a quote never spans lines, so every line + * start is a clean token boundary and the tokens from any `{` the directive scanner reaches + * are the same as the tokens of the pass from the start of the text. + */ +class BraceBlocks { + private ends: Map | undefined; + + constructor(private readonly text: string) {} + + /** The index just after the `}` closing the block opened at `open`; `-1` when it never closes. */ + endOf(open: number): number { + this.ends ??= this.index(); + return this.ends.get(open) ?? -1; + } + + private index(): Map { + const { text } = this; + const ends = new Map(); + const open: number[] = []; + for (let i = 0; i < text.length; ) { + const ch = text[i]; + if (ch === '#') { + i = skipLineComment(text, i); + } else if (isQuote(ch)) { + const end = skipQuoted(text, i); + if (end !== -1) { + i = end; + continue; + } + // An unterminated quote aborts every block open here; resume on the next line. + open.length = 0; + const newline = text.indexOf('\n', i); + i = newline === -1 ? text.length : newline + 1; + } else { + if (ch === '{') open.push(i); + else if (ch === '}') { + const start = open.pop(); + if (start !== undefined) ends.set(start, i + 1); + } + i++; + } + } + return ends; + } +} + +/** + * Skip one statement following an `if (…)` / `else`: either a `{ … }` block or a + * single directive call. Returns the index to resume scanning from. + */ +function skipConditionalBody(text: string, from: number, blocks: BraceBlocks): number { + let i = from; + for (;;) { + while (i < text.length && /\s/.test(text[i])) i++; + // `else if (cond) body` and `if (a) if (b) body`: the nested `if` owns the body that + // follows its condition, so step over `if (cond)` and skip that body instead + // (iteratively: an `if` chain can be arbitrarily long). + const nested = nestedIfCondition(text, i); + if (nested === null) break; + if (nested === -1) return resyncAfterMalformed(text, i); + i = nested; + } + if (i >= text.length) return i; + if (text[i] === '{') { + const end = blocks.endOf(i); + return end === -1 ? resyncAfterMalformed(text, i) : end; // -1: unterminated `{ … }` + } + if (isIdentStart(text[i])) { + let j = i + 1; + while (j < text.length && isIdentPart(text[j])) j++; + const head = text.slice(i, j); + if (CONDITIONAL_HEADS.has(head)) return j; // a bare `else` here has no body to skip + while (j < text.length && /\s/.test(text[j])) j++; + if (text[j] === '(') { + const group = parseArgs(text, j); + return group ? group.end : resyncAfterMalformed(text, i); + } + return j; + } + return i; +} + +/** One top-level NAMESPACE directive: its head and its raw (comment-stripped) arguments. */ +interface RawDirective { + readonly head: string; + readonly args: readonly string[]; +} + +/** + * Tokenise NAMESPACE text into the top-level directives whose head is in + * `heads`, in file order. Conditional (`if` / `else`) bodies are skipped, and + * directives nested inside another directive's arguments are never emitted. + * Never throws. A malformed directive (unbalanced group, unterminated quote, + * unterminated `{` block) is discarded and scanning resumes at the next line + * starting `identifier(` at column 0 ({@link resyncAfterMalformed}); everything + * before and after is kept. Directives indented after a malformed one are + * dropped with it. + */ +function scanRNamespaceDirectives( + text: string, + heads: ReadonlySet, + hiddenHead: RegExp = EXPORT_HEAD_IN_TEXT, +): { directives: RawDirective[]; hidesExport: boolean } { + const directives: RawDirective[] = []; + const blocks = new BraceBlocks(text); + // Set when the scan skipped or discarded text that names a directive `hiddenHead` matches (by + // default the export family). The regions are disjoint (every skip moves `i` forward), so the + // tests cost one pass in total. + let hidesExport = false; + const skipped = (from: number, to: number): void => { + if (!hidesExport && hiddenHead.test(text.slice(from, to))) hidesExport = true; + }; + let i = 0; + while (i < text.length) { + const ch = text[i]; + if (ch === '#') { + i = skipLineComment(text, i); + continue; + } + if (isQuote(ch)) { + const end = skipQuoted(text, i); + if (end === -1) { + const resume = resyncAfterMalformed(text, i); // stray unterminated quote + skipped(i, resume); + i = resume; + continue; + } + i = end; + continue; + } + if (!isIdentStart(ch)) { + i++; + continue; + } + + let j = i + 1; + while (j < text.length && isIdentPart(text[j])) j++; + const head = text.slice(i, j); + let k = j; + while (k < text.length && /\s/.test(text[k])) k++; + + if (text[k] !== '(') { + // `else` may be followed directly by a directive with no parens of its own. + if (head === 'else') { + const end = skipConditionalBody(text, k, blocks); + skipped(k, end); + i = end; + continue; + } + i = j; + continue; + } + + const group = parseArgs(text, k); + if (!group) { + const resume = resyncAfterMalformed(text, i); // discard this directive, resume at the next one + skipped(i, resume); + i = resume; + continue; + } + i = group.end; + + if (CONDITIONAL_HEADS.has(head)) { + const end = skipConditionalBody(text, i, blocks); + skipped(i, end); + i = end; + continue; + } + if (heads.has(head)) directives.push({ head, args: group.args }); + } + return { directives, hidesExport }; +} + +/** + * Extract every `importFrom(pkg, name, ...)` pair from NAMESPACE text, in file + * order. Never throws; returns `[]` for empty, garbled or unbalanced input. + */ +export function parseRNamespaceImportFrom(text: string): RNamespaceImportFromEntry[] { + const entries: RNamespaceImportFromEntry[] = []; + for (const { args } of scanRNamespaceDirectives(text, IMPORT_FROM_DIRECTIVES).directives) { + const values = args.map(normaliseArg); + const pkg = values[0]; + if (!pkg) continue; + for (let n = 1; n < values.length; n++) { + if (values[n]) entries.push({ pkg, name: values[n] }); + } + } + return entries; +} + +/** The packages a NAMESPACE file imports from, as read by {@link parseRNamespaceImportPackages}. */ +export interface RNamespaceImportPackages { + /** Packages of `import()`, `importClassesFrom()` and `importMethodsFrom()`, in file order. */ + readonly packages: readonly string[]; + /** + * True when the reader skipped or could not read something that may hide an import: a + * conditional (`if` / `else`) body or a malformed directive that names an import-family + * directive, or an argument of an `import()` or `importFrom()` it cannot read. + */ + readonly unreadable: boolean; +} + +/** + * Read the packages a NAMESPACE imports from without naming each object: `import(a, b)` (a named + * argument such as `except = c(...)` is skipped, the packages are still imported) and the package + * of `importClassesFrom()` / `importMethodsFrom()`. `importFrom()` entries are read by + * {@link parseRNamespaceImportFrom}; here they only count toward {@link + * RNamespaceImportPackages.unreadable}. Never throws. + */ +export function parseRNamespaceImportPackages(text: string): RNamespaceImportPackages { + const packages: string[] = []; + const scan = scanRNamespaceDirectives(text, IMPORT_DIRECTIVES, IMPORT_HEAD_IN_TEXT); + let unreadable = scan.hidesExport; + for (const { head, args } of scan.directives) { + if (head === 'import') { + for (const raw of args) { + if (/^\s*[A-Za-z_.][A-Za-z0-9_.]*\s*=/.test(raw)) continue; // except = ... + const pkg = normaliseArg(raw); + if (pkg) packages.push(pkg); + else if (raw.trim()) unreadable = true; + } + } else if (head === 'importFrom') { + for (const raw of args) if (!normaliseArg(raw) && raw.trim()) unreadable = true; + } else { + const pkg = normaliseArg(args[0] ?? ''); + if (pkg) packages.push(pkg); + else unreadable = true; + } + } + return { packages, unreadable }; +} + +/** What a NAMESPACE file exports, as read by {@link parseRNamespaceExports}. */ +export interface RNamespaceExports { + /** Names from `export()`, `exportClasses()`, `exportMethods()` and `S3method()` (the third argument, else `generic.class`), in file order. */ + readonly namedExports: readonly string[]; + /** `exportPattern()` arguments, R-unescaped (an R string `"\\."` yields the regex source `\.`), in file order. */ + readonly exportPatterns: readonly string[]; + /** + * True when the reader skipped, dropped or could not read something that may hide an export: + * a conditional (`if` / `else`) body or a malformed directive that names an export-family + * directive, or an argument of one it cannot read (an R 4.0 raw string, a string that spans + * lines, a name given as an expression, an `S3method()` without a readable generic and class). + * The lists above are then a lower bound, so the absence of a name from them does not show + * that it is private. + */ + readonly unreadable: boolean; + /** + * The `S3method()` names (as in {@link namedExports}) that no `export()`, `exportClasses()` or + * `exportMethods()` entry also lists, in file order. `S3method()` registers a method for S3 + * dispatch; it does not export the function (`pkg::name` errors and `library(pkg)` does not + * attach it), so `loadRPackageConfig` takes these out of the exports resolution sees. Optional + * so that hand-built values read as "no registration-only name". + */ + readonly s3RegisteredOnly?: readonly string[]; + /** Every `S3method()` name (as in {@link namedExports}), in file order. Optional, as above. */ + readonly s3Registered?: readonly string[]; +} + +/** + * The single-token string argument of `exportPattern()`, R-unescaped, or `''` + * when the argument is empty, not a quoted string, or unterminated. Only `"` + * and `'` quote a string in R (a backticked name is a symbol, not a pattern). + */ +function exportPatternArg(raw: string): string { + const trimmed = raw.trim(); + const quote = trimmed[0]; + if (quote !== '"' && quote !== "'") return ''; + if (skipQuoted(trimmed, 0) !== trimmed.length) return ''; + return unescapeRString(trimmed.slice(1, -1)); +} + +/** + * Extract the export directives from NAMESPACE text: `export`, `exportClasses` + * and `exportMethods` names, `S3method(generic, class[, method])` as `method` + * (else `generic.class`; `pkg::generic` names the generic after the `::`), and `exportPattern("…")` sources. Multi-line, quoted, + * backticked, commented and CRLF input is handled like {@link + * parseRNamespaceImportFrom}. Non-simple arguments (named, nested calls) are + * skipped, and so are conditional bodies and malformed directives; each of those that may hide + * an export sets {@link RNamespaceExports.unreadable}. Never throws; returns empty lists for + * empty or garbled input. + */ +export function parseRNamespaceExports(text: string): RNamespaceExports { + const namedExports: string[] = []; + const s3Registered: string[] = []; + const exportedByName = new Set(); + const exportPatterns: string[] = []; + const scan = scanRNamespaceDirectives(text, EXPORT_DIRECTIVES); + let unreadable = scan.hidesExport; + for (const { head, args } of scan.directives) { + if (head === 'exportPattern') { + for (const raw of args) { + const pattern = exportPatternArg(raw); + if (pattern) exportPatterns.push(pattern); + else if (raw.trim()) unreadable = true; + } + } else if (head === 'S3method') { + const generic = s3GenericArg(args[0] ?? ''); + const cls = normaliseArg(args[1] ?? ''); + if (generic && cls) { + // The optional third argument names the function registered as the method; + // without it the method is `generic.class`. + const method = normaliseArg(args[2] ?? ''); + namedExports.push(method || `${generic}.${cls}`); + s3Registered.push(method || `${generic}.${cls}`); + // A third argument that is not a plain name (`method = impl`) registers a function + // this reader does not name. + if (!method && (args[2] ?? '').trim()) unreadable = true; + } else { + unreadable = true; + } + } else { + for (const raw of args) { + const name = normaliseArg(raw); + if (name) { + namedExports.push(name); + exportedByName.add(name); + } else if (raw.trim()) unreadable = true; + } + } + } + const s3RegisteredOnly = s3Registered.filter((name) => !exportedByName.has(name)); + return { namedExports, exportPatterns, unreadable, s3RegisteredOnly, s3Registered }; +} + +// ─── Package lookup ───────────────────────────────────────────────────────── + +/** A local R package located by {@link rPackageDirForFile}. */ +export interface RPackageLocation { + /** `Package:` name from `DESCRIPTION`. */ + readonly name: string; + /** Package directory relative to the repo root; `''` for a root-level package. */ + readonly dir: string; +} + +/** `R/` directory prefix of a package dir (`'R/'` for a root-level package). */ +export function rRDirPrefix(pkgDir: string): string { + return pkgDir === '' ? 'R/' : `${pkgDir}/R/`; +} + +/** + * The local package whose `/R/**` contains `filePath`, or `undefined`. + * + * NAMESPACE effects apply only to a package's `R/` sources (R loads nothing + * else into the namespace), so files under `tests/`, `scripts/`, `inst/`, a + * root-level `plumber.R`, … belong to no package here. When package dirs nest, + * the longest matching dir wins. A root-level package (`dir === ''`) is + * handled without ever building a `'' + '/'` prefix. + */ +export function rPackageDirForFile( + filePath: string, + cfg: RPackageConfig | null | undefined, +): RPackageLocation | undefined { + if (cfg === null || cfg === undefined) return undefined; + const normalized = filePath.replace(/\\/g, '/'); + let best: RPackageLocation | undefined; + for (const [name, dir] of cfg.packages) { + if (!normalized.startsWith(rRDirPrefix(dir))) continue; + if (best === undefined || dir.length > best.dir.length) best = { name, dir }; + } + return best; +} + +/** + * The directory of the package whose `R/` sources contain `filePath`, or `undefined`. + * + * Unlike {@link rPackageDirForFile} this considers every directory that has a NAMESPACE as + * well as every registered package, so the nearest enclosing package owns a file and a nested + * package without a NAMESPACE never inherits the enclosing package's. The longest matching + * directory wins; the root package (`''`) is matched without building a `'' + '/'` prefix. + */ +export function rOwningPackageDir( + filePath: string, + cfg: RPackageConfig | null | undefined, +): string | undefined { + if (cfg === null || cfg === undefined) return undefined; + const normalized = filePath.replace(/\\/g, '/'); + let best: string | undefined; + const consider = (dir: string): void => { + if (!normalized.startsWith(rRDirPrefix(dir))) return; + if (best === undefined || dir.length > best.length) best = dir; + }; + for (const dir of cfg.packages.values()) consider(dir); + for (const dir of cfg.namespaceInfoByPackageDir.keys()) consider(dir); + return best; +} + +/** + * The directory of the package whose TREE contains `filePath`, or `undefined`: the same + * candidate directories as {@link rOwningPackageDir} (registered packages and NAMESPACE + * directories), but a file counts as inside a directory when it sits anywhere under it (prefix + * `dir + '/'`, every path when `dir` is `''`), so `tests/`, `inst/` and `vignettes/` belong to + * the package too. The longest matching directory wins, so a nested package claims its own files + * and the enclosing package the rest. For a root-level package every file outside a nested + * package is inside, so what the caller-inside-the-package exemption covers is broad there. + * + * A caller is "inside package P" iff this equals P's directory. {@link rOwningPackageDir}, which + * decides whose NAMESPACE governs a DEFINITION (only `R/` files), is unchanged. + */ +export function rEnclosingPackageDir( + filePath: string, + cfg: RPackageConfig | null | undefined, +): string | undefined { + if (cfg === null || cfg === undefined) return undefined; + const normalized = filePath.replace(/\\/g, '/'); + let best: string | undefined; + const consider = (dir: string): void => { + if (dir !== '' && !normalized.startsWith(`${dir}/`)) return; + if (best === undefined || dir.length > best.length) best = dir; + }; + for (const dir of cfg.packages.values()) consider(dir); + for (const dir of cfg.namespaceInfoByPackageDir.keys()) consider(dir); + return best; +} + +/** + * True when `filePath` sits under the `R/` directory of any package discovery read a + * `DESCRIPTION` for: a registered package, a directory with a NAMESPACE, or a same-name copy + * recorded only in {@link RPackageConfig.packageDirs}. Such a file belongs to the package its + * `DESCRIPTION` names, whatever its directory is called. + */ +export function rFileBelongsToDiscoveredPackage( + filePath: string, + cfg: RPackageConfig | null | undefined, +): boolean { + if (cfg === null || cfg === undefined) return false; + if (rOwningPackageDir(filePath, cfg) !== undefined) return true; + if (cfg.packageDirs === undefined) return false; + const normalized = filePath.replace(/\\/g, '/'); + for (const dir of cfg.packageDirs) if (normalized.startsWith(rRDirPrefix(dir))) return true; + return false; +} + +// ─── Top-level definitions of a parsed file ──────────────────────────────── + +/** Def kinds an R `importFrom()` (or a `pkg::name()` call) can bind to (functions and R6/R5/S4 classes). */ +export const TOP_LEVEL_BINDABLE_TYPES: ReadonlySet = new Set(['Function', 'Class']); + +/** What one R file defines at top level, read from its Module scope. */ +export interface RFileTopLevel { + /** + * Symbols of the top-level function/class definitions: the name without the backticks it may + * be written in (`` `%||%` `` is `%||%`), which is how NAMESPACE entries and `importFrom()` spell it. + */ + readonly names: ReadonlySet; + /** + * For each dotted definition, the text after the last `.` of its name as bound (backticks + * kept): the key the shared finalize pass indexes the definition under. + */ + readonly dottedTails: ReadonlySet; + /** + * For each symbol in {@link names} written with backticks somewhere, every spelling bound in + * this file (the plain one too, when it is also bound). Empty for a file with no such name. + */ + readonly writtenAs: ReadonlyMap; +} + +const EMPTY_TOP_LEVEL: RFileTopLevel = { + names: new Set(), + dottedTails: new Set(), + writtenAs: new Map(), +}; +const topLevelCache = new WeakMap(); + +/** + * Top-level definitions of `parsed`, read from the Module scope's hoisted + * bindings. + * + * Since R scope queries anchor a `@scope.function` on every named function, a top-level function def is + * owned by its *own* Function scope and only its binding is hoisted into the + * Module scope, so neither the Module scope's `ownedDefs` nor the flat + * `localDefs` (which also lists nested functions and R6/R5 members) answers + * "is `name` defined at top level?". A `local`-origin Module binding to a + * function/class def does. Bindings are read at extraction time, before + * `finalizeScopeModel` adds import/wildcard bindings. + * + * A binding key keeps the backticks the definition was written with; {@link RFileTopLevel.names} + * is keyed by the symbol (see `rBareName`) so it can be compared with a NAMESPACE entry, and + * {@link RFileTopLevel.writtenAs} keeps the spelling a call site and the file's exports use. + */ +export function rFileTopLevel(parsed: ParsedFile): RFileTopLevel { + const cached = topLevelCache.get(parsed); + if (cached !== undefined) return cached; + const moduleScope = parsed.scopes.find((s) => s.id === parsed.moduleScope); + let result = EMPTY_TOP_LEVEL; + if (moduleScope !== undefined) { + const names = new Set(); + const dottedTails = new Set(); + const writtenAs = new Map(); + const bindsTopLevel = (refs: readonly { origin: string; def: { type: string } }[]): boolean => + refs.some((r) => r.origin === 'local' && TOP_LEVEL_BINDABLE_TYPES.has(r.def.type)); + for (const [name, refs] of moduleScope.bindings) { + if (!bindsTopLevel(refs)) continue; + const symbol = rBareName(name); + names.add(symbol); + if (symbol !== name) { + const spellings = writtenAs.get(symbol); + if (spellings === undefined) writtenAs.set(symbol, [name]); + else spellings.push(name); + } + const dot = name.lastIndexOf('.'); + if (dot >= 0) dottedTails.add(name.slice(dot + 1)); + } + for (const [symbol, spellings] of writtenAs) { + const plain = moduleScope.bindings.get(symbol); + if (plain !== undefined && bindsTopLevel(plain)) spellings.unshift(symbol); + } + result = { names, dottedTails, writtenAs }; + } + topLevelCache.set(parsed, result); + return result; +} + +interface RDefIds { + /** Ids of the function and class definitions the file binds at top level. */ + readonly topLevel: ReadonlySet; + /** Ids of every definition the file's scope model holds, nested or not. */ + readonly all: ReadonlySet; +} + +const defIdsCache = new WeakMap(); + +/** + * The function and class definitions of `parsed` that are bound at top level: the `local` + * bindings of its Module scope. A function written inside another function is owned by that + * function's scope (and `localDefs` lists it all the same), so its id is in `all` only. + */ +function rDefIds(parsed: ParsedFile): RDefIds { + const cached = defIdsCache.get(parsed); + if (cached !== undefined) return cached; + const topLevel = new Set(); + const moduleScope = parsed.scopes.find((s) => s.id === parsed.moduleScope); + if (moduleScope !== undefined) { + for (const refs of moduleScope.bindings.values()) { + for (const ref of refs) { + if (ref.origin === 'local' && TOP_LEVEL_BINDABLE_TYPES.has(ref.def.type)) { + topLevel.add(ref.def.nodeId); + } + } + } + } + const result = { topLevel, all: new Set(parsed.localDefs.map((d) => d.nodeId)) }; + defIdsCache.set(parsed, result); + return result; +} + +// ─── Import synthesis (populateWorkspaceReferences) ───────────────────────── + +/** + * Per local package, the set of symbols it defines at top level in its `R/` + * files (a backtick-quoted definition under its bare symbol), recorded by {@link populateRNamespaceImports} for the packages that + * take part in an `importFrom()` (as caller or as provider). Keyed by the + * `resolutionConfig` object, which reaches both this hook and the + * free-call-fallback veto. A missing entry means "cannot decide". + */ +const packageTopLevelNames = new WeakMap>>(); + +/** Recorded top-level names per package for `resolutionConfig`, if the hook ran with imports. */ +export function rRecordedPackageTopLevelNames( + resolutionConfig: unknown, +): ReadonlyMap> | undefined { + if (typeof resolutionConfig !== 'object' || resolutionConfig === null) return undefined; + return packageTopLevelNames.get(resolutionConfig); +} + +interface CallerPlan { + readonly dir: string; + /** Surviving `name → provider package` after last-wins / self-drop / external-skip. */ + readonly bindable: ReadonlyMap; +} + +/** + * Synthesise `named` imports for NAMESPACE `importFrom(pkg, name)` entries + * whose `pkg` is a local package, into every `R/` file of the caller package. + * + * NOTE: `populateWorkspaceReferences` is documented as reference-site + * enrichment; it is reused here (same call site and ctx as the Zig static + * gating precedent) because it runs before `finalizeScopeModel`, receives the + * mutable `ParsedFile[]`, and also runs over warm-cache ParsedFiles, so the + * synthesised imports are never persisted stale. If this is ever upstreamed, + * propose widening that contract's doc string. + * + * Order of operations per caller package: + * 1. last-wins over ALL entries for a name, in file order, external + * packages included (R replaces the earlier `importFrom` binding); + * 2. drop the surviving entry when its `pkg` is the caller itself; + * 3. skip it when its `pkg` is not a local package (an earlier local entry + * never resurrects); + * 4. skip it when the caller's own package defines `name` at top level in + * any `R/` file (the own namespace masks its imports; `rMergeBindings` + * only shadows within a single scope); + * 5. append a `named` import to every `R/` file of the caller package, one per spelling + * the provider binds the symbol under (a definition written in backticks keeps them, so + * the call site and the provider file's own export both match the import). + */ +export function populateRNamespaceImports( + parsedFiles: ParsedFile[], + ctx: { readonly resolutionConfig?: unknown }, +): void { + const cfg = (ctx.resolutionConfig as RPackageConfig | null | undefined) ?? null; + if (cfg === null || cfg.packages.size === 0) return; + + // Steps 1-3: collapse each caller's entries; note which packages we need names for. + const plans = new Map(); + const needNamesFor = new Set(); + for (const [callerName, dir] of cfg.packages) { + const info = cfg.namespaceInfoByPackageDir.get(dir); + if (info === undefined || info.importFrom.length === 0) continue; + const last = new Map(); + for (const entry of info.importFrom) last.set(rBareName(entry.name), entry.pkg); + const bindable = new Map(); + for (const [name, pkg] of last) { + if (cfg.packages.has(pkg)) needNamesFor.add(pkg); // the fallback veto reads the provider's names too + if (pkg === callerName || !cfg.packages.has(pkg)) continue; + bindable.set(name, pkg); + } + if (bindable.size === 0) continue; + plans.set(callerName, { dir, bindable }); + needNamesFor.add(callerName); + } + if (plans.size === 0) return; + + // One pass over the parsed files: group caller files, collect top-level names. + const namesByPackage = new Map>(); + const spellingsByPackage = new Map>(); + const callerFiles = new Map(); + for (let i = 0; i < parsedFiles.length; i++) { + const location = rPackageDirForFile(parsedFiles[i].filePath, cfg); + if (location === undefined) continue; + if (needNamesFor.has(location.name)) { + let names = namesByPackage.get(location.name); + if (names === undefined) namesByPackage.set(location.name, (names = new Set())); + const topLevel = rFileTopLevel(parsedFiles[i]); + for (const name of topLevel.names) names.add(name); + for (const [symbol, written] of topLevel.writtenAs) { + let bySymbol = spellingsByPackage.get(location.name); + if (bySymbol === undefined) spellingsByPackage.set(location.name, (bySymbol = new Map())); + let known = bySymbol.get(symbol); + if (known === undefined) bySymbol.set(symbol, (known = [])); + for (const spelling of written) if (!known.includes(spelling)) known.push(spelling); + } + } + if (plans.has(location.name)) { + const indices = callerFiles.get(location.name); + if (indices === undefined) callerFiles.set(location.name, [i]); + else indices.push(i); + } + } + for (const pkg of needNamesFor) if (!namesByPackage.has(pkg)) namesByPackage.set(pkg, new Set()); + packageTopLevelNames.set(cfg, namesByPackage); + + // A symbol written with backticks in one file may be written plainly in another: complete the + // spellings before importing them. + if (spellingsByPackage.size > 0) { + for (const parsed of parsedFiles) { + const owner = rPackageDirForFile(parsed.filePath, cfg)?.name; + const bySymbol = owner === undefined ? undefined : spellingsByPackage.get(owner); + if (bySymbol === undefined) continue; + const topLevel = rFileTopLevel(parsed); + for (const [symbol, known] of bySymbol) { + if ( + topLevel.names.has(symbol) && + !topLevel.writtenAs.has(symbol) && + !known.includes(symbol) + ) { + known.unshift(symbol); + } + } + } + } + + // Steps 4-5. + for (const [callerName, plan] of plans) { + const ownNames = namesByPackage.get(callerName); + const synthesised: Extract[] = []; + for (const [name, pkg] of plan.bindable) { + if (ownNames?.has(name) === true) continue; + for (const spelling of spellingsByPackage.get(pkg)?.get(name) ?? [name]) { + // `null` means the name can never bind. A synthesised import for it would stay unresolved + // yet count as an explicit name claim, suppressing the documented `global-name-fallback`. + // (Importing the resolver forms a cycle with languages/r/import-target.ts; every use is deferred + // to call time, so it is safe at runtime.) + if (resolveRNamedImport(cfg, pkg, spelling, parsedFiles) === null) continue; + synthesised.push({ + kind: 'named', + localName: spelling, + importedName: spelling, + targetRaw: pkg, + }); + } + } + if (synthesised.length === 0) continue; + for (const index of callerFiles.get(callerName) ?? []) { + const parsed = parsedFiles[index]; + const fresh = synthesised.filter( + (imp) => + !parsed.parsedImports.some( + (existing) => + existing.kind === 'named' && + existing.localName === imp.localName && + existing.targetRaw === imp.targetRaw, + ), + ); + if (fresh.length === 0) continue; + parsedFiles[index] = Object.freeze({ + ...parsed, + parsedImports: Object.freeze([...parsed.parsedImports, ...fresh]), + }); + } + } +} + +// ─── `pkg::name()` qualifier helpers ──────────────────────────────────────── + +/** + * The package named by the text of an `@reference.qualified-name` capture (the + * whole `namespace_operator`, e.g. `pkg::f`, `pkg:::f`, `pkg ::: f`, + * `"pkg"::f`, `` `pkg`::f ``): the text before the first `::` / `:::`, + * trimmed, with one pair of surrounding `"`, `'` or backticks removed. + * `undefined` when there is no `::` or the package part is empty. The + * function name is not parsed here: it is the site's own `name`. + * + * Lives here (not in `qualified-call.ts`, which re-exports it) because the + * veto below needs it and `qualified-call.ts` imports this file. + */ +export function parseRQualifier(raw: string): string | undefined { + // Not a lazy `^([\s\S]*?)\s*::`, which is quadratic on a long run of blanks. + const colons = raw.indexOf('::'); + if (colons === -1) return undefined; + let pkg = raw.slice(0, colons).trim(); + if (pkg.length >= 2 && /^(["'`])[\s\S]*\1$/.test(pkg)) pkg = pkg.slice(1, -1); + return pkg.length === 0 ? undefined : pkg; +} + +/** + * The namespace operator of an `@reference.qualified-name` capture: `:::` when the first `::` + * is followed by a third colon (`pkg:::f`, `pkg ::: f`), `::` otherwise, `undefined` when the + * text has no `::` at all. {@link parseRQualifier} reads the package from the same text. + */ +export function parseRQualifierOperator(raw: string): '::' | ':::' | undefined { + const colons = raw.indexOf('::'); + if (colons === -1) return undefined; + return raw[colons + 2] === ':' ? ':::' : '::'; +} + +/** + * The package a package's NAMESPACE imports `symbol` from (`importFrom(origin, symbol)`; the + * LAST such entry wins, as the synthesised imports and the fallback veto read it), or + * `undefined` when there is none or the package imports from itself. One hop: the origin's own + * imports are not followed. Whether the package also exports the name is the caller's question. + */ +export function rReexportOrigin( + cfg: RPackageConfig, + pkgDir: string, + pkgName: string, + symbol: string, +): string | undefined { + const importFrom = cfg.namespaceInfoByPackageDir.get(pkgDir)?.importFrom; + if (importFrom === undefined) return undefined; + let last: RNamespaceImportFromEntry | undefined; + for (const entry of importFrom) if (rBareName(entry.name) === symbol) last = entry; + return last === undefined || last.pkg === pkgName ? undefined : last.pkg; +} + +/** Whether a name is reachable through a package's NAMESPACE (see {@link rSymbolVisibility}). */ +export type RSymbolVisibility = 'public' | 'private' | 'unknown'; + +/** + * Per package directory, the visibility of each top-level name (and each name the package + * imports, i.e. may re-export), decided once per pass by `rBuildSymbolVisibilityMemo` + * (`package-config.ts`) so that it does not depend on the order sites are judged in: whether a + * name is `private` or `unknown` reads the export-pattern budget, which every pattern match + * spends. Keyed by the `resolutionConfig` object, like the other per-pass registries here. + */ +const symbolVisibilityMemo = new WeakMap< + object, + ReadonlyMap> +>(); + +/** Record (replacing any earlier record) the visibility memo for `resolutionConfig`. */ +export function rRecordSymbolVisibilityMemo( + resolutionConfig: object, + memo: ReadonlyMap>, +): void { + symbolVisibilityMemo.set(resolutionConfig, memo); +} + +/** + * Visibility of `symbol` (the name without backticks) for a package-level reader, in the package + * whose directory is `pkgDir`: + * + * - `public`: the package has no NAMESPACE file (nothing is declared, so every name stays + * visible), or its NAMESPACE exports the name; + * - `private`: the NAMESPACE is present, readable and complete, and does not export the name; + * - `unknown`: not exported, but the NAMESPACE is unreadable or an `exportPattern()` was dropped + * or cut short, so the name may still be exported; also any name the memo does not hold. + */ +export function rPackageSymbolVisibility( + cfg: RPackageConfig, + pkgDir: string, + symbol: string, +): RSymbolVisibility { + const info = cfg.namespaceInfoByPackageDir.get(pkgDir); + if (info === undefined || !info.hasNamespaceFile) return 'public'; + return symbolVisibilityMemo.get(cfg)?.get(pkgDir)?.get(symbol) ?? 'unknown'; +} + +/** + * Visibility of the definition of `symbol` in `filePath`. Only a definition under a package's + * `R/` directory is judged ({@link rOwningPackageDir}; R loads nothing else into a namespace), + * so a file outside every package reads `public`. + */ +export function rSymbolVisibility( + cfg: RPackageConfig | null | undefined, + filePath: string, + symbol: string, +): RSymbolVisibility { + if (cfg === null || cfg === undefined) return 'public'; + const dir = rOwningPackageDir(filePath, cfg); + return dir === undefined ? 'public' : rPackageSymbolVisibility(cfg, dir, rBareName(symbol)); +} + +/** + * Per local package, how many top-level definitions of each symbol (the name without its + * backticks) it has in its + * `R/` files, recorded by `populateRQualifiedCalls` for the packages that some + * `pkg::name()` call in the workspace names. Keyed by the `resolutionConfig` + * object (like {@link packageTopLevelNames}), which reaches both that hook and + * the free-call-fallback veto. A missing entry means "cannot decide". + */ +const qualifiedDefinitionCounts = new WeakMap< + object, + ReadonlyMap> +>(); + +/** Record (replacing any earlier record) the per-package definition counts for `resolutionConfig`. */ +export function rRecordQualifiedDefinitionCounts( + resolutionConfig: object, + counts: ReadonlyMap>, +): void { + qualifiedDefinitionCounts.set(resolutionConfig, counts); +} + +/** + * Number of top-level definitions of `name` recorded for local package `pkg`, + * or `undefined` when nothing was recorded (hook did not run for that package). + */ +export function rQualifiedDefinitionCount( + resolutionConfig: object, + pkg: string, + name: string, +): number | undefined { + const byName = qualifiedDefinitionCounts.get(resolutionConfig)?.get(pkg); + return byName === undefined ? undefined : (byName.get(name) ?? 0); +} + +// ─── Global-name fallback veto (isGlobalNameFallbackPlausible) ────────────── + +/** + * Structural subset of the hook context `isRGlobalNameFallbackPlausible` + * reads; the shared contract's full context is assignable to it. + */ +export interface RGlobalNameFallbackContext { + readonly callerParsed: { + readonly filePath: string; + /** The caller file's imports; absent in hand-built contexts (no `library()` / `source()`). */ + readonly parsedImports?: readonly { readonly kind: string; readonly targetRaw: string }[]; + /** The caller file's reference sites; absent in hand-built contexts (no call). */ + readonly referenceSites?: readonly { readonly name: string; readonly kind: string }[]; + }; + readonly candidate: { + readonly filePath: string; + readonly nodeId?: string; + readonly type?: string; + }; + /** Reaches the candidate's parse; absent in hand-built contexts, which read as "cannot decide". */ + readonly parsedFileOf?: (filePath: string) => ParsedFile | undefined; + /** Opaque `loadResolutionConfig` result; an {@link RPackageConfig} for R. */ + readonly resolutionConfig?: unknown; + readonly site: { readonly name: string; readonly rawQualifiedName?: string }; +} + +/** + * True when the candidate is a function or class that its file's scope model holds and binds + * below the top level. A candidate the scope model does not hold (the semantic model's own + * entry for a dotted name carries a graph id, not a scope definition id) is not judged. + */ +function isClosureLocalCandidate(ctx: RGlobalNameFallbackContext): boolean { + const { candidate } = ctx; + if (candidate.nodeId === undefined || candidate.type === undefined) return false; + if (!TOP_LEVEL_BINDABLE_TYPES.has(candidate.type)) return false; + const parsed = ctx.parsedFileOf?.(candidate.filePath); + if (parsed === undefined) return false; + const ids = rDefIds(parsed); + return ids.all.has(candidate.nodeId) && !ids.topLevel.has(candidate.nodeId); +} + +/** A file's own `source()`, `system.file()` or `test_path()` call: it may load a script itself. */ +function callerLoadsScripts(callerParsed: RGlobalNameFallbackContext['callerParsed']): boolean { + if (callerParsed.parsedImports?.some((imp) => imp.kind === 'side-effect') === true) return true; + return ( + callerParsed.referenceSites?.some( + (site) => + site.kind === 'call' && + (rBareName(site.name) === 'system.file' || rBareName(site.name) === 'test_path'), + ) === true + ); +} + +/** + * Rule D of the package-visibility rules: a caller under `

/tests/` and a candidate under + * `

/data-raw/` or `

/inst/`. Tests run against the loaded package; `data-raw/` is never + * installed and `inst/` files are not sourced into the namespace, so a test reaches such a script + * only by loading it itself. Not applied when the caller's file calls `source()`, `system.file()` + * or `test_path()` (it may do exactly that). + */ +function isRTestReachingUnloadedScript( + ctx: RGlobalNameFallbackContext, + cfg: RPackageConfig, +): boolean { + const tree = rEnclosingPackageDir(ctx.callerParsed.filePath, cfg); + if (tree === undefined) return false; + const base = tree === '' ? '' : `${tree}/`; + const callerFile = ctx.callerParsed.filePath.replace(/\\/g, '/'); + const candidateFile = ctx.candidate.filePath.replace(/\\/g, '/'); + if (!callerFile.startsWith(`${base}tests/`)) return false; + if (!candidateFile.startsWith(`${base}data-raw/`) && !candidateFile.startsWith(`${base}inst/`)) { + return false; + } + // A nested package claims its own files: the candidate must be in the caller's package tree. + if (rEnclosingPackageDir(ctx.candidate.filePath, cfg) !== tree) return false; + return !callerLoadsScripts(ctx.callerParsed); +} + +/** + * Whether `pkg` is attached or imported by package P (`callerInfo`) or the caller's file: `import(pkg)` + * and the S4 import forms, a `library(pkg)` / `require(pkg)` in the caller's file, or `pkg` in the + * transitive `Depends:` closure of P over repository packages (attaching a package attaches its + * `Depends:` too). `undefined` when the closure reaches a repository package whose `Depends:` was + * not recorded (it has no NAMESPACE): cannot decide. + */ +function rPackageDeclaresLink( + ctx: RGlobalNameFallbackContext, + cfg: RPackageConfig, + callerInfo: RNamespaceInfo, + pkg: string, +): boolean | undefined { + if (callerInfo.importPackages?.has(pkg) === true) return true; + const attaches = ctx.callerParsed.parsedImports?.some( + (imp) => imp.kind === 'wildcard' && imp.targetRaw.replace(/^["']|["']$/g, '') === pkg, + ); + if (attaches === true) return true; + const seen = new Set(); + const queue = [...(callerInfo.depends ?? [])]; + let undecided = false; + for (let next = queue.pop(); next !== undefined; next = queue.pop()) { + if (seen.has(next)) continue; + seen.add(next); + if (next === pkg) return true; + const dir = cfg.packages.get(next); + if (dir === undefined) continue; // an external package: its Depends are not known here + const info = cfg.namespaceInfoByPackageDir.get(dir); + if (info === undefined) { + undecided = true; + continue; + } + for (const dep of info.depends ?? []) if (!seen.has(dep)) queue.push(dep); + } + return undecided ? undefined : false; +} + +/** + * Rules A and B of the package-visibility rules, for a bare call in `P/R/` of a package whose + * NAMESPACE was read completely (imports and exports). True (refuse) when the candidate is in + * another package Q's `R/` and P declares no link to Q (Rule A: no `importFrom()` entry for the + * name, no `import(Q)`, Q not in the transitive `Depends:`, no `library(Q)` in the caller's file), + * or in no package's `R/` at all, a loose script (Rule B: no `importFrom()` entry for the name). + * + * Such a call is not impossible in R: an undeclared use is an R CMD check NOTE ("no visible global + * function definition"), and package code falls through to the global environment and the search + * path, where a sibling package may be attached (`devtools::load_all()`, monorepo sessions). The + * rules refuse calls that are unsupported for check-clean package code, which the package's + * declared dependencies cannot justify: an unresolved call is preferred to an unsupported + * cross-package edge. That is stricter than the hook contract's "impossible", a trade stated in + * the R provider guide. With an `importFrom()` entry for the name the importFrom rules below + * decide, as before. + */ +function isRPackageCallUndeclared( + ctx: RGlobalNameFallbackContext, + cfg: RPackageConfig, + caller: RPackageLocation, + callerInfo: RNamespaceInfo, + symbol: string, +): boolean { + if (!callerInfo.hasNamespaceFile) return false; + if (callerInfo.exportsUnreadable === true || callerInfo.importsUnreadable === true) return false; + if (callerInfo.importFrom.some((entry) => rBareName(entry.name) === symbol)) return false; + const candidatePkg = rPackageDirForFile(ctx.candidate.filePath, cfg); + if (candidatePkg === undefined) { + // In the `R/` of a package discovery read but did not register (a same-name copy): its name + // is not known here, so the link cannot be checked. Cannot decide: allow. + if (rFileBelongsToDiscoveredPackage(ctx.candidate.filePath, cfg)) return false; + return true; // Rule B + } + if (candidatePkg.name === caller.name) return false; + return rPackageDeclaresLink(ctx, cfg, callerInfo, candidatePkg.name) === false; // Rule A +} + +/** + * Refuse a `global-name-fallback` guess that R cannot make, or that check-clean package code + * cannot make. Returns `false` (veto) when the candidate is a function or class bound inside + * another function or a block (not at top level), when its package's NAMESPACE proves it private + * to a caller outside that package's directory tree, when the package-visibility rules refuse it + * (a bare call from a package's `R/` into a package it declares no link to, or into a loose + * script, see {@link isRPackageCallUndeclared}; a call from `tests/` into the package's own + * `data-raw/` or `inst/`, see {@link isRTestReachingUnloadedScript}), or when the caller + * package's NAMESPACE contradicts the guess. + * + * The private-candidate veto applies when the candidate is a top-level function or class under + * package P's `R/`, {@link rSymbolVisibility} says `private` (P has a readable, complete + * NAMESPACE that does not export the name), and the caller is outside P's tree + * ({@link rEnclosingPackageDir}: `tests/` and `inst/` of P count as inside). `unknown` + * (unreadable NAMESPACE, a cut-short `exportPattern()`) is not vetoed: a guess is vetoed only + * when impossible. A `source()` of the private file is not honoured (R puts sourced names in + * the global environment); that is a documented gap. + * + * The NAMESPACE veto applies only when + * ALL of these hold: + * + * - the caller file lives under `/R/**` of a package with a NAMESPACE + * that has an `importFrom(P, )` entry (the LAST such entry over + * all entries, external packages included — the same last-wins rule the synthesised imports follow); + * - `P` is not the caller's own package (a self-import binds nothing); + * - the candidate lives under some package's `R/**`, and that package is + * neither the caller's own (own namespace masks; a plain 0.5 fallback edge + * until own-package binding exists) nor `P`; + * - if `P` is local, it defines `site.name` at top level; otherwise `P` + * re-exports the name from a third package and the candidate may be the + * real origin (unknown per-package names = cannot decide = allow). + * + * Everything else — no config, no parse of the candidate's file, caller outside a package `R/` + * (`tests/`, `scripts/`, root `plumber.R`) other than Rule D, a call the package-visibility rules + * allow with no `importFrom()` entry for the name, a candidate outside any package `R/` reached + * through an entry — answers `true`, so `source()`/`library()` script flows are untouched. + * + * A qualified call `pkg::name()` (site carries `rawQualifiedName`) is judged + * by its qualifier first: with `pkg` a local package, a candidate + * inside `pkg` is plausible whatever the NAMESPACE says, and a candidate + * elsewhere is vetoed when `pkg` itself defines `name`, or when its NAMESPACE proves `name` + * private (a re-export is an `export()` entry plus an `importFrom()`). A `pkg` that is not a + * local package falls through to the NAMESPACE rules above. (`populateRQualifiedCalls` resolves or + * drops every qualified site before the fallback runs, so this branch is a second line.) + * + * Known unsound shape, pinned by a characterisation test: an `import(pkg)` is read only as a link + * to `pkg` for the package-visibility rules; it does not take part in the `importFrom()` rules, so + * a later `import(pkg)` that R would prefer to an `importFrom()` entry is invisible to them. + */ +export function isRGlobalNameFallbackPlausible(ctx: RGlobalNameFallbackContext): boolean { + // A function or class bound inside another function lives in that call's frame: no other + // function, in this file or another, can reach it by its bare name. Only top-level + // definitions are candidates for a unique-name guess. + if (isClosureLocalCandidate(ctx)) return false; + + const cfg = ctx.resolutionConfig as RPackageConfig | null | undefined; + if ( + typeof cfg !== 'object' || + cfg === null || + !(cfg.packages instanceof Map) || + !(cfg.namespaceInfoByPackageDir instanceof Map) + ) { + return true; + } + + // Qualified call `pkg::name()`: the source named the package, so the package + // outranks the caller's NAMESPACE. A candidate inside a local `pkg` is right; + // a candidate elsewhere contradicts the qualifier when `pkg` defines `name` + // itself. A `pkg` that defines nothing of that name (a re-export) or that has + // no recorded counts cannot be judged from here: fall through to the rules + // below, i.e. today's behaviour. An unknown or external `pkg` falls through + // too (external sites are dropped upstream, before any fallback tier). + const qualifier = + ctx.site.rawQualifiedName === undefined + ? undefined + : parseRQualifier(ctx.site.rawQualifiedName); + const symbol = rBareName(ctx.site.name); + if (qualifier !== undefined && cfg.packages.has(qualifier)) { + if (rPackageDirForFile(ctx.candidate.filePath, cfg)?.name === qualifier) return true; + if ((rQualifiedDefinitionCount(cfg, qualifier, symbol) ?? 0) > 0) return false; + const qualifierDir = cfg.packages.get(qualifier); + if ( + qualifierDir !== undefined && + rPackageSymbolVisibility(cfg, qualifierDir, symbol) === 'private' + ) { + return false; + } + } + + // A candidate a readable, complete NAMESPACE proves private is not reachable by its bare name + // from outside its package's tree. + if ( + ctx.candidate.type !== undefined && + TOP_LEVEL_BINDABLE_TYPES.has(ctx.candidate.type) && + rSymbolVisibility(cfg, ctx.candidate.filePath, symbol) === 'private' && + rEnclosingPackageDir(ctx.callerParsed.filePath, cfg) !== + rOwningPackageDir(ctx.candidate.filePath, cfg) + ) { + return false; + } + + // The package-visibility rules judge a bare name; a qualified site was decided above (and by + // `populateRQualifiedCalls`, which runs first). + if (ctx.site.rawQualifiedName === undefined && isRTestReachingUnloadedScript(ctx, cfg)) { + return false; + } + + const caller = rPackageDirForFile(ctx.callerParsed.filePath, cfg); + if (caller === undefined) return true; + const callerInfo = cfg.namespaceInfoByPackageDir.get(caller.dir); + if ( + ctx.site.rawQualifiedName === undefined && + callerInfo !== undefined && + isRPackageCallUndeclared(ctx, cfg, caller, callerInfo, symbol) + ) { + return false; + } + const importFrom = callerInfo?.importFrom; + if (importFrom === undefined) return true; + + let lastEntry: RNamespaceImportFromEntry | undefined; + for (const entry of importFrom) if (rBareName(entry.name) === symbol) lastEntry = entry; + if (lastEntry === undefined) return true; + const importedFrom = lastEntry.pkg; + if (importedFrom === caller.name) return true; + + const candidatePkg = rPackageDirForFile(ctx.candidate.filePath, cfg); + if (candidatePkg === undefined) return true; + if (candidatePkg.name === caller.name || candidatePkg.name === importedFrom) return true; + + if (cfg.packages.has(importedFrom)) { + const providerNames = rRecordedPackageTopLevelNames(cfg)?.get(importedFrom); + if (providerNames === undefined || !providerNames.has(symbol)) return true; + } + return false; +} diff --git a/gitnexus/src/core/ingestion/languages/r/naming-argument.ts b/gitnexus/src/core/ingestion/languages/r/naming-argument.ts new file mode 100644 index 000000000..f379171d6 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/naming-argument.ts @@ -0,0 +1,187 @@ +/** + * Which argument of an R call names what the call defines or imports. + * + * `R_QUERIES` captures every string-valued argument of `setClass`, + * `setRefClass`, `setGeneric` and `setMethod`, and the scope query captures every + * identifier- or string-valued argument of `library`, `require` and `source` and + * every string argument of `setClass`/`setRefClass`. Only one of them is the + * definition's name or the import's source; the rest are `contains = "VIRTUAL"`, + * `valueClass = "numeric"`, the class slot of `setMethod("show", "Foo", ...)`, + * `lib.loc = libO`, and so on. The choice is made here, in code, rather than in the + * query: a tree-sitter query can only skip an unbounded run of leading + * comments and named arguments with a repeated sibling group, and that costs + * time quadratic in the argument count on every call in the file (a single + * `list(a0 = 0, ...)` with 16000 arguments took 18s). This check is linear: it + * reads each argument list once and caches the answer. + * + * R matches arguments by name first, then positionally. The naming argument is + * therefore the argument spelled with the call's first formal (`Class = "A"`), + * or, when no argument anywhere in the list is spelled that way, the first + * unnamed argument. Comments, commas and other named arguments may precede it, + * and a `Class = ...` after an unnamed argument takes the formal from it, so + * `setClass(a = "Q", "B", Class = "A")` names `A` only. + * + * A formal may be spelled bare, in backticks or in quotes (`Class`, `` `Class` ``, + * `"Class"`); all name the same argument. Partial matching of an abbreviated + * name (`Cl = "A"`) is not modelled. + */ + +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import type { CaptureMap } from '../../language-provider.js'; + +/** The first formal of each call whose first argument names what it defines or imports. */ +const NAMING_FORMAL: Readonly> = { + setClass: 'Class', + setRefClass: 'Class', + setGeneric: 'name', + setMethod: 'f', + library: 'package', + require: 'package', + source: 'file', +}; + +/** An argument's name with one pair of surrounding backticks or quotes removed. */ +function argumentNameText(name: SyntaxNode): string { + const text = name.text; + if (text.length >= 2) { + const first = text[0]; + if ((first === '`' || first === '"' || first === "'") && text[text.length - 1] === first) { + return text.slice(1, -1); + } + } + return text; +} + +/** + * Per tree, per (argument list, formal): the id of the unnamed argument that takes + * the formal, or null when none does. One pass over the argument list answers every + * later match in that list in constant time. Looking backwards from each match + * instead is quadratic in the number of preceding comments (sibling lookup is + * linear in the sibling's position), so the list is read forwards, once. + */ +const namingArgumentIds = new WeakMap>(); + +function namingUnnamedArgumentId(argumentList: SyntaxNode, formal: string): number | null { + let perTree = namingArgumentIds.get(argumentList.tree); + if (perTree === undefined) { + perTree = new Map(); + namingArgumentIds.set(argumentList.tree, perTree); + } + const key = `${argumentList.id}:${formal}`; + const cached = perTree.get(key); + if (cached !== undefined) return cached; + + // The first unnamed argument takes the formal, unless an argument anywhere in the + // list is spelled with it (R matches by name first), which retracts that choice. + let found: number | null = null; + for (const child of argumentList.namedChildren) { + if (child.type !== 'argument') continue; // comment, comma + const name = child.childForFieldName('name'); + if (name === null) { + found ??= child.id; + } else if (argumentNameText(name) === formal) { + found = null; + break; + } + } + perTree.set(key, found); + return found; +} + +/** + * The argument R would match to `target` in a call to a function whose formals are + * `formals`, in order. R matches named arguments first (a name that spells a formal + * binds it) and then fills the formals that remain from the unnamed arguments, in + * order. Partial matching of abbreviated names (`sig =`) is not modelled. Returns + * null when no argument reaches `target`. Linear in the argument count; meant for + * the one call a definition hook is looking at, not for every call in a file. + */ +export function getRMatchedArgument( + argumentList: SyntaxNode, + formals: readonly string[], + target: string, +): SyntaxNode | null { + const unnamed: SyntaxNode[] = []; + const named = new Set(); + for (const child of argumentList.namedChildren) { + if (child.type !== 'argument') continue; // comment, comma + const name = child.childForFieldName('name'); + if (name === null) { + unnamed.push(child); + continue; + } + const text = argumentNameText(name); + if (text === target) return child; + named.add(text); + } + const remaining = formals.filter((formal) => !named.has(formal)); + const index = remaining.indexOf(target); + return index === -1 ? null : (unnamed[index] ?? null); +} + +/** + * True when `argument` (an `argument` node) is the one that names the call's + * definition, given the call's first `formal`. + */ +export function isRNamingArgument(argument: SyntaxNode, formal: string): boolean { + const ownName = argument.childForFieldName('name'); + if (ownName !== null) return argumentNameText(ownName) === formal; + const argumentList = argument.parent; + if (argumentList === null) return false; + return namingUnnamedArgumentId(argumentList, formal) === argument.id; +} + +/** + * True when a `R_QUERIES` match captured an argument of one of the calls above + * that is NOT the naming argument, so the definition it would mint is spurious. + * Matches of any other shape (R6, functions, slots, calls, imports) are never + * skipped. The legacy `@import` matches are discarded by the parse worker, so + * they are not considered here. + */ +export function isRNonNamingArgumentCapture(captureMap: CaptureMap): boolean { + const fn = captureMap['_fn']; + if (fn === undefined) return false; + const formal = NAMING_FORMAL[fn.text]; + if (formal === undefined) return false; + + // Definitions capture the string's content. Slot matches capture a named + // argument's identifier and are not ours. + const captured = captureMap['name']; + if (captured === undefined || captured.type !== 'string_content') return false; + const argument = captured.parent?.parent; + if (argument === undefined || argument === null || argument.type !== 'argument') return false; + + return !isRNamingArgument(argument, formal); +} + +/** The captures of a raw tree-sitter match, as the scope emitter sees them. */ +interface RawCapture { + readonly name: string; + readonly node: SyntaxNode; +} + +/** Callee captures of the scope query's naming-argument patterns (`r/query.ts`). */ +const SCOPE_CALLEE_CAPTURES: ReadonlySet = new Set(['_s4_decl', '_srcfn', '_libfn']); + +/** + * True when a scope-query match captured an argument of `setClass` / + * `setRefClass` (`@declaration.name`), `library` / `require` or `source` + * (`@import.source`) that is NOT the naming argument. The scope query captures + * every such argument; the emitter drops these matches. + */ +export function isRNonNamingScopeMatch(captures: readonly RawCapture[]): boolean { + const fn = captures.find((c) => SCOPE_CALLEE_CAPTURES.has(c.name)); + if (fn === undefined) return false; + const formal = NAMING_FORMAL[fn.node.text]; + if (formal === undefined) return false; + + const captured = captures.find( + (c) => c.name === 'declaration.name' || c.name === 'import.source', + ); + if (captured === undefined) return false; + const argument = + captured.node.type === 'string_content' ? captured.node.parent?.parent : captured.node.parent; + if (argument === undefined || argument === null || argument.type !== 'argument') return false; + + return !isRNamingArgument(argument, formal); +} diff --git a/gitnexus/src/core/ingestion/languages/r/owner-hooks.ts b/gitnexus/src/core/ingestion/languages/r/owner-hooks.ts new file mode 100644 index 000000000..efe771d5a --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/owner-hooks.ts @@ -0,0 +1,55 @@ +/** + * R owner hooks for the shared ingestion pipeline. + * + * R has no class syntax: R6 / RefClass members belong to a call or assignment + * (`Foo <- R6::R6Class(...)`, `setRefClass(...)`), not to an enclosing + * container node. `resolveMemberOwnerNode` hands the member extractors that + * owner node. Members whose owner is a string (`setMethod("area", "Circle", + * ...)`) or a top-level R6/RefClass call are instead tied to their class after + * all chunks merge: `definitionPropertiesExtractor` records the owner name as + * an `ownerNameHint`, resolved by R's `postParse` hook. + */ + +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import type { DefinitionPropertiesContext } from '../../language-provider.js'; +import { + findRFieldOwnerNode, + getRClassNameOfOwnerNode, + getRTopLevelPropertyOwnerName, +} from '../../field-extractors/r.js'; +import { getRTopLevelMethodOwnerName } from '../../method-extractors/r.js'; + +/** `LanguageProvider.resolveMemberOwnerNode` for R. */ +export const rResolveMemberOwnerNode = (definitionNode: SyntaxNode): SyntaxNode | null => + findRFieldOwnerNode(definitionNode); + +/** + * `LanguageProvider.resolveDetachedOwnerName` for R: the class name an ancestor + * call or assignment declares (`Foo <- R6Class(...)`, `setClass("Foo", ...)`, + * `setRefClass("Foo", ...)`) or the class a `setMethod("m", "Foo", ...)` call + * names in its signature. It qualifies the member's id (`Method::Foo.m`, + * `Property::Foo.x`), so two classes in one file that share a member name + * stay distinct nodes. It carries no graph id: the owner edge is still attached + * by `postParse` through `ownerNameHint`. + */ +export const rResolveDetachedOwnerName = (ancestor: SyntaxNode): string | undefined => + getRClassNameOfOwnerNode(ancestor) ?? getRTopLevelMethodOwnerName(ancestor) ?? undefined; + +/** + * `LanguageProvider.definitionPropertiesExtractor` for R: the deferred owner + * name (`ownerNameHint`) of a `Method`/`Property` whose owning class is named by + * a string argument or a top-level R6/RefClass call rather than a syntactic + * ancestor. + */ +export const rDefinitionProperties = ( + ctx: DefinitionPropertiesContext, +): Readonly> | undefined => { + const ownerNameHint = + ctx.nodeLabel === 'Method' + ? (getRTopLevelMethodOwnerName(ctx.definitionNode) ?? + getRTopLevelPropertyOwnerName(ctx.definitionNode)) + : ctx.nodeLabel === 'Property' + ? getRTopLevelPropertyOwnerName(ctx.definitionNode) + : null; + return ownerNameHint ? { ownerNameHint } : undefined; +}; diff --git a/gitnexus/src/core/ingestion/languages/r/package-config.ts b/gitnexus/src/core/ingestion/languages/r/package-config.ts new file mode 100644 index 000000000..faaad98b5 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/package-config.ts @@ -0,0 +1,724 @@ +import { constants, type Dirent } from 'fs'; +import fs from 'fs/promises'; +import path from 'path'; +import { + parseRNamespaceExports, + parseRNamespaceImportFrom, + parseRNamespaceImportPackages, + rFileTopLevel, + rOwningPackageDir, + rRecordSymbolVisibilityMemo, + type RNamespaceImportFromEntry, + type RSymbolVisibility, +} from './namespace-imports.js'; +import { rBareName } from './bare-name.js'; +import { compileRExportPatternDetailed, type RExportMatcher } from './export-pattern.js'; +import { SharedWorkBudget } from './linear-regex.js'; +import type { RUnindexedSource } from './unindexed-sources.js'; +import type { ParsedFile } from 'gitnexus-shared'; +import { isDev } from '../../utils/env.js'; +import { isHardcodedIgnoredDirectory } from '../../../../config/ignore-service.js'; +import { logger } from '../../../logger.js'; + +/** An `exportPattern()` argument that was not compiled, and why. */ +export interface RDroppedExportPattern { + /** The pattern text as R would see it (the string value, after unescaping). */ + readonly pattern: string; + readonly reason: string; +} + +/** R package config parsed from DESCRIPTION files in a multi-package repo */ +export interface RPackageConfig { + /** + * Map of package name to directory path relative to repo root. When several + * directories declare the same `Package:` name the shallowest directory wins + * and, among equal depths, the one whose whole path string sorts first by + * code unit (so `a-b/x` beats `a/x`: `-` sorts before `/`); the others are + * not entries here (their NAMESPACE is still recorded in + * {@link namespaceInfoByPackageDir}, which is keyed by directory). Installed + * libraries and check output (`renv`, `packrat`, `revdep`, `.Rcheck`), dot-directories + * and the directories the indexer ignores are not searched, and a `DESCRIPTION` with a + * `Built:` field (an installed package) is not read, so none of their packages are entries here. + */ + packages: Map; + /** Package-scoped NAMESPACE config keyed by package dir relative to repo root. */ + namespaceInfoByPackageDir: Map; + /** + * Every directory (relative to the repo root, `''` for the root) in which discovery read a + * `DESCRIPTION` with a `Package:` line: the directories of {@link packages} plus the + * directories of same-name copies that lost to them, with or without a NAMESPACE. The + * qualifier-locality decision reads it, so that a file under such a directory's `R/` is not + * taken as evidence for a package named after its directory. Optional so that hand-built + * configs read as "the directories of {@link packages} and {@link namespaceInfoByPackageDir}". + */ + packageDirs?: ReadonlySet; + /** + * True when discovery cannot rule out an undiscovered package: a directory + * of the main walk could not be read, or a subtree the walk skipped (below + * the depth limit, or still queued when the directory cap stopped it) may + * hide a package. Skipped subtrees are scanned once more for `DESCRIPTION` + * files (see {@link hidesUndiscoveredPackage}); the flag is set only when + * that scan finds a `Package:` name absent from {@link packages}, exceeds + * its directory budget, or hits an unreadable path. A skipped subtree with + * no package inside (`tests/testthat/fixtures/...`, `inst/extdata/...`) or + * only a same-name copy of an already-discovered package does NOT truncate. + * A package missing from {@link packages} is then not proven absent from the + * repo, so `rQualifierLocality` answers `unknown` rather than `external`. + * This holds for an empty {@link packages} map too: a complete scan that found + * no package at all proves every qualifier external. + * Optional so that hand-built configs (tests, callers that never scan) read + * as "complete". + */ + truncated?: boolean; + /** + * R files of the repository that are not indexed sources (skipped directories, `.Rprofile`, + * `R/*.S`), with the identifier tokens each contains. Set only by the rename-preview reload + * (`refreshResolutionConfig`), so the rename guard can refuse a name such a file uses; absent + * means the repository was not scanned. + */ + unindexedRSources?: readonly RUnindexedSource[]; +} + +export interface RNamespaceInfo { + /** True when the package has a readable NAMESPACE file. */ + hasNamespaceFile: boolean; + /** Explicit named exports from export()/exportClasses()/exportMethods()/S3method(). */ + namedExports: Set; + /** Precompiled matchers from exportPattern("..."), compiled once here so + * `refineRExportStatus` doesn't recompile per node it checks. POSIX bracket + * classes such as `[[:alpha:]]` are translated to JS equivalents first + * (see `compileRExportPattern`). The matchers run in linear time, so a hostile + * pattern cannot stall analysis. Patterns that fail to compile, or that the + * linear-time matcher cannot handle, are dropped at this stage (they never match) + * and listed in {@link droppedExportPatterns}. */ + exportPatterns: RExportMatcher[]; + /** + * Non-empty `exportPattern()` arguments that could not be compiled, with the reason. + * A dropped pattern exports nothing, so every name the pattern was meant to cover + * reads as unexported; `reportDroppedRExportPatterns` turns these into warnings. + * Optional so that hand-built configs read as "nothing dropped". + */ + droppedExportPatterns?: readonly RDroppedExportPattern[]; + /** + * How many further non-empty `exportPattern()` arguments were dropped but not listed in + * {@link droppedExportPatterns}, which holds at most {@link MAX_LISTED_DROPPED_PATTERNS} + * per package so that a NAMESPACE of hundreds of thousands of refused patterns cannot + * produce hundreds of thousands of warnings. Reported as one line. + */ + omittedDroppedExportPatterns?: number; + /** + * The work budget this package's matchers draw on: a sub-budget of the load's. Absent on + * hand-built configs. {@link rExportPatternMatches} stops consulting the matchers once it + * is spent. + */ + exportPatternBudget?: SharedWorkBudget; + /** + * True when the NAMESPACE reader skipped, dropped or could not read something that may hide an + * export (see {@link RNamespaceExports.unreadable}): a conditional export, a raw-string or + * multi-line argument, a malformed directive. A name absent from the export lists is then not + * proven private. Optional so that hand-built configs read as "nothing hidden". + */ + exportsUnreadable?: boolean; + /** `importFrom(pkg, name)` pairs in NAMESPACE file order (all entries, incl. self-imports and duplicates). */ + importFrom: readonly RNamespaceImportFromEntry[]; + /** + * Every name an `S3method()` directive registers (bare, as {@link rBareName} spells it). A + * registration is not an export: it makes the method reachable through S3 dispatch only. The + * graph keeps such a method public (`isExported`) and `rename` refuses it, but resolution does + * not treat it as exported (see {@link s3RegisteredOnly}). Optional so that hand-built configs + * read as "no registration". + */ + s3Registered?: ReadonlySet; + /** + * The {@link s3Registered} names that no `export()`, `exportClasses()` or `exportMethods()` + * entry lists. They stay in {@link namedExports} (which records what the directives name), and + * {@link rNamespaceExports} answers false for them unless an `exportPattern()` matches, so + * `library()`, `pkg::name`, `importFrom()` and the visibility memo see real exports only. + * Optional, absent reads as empty. + */ + s3RegisteredOnly?: ReadonlySet; + /** + * Packages the NAMESPACE declares an import from other than through `importFrom()`: + * `import(Q)` (a named `except =` argument is skipped; the package is still imported) and the + * S4 forms `importClassesFrom(Q, ...)` and `importMethodsFrom(Q, ...)`. Read by the + * package-visibility rule of the unique-name guess. Optional, absent reads as empty. + */ + importPackages?: ReadonlySet; + /** + * Package names of the DESCRIPTION `Depends:` field (continuation lines included, `R` itself + * dropped): the packages attaching this one also attaches. Optional, absent reads as empty. + */ + depends?: ReadonlySet; + /** + * True when the NAMESPACE reader skipped or could not read something that may hide an import: + * a conditional (`if` / `else`) body or a malformed directive that names an import-family + * directive, or an import argument it cannot read. The imports above are then a lower bound, + * so the package-visibility rule is not applied. Optional, absent reads as false. + */ + importsUnreadable?: boolean; +} + +/** + * Most patterns named in warnings per package, for dropped patterns (the rest are counted, + * see `omittedDroppedExportPatterns`) and for patterns cut short by the budget alike, so + * that a NAMESPACE of tens of thousands of patterns cannot produce as many warnings. + */ +export const MAX_LISTED_DROPPED_PATTERNS = 50; + +/** Most names remembered as matched per package, so a hostile name list cannot grow the set without bound. */ +const MAX_REMEMBERED_MATCHES = 100_000; + +/** Names a pattern list matched while its budget lasted (only for lists that draw on a budget). */ +const matchedWhileBudgetLasted = new WeakMap>(); + +/** + * True when one of the package's `exportPattern()` matchers matches `name`. Once the + * package's work budget (or the load's) is spent it stops asking the matchers, so the cost no + * longer grows as patterns x names. A name it matched while the budget lasted keeps that + * answer (one set lookup, so a duplicate-named definition does not flip from exported to + * private), but only for the first {@link MAX_REMEMBERED_MATCHES} (100,000) distinct matched + * names per package: a name matched after that is not remembered and answers false once the + * budget is spent. A name it had not decided answers false, and the patterns it did not get to run are + * then reported by {@link reportRExportPatternProblems}. + */ +export function rExportPatternMatches( + info: Pick, + name: string, +): boolean { + const budget = info.exportPatternBudget; + let remembered = matchedWhileBudgetLasted.get(info.exportPatterns); + for (const pattern of info.exportPatterns) { + if (budget?.isSpent === true) { + budget.cutShort = true; + return remembered?.has(name) === true; + } + if (pattern.test(name)) { + if (budget !== undefined) { + if (remembered === undefined) { + remembered = new Set(); + matchedWhileBudgetLasted.set(info.exportPatterns, remembered); + } + if (remembered.size < MAX_REMEMBERED_MATCHES) remembered.add(name); + } + return true; + } + } + return false; +} + +/** + * True when the package's NAMESPACE exports `symbol`: an explicit `export()`, `exportClasses()` + * or `exportMethods()` entry, or an `exportPattern()` match. An `S3method()` registration alone + * does not export ({@link RNamespaceInfo.s3RegisteredOnly}). The one test resolution, the + * visibility memo, the indexer's export refinement and the rename guard use; the refinement and + * the guard also keep a registered method public ({@link RNamespaceInfo.s3Registered}). + */ +export function rNamespaceExports( + info: Pick< + RNamespaceInfo, + 'namedExports' | 'exportPatterns' | 'exportPatternBudget' | 's3RegisteredOnly' + >, + symbol: string, +): boolean { + return ( + (info.namedExports.has(symbol) && info.s3RegisteredOnly?.has(symbol) !== true) || + rExportPatternMatches(info, symbol) + ); +} + +/** + * True when part of the package's `exportPattern()` entries is not in effect: a pattern that + * was dropped at load, or one that ran out of work budget. {@link rNamespaceExports} answers + * false for a name such a pattern was meant to cover, so a false answer is not proof that the + * name is private. + */ +export function rExportPatternsIncomplete( + info: Pick< + RNamespaceInfo, + | 'exportPatterns' + | 'exportPatternBudget' + | 'droppedExportPatterns' + | 'omittedDroppedExportPatterns' + >, +): boolean { + return ( + (info.droppedExportPatterns?.length ?? 0) > 0 || + (info.omittedDroppedExportPatterns ?? 0) > 0 || + info.exportPatternBudget?.isSpent === true || + info.exportPatternBudget?.cutShort === true || + info.exportPatterns.some((matcher) => matcher.exhausted === true) + ); +} + +export { rSymbolVisibility, type RSymbolVisibility } from './namespace-imports.js'; + +/** + * Decide, once for the whole pass, the {@link RSymbolVisibility} of every top-level name of every + * package that has a NAMESPACE (and of every name that package imports, which it may re-export), + * and record the result for `rSymbolVisibility`. + * + * Determinism: `rNamespaceExports` spends the package's `exportPattern()` budget, and + * {@link rExportPatternsIncomplete} reads that budget, so judging a name at the moment a site asks + * could give a different answer depending on which site (or file) came first. Here every name is + * tested in one batch, packages in directory order and names in code-unit order, and completeness + * is read only AFTER the whole batch: a name no pattern matched is `private` when no pattern was + * dropped or cut short and the NAMESPACE was fully read, `unknown` otherwise. A name outside the + * memo reads `unknown`. Replaces any earlier memo for `cfg`. + */ +export function rBuildSymbolVisibilityMemo( + parsedFiles: readonly ParsedFile[], + cfg: RPackageConfig, +): void { + const namesByDir = new Map>(); + const namesOf = (dir: string): Set => { + let names = namesByDir.get(dir); + if (names === undefined) namesByDir.set(dir, (names = new Set())); + return names; + }; + for (const parsed of parsedFiles) { + const dir = rOwningPackageDir(parsed.filePath, cfg); + if (dir === undefined || cfg.namespaceInfoByPackageDir.get(dir)?.hasNamespaceFile !== true) { + continue; + } + const names = namesOf(dir); + for (const name of rFileTopLevel(parsed).names) names.add(name); + } + for (const [dir, info] of cfg.namespaceInfoByPackageDir) { + if (!info.hasNamespaceFile) continue; + const names = namesOf(dir); + for (const entry of info.importFrom) names.add(rBareName(entry.name)); + } + + // Pass 1 (the batch): test every name, packages in directory order, names in code-unit order. + const verdicts: { info: RNamespaceInfo; dir: string; name: string; exported: boolean }[] = []; + for (const dir of [...namesByDir.keys()].sort()) { + const info = cfg.namespaceInfoByPackageDir.get(dir); + if (info === undefined) continue; + for (const name of [...namesOf(dir)].sort()) { + verdicts.push({ info, dir, name, exported: rNamespaceExports(info, name) }); + } + } + + // Pass 2: completeness is read only now, after the whole batch has spent what it will spend. + const memo = new Map>(); + for (const { info, dir, name, exported } of verdicts) { + const notProvenPrivate = info.exportsUnreadable === true || rExportPatternsIncomplete(info); + let byName = memo.get(dir); + if (byName === undefined) memo.set(dir, (byName = new Map())); + byName.set(name, exported ? 'public' : notProvenPrivate ? 'unknown' : 'private'); + } + rRecordSymbolVisibilityMemo(cfg, memo); +} + +/** + * The package config each parsed file was resolved with. The resolution pass records it + * (`recordRPackageConfig`, from the scope resolver's workspace hook) so that the rename guard, + * which receives the parsed files but not the config, can read the NAMESPACE the planner loaded + * from disk for this very plan rather than the export verdict persisted at index time. + */ +const packageConfigOfParsedFile = new WeakMap(); + +/** Record `resolutionConfig` (an {@link RPackageConfig}, if it is one) against each parsed file. */ +export function recordRPackageConfig( + parsedFiles: readonly object[], + resolutionConfig: unknown, +): void { + if (typeof resolutionConfig !== 'object' || resolutionConfig === null) return; + const config = resolutionConfig as RPackageConfig; + if (!(config.namespaceInfoByPackageDir instanceof Map)) return; + for (const parsed of parsedFiles) packageConfigOfParsedFile.set(parsed, config); +} + +/** The config recorded for `parsed`, or undefined when no resolution pass recorded one. */ +export function recordedRPackageConfig(parsed: object | undefined): RPackageConfig | undefined { + return parsed === undefined ? undefined : packageConfigOfParsedFile.get(parsed); +} + +/** Longest pattern prefix quoted in a warning. */ +const QUOTED_PATTERN_LENGTH = 60; + +const quotePattern = (pattern: string): string => + JSON.stringify( + pattern.length > QUOTED_PATTERN_LENGTH + ? `${pattern.slice(0, QUOTED_PATTERN_LENGTH)}...` + : pattern, + ); + +/** + * Warn, once per pattern per package, about every `exportPattern()` that is not in + * effect: one dropped at load ({@link RNamespaceInfo.droppedExportPatterns}) or one + * whose matcher ran out of work budget since (names it had not yet decided were read as + * unexported). Call after the matchers have been used. A dropped pattern exports + * nothing, which would otherwise look exactly like a package that exports few names. + */ +export function reportRExportPatternProblems(config: RPackageConfig): void { + for (const [pkgDir, info] of config.namespaceInfoByPackageDir) { + const where = pkgDir === '' ? '.' : pkgDir; + for (const { pattern, reason } of info.droppedExportPatterns ?? []) { + logger.warn( + `R package ${where}: exportPattern(${quotePattern(pattern)}) was ignored (${reason}); ` + + `names it would export are treated as not exported`, + ); + } + if ((info.omittedDroppedExportPatterns ?? 0) > 0) { + logger.warn( + `R package ${where}: ${info.omittedDroppedExportPatterns} further exportPattern() ` + + `arguments were ignored as well and are not listed; names they would export are ` + + `treated as not exported`, + ); + } + const cutShort = info.exportPatternBudget?.cutShort === true; + let listed = 0; + let unlisted = 0; + for (const matcher of info.exportPatterns) { + if (matcher.exhausted !== true && !cutShort) continue; + if (listed >= MAX_LISTED_DROPPED_PATTERNS) { + unlisted++; + continue; + } + listed++; + logger.warn( + `R package ${where}: exportPattern(${quotePattern(matcher.source)}) exceeded its work ` + + `budget (its own, its package's or the repository's); ` + + `names it had not yet been tested against are treated as not exported`, + ); + } + if (unlisted > 0) { + logger.warn( + `R package ${where}: ${unlisted} further exportPattern() arguments exceeded their work ` + + `budget as well and are not listed; names they had not yet been tested against are ` + + `treated as not exported`, + ); + } + } +} + +/** Directories never descended into, by the main walk and the hidden-package scan alike. */ +const SKIPPED_DIR_NAMES: ReadonlySet = new Set(['node_modules', '.git', '.Rproj.user']); + +/** + * Directories R tooling fills with installed packages or check output: the renv project + * library, the packrat private library and the revdepcheck workspace. Each holds one + * `DESCRIPTION` per third-party package, none of which is part of the repository. + */ +const VENDORED_LIBRARY_DIR_NAMES: ReadonlySet = new Set(['renv', 'packrat', 'revdep']); + +/** Suffix of the `.Rcheck` directories `R CMD check` writes. */ +const RCHECK_SUFFIX = '.Rcheck'; + +/** + * True for a directory that discovery must not enter: `renv`, `packrat` or `revdep`, + * `.Rcheck` (a name that is only the suffix is not one), any dot-directory + * (`.Rlibs`, `.git`, `.Rproj.user`), and every directory the indexer itself never reads + * ({@link isHardcodedIgnoredDirectory}: `vendor`, `build`, `dist`, `lib64`, `fixtures`, ...). + * The indexer indexes no file under those, so a package in one is not part of the repository, + * and a tree of them would otherwise spend the scan budgets. The match is on the directory's + * own name, exact and case-sensitive: `renv-tools`, `Renv`, `revdeps` or `pkg.rcheck` are + * ordinary directories and may hold a real package. The indexer's rules that need the + * directory's location (a root-level `env` or `generated`, an `env` with `pyvenv.cfg`) are not + * applied; neither holds an R package in practice. + */ +function isSkippedDirName(name: string): boolean { + return ( + SKIPPED_DIR_NAMES.has(name) || + VENDORED_LIBRARY_DIR_NAMES.has(name) || + name.startsWith('.') || + isHardcodedIgnoredDirectory(name) || + (name.length > RCHECK_SUFFIX.length && name.endsWith(RCHECK_SUFFIX)) + ); +} + +/** + * Directory entries in code-unit name order. `readdir` order is filesystem-dependent + * (and `localeCompare` is locale-dependent), so every walk below reads through this to + * make what it visits, and what it finds first, the same on every machine. + */ +async function readDirSorted(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + return entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); +} + +/** + * Sort key that ranks the directories declaring one package name: fewer path segments + * first, then the whole `/`-separated relative path compared by code unit. Comparing keys + * with `<` therefore needs no help from the walk order. The depth is zero-padded so that + * it compares as a number; `''` (the repo root) has depth 0. + */ +function preferredPackageDir(pkgDir: string): string { + const depth = pkgDir === '' ? 0 : pkgDir.split('/').length; + return `${String(depth).padStart(6, '0')}\0${pkgDir}`; +} + +/** Global directory budget for {@link hidesUndiscoveredPackage} across all its roots. */ +const HIDDEN_SCAN_BUDGET = 5000; + +/** + * The `Package:` field of a `DESCRIPTION` file's text, or `undefined` when it has none. A leading + * UTF-8 byte-order mark is not part of the first field name, and a field with an empty value + * names nothing: `[ \t]*`, not `\s*`, so the value never begins on the next line. + */ +function descriptionPackageName(content: string): string | undefined { + const text = content.charCodeAt(0) === 0xfeff ? content.slice(1) : content; + return /^Package:[ \t]*(\S+)/m.exec(text)?.[1]; +} + +/** + * The package names of a `DESCRIPTION` file's `Depends:` field: comma-separated entries, each a + * name optionally followed by a version requirement (`pkgB (>= 1.0)`), over the field's + * continuation lines (lines that start with a space or a tab). `R` itself is not a package. + */ +function descriptionDepends(content: string): Set { + const text = content.charCodeAt(0) === 0xfeff ? content.slice(1) : content; + const out = new Set(); + const lines = text.split('\n'); + let start = lines.findIndex((line) => line.startsWith('Depends:')); + if (start === -1) return out; + let field = lines[start].slice('Depends:'.length); + while (++start < lines.length && /^[ \t]/.test(lines[start])) field += `\n${lines[start]}`; + for (const entry of field.split(',')) { + const name = /^\s*([A-Za-z][A-Za-z0-9.]*)/.exec(entry)?.[1]; + if (name !== undefined && name !== 'R') out.add(name); + } + return out; +} + +/** + * True when a `DESCRIPTION` file's text has a `Built:` field. `R CMD INSTALL` adds it to the + * copy it installs; a source package never has one, so a `DESCRIPTION` with it belongs to an + * installed library package (`lib/`, `library/`, a site library checked in), not to the + * repository's own packages. + */ +function isInstalledPackageDescription(content: string): boolean { + const text = content.charCodeAt(0) === 0xfeff ? content.slice(1) : content; + return /^Built:/m.test(text); +} + +/** + * Scan the subtrees the main walk skipped for a package it never discovered. + * + * Iterative DFS over `roots` under one global `budget` of directories (skipping + * the directories {@link isSkippedDirName} names, as the main walk does). Returns + * `true` (cannot rule a hidden package out) when the budget is exceeded, when a + * directory or `DESCRIPTION` cannot be read, or when a `DESCRIPTION` declares a + * `Package:` name that is not in `knownPackageNames`. Returns `false` when every + * subtree was scanned and holds no such package. Deterministic and read-only. + * The answer is a property of the set of directories, not of the order they are + * visited in (every `true` condition is order-independent and a `false` means the + * whole forest was read), but the walk still reads sorted so that it is reproducible. + */ +export async function hidesUndiscoveredPackage( + roots: readonly string[], + knownPackageNames: ReadonlySet, + budget: number = HIDDEN_SCAN_BUDGET, +): Promise { + const stack = [...roots]; + let remaining = budget; + for (let dir = stack.pop(); dir !== undefined; dir = stack.pop()) { + if (--remaining < 0) return true; + let entries; + try { + entries = await readDirSorted(dir); + } catch { + return true; + } + for (const entry of entries) { + if (entry.isDirectory()) { + if (!isSkippedDirName(entry.name)) stack.push(path.join(dir, entry.name)); + } else if (entry.isFile() && entry.name === 'DESCRIPTION') { + try { + const content = await readManifestFile(path.join(dir, entry.name)); + const pkgName = descriptionPackageName(content); + if ( + pkgName !== undefined && + !knownPackageNames.has(pkgName) && + !isInstalledPackageDescription(content) + ) { + return true; + } + } catch { + return true; + } + } + } + } + return false; +} + +/** + * Largest DESCRIPTION or NAMESPACE read (4 MiB). Neither is a parsed source file, so the walker's + * per-file cap does not apply; a real one is a few kilobytes. A larger file is treated like one + * that cannot be read: a DESCRIPTION leaves its package undiscovered (discovery is then marked + * truncated), a NAMESPACE leaves the package without NAMESPACE information (rename refuses). + */ +const MAX_MANIFEST_BYTES = 4 * 1024 * 1024; + +/** + * Read a DESCRIPTION or NAMESPACE as UTF-8. Rejects when the file cannot be read, is not a regular + * file, or is larger than {@link MAX_MANIFEST_BYTES}. + * + * The file is opened once and the type check, the size check and the read all use that one handle, + * so the file cannot be swapped between the check and the read (a `stat` by path followed by a + * `readFile` by path can). `O_NOFOLLOW` makes the open itself refuse a symlink where the platform + * has it; where it does not (Windows) the callers' directory read has already admitted only a + * regular file. `O_NONBLOCK` stops a FIFO blocking the open; it is then refused as not a file. At + * most one byte past the checked size is read, so a file that grows after the check is refused + * rather than read whole (or truncated). + */ +export async function readManifestFile(file: string): Promise { + const handle = await fs.open( + file, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0), + ); + try { + const opened = await handle.stat(); + if (!opened.isFile()) throw new Error(`${path.basename(file)} is not a file`); + const { size } = opened; + if (size > MAX_MANIFEST_BYTES) throw new Error(`${path.basename(file)} is too large to read`); + // One byte more than the size checked above: reading it means the file grew. + const buffer = Buffer.allocUnsafe(size + 1); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const chunk = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead); + if (chunk.bytesRead === 0) break; + bytesRead += chunk.bytesRead; + } + if (bytesRead > size) throw new Error(`${path.basename(file)} changed while it was read`); + return buffer.toString('utf-8', 0, bytesRead); + } finally { + await handle.close(); + } +} + +/** + * Discover the R packages of a repo. Always returns a config: a scan that finds + * no package yields an empty {@link RPackageConfig.packages} map, and its + * {@link RPackageConfig.truncated} flag says whether that result is complete + * (no `DESCRIPTION` anywhere, so no `pkg::fn()` qualifier names a local package) + * or merely what the bounded scan managed to see. + */ +export async function loadRPackageConfig(repoRoot: string): Promise { + const packages = new Map(); + const namespaceInfoByPackageDir = new Map(); + const packageDirs = new Set(); + // One allowance (matching work and NFA states) for every exportPattern of every package + // found by this load, split into a sub-budget per package: the number of patterns a + // repository carries must not multiply the per-matcher cost. + const loadBudget = new SharedWorkBudget(); + const scanQueue: { dir: string; depth: number }[] = [{ dir: repoRoot, depth: 0 }]; + const maxDepth = 3; + const maxDirs = 200; + let dirsScanned = 0; + let truncated = false; + // Directories the walk never visits (depth limit, cap leftovers): scanned at the end. + const unvisited: string[] = []; + + while (scanQueue.length > 0 && dirsScanned < maxDirs) { + const { dir, depth } = scanQueue.shift()!; + dirsScanned++; + try { + const entries = await readDirSorted(dir); + for (const entry of entries) { + if (entry.isDirectory()) { + if (isSkippedDirName(entry.name)) continue; + const child = path.join(dir, entry.name); + if (depth < maxDepth) { + scanQueue.push({ dir: child, depth: depth + 1 }); + } else { + // Below the depth limit: not visited by the walk; scanned below. + unvisited.push(child); + } + } + if (entry.isFile() && entry.name === 'DESCRIPTION') { + try { + const descPath = path.join(dir, entry.name); + const content = await readManifestFile(descPath); + const pkgName = descriptionPackageName(content); + if (pkgName !== undefined && !isInstalledPackageDescription(content)) { + const pkgDir = path.relative(repoRoot, dir).replace(/\\/g, '/'); + packageDirs.add(pkgDir); + // Same name in several directories: keep the preferred one, whatever the + // order the walk met them in (see preferredPackageDir). + const current = packages.get(pkgName); + if ( + current === undefined || + preferredPackageDir(pkgDir) < preferredPackageDir(current) + ) { + packages.set(pkgName, pkgDir); + if (isDev) { + logger.info(`📦 Found R package: ${pkgName} at ${pkgDir}`); + } + } + + // The same test as DESCRIPTION: a regular file listed by the directory read, never + // a symlink (which could point outside the repository). + const nsEntry = entries.find((candidate) => candidate.name === 'NAMESPACE'); + try { + if (nsEntry === undefined || !nsEntry.isFile()) + throw new Error('no NAMESPACE file'); + const nsContent = await readManifestFile(path.join(dir, 'NAMESPACE')); + const parsedExports = parseRNamespaceExports(nsContent); + const namedExports = new Set(parsedExports.namedExports); + const imports = parseRNamespaceImportPackages(nsContent); + // POSIX classes are translated; an uncompilable pattern is skipped (never + // matches) and recorded so the caller can say so. + const exportPatterns: RExportMatcher[] = []; + const droppedExportPatterns: RDroppedExportPattern[] = []; + let omittedDropped = 0; + // This package's share of the load's budget: one hostile package cannot + // take all of it and leave the packages after it without their patterns. + const packageBudget = loadBudget.child(); + for (const pattern of new Set(parsedExports.exportPatterns)) { + const compiled = compileRExportPatternDetailed(pattern, packageBudget); + if ('reason' in compiled) { + if (droppedExportPatterns.length < MAX_LISTED_DROPPED_PATTERNS) { + droppedExportPatterns.push({ pattern, reason: compiled.reason }); + } else { + omittedDropped++; + } + } else { + exportPatterns.push(compiled.matcher); + } + } + + namespaceInfoByPackageDir.set(pkgDir, { + hasNamespaceFile: true, + namedExports, + exportPatterns, + droppedExportPatterns, + omittedDroppedExportPatterns: omittedDropped, + exportPatternBudget: packageBudget, + exportsUnreadable: parsedExports.unreadable, + importFrom: parseRNamespaceImportFrom(nsContent), + s3Registered: new Set((parsedExports.s3Registered ?? []).map(rBareName)), + s3RegisteredOnly: new Set((parsedExports.s3RegisteredOnly ?? []).map(rBareName)), + importPackages: new Set(imports.packages), + depends: descriptionDepends(content), + importsUnreadable: imports.unreadable, + }); + } catch { + // No NAMESPACE file or can't read it + } + } + } catch { + // Can't read DESCRIPTION: the package it declares was not discovered. + truncated = true; + } + } + } + } catch { + // Can't read directory: whatever it holds was not discovered. + truncated = true; + } + } + + // Directories still queued when the cap stopped the walk were never visited either. + for (const { dir } of scanQueue) unvisited.push(dir); + if ( + !truncated && + unvisited.length > 0 && + (await hidesUndiscoveredPackage(unvisited, new Set(packages.keys()))) + ) { + truncated = true; + } + return { packages, namespaceInfoByPackageDir, packageDirs, truncated }; +} diff --git a/gitnexus/src/core/ingestion/languages/r/post-parse.ts b/gitnexus/src/core/ingestion/languages/r/post-parse.ts new file mode 100644 index 000000000..92ecd7e4b --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/post-parse.ts @@ -0,0 +1,230 @@ +/** + * R post-parse hooks. + * + * R classes are expressed as function calls (`R6::R6Class("Name", ...)`, + * `setClass("Name", ...)`, `setMethod("fn", "ClassName", ...)`) rather than + * tree-sitter class syntax, so the worker cannot always find the enclosing + * class node via a parent walk. When that happens, the worker stores the + * owner name as an `ownerNameHint` string on the symbol/graph node. + * + * After parsing is complete and the TypeRegistry is fully populated, we + * resolve every `ownerNameHint` to a real `ownerId` and: + * 1. Patch `node.properties.ownerId` + * 2. Mutate the SymbolDefinition's `ownerId` (shared ref across registries) + * 3. Register the symbol into MutableMethodRegistry / MutableFieldRegistry + * 4. Emit the HAS_METHOD / HAS_PROPERTY graph edge + * + * NAMESPACE-aware export refinement also runs here — ExportChecker doesn't have + * a file-path context, so it defaults all R symbols to public. Once we know + * which package's `R/` directory each file lives in, we flip `isExported` to false + * for symbols that aren't in the NAMESPACE exports. + */ + +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { MutableSemanticModel } from '../../model/semantic-model.js'; +import type { PostParseContext } from '../../language-provider.js'; +import { + loadRPackageConfig, + reportRExportPatternProblems, + rNamespaceExports, + type RNamespaceInfo, + type RPackageConfig, +} from './package-config.js'; +import { rBareName } from './bare-name.js'; +import { rOwningPackageDir } from './namespace-imports.js'; +import type { GraphNode, SymbolDefinition } from 'gitnexus-shared'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { generateId } from '../../../../lib/utils.js'; +import { logger } from '../../../logger.js'; + +type EdgeLabel = 'HAS_METHOD' | 'HAS_PROPERTY'; + +/** Most class names quoted in the warning for owner hints that stay ambiguous. */ +const MAX_LISTED_AMBIGUOUS_OWNERS = 5; + +/** + * The R classes named `name`, one per graph node. The class registry is shared by every + * language, so a Python `Circle` next to an R `Circle` must not make the name ambiguous, nor + * stand in for an R class that is absent. It also appends without de-duplication: a class + * defined twice in one file (a script re-defining it, `if (x) Foo <- R6Class(...) else Foo <- + * R6Class(...)`, a factory called from two places) registers the same node twice, which is + * one class, not an ambiguity. + */ +const distinctRClassDefs = ( + graph: KnowledgeGraph, + model: MutableSemanticModel, + name: string, +): SymbolDefinition[] => { + const byNode = new Map(); + for (const def of model.types.lookupClassByName(name)) { + if (byNode.has(def.nodeId)) continue; + if (graph.getNode(def.nodeId)?.properties.language !== SupportedLanguages.R) continue; + byNode.set(def.nodeId, def); + } + return [...byNode.values()]; +}; + +/** + * The class a member's owner hint names. A single class wins. When several distinct classes + * share the name (the same helper in `R/` and `tests/`, a package copied into a second + * directory), the one defined in the member's own file wins: a method or field written inside + * `Foo <- R6Class(...)` always belongs to the `Foo` of that file, and a `setMethod()` next to + * its `setClass()` does too. When the member's file defines none of them, or several, the owner is + * undecidable from the name and the member stays unowned. + */ +const pickOwnerClass = ( + graph: KnowledgeGraph, + classDefs: readonly SymbolDefinition[], + memberFilePath: string, +): SymbolDefinition | undefined => { + if (classDefs.length === 1) return classDefs[0]; + const sameFile = classDefs.filter( + (def) => graph.getNode(def.nodeId)?.properties.filePath === memberFilePath, + ); + return sameFile.length === 1 ? sameFile[0] : undefined; +}; + +/** + * Resolve R deferred owner hints into ownerIds. Runs post-parse, after all + * Class symbols have been registered into the TypeRegistry. + */ +export const attachDeferredROwners = ( + graph: KnowledgeGraph, + model: MutableSemanticModel, + nodeLabel: 'Method' | 'Property', + edgeType: EdgeLabel, +): void => { + const ambiguousOwners = new Set(); + graph.forEachNode((node: GraphNode) => { + // Skip immediately for non-R nodes — avoids doing any further work per node + // in this full-graph traversal for repos with no R files at all. + if (node.properties.language !== SupportedLanguages.R) return; + if (node.label !== nodeLabel) return; + + // If we already have a resolved ownerId, the hint is redundant — drop it. + if (typeof node.properties.ownerId === 'string') { + delete node.properties.ownerNameHint; + return; + } + + const ownerNameHint = + typeof node.properties.ownerNameHint === 'string' ? node.properties.ownerNameHint : null; + if (!ownerNameHint) return; + + const filePath = typeof node.properties.filePath === 'string' ? node.properties.filePath : ''; + const classDefs = distinctRClassDefs(graph, model, ownerNameHint); + const owner = pickOwnerClass(graph, classDefs, filePath); + if (owner === undefined) { + // Not found, or several classes of this name that the member's own file cannot tell apart: + // leave the hint for debugging and report the second case once per call. + if (classDefs.length > 1) ambiguousOwners.add(ownerNameHint); + return; + } + const ownerId = owner.nodeId; + + // 1. Patch the graph node. + node.properties.ownerId = ownerId; + delete node.properties.ownerNameHint; + + // 2. Mutate the SymbolDefinition in place (shared ref across indexes). + const name = typeof node.properties.name === 'string' ? node.properties.name : ''; + const defs = model.symbols.lookupExactAll(filePath, name); + const def = defs.find((d) => d.nodeId === node.id); + if (def) { + def.ownerId = ownerId; + + // 3. Explicitly register into the owner-scoped registry so resolveMemberCall + // can find the symbol via MethodRegistry / FieldRegistry. + if (nodeLabel === 'Method') { + model.methods.register(ownerId, name, def); + } else { + model.fields.register(ownerId, name, def); + } + } + + // 4. Emit the HAS_METHOD / HAS_PROPERTY edge. + graph.addRelationship({ + id: generateId(edgeType, `${ownerId}->${node.id}`), + sourceId: ownerId, + targetId: node.id, + type: edgeType, + confidence: 1.0, + reason: '', + }); + }); + + if (ambiguousOwners.size > 0) { + const names = [...ambiguousOwners].sort(); + const listed = names.slice(0, MAX_LISTED_AMBIGUOUS_OWNERS).map((n) => JSON.stringify(n)); + const more = names.length - listed.length; + logger.warn( + `R: ${nodeLabel} nodes of ${names.length} class name(s) were left without an owner because ` + + `several classes share the name and the member's own file does not define exactly one ` + + `of them (${listed.join(', ')}${more > 0 ? `, and ${more} more` : ''})`, + ); + } +}; + +/** + * Refine R export status based on NAMESPACE files. A file belongs to the package whose + * `R/` directory contains it (the nearest enclosing discovered package; R loads nothing else + * into a package namespace), so only `R/` symbols are tested against that package's NAMESPACE: + * a symbol that package does not export is flipped to `isExported: false`. Files outside every + * package's `R/` (`tests/`, `vignettes/`, `inst/`, scripts, ...) and packages without a + * NAMESPACE file keep the default public behaviour; a nested package without a NAMESPACE does + * not inherit its enclosing package's. + */ +export const refineRExportStatus = ( + graph: KnowledgeGraph, + rPackageConfig: RPackageConfig | null, +): void => { + if (!rPackageConfig || rPackageConfig.namespaceInfoByPackageDir.size === 0) return; + + // Nodes of one file share an owner; resolve it once per file. + const ownerByFile = new Map(); + const namespaceInfoFor = (filePath: string): RNamespaceInfo | undefined => { + if (ownerByFile.has(filePath)) return ownerByFile.get(filePath); + const pkgDir = rOwningPackageDir(filePath, rPackageConfig); + const info = + pkgDir === undefined ? undefined : rPackageConfig.namespaceInfoByPackageDir.get(pkgDir); + ownerByFile.set(filePath, info); + return info; + }; + + graph.forEachNode((node: GraphNode) => { + if (node.properties.language !== SupportedLanguages.R) return; + const filePath = typeof node.properties.filePath === 'string' ? node.properties.filePath : ''; + if (!filePath) return; + + const nsInfo = namespaceInfoFor(filePath); + if (!nsInfo || !nsInfo.hasNamespaceFile) return; + + const name = typeof node.properties.name === 'string' ? node.properties.name : ''; + if (!name) return; + + // NAMESPACE writes the symbol (`export(`%+%`)`); the node keeps the definition's spelling. + // An explicit export or an exportPattern match keeps the node public. The patterns are + // precompiled linear-time matchers (see loadRPackageConfig): no per-node recompile. An + // `S3method()` registration keeps it public too: resolution does not treat the method as + // exported (`pkg::name` and `library()` do not reach it), but it is public API reached + // through dispatch, so entry-point scoring and process detection see it as before. + const bare = rBareName(name); + if (rNamespaceExports(nsInfo, bare) || nsInfo.s3Registered?.has(bare) === true) return; + + // Not in NAMESPACE → not exported. + node.properties.isExported = false; + }); +}; + +/** + * R's `LanguageProvider.postParse` hook: deferred owner attach for methods and + * properties, then NAMESPACE-driven export refinement. Runs once per analyze, + * after every chunk is merged and before scope resolution. + */ +export const rPostParse = async (ctx: PostParseContext): Promise => { + attachDeferredROwners(ctx.graph, ctx.model, 'Method', 'HAS_METHOD'); + attachDeferredROwners(ctx.graph, ctx.model, 'Property', 'HAS_PROPERTY'); + const config = await loadRPackageConfig(ctx.repoPath); + refineRExportStatus(ctx.graph, config); + reportRExportPatternProblems(config); +}; diff --git a/gitnexus/src/core/ingestion/languages/r/qualified-call.ts b/gitnexus/src/core/ingestion/languages/r/qualified-call.ts new file mode 100644 index 000000000..b2b0528a4 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/qualified-call.ts @@ -0,0 +1,353 @@ +/** + * R `pkg::name()` / `pkg:::name()` qualified calls. + * + * The scope query captures the whole `namespace_operator` as + * `@reference.qualified-name`, so a call site carries `rawQualifiedName` + * (`pkg::f`). Without it every tier discards the qualifier and guesses by + * name: `dplyr::filter()` binds to an unrelated local `filter`, and + * `mypkg::tidy()` inside a wrapper named `tidy` binds to the wrapper itself. + * + * {@link populateRQualifiedCalls} runs at `populateWorkspaceReferences` time + * (before `finalizeScopeModel`, on cold and warm-cache runs alike, with the + * workspace's `RPackageConfig`) and decides each qualified site by the + * {@link rQualifierLocality} of its package: + * + * - `external` (not local, and package discovery completed): the site is + * dropped from `referenceSites`, so no tier can guess an edge for it; + * - `local` with exactly one top-level definition of the name in the + * package's `R/` files: that definition is recorded per call site and + * returned by {@link resolveRQualifiedFreeCall}, which the free-call + * fallback consults before the lexical tiers (a precise 0.85 edge), when + * the operator is `:::`, or `::` and the name is public (the package has no + * NAMESPACE, or exports the name) or the caller is inside the package's + * directory tree ({@link rEnclosingPackageDir}). A `::` site naming a + * private or not-provably-exported name from outside is dropped (R errors + * there). The fallback resolves the hook before it applies a lexical name claim, so a + * caller's parameter, `for` variable or local assignment of the same name does not + * suppress the edge: `::` never consults local bindings; + * - `local` with two or more such definitions (same file or across files): + * the site is dropped (every valid target is inside the named package, so + * any edge would be provably wrong); + * - `local` with no definition: a re-export is the one case that resolves. When + * the package's NAMESPACE exports the name and has `importFrom(origin, name)` for + * a local `origin` that defines the name exactly once, the site is recorded against + * that definition (one hop). Anything else drops the site: a name not exported, + * no `importFrom`, an external or multiply-defining origin, or `:::`; + * - `unknown` locality (discovery truncated, or no config): the site is dropped. + * + * Invariant: after this pass no qualified call site survives without a recorded + * precise target. The free-call fallback never reads `rawQualifiedName` between its + * explicit-path hook and the scope-chain walk, so a kept site without a target would + * walk the lexical scope on the bare tail and bind a caller-local decoy + * (`pkg::missing()` to the caller's own `missing`). + * + * The per-package definition counts are also recorded for the + * global-name-fallback veto (`isRGlobalNameFallbackPlausible`). + * + * Import direction: this file imports `namespace-imports.ts`, never the other + * way round, so `parseRQualifier` and the definition-count registry live there + * and are re-exported here. + */ + +import type { ParsedFile, ReferenceSite, SymbolDefinition } from 'gitnexus-shared'; +import { rBuildSymbolVisibilityMemo, type RPackageConfig } from './package-config.js'; +import { rBareName } from './bare-name.js'; +import { + parseRQualifier, + parseRQualifierOperator, + rEnclosingPackageDir, + rFileBelongsToDiscoveredPackage, + rOwningPackageDir, + rPackageDirForFile, + rPackageSymbolVisibility, + rRecordQualifiedDefinitionCounts, + rReexportOrigin, + rSymbolVisibility, + TOP_LEVEL_BINDABLE_TYPES, +} from './namespace-imports.js'; + +export { parseRQualifier } from './namespace-imports.js'; + +/** Whether a qualifier's package is a package of the analysed repo. */ +export type RQualifierLocality = 'local' | 'external' | 'unknown'; + +/** + * True when `filePath` sits under a `/R/` directory: a `/`-delimited path + * segment equal to `pkg` immediately followed by `R/` (`pkg/R/a.R`, + * `deep/x/pkg/R/a.R`; not `mypkg/R/a.R`, not `pkg/scripts/a.R`). + * + * The directory name stands in for the package name only for a file that no discovered + * package owns (the fallback for a package whose `DESCRIPTION` discovery missed or that has + * none). A file under a discovered package's `R/` belongs to the package its `DESCRIPTION` + * names, whatever its directory is called, so it is never evidence for another name. + */ +function isUnderPackageRDir( + pkg: string, + filePath: string, + cfg: RPackageConfig | null | undefined, +): boolean { + const normalized = filePath.replace(/\\/g, '/'); + if (!normalized.startsWith(`${pkg}/R/`) && !normalized.includes(`/${pkg}/R/`)) return false; + return !rFileBelongsToDiscoveredPackage(normalized, cfg); +} + +/** + * The directory names `` of the `...//R/` paths in `filePaths` that no + * discovered package owns: the names {@link isUnderPackageRDir} would accept, read from all the + * paths once. {@link rQualifierLocality} asks about every qualifier of a run; answering each from + * this set instead of a scan of every path makes the work linear in the paths plus the + * qualifiers instead of their product. + */ +export function rUnownedPackageDirNames( + filePaths: Iterable, + cfg: RPackageConfig | null | undefined, +): ReadonlySet { + const names = new Set(); + for (const filePath of filePaths) { + const parts = filePath.replace(/\\/g, '/').split('/'); + let owned: boolean | undefined; + // `R` must be a directory (not the last segment), with the package directory before it. + for (let i = 0; i + 2 < parts.length; i++) { + if (parts[i + 1] !== 'R' || names.has(parts[i])) continue; + owned ??= rFileBelongsToDiscoveredPackage(parts.join('/'), cfg); + if (owned) break; + names.add(parts[i]); + } + } + return names; +} + +/** + * Locality of the package named by a qualifier. + * + * - `local`: `pkg` is in `cfg.packages`, or a parsed file sits under a + * `/R/` directory (a package whose `DESCRIPTION` discovery missed or + * that has none; root-level packages are found through `cfg` only); + * - `external`: not local, and `cfg` exists and its discovery was complete + * (`!cfg.truncated`), so `pkg` provably is not part of the repo. This + * includes a complete scan that found no package at all (no `DESCRIPTION` + * anywhere): the config is then empty, not absent; + * - `unknown`: not found, but discovery was truncated or there is no config + * (hand-built callers), so `pkg` may still be a local package that was not seen. + * + * One predicate, so the populate step and the veto cannot disagree on "local". + */ +export function rQualifierLocality( + pkg: string, + cfg: RPackageConfig | null | undefined, + parsedFilePaths: ReadonlySet, + /** {@link rUnownedPackageDirNames} of `parsedFilePaths`, when the caller asks many times. */ + unownedPackageDirNames?: ReadonlySet, +): RQualifierLocality { + if (cfg?.packages.has(pkg) === true) return 'local'; + if (unownedPackageDirNames !== undefined) { + if (unownedPackageDirNames.has(pkg)) return 'local'; + } else { + for (const filePath of parsedFilePaths) { + if (isUnderPackageRDir(pkg, filePath, cfg)) return 'local'; + } + } + if (cfg === null || cfg === undefined) return 'unknown'; + return cfg.truncated === true ? 'unknown' : 'external'; +} + +/** + * Call site → the single definition its `pkg::name()` qualifier names, for the + * current run. Module state, replaced wholesale by every + * {@link populateRQualifiedCalls} call. Keyed by file path and start position + * (the key `free-call-fallback.ts` uses for its handled sites), not by object + * identity, so it survives the frozen `ParsedFile` copies made after populate. + */ +let preciseBySite: Map = new Map(); + +const siteKey = (filePath: string, atRange: { startLine: number; startCol: number }): string => + `${filePath}:${atRange.startLine}:${atRange.startCol}`; + +function isQualifiedCall(site: ReferenceSite): site is ReferenceSite & { + readonly rawQualifiedName: string; +} { + return site.kind === 'call' && site.rawQualifiedName !== undefined; +} + +/** + * Per symbol, every top-level function/class definition (one entry per binding + * ref, so a same-file redefinition counts twice) of the given package files. + * Read from Module-scope `local` bindings at extraction time, like + * `rFileTopLevel`, but keeping the definitions rather than the names. The symbol + * is the name without the backticks it may be written in, so `pkg::f()` finds + * `` `f` <- `` and the reverse; definitions spelled both ways count together. + */ +function collectTopLevelDefinitions(files: readonly ParsedFile[]): Map { + const byName = new Map(); + for (const parsed of files) { + const moduleScope = parsed.scopes.find((s) => s.id === parsed.moduleScope); + if (moduleScope === undefined) continue; + for (const [name, refs] of moduleScope.bindings) { + for (const ref of refs) { + if (ref.origin !== 'local' || !TOP_LEVEL_BINDABLE_TYPES.has(ref.def.type)) continue; + const symbol = rBareName(name); + const list = byName.get(symbol); + if (list === undefined) byName.set(symbol, [ref.def]); + else list.push(ref.def); + } + } + } + return byName; +} + +/** + * Decide every qualified call site of `parsedFiles` as described in the file + * header: drop external, unknown-locality, duplicate-definition and not-reachable sites (in + * place, through frozen `ParsedFile` copies), record the precise target of every other site. + * Resets all module state first, so a run never sees the previous run's answers, and builds + * the pass's export-visibility memo (`rBuildSymbolVisibilityMemo`) before judging any site. + */ +export function populateRQualifiedCalls( + parsedFiles: ParsedFile[], + ctx: { readonly resolutionConfig?: unknown }, +): void { + preciseBySite = new Map(); + const cfg = (ctx.resolutionConfig as RPackageConfig | null | undefined) ?? null; + if (cfg !== null) { + rRecordQualifiedDefinitionCounts(cfg, new Map()); + rBuildSymbolVisibilityMemo(parsedFiles, cfg); + } + + // Packages named by some qualified call, and how local each one is. + let filePaths: Set | undefined; + let unownedDirNames: ReadonlySet | undefined; + const localityByPackage = new Map(); + const localityOf = (pkg: string): RQualifierLocality => { + let locality = localityByPackage.get(pkg); + if (locality === undefined) { + filePaths ??= new Set(parsedFiles.map((p) => p.filePath)); + unownedDirNames ??= rUnownedPackageDirNames(filePaths, cfg); + locality = rQualifierLocality(pkg, cfg, filePaths, unownedDirNames); + localityByPackage.set(pkg, locality); + } + return locality; + }; + let hasQualifiedSite = false; + for (const parsed of parsedFiles) { + for (const site of parsed.referenceSites) { + if (!isQualifiedCall(site)) continue; + hasQualifiedSite = true; + const pkg = parseRQualifier(site.rawQualifiedName); + if (pkg !== undefined) localityOf(pkg); + } + } + if (!hasQualifiedSite) return; + + // Top-level definitions of a local package, by name (computed on first use, once per package). + const definitionsByPackage = new Map>(); + const definitionsOf = (pkg: string): Map => { + let byName = definitionsByPackage.get(pkg); + if (byName === undefined) { + const files: ParsedFile[] = []; + for (const parsed of parsedFiles) { + // In the config a package owns its `R/` files (longest-dir wins); otherwise + // it is local only through the `/R/` path, as `rQualifierLocality` decided. + const inPackage = + cfg?.packages.has(pkg) === true + ? rPackageDirForFile(parsed.filePath, cfg)?.name === pkg + : isUnderPackageRDir(pkg, parsed.filePath, cfg); + if (inPackage) files.push(parsed); + } + byName = collectTopLevelDefinitions(files); + definitionsByPackage.set(pkg, byName); + } + return byName; + }; + + // Decide each site. + for (let i = 0; i < parsedFiles.length; i++) { + const parsed = parsedFiles[i]; + const kept: ReferenceSite[] = []; + for (const site of parsed.referenceSites) { + if (!isQualifiedCall(site)) { + kept.push(site); + continue; + } + const pkg = parseRQualifier(site.rawQualifiedName); + const target = + pkg === undefined || localityOf(pkg) !== 'local' + ? undefined + : resolveLocalQualifiedSite( + pkg, + rBareName(site.name), + parseRQualifierOperator(site.rawQualifiedName) === ':::', + parsed.filePath, + ); + if (target === undefined) continue; + preciseBySite.set(siteKey(parsed.filePath, site.atRange), target); + kept.push(site); + } + if (kept.length !== parsed.referenceSites.length) { + parsedFiles[i] = Object.freeze({ ...parsed, referenceSites: Object.freeze(kept) }); + } + } + + if (cfg !== null) { + rRecordQualifiedDefinitionCounts( + cfg, + new Map( + [...definitionsByPackage].map(([pkg, byName]) => [ + pkg, + new Map([...byName].map(([name, defs]) => [name, defs.length])), + ]), + ), + ); + } + + /** + * The definition `pkg::symbol` (or `pkg:::symbol`) reaches from `callerPath`, or `undefined` + * when the site must be dropped: no definition and no resolvable re-export, several + * definitions, or a `::` to a name the package does not make public that the caller, outside + * the package's tree, cannot reach. + */ + function resolveLocalQualifiedSite( + pkg: string, + symbol: string, + internal: boolean, + callerPath: string, + ): SymbolDefinition | undefined { + const found = definitionsOf(pkg).get(symbol) ?? []; + if (found.length >= 2) return undefined; + if (found.length === 1) { + if (internal) return found[0]; + const visibility = rSymbolVisibility(cfg, found[0].filePath, symbol); + if (visibility === 'public') return found[0]; + const owner = rOwningPackageDir(found[0].filePath, cfg); + return owner !== undefined && rEnclosingPackageDir(callerPath, cfg) === owner + ? found[0] + : undefined; + } + // No definition: only a `::` re-export (`export(name)` + `importFrom(origin, name)`) resolves. + const pkgDir = cfg?.packages.get(pkg); + if (internal || cfg === null || pkgDir === undefined) return undefined; + if (rPackageSymbolVisibility(cfg, pkgDir, symbol) !== 'public') return undefined; + if (cfg.namespaceInfoByPackageDir.get(pkgDir)?.hasNamespaceFile !== true) return undefined; + const origin = rReexportOrigin(cfg, pkgDir, pkg, symbol); + if (origin === undefined || localityOf(origin) !== 'local') return undefined; + const originDefs = definitionsOf(origin).get(symbol) ?? []; + return originDefs.length === 1 ? originDefs[0] : undefined; + } +} + +/** + * `ScopeResolver.resolveQualifiedFreeCall` for R: the definition + * {@link populateRQualifiedCalls} recorded for this call site, if any. + * + * The contract types `site` without its position; the runtime object is the + * `ReferenceSite`, which has `atRange`. That is read through a narrow cast + * here instead of widening the shared contract. + */ +export function resolveRQualifiedFreeCall( + site: object, + callerParsed: ParsedFile, +): SymbolDefinition | undefined { + const atRange = (site as { readonly atRange?: { startLine: number; startCol: number } }).atRange; + return atRange === undefined + ? undefined + : preciseBySite.get(siteKey(callerParsed.filePath, atRange)); +} diff --git a/gitnexus/src/core/ingestion/languages/r/query.ts b/gitnexus/src/core/ingestion/languages/r/query.ts new file mode 100644 index 000000000..2c941cc09 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/query.ts @@ -0,0 +1,415 @@ +/** + * Tree-sitter query for R scope captures (RFC #909 Ring 3). + * + * R has no dedicated class syntax — R6 classes are `R6Class(...)` calls, S4/R5 + * classes are `setClass(...)`/`setRefClass(...)` calls — so this query mirrors + * the shapes the legacy `R_QUERIES` (`tree-sitter-queries.ts`) already proved + * queryable for structural extraction (the heritage shapes below have no + * counterpart there), renamed onto the scope-resolution capture vocabulary + * (`@scope.*`, `@declaration.*`, `@reference.*`, `@import.*`, + * `@type-binding.*`). + * + * R specifics encoded here: + * + * - Every `binary_operator` pattern is an assignment: `operator: ["<-" "<<-" "="]` + * (see `assignment.ts`). The grammar uses one node for every infix operator, so + * without it `x + R6Class(...)` or `x == function() 1` would declare something. + * + * - A Class scope spans the ENTIRE `R6Class(...)`/`setClass(...)` call + * (the whole `binary_operator` for the `name <- R6Class(...)` shape), + * not just `public = list(...)` — the `inherit=`/`contains=` argument + * must fall inside the scope's range so the `@reference.inherits` sites + * resolve against the deriving class's own scope. + * + * - `@declaration.function`/`@declaration.class` anchor on the WHOLE + * `name <- ` assignment (matching the legacy `@definition.*` + * anchors exactly), so `resolveDefGraphId`'s position-based (Function) + * and qualifiedName-keyed (Class) reconciliation lands on the SAME + * graph node the legacy extractor already created. + * + * - `@declaration.method` anchors on the `name = function(...)` ARGUMENT + * inside `public =`/`private =`/`active =`/`methods = list(...)`. + * + * - A def is owned by the innermost scope whose range EQUALS its anchor + * range (`scope-extractor.ts` Pass 2), and `resolveCallerGraphId` walks + * from a call site up to the first scope that owns a callable def. The + * blanket `(function_definition) @scope.function` is strictly narrower + * than the assignment/argument anchors, so on its own it never owns a + * def: every named function/method would be owned by the module or + * Class scope and every call in the file would be credited to the + * FIRST callable in it. So each named-callable anchor also gets a + * `@scope.function` on the SAME node, placed directly beside its + * `@declaration.*` pattern below (the two must stay in lockstep — a + * declaration anchor without a matching scope silently reverts that + * shape to first-callable attribution). The def's own binding still + * hoists to the PARENT scope (anchor range == scope range), so name + * lookup is unchanged. The blanket rule is kept: it still covers + * anonymous functions (`lapply(x, function(z) ...)`), which own nothing. + * Calls in a Class body outside any method (an R6 field default, a + * `setClass(validity = function...)` body) are credited to the Class. + * + * - No `@declaration.qualified_name` is emitted for `@declaration.class` + * — R's legacy extractor has no `classExtractor`, so a Class node's + * `qualifiedName` today is the bare class-name text. Omitting the + * capture lets `buildDefFromDeclarationMatch` default `qualifiedName` + * to the bare name, matching the legacy node so `resolveDefGraphId`'s + * name-keyed fallback finds it. + * + * - S4/setGeneric/setMethod declarations are deliberately NOT captured + * here — they are already produced by the legacy extractor via + * R_QUERIES, independent of scope-resolution, and none of the target + * scenarios needs a NEW declaration for them. Their inner + * `function_definition` bodies still get a `@scope.function` from the + * blanket rule below (an unnamed scope, harmless). + * + * - `library()`/`require()` are wildcard imports (`@import.wildcard`) — + * the named local package's exported names become visible (every + * top-level name when it has no NAMESPACE file; see + * `rScopeResolver.expandsWildcardTo`, which filters by NAMESPACE). + * `source()` is a side-effect import (`@import.side-effect`) — it + * establishes a file dependency but binds no name; cross-file free + * calls into a `source()`d file resolve via `allowGlobalFreeCallFallback` + * instead (`rScopeResolver`). + * + * - `obj$method()` (the `extract_operator`, R's `$`) is a member call with + * an explicit receiver — the same shape as Ruby's `obj.method`. + * `Type$new(...)` is R6's constructor spelling: captured separately as + * `@type-binding.constructor` so `x <- Type$new(...)` types `x`. + * + * - `a$b$m()` and `Cls$new()$m()` (the receiver is itself an `extract_operator` or a `call`) + * are chained member calls. R never types through a chain (R6 fields and results carry no + * declared type), so `emitRScopeCaptures` mints `@reference.receiver-chain` with the shared + * codec's `truncated` marker: the shared fold refuses it and the call ends as an unresolved + * receiver (an outcome, no edge) instead of reaching a same-named free function. A chain + * with no variable base mints nothing at all (an invisible gap, see `rReceiverChain`). + * + * Exposes lazy `Parser` and `Query` singletons so callers don't pay + * tree-sitter init cost per file. + */ + +import Parser from 'tree-sitter'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { getLanguageGrammar } from '../../../tree-sitter/parser-loader.js'; + +/** + * Exported for `value-ref-dispatchability.test.ts`-style whole-query readers. + * Not part of the provider surface — nothing else should import it. + */ +export const R_SCOPE_QUERY = ` +;; ── Scopes ─────────────────────────────────────────────────────────────── + +(program) @scope.module + +(function_definition) @scope.function + +;; R6 class via namespace call (ClassName <- R6::R6Class(...)) — the scope +;; spans the WHOLE assignment, including inherit=, so heritage sites resolve +;; inside the deriving class's own scope. +(binary_operator + lhs: (identifier) + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_r6ns_scope + (#match? @_r6ns_scope "^R6Class$")))) @scope.class + +;; R6 class via bare call (ClassName <- R6Class(...)) +(binary_operator + lhs: (identifier) + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6bare_scope + (#match? @_r6bare_scope "^R6Class$"))) @scope.class + +;; S4/R5 class (setClass(...) / setRefClass(...)) — scope spans the whole call. +(call + function: (identifier) @_s4_scope + (#match? @_s4_scope "^(setClass|setRefClass)$")) @scope.class + +;; Braces that R evaluates in an environment of their own (\`local({...})\`, \`with(data, {...})\`, +;; \`within(data, {...})\`, testthat's \`test_that(desc, {...})\`, \`describe()\` and \`it()\`): a +;; definition inside binds in the block, not in the module or the enclosing function, so it is +;; reachable from those braces only. \`withr::with_*()\` and \`suppressWarnings({...})\` evaluate +;; their braces in the caller's frame and get no scope. Bare or \`pkg::\`-qualified. +(call + function: (identifier) @_blkfn + (#match? @_blkfn "^(local|test_that|describe|it|with|within)$") + arguments: (arguments + (argument + value: (braced_expression) @scope.block))) + +(call + function: (namespace_operator + rhs: (identifier) @_blkfn_ns + (#match? @_blkfn_ns "^(local|test_that|describe|it|with|within)$")) + arguments: (arguments + (argument + value: (braced_expression) @scope.block))) + +;; ── Declarations — functions ────────────────────────────────────────────── + +(binary_operator + lhs: (identifier) @declaration.name + operator: ["<-" "<<-" "="] + rhs: (function_definition)) @declaration.function + +;; Function scope on the SAME assignment node as the declaration above, so the +;; def is owned by its own scope (and calls inside the body attribute to it). +;; This pattern and the \`@declaration.function\` pattern above must stay in +;; lockstep — identical structure, identical anchor node (see file header). +(binary_operator + lhs: (identifier) + operator: ["<-" "<<-" "="] + rhs: (function_definition)) @scope.function + +;; ── Declarations — classes (name only; no qualified_name, see file header) ─ +;; +;; The setClass/setRefClass declaration and the source/library/require imports +;; below capture EVERY string (imports: identifier or string) argument of the call. +;; \`emitRScopeCaptures\` keeps only the one that names the class or import +;; (\`isRNonNamingScopeMatch\`): the argument spelled with the first formal, else the +;; first unnamed one, so a leading comment or a named \`lib.loc =\` / \`local =\` does +;; not change the answer. A query-level anchor cannot skip an unbounded run of +;; leading comments and named arguments without a repeated sibling group, which +;; is quadratic in the argument count on every call. + +(binary_operator + lhs: (identifier) @declaration.name + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_r6ns_decl + (#match? @_r6ns_decl "^R6Class$")))) @declaration.class + +(binary_operator + lhs: (identifier) @declaration.name + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6bare_decl + (#match? @_r6bare_decl "^R6Class$"))) @declaration.class + +(call + function: (identifier) @_s4_decl + (#match? @_s4_decl "^(setClass|setRefClass)$") + arguments: (arguments + (argument + value: (string + content: (string_content) @declaration.name)))) @declaration.class + +;; ── Declarations — R6/R5 methods ────────────────────────────────────────── + +;; R6 methods via namespace call (inside public/private/active = list(...)) +(binary_operator + lhs: (identifier) @_class1 + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_r6ns_m + (#match? @_r6ns_m "^R6Class$")) + arguments: (arguments + (argument + name: (identifier) @_section1 + (#match? @_section1 "^(public|private|active)$") + value: (call + function: (identifier) @_listfn1 + (#match? @_listfn1 "^list$") + arguments: (arguments + (argument + name: (identifier) @declaration.name + value: (function_definition)) @declaration.method)))))) + +;; R6 methods via bare call (inside public/private/active = list(...)) +(binary_operator + lhs: (identifier) @_class2 + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6bare_m + (#match? @_r6bare_m "^R6Class$") + arguments: (arguments + (argument + name: (identifier) @_section2 + (#match? @_section2 "^(public|private|active)$") + value: (call + function: (identifier) @_listfn2 + (#match? @_listfn2 "^list$") + arguments: (arguments + (argument + name: (identifier) @declaration.name + value: (function_definition)) @declaration.method)))))) + +;; R5 methods (inside setRefClass(... methods = list(...))) — the call itself, assigned +;; or not, so \`setRefClass("A", methods = list(...))\` defines them like +;; \`A <- setRefClass(...)\`. +(call + function: (identifier) @_refFn_m + (#match? @_refFn_m "^setRefClass$") + arguments: (arguments + (argument + name: (identifier) @_methods3 + (#match? @_methods3 "^methods$") + value: (call + function: (identifier) @_listfn3 + (#match? @_listfn3 "^list$") + arguments: (arguments + (argument + name: (identifier) @declaration.name + value: (function_definition)) @declaration.method))))) + +;; Function scope on the SAME \`name = function(...)\` argument node that the +;; three \`@declaration.method\` patterns above anchor on, so each R6/R5 method +;; def is owned by its own Function scope (a child of the Class scope) rather +;; than by the Class scope. Deliberately generic — any named function-valued +;; argument — so it cannot drift from the three long nesting patterns above; it +;; is inert for non-method arguments (\`FUN = function...\`, \`error = function(e)\` +;; own no defs, and calls inside them walk up to the enclosing callable). The +;; declaration patterns above and this pattern must stay in lockstep. +(argument + name: (identifier) + value: (function_definition)) @scope.function + +;; ── Imports ──────────────────────────────────────────────────────────────── + +;; source("path.R") — side-effect: establishes a file dependency, binds no name. +(call + function: (identifier) @_srcfn + (#match? @_srcfn "^source$") + arguments: (arguments + (argument + value: [(identifier) (string)] @import.source))) @import.side-effect + +;; library("pkg") / require("pkg") — wildcard: the exported names of the +;; named local package become visible (rScopeResolver.expandsWildcardTo). +(call + function: (identifier) @_libfn + (#match? @_libfn "^(library|require)$") + arguments: (arguments + (argument + value: [(identifier) (string)] @import.source))) @import.wildcard + +;; ── Type bindings — R6 constructor (x <- Type$new(...)) ─────────────────── + +(binary_operator + lhs: (identifier) @type-binding.name + operator: ["<-" "<<-" "="] + rhs: (call + function: (extract_operator + lhs: (identifier) @type-binding.type + rhs: (identifier) @_newfn + (#match? @_newfn "^new$")))) @type-binding.constructor + +;; ── References — heritage: R6 inherit= ──────────────────────────────────── + +(binary_operator + lhs: (identifier) + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_r6ns_h + (#match? @_r6ns_h "^R6Class$")) + arguments: (arguments + (argument + name: (identifier) @_inh1 + (#match? @_inh1 "^inherit$") + value: (identifier) @reference.inherits)))) + +(binary_operator + lhs: (identifier) + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6bare_h + (#match? @_r6bare_h "^R6Class$") + arguments: (arguments + (argument + name: (identifier) @_inh2 + (#match? @_inh2 "^inherit$") + value: (identifier) @reference.inherits)))) + +;; ── References — heritage: S4 contains= (single parent, incl. "VIRTUAL") ── + +(call + function: (identifier) @_s4h1 + (#match? @_s4h1 "^(setClass|setRefClass)$") + arguments: (arguments + (argument + name: (identifier) @_contains1 + (#match? @_contains1 "^(contains)$") + value: (string + content: (string_content) @reference.inherits)))) + +;; ── References — heritage: S4 contains=c(...) (multiple parents) ────────── + +(call + function: (identifier) @_s4h2 + (#match? @_s4h2 "^(setClass|setRefClass)$") + arguments: (arguments + (argument + name: (identifier) @_contains2 + (#match? @_contains2 "^(contains)$") + value: (call + function: (identifier) @_cfn2 + (#match? @_cfn2 "^c$") + arguments: (arguments + (argument + value: (string + content: (string_content) @reference.inherits))))))) + +;; ── References — calls ──────────────────────────────────────────────────── + +;; Free calls: foo(x). emitRScopeCaptures renames the call to its setter (foo<-) when it sits in +;; an assignment target (foo(x) <- v), see isRReplacementCall in assignment.ts. +(call + function: (identifier) @reference.name) @reference.call.free + +;; Namespaced calls: pkg::func() and pkg:::func(). The whole namespace_operator +;; is captured as the qualified name so the qualifier survives the parse cache. +(call + function: (namespace_operator + rhs: (identifier) @reference.name) @reference.qualified-name) @reference.call.free + +;; Member calls via $: obj$method(). Only $: the grammar uses extract_operator for @ too, and +;; u@save() on an R6 instance is an error in R, not a member call (no site). +(call + function: (extract_operator + lhs: (identifier) @reference.receiver + operator: "$" + rhs: (identifier) @reference.name)) @reference.call.member + +;; Chained member calls: a$b$m(), Cls$new()$m(). The receiver is itself an extract_operator or a +;; call; the pattern is disjoint from the one above (an identifier receiver), so no call is +;; matched twice. emitRScopeCaptures turns the receiver into a truncated chain, or drops the match +;; when the chain has no variable base (get_user()$m(), pkg::Cls$new()$m(), x[[1]]$f$m()). +(call + function: (extract_operator + lhs: [(extract_operator) (call)] @reference.receiver + operator: "$" + rhs: (identifier) @reference.name)) @reference.call.member +`; + +let _parser: Parser | null = null; +let _query: Parser.Query | null = null; + +/** + * The R grammar, loaded on first use through the shared loader rather than by a static import: + * a native binding that fails to load must disable R parsing, not every language that shares + * this module graph. + */ +const getRGrammar = (): Parameters[0] => + getLanguageGrammar(SupportedLanguages.R) as Parameters[0]; + +export function getRParser(): Parser { + if (_parser === null) { + // Cache the parser only once its language is set: a failed grammar load must leave nothing + // behind, so the next call retries (or throws again) instead of returning a language-less parser. + const parser = new Parser(); + parser.setLanguage(getRGrammar()); + _parser = parser; + } + return _parser; +} + +export function getRScopeQuery(): Parser.Query { + if (_query === null) { + _query = new Parser.Query(getRGrammar(), R_SCOPE_QUERY); + } + return _query; +} diff --git a/gitnexus/src/core/ingestion/languages/r/r6-self.ts b/gitnexus/src/core/ingestion/languages/r/r6-self.ts new file mode 100644 index 000000000..ad026cd80 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/r6-self.ts @@ -0,0 +1,230 @@ +/** + * What `self` is inside an R6 method. + * + * R6 methods are `name = function(...)` arguments of the `public`/`private`/`active` list of an + * `R6Class(...)` call, and `self` in their body is the instance of that class. The class is the + * name the `R6Class` call is assigned to (`Child <- R6::R6Class(...)`), the same name the + * `Class` definition carries, so `self$m()` resolves through that class and its parents. + * Anywhere else `self` is no R6 instance: a free function, an `R6Class` call that is assigned to + * nothing, a `setRefClass` method (which uses `.self`). The scope there owns `self` without a + * type, and a `self$m()` call is left unresolved rather than matched to a free function `m`. + * + * `self` is the public interface of the instance: a member of the `private` list is reached + * through `private$m()`, never through `self$m()`. The definitions carry no section, so the + * capture pass records where the private members are (`recordRR6PrivateMembers`), per file, and + * the record crosses the worker boundary as the file's capture side channel. + */ +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import { definitionIdPosition } from '../../scope-resolution/utils/definition-id.js'; +import { assertCloneable } from '../../workers/clone-safety.js'; +import { isRLeftAssignment, rBoundName } from './assignment.js'; + +const R6_SECTIONS: ReadonlySet = new Set(['public', 'private', 'active']); + +/** The section name of an `argument` node (`public = list(...)`), or null. */ +const argumentName = (argument: SyntaxNode): string | null => { + const name = argument.childForFieldName('name'); + return name?.type === 'identifier' ? name.text : null; +}; + +/** Whether `call` is `R6Class(...)` or `R6::R6Class(...)`. */ +const isR6ClassCall = (call: SyntaxNode): boolean => { + if (call.type !== 'call') return false; + const fn = call.childForFieldName('function'); + if (fn?.type === 'identifier') return fn.text === 'R6Class'; + return fn?.type === 'namespace_operator' && fn.childForFieldName('rhs')?.text === 'R6Class'; +}; + +/** The `call` an `arguments` node's `argument` belongs to, or null. */ +const callOf = (argument: SyntaxNode): SyntaxNode | null => { + const list = argument.parent; + const call = list?.type === 'arguments' ? list.parent : null; + return call?.type === 'call' ? call : null; +}; + +/** + * The class a method's `self` is, or null when `argument` is not an R6 method. + * + * `argument` is a `name = function(...)` argument of `list(...)`, itself the value of a + * `public`/`private`/`active` argument of an `R6Class(...)` call that is the right-hand side of + * an assignment to a plain identifier. The name is returned as written, backticks included, as + * the `Class` definition keeps it. + */ +export function rR6MethodClassName(argument: SyntaxNode): string | null { + return rR6Method(argument)?.className ?? null; +} + +/** {@link rR6MethodClassName} and the section (`public`, `private`, `active`) the method is in. */ +export function rR6Method( + argument: SyntaxNode, +): { readonly className: string; readonly section: string } | null { + if (argument.type !== 'argument') return null; + if (argument.childForFieldName('name')?.type !== 'identifier') return null; + if (argument.childForFieldName('value')?.type !== 'function_definition') return null; + + const listCall = callOf(argument); + if (listCall === null) return null; + const listFn = listCall.childForFieldName('function'); + if (listFn?.type !== 'identifier' || listFn.text !== 'list') return null; + + const section = listCall.parent; + if (section?.type !== 'argument') return null; + if (section.childForFieldName('value')?.id !== listCall.id) return null; + const sectionName = argumentName(section); + if (sectionName === null || !R6_SECTIONS.has(sectionName)) return null; + + const r6Call = callOf(section); + if (r6Call === null || !isR6ClassCall(r6Call)) return null; + + const assignment = r6Call.parent; + if (assignment === null || !isRLeftAssignment(assignment)) return null; + if (assignment.childForFieldName('rhs')?.id !== r6Call.id) return null; + const target = assignment.childForFieldName('lhs'); + if (target?.type !== 'identifier' || target.text.length === 0) return null; + return { className: target.text, section: sectionName }; +} + +/** + * Whether the file binds the name `self` at top level (`self <- Foo$new()`, `self = ...`, + * `... -> self`). Its free functions then read that variable, which may be typed, so they are not + * frames that own `self` untyped. + */ +export function fileBindsSelf(program: SyntaxNode): boolean { + for (let i = 0; i < program.namedChildCount; i++) { + const child = program.namedChild(i); + if (child !== null && rBoundName(child) === 'self') return true; + } + return false; +} + +/** + * A per-file classifier for {@link rFunctionOwnsSelf}-style questions: whether a + * `function_definition` starts a frame in which `self` is not an R6 instance. That holds when it + * is neither the function of an R6 method nor nested inside another function (a closure in a + * method sees the method's `self`; a closure in a free function sees the free function's, which + * is none). The answer for an ancestor is remembered, so the work over a file is linear in its + * size however deeply functions nest. + */ +export function createSelfOwnerClassifier(): (fn: SyntaxNode) => boolean { + const insideFunction = new Map(); + const isInsideFunction = (node: SyntaxNode): boolean => { + const path: number[] = []; + let answer = false; + for (let up = node.parent; up !== null; up = up.parent) { + if (up.type === 'function_definition') { + answer = true; + break; + } + const known = insideFunction.get(up.id); + if (known !== undefined) { + answer = known; + break; + } + path.push(up.id); + } + for (const id of path) insideFunction.set(id, answer); + return answer; + }; + return (fn) => { + if (fn.type !== 'function_definition') return false; + const parent = fn.parent; + if (parent !== null && rR6MethodClassName(parent) !== null) return false; + return !isInsideFunction(fn); + }; +} + +/** The capture side channel of an R file: where its R6 private members are defined. */ +interface RCaptureSideChannel { + readonly kind: 'r'; + /** `line:column` of each private member's definition anchor, as its definition id encodes it. */ + readonly r6PrivateMembers: readonly string[]; +} + +const r6PrivateMembersByFile = new Map>(); + +/** The `line:column` key of a node, as `nodeToCapture` and the definition id write it. */ +export const rDefinitionPositionKey = (node: SyntaxNode): string => + `${node.startPosition.row + 1}:${node.startPosition.column}`; + +/** + * Record the private R6 members of `filePath` (keys from {@link rDefinitionPositionKey}), + * replacing what an earlier parse of a file at that path recorded. + */ +export function recordRR6PrivateMembers(filePath: string, keys: ReadonlySet): void { + if (keys.size === 0) r6PrivateMembersByFile.delete(filePath); + else r6PrivateMembersByFile.set(filePath, keys); +} + +/** `LanguageProvider.collectCaptureSideChannel` for R. */ +export function collectRCaptureSideChannel(filePath: string): RCaptureSideChannel | undefined { + const keys = r6PrivateMembersByFile.get(filePath); + if (keys === undefined) return undefined; + return assertCloneable({ kind: 'r', r6PrivateMembers: [...keys] }); +} + +/** `ScopeResolver.applyCaptureSideChannel` for R: restore the record of a reused file. */ +export function applyRCaptureSideChannel(parsed: ParsedFile): void { + const facts = parsed.captureSideChannel as RCaptureSideChannel | undefined; + recordRR6PrivateMembers( + parsed.filePath, + new Set(facts?.kind === 'r' ? facts.r6PrivateMembers : []), + ); +} + +/** Whether `def` is a member of an R6 class's `private` list. */ +function isRR6PrivateMember(def: SymbolDefinition): boolean { + const position = definitionIdPosition(def.nodeId, def.filePath); + if (position === undefined) return false; + const keys = r6PrivateMembersByFile.get(def.filePath); + return keys?.has(`${position.line}:${position.column}`) === true; +} + +/** + * `ScopeResolver.resolveReceiverMember` for R: every typed receiver, `self` in an R6 method and a + * local typed by a constructor (`u <- User$new(); u$m()`) alike. The name is historical. + * + * `x$m()` reads `m` off the public interface of the instance: the public (and active) members of + * the class and its parents. A private member is not on the instance, so it is skipped, and a + * parent's public member of that name is the answer even when the class has a private one. When + * no public member is found, the lookup is reported ambiguous with no candidate. + * + * The generic lookup keeps a lexical route for `self` (a class body binds its own members, so the + * name is also looked up in the enclosing scopes), and in R that walk reaches a free function of + * the same name: `self$nothere()` would bind a top-level `nothere`. For any typed receiver, a + * member the owner lookup misses (a dotted member such as `to.json`, which the registry keys by + * its last segment) would fall to a global lookup of the bare name that ignores the receiver. R + * reads the member off the instance, so a free function is never the answer. Answering + * `ambiguous` makes the pass record a suppressed outcome and mark the site handled, which stops + * those fallbacks; the outcome reason is `member-lookup-ambiguous`, where `member-not-found` would + * be more exact. A member found before any private one was skipped is left to the generic MRO walk + * (`undefined`), which finds the same one; past a skipped private member the answer is given here, + * because that walk would stop at it. + */ +export const rResolveSelfMember: NonNullable = ( + owner, + name, + site, + scopes, + model, +) => { + if (site.kind !== 'call') return undefined; + let skippedPrivate = false; + for (const ownerId of [owner.nodeId, ...scopes.methodDispatch.mroFor(owner.nodeId)]) { + const methods = model.methods.lookupAllByOwner(ownerId, name); + const visible = methods.filter((def) => !isRR6PrivateMember(def)); + if (visible.length < methods.length) skippedPrivate = true; + if (visible.length > 0) { + if (!skippedPrivate) return undefined; + const [only] = visible; + return visible.length === 1 && only !== undefined + ? { kind: 'resolved', definition: only } + : { kind: 'ambiguous', candidateIds: visible.map((def) => def.nodeId) }; + } + if (model.fields.lookupAllByOwner(ownerId, name).length > 0) { + return skippedPrivate ? { kind: 'ambiguous', candidateIds: [] } : undefined; + } + } + return { kind: 'ambiguous', candidateIds: [] }; +}; diff --git a/gitnexus/src/core/ingestion/languages/r/receiver-agreement.ts b/gitnexus/src/core/ingestion/languages/r/receiver-agreement.ts new file mode 100644 index 000000000..372057d1d --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/receiver-agreement.ts @@ -0,0 +1,266 @@ +/** + * Which constructor type bindings (`x <- T$new()`) may type a receiver: only those whose name the + * frame binds to nothing but that constructor. + * + * The scope query emits one `@type-binding.constructor` per `x <- T$new()`, `x = T$new()` and + * `x <<- T$new()`, and the shared extractor keeps one type per name per scope, the later binding + * winning. On its own that types `x` as `T` however else the frame binds `x`: a second class, an + * untyped value, a loop variable, a formal, `rm(x)`. R calls the method of whatever `x` holds at + * the call, which the order of bindings alone does not decide (branches, loops, closures). So a + * constructor types `x` only when every binding of `x` in its frame agrees with it, and otherwise + * the receiver is left untyped: an unresolved call, never an edge to another class's method. + * + * A frame is the file's top level (`program`) or a `function_definition`. The braces of + * `local({...})` and of every other call belong to the enclosing frame. Frames are per file: a + * package namespace spans files, so a rebinding in another file is not seen. + * + * Each binding of a name gives one signature in its frame: + * - `ctor:T` for `x <- T$new()`, `x = T$new()`, `x <<- T$new()` and the right-assignment forms + * (`T$new() -> x` binds no type, but it still counts), with `T` read as the query reads it; + * - neutral for `NULL` and a constant (`x$m()` on one always errors, so it never names another + * class for a call that runs) and for a write through `x` that leaves it by `$`, `@`, `[[` or + * `[` (`x$a <- v`, `x[[i]] <- v`, `names(x$a) <- v`): an R6 instance is a reference, so the + * write keeps it the same object, as for `self`; + * - `other` for anything else: another value, a formal, a `for` variable, `rm(x)` or + * `remove(x)` (a name, a string, or `list = "x"`), `assign("x", ...)` and + * `delayedAssign("x", ...)` with a literal name, and a replacement call applied to `x` itself + * (`class(x) <- v` binds `x` to what the setter returns). + * + * `<<-` and `->>` never bind in their own frame: they record their signature into every strictly + * enclosing frame up to the top level (at the top level, into the top level). So in + * `function() { x <<- User$new(); x$save() }` the name is free in the function and the call stays + * untyped. + * + * `with()` and `within()` are not modelled, nor `assign()` with a computed name or another + * environment (read as the current frame). + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { rBareName } from './bare-name.js'; +import { isRConstantValue } from './assignment.js'; + +const OTHER = 'other'; +const NEUTRAL = 'neutral'; + +/** `T$new()` (any extract operator, as the type-binding query reads it): `ctor:T`, else null. */ +function constructorSignature(value: SyntaxNode): string | null { + if (value.type !== 'call') return null; + const fn = value.childForFieldName('function'); + if (fn?.type !== 'extract_operator') return null; + const lhs = fn.childForFieldName('lhs'); + const rhs = fn.childForFieldName('rhs'); + if (lhs?.type !== 'identifier' || rhs?.type !== 'identifier' || rhs.text !== 'new') return null; + return 'ctor:' + rBareName(lhs.text); +} + +/** The signature an assigned value gives its name. */ +function valueSignature(value: SyntaxNode | null): string { + if (value === null) return OTHER; + if (value.type === 'null' || isRConstantValue(value)) return NEUTRAL; + return constructorSignature(value) ?? OTHER; +} + +/** The first argument value of a call (`x` of `class(x)`), or null. */ +function firstArgumentValue(call: SyntaxNode): SyntaxNode | null { + const args = call.childForFieldName('arguments'); + if (args === null) return null; + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (arg?.type === 'argument') return arg.childForFieldName('value'); + } + return null; +} + +/** + * The base identifier a complex assignment target writes through, and whether the step that + * leaves it is a replacement call (`class(x)`: `x` is rebound to what the setter returns) rather + * than a field step (`x$a`, `x@a`, `x[[i]]`, `x[i]`, also under a call: `names(x$a)`). + */ +function complexTargetBase(target: SyntaxNode): { base: SyntaxNode; viaCall: boolean } | null { + let viaCall = false; + for (let node: SyntaxNode | null = target; node !== null; ) { + switch (node.type) { + case 'identifier': + return { base: node, viaCall }; + case 'extract_operator': + viaCall = false; + node = node.childForFieldName('lhs'); + break; + case 'subset': + case 'subset2': + viaCall = false; + node = node.childForFieldName('function'); + break; + case 'call': + viaCall = true; + node = firstArgumentValue(node); + break; + default: + return null; + } + } + return null; +} + +/** The name a string literal spells (its content as written), or null. */ +function stringName(node: SyntaxNode): string | null { + if (node.type !== 'string') return null; + return node.childForFieldName('content')?.text ?? null; +} + +/** The names `rm()`/`remove()` unbind, and the one `assign()`/`delayedAssign()` binds. */ +function unboundOrAssignedNames(call: SyntaxNode): string[] { + const fn = call.childForFieldName('function'); + if (fn?.type !== 'identifier') return []; + const callee = fn.text; + const isRemove = callee === 'rm' || callee === 'remove'; + if (!isRemove && callee !== 'assign' && callee !== 'delayedAssign') return []; + const args = call.childForFieldName('arguments'); + if (args === null) return []; + const names: string[] = []; + let firstUnnamed: SyntaxNode | null = null; + let namedX: SyntaxNode | null = null; + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (arg?.type !== 'argument') continue; + const value = arg.childForFieldName('value'); + if (value === null) continue; + const argName = arg.childForFieldName('name')?.text; + if (isRemove) { + if (argName === undefined) { + const name = value.type === 'identifier' ? rBareName(value.text) : stringName(value); + if (name !== null) names.push(name); + } else if (argName === 'list') { + const name = stringName(value); + if (name !== null) names.push(name); + } + } else if (argName === 'x') { + namedX = value; + } else if (argName === undefined && firstUnnamed === null) { + firstUnnamed = value; + } + } + const literal = namedX ?? firstUnnamed; + if (!isRemove && literal !== null) { + const name = stringName(literal); + if (name !== null) names.push(name); + } + return names; +} + +interface Anchor { + readonly id: number; + readonly name: string; + readonly signature: string; + readonly frame: number; + /** For `<<-`: the frames it records into, which must agree too. */ + readonly enclosing: readonly number[]; +} + +/** + * The ids of the `@type-binding.constructor` anchors (the assignment nodes) whose type binding + * is kept: every non-neutral binding of the name in the anchor's frame is the same constructor + * (and, for `<<-`, in every frame it assigns into). One top-down pass with an explicit stack; + * no `.parent` call, so the work is linear in the size of the tree. + */ +export function rAgreedConstructorAnchors(root: SyntaxNode): Set { + /** frame id -> name -> signatures. */ + const signatures = new Map>>(); + /** frame id -> enclosing frame id (absent for the top level). */ + const enclosingFrame = new Map(); + const record = (frame: number, name: string, signature: string): void => { + let names = signatures.get(frame); + if (names === undefined) signatures.set(frame, (names = new Map())); + let set = names.get(name); + if (set === undefined) names.set(name, (set = new Set())); + set.add(signature); + }; + /** The frames `<<-` assigns into from `frame`: every strictly enclosing one, or the top level. */ + const outwardFrames = (frame: number): number[] => { + const frames: number[] = []; + for (let f = enclosingFrame.get(frame); f !== undefined; f = enclosingFrame.get(f)) { + frames.push(f); + } + return frames.length > 0 ? frames : [frame]; + }; + const anchors: Anchor[] = []; + + const stack: { node: SyntaxNode; frame: number }[] = [{ node: root, frame: root.id }]; + for (let item = stack.pop(); item !== undefined; item = stack.pop()) { + const { node } = item; + let frame = item.frame; + if (node.type === 'function_definition') { + enclosingFrame.set(node.id, frame); + frame = node.id; + const parameters = node.childForFieldName('parameters'); + for (let i = 0; parameters !== null && i < parameters.namedChildCount; i++) { + const parameter = parameters.namedChild(i); + if (parameter?.type !== 'parameter') continue; + const name = parameter.childForFieldName('name'); + if (name?.type === 'identifier') record(frame, rBareName(name.text), OTHER); + } + } else if (node.type === 'for_statement') { + const variable = node.childForFieldName('variable'); + if (variable?.type === 'identifier') record(frame, rBareName(variable.text), OTHER); + } else if (node.type === 'binary_operator') { + const operator = node.childForFieldName('operator')?.text; + let target: SyntaxNode | null = null; + let value: SyntaxNode | null = null; + if (operator === '<-' || operator === '=' || operator === '<<-') { + target = node.childForFieldName('lhs'); + value = node.childForFieldName('rhs'); + } else if (operator === '->' || operator === '->>') { + target = node.childForFieldName('rhs'); + value = node.childForFieldName('lhs'); + } + if (target !== null) { + const outward = operator === '<<-' || operator === '->>'; + const into = outward ? outwardFrames(frame) : [frame]; + if (target.type === 'identifier') { + const name = rBareName(target.text); + const signature = valueSignature(value); + for (const f of into) record(f, name, signature); + // The query emits a constructor binding for the left-assignment forms only. + if (operator !== '->' && operator !== '->>' && signature.startsWith('ctor:')) { + anchors.push({ id: node.id, name, signature, frame, enclosing: outward ? into : [] }); + } + } else { + const written = complexTargetBase(target); + if (written?.viaCall === true) { + for (const f of into) record(f, rBareName(written.base.text), OTHER); + } + } + } + } else if (node.type === 'call') { + for (const name of unboundOrAssignedNames(node)) record(frame, name, OTHER); + } + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child !== null) stack.push({ node: child, frame }); + } + } + + /** The non-neutral signatures of `name` in `frame`. */ + const real = (frame: number, name: string): string[] => { + const set = signatures.get(frame)?.get(name); + return set === undefined ? [] : [...set].filter((signature) => signature !== NEUTRAL); + }; + const kept = new Set(); + for (const anchor of anchors) { + const own = real(anchor.frame, anchor.name); + if (own.length !== 1 || own[0] !== anchor.signature) continue; + if (anchor.enclosing.some((f) => real(f, anchor.name).some((s) => s !== anchor.signature))) { + continue; + } + kept.add(anchor.id); + } + return kept; +} + +/** + * A predicate over `@type-binding.constructor` anchor nodes, built on its first use: the tree is + * walked once per file, and only for a file that has a constructor binding at all. + */ +export function rConstructorAnchorTest(root: SyntaxNode): (anchor: SyntaxNode) => boolean { + let kept: Set | undefined; + return (anchor) => (kept ??= rAgreedConstructorAnchors(root)).has(anchor.id); +} diff --git a/gitnexus/src/core/ingestion/languages/r/rename.ts b/gitnexus/src/core/ingestion/languages/r/rename.ts new file mode 100644 index 000000000..6e78aa06c --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/rename.ts @@ -0,0 +1,1014 @@ +import type Parser from 'tree-sitter'; +import type { NameClaim, ParsedFile, Range } from 'gitnexus-shared'; +import type { + RenameIssue, + RenameProvider, + RenameSemanticContext, + RenameSource, +} from '../../rename-provider.js'; +import { unescapeRString } from './export-pattern.js'; +import { rBareName } from './bare-name.js'; +import { assignmentTarget, isCallee, rBoundName, rLexicalAssignedNames } from './assignment.js'; +import { rOwningPackageDir } from './namespace-imports.js'; +import { + rExportPatternsIncomplete, + rNamespaceExports, + recordedRPackageConfig, +} from './package-config.js'; +import { + canonicalizeRenameBindings, + createTreeSitterRenameSource, + innermostRenameScope, + innermostRenameScopeAt, + nodeRange, + walkNamedNodes, +} from '../rename-tree-sitter.js'; + +/** + * Conservative semantic rename for R. Plain functions are renamed through the + * scope-resolution graph. R binds names dynamically (S3/S4/R6 dispatch, strings, + * NAMESPACE), so every declaration or name that the graph cannot prove is refused. + */ +const RESERVED = new Set( + 'if else repeat while function for next break TRUE FALSE NULL Inf NaN NA NA_integer_ NA_real_ NA_character_ NA_complex_ in'.split( + ' ', + ), +); + +/** `r"(...)"`, `R'[...]'`, `r"---{...}---"`: no escapes; the body is taken as written. */ +const RAW_STRING = /^[rR](["'])(-*)(?:\(([\s\S]*)\)|\[([\s\S]*)\]|\{([\s\S]*)\})\2\1$/; + +/** + * The value R holds for a string literal's source text, or null when it is not a + * recognisable literal. Escapes are decoded (`"f\u006fo"` is `foo`), raw strings are + * taken verbatim, so the guard compares what the string means, not how it is spelled. + */ +function rStringValue(text: string): string | null { + const raw = RAW_STRING.exec(text); + if (raw) return raw[3] ?? raw[4] ?? raw[5] ?? ''; + const quote = text[0]; + if ((quote !== '"' && quote !== "'") || text.length < 2 || text[text.length - 1] !== quote) + return null; + return unescapeRString(text.slice(1, -1)); +} + +/** + * Generics of base R and its default packages that an S3 method name `generic.class` can register + * against, whether or not the indexed sources define them: the S3 group generics (`Ops`, `Math`, + * `Summary`, `Complex`, `matrixOps`) and the functions each group dispatches, the `is.*` and `as.*` + * generics, and the common generics of base, stats, utils and graphics. Not exhaustive, and it + * names no other package's generics (`tidy`, `filter`, ...); the repository's own functions are + * checked separately. + */ +const BASE_GENERICS: ReadonlySet = new Set( + [ + // Group generics, and the members of the Math, Summary and Complex groups + 'Ops Math Summary Complex matrixOps', + 'abs sign sqrt floor ceiling trunc round signif exp log expm1 log1p cos sin tan cospi sinpi tanpi', + 'acos asin atan cosh sinh tanh acosh asinh atanh lgamma gamma digamma trigamma', + 'cumsum cumprod cummax cummin', + 'all any sum prod max min range', + 'Arg Conj Im Mod Re', + // is.* and as.* generics + 'is.na is.nan is.finite is.infinite is.numeric is.array is.matrix is.unsorted is.list is.vector', + 'is.character is.logical is.integer is.double is.complex is.function is.environment', + 'as.character as.numeric as.integer as.logical as.double as.complex as.raw as.vector as.list', + 'as.data.frame as.matrix as.array as.Date as.POSIXct as.POSIXlt as.difftime as.factor as.table', + 'as.function as.environment as.name as.symbol as.call as.expression as.formula as.pairlist', + 'as.numeric_version as.package_version as.hexmode as.octmode as.roman as.person as.personList', + 'as.ts as.dist as.hclust as.dendrogram as.stepfun as.single', + // base + 'print format summary length names dim dimnames levels rev sort rep seq unique duplicated', + 'anyDuplicated anyNA unlist c t cbind rbind toString all.equal labels diff scale cut split', + 'merge subset transform with within by aggregate droplevels relevel reorder xtfrm solve', + 'determinant rowsum row.names julian months weekdays quarters units open close flush seek', + 'truncate mean median quantile sort_by unsplit stack unstack kronecker aperm', + // stats + 'update predict residuals coef fitted anova AIC BIC logLik nobs simulate terms formula family', + 'weights vcov deviance confint profile model.frame model.matrix sigma df.residual extractAIC', + 'add1 drop1 effects alias case.names variable.names dummy.coef proj influence rstandard', + 'rstudent cooks.distance hatvalues dfbeta dfbetas covratio density lag window time start end', + 'frequency cycle deltat na.omit na.exclude na.fail na.action naprint naresid napredict', + 'biplot screeplot qqnorm termplot knots cophenetic loadings smooth qr chol', + // graphics and utils + 'plot lines points text image hist barplot boxplot pairs persp head tail str', + 'toBibtex toLatex', + // pretty() (base; grDevices registers pretty.Date) and stats' weighted.mean() + 'pretty weighted.mean', + ] + .join(' ') + .split(' '), +); + +/** + * Whether `newName` reads as an S3 method `generic.class`: some dot-separated prefix of it is a + * function the indexed sources define or a known base generic. Every dot is a possible split + * (`to.json.foo` is a method of `to.json`; `as.data.frame.foo` of `as.data.frame`). + */ +export function registersS3Method(newName: string, defined: ReadonlySet): boolean { + for (let dot = newName.indexOf('.'); dot >= 0; dot = newName.indexOf('.', dot + 1)) { + if (dot === 0) continue; + const prefix = newName.slice(0, dot); + if (defined.has(prefix) || BASE_GENERICS.has(prefix)) return true; + } + return false; +} + +/** + * Whether `node` is the name of a parameter. A default proves nothing about the argument a caller + * passes, so every named formal is treated alike: a call of its name may reach the parameter (the + * argument is a function) or a function of that name outside (it is not; R skips a non-function + * binding when it looks up a callee), which cannot be decided statically. The provider therefore + * + * - claims the formal as a binding, so a call of the name under it is not renamed with the + * function (`role` below), and refuses the rename when such a call exists (`checkSafety`); + * - reads a call of the NEW name under a formal of that name as a read of the new name, which the + * renamed function would capture (`readsNewName`); + * - leaves a plain read of the formal alone: it is a read of the argument, not of a function. + */ +export function isRParameterName(node: Parser.SyntaxNode): boolean { + const parameter = node.parent; + if (parameter?.type !== 'parameter' || parameter.childForFieldName('name')?.id !== node.id) { + return false; + } + return parameter.parent?.parent?.type === 'function_definition'; +} + +/** Whether some function around `node` has a parameter called `name`. */ +function hasEnclosingParameter(node: Parser.SyntaxNode, name: string): boolean { + for (let ancestor = node.parent; ancestor !== null; ancestor = ancestor.parent) { + if (ancestor.type !== 'function_definition') continue; + const parameters = ancestor.childForFieldName('parameters'); + if (parameters === null) continue; + for (const parameter of parameters.namedChildren) { + if (parameter.type !== 'parameter') continue; + const written = parameter.childForFieldName('name'); + if (written !== null && rBareName(written.text) === name) return true; + } + } + return false; +} + +/** The name a `<<-` or `->>` assignment writes to (backticks removed), or null for any other node. */ +function superassignedName(node: Parser.SyntaxNode): string | null { + if (node.type !== 'binary_operator') return null; + const operator = node.childForFieldName('operator')?.text; + if (operator !== '<<-' && operator !== '->>') return null; + const target = node.childForFieldName(operator === '<<-' ? 'lhs' : 'rhs'); + return target?.type === 'identifier' ? rBareName(target.text) : null; +} + +/** + * Whether a call of `name` sits in the target of an assignment: `helper(z) <- 5`, + * `helper(z)$a <- 1`, `names(helper(z)) <- v`. R runs the replacement function `helper<-` + * there, not `helper`, so renaming the getter would strand the setter. + */ +function callsInAssignmentTarget(node: Parser.SyntaxNode, name: string): boolean { + const target = assignmentTarget(node); + // A plain name or a string names the variable being bound, not a function being called. + if (target === null || target.type === 'identifier' || target.type === 'string') return false; + for (const inner of walkNamedNodes(target)) { + if (inner.type === 'identifier' && isCallee(inner) && rBareName(inner.text) === name) + return true; + } + return false; +} + +/** Whether `node` sits inside a function body or its parameter defaults. */ +function insideFunction(node: Parser.SyntaxNode): boolean { + for (let ancestor = node.parent; ancestor !== null; ancestor = ancestor.parent) { + if (ancestor.type === 'function_definition') return true; + } + return false; +} + +/** + * Whether `node`, an identifier spelled like the new name, is used to READ a binding: a value + * (`sapply(x, paste)`, `x <- letters`, a default `f = assist`) or a callee. After the rename + * the renamed function is the global of that name, so a read that reached something else + * (a base or attached object, another package's function) would be captured by it. Not reads: + * a bound name (`assist <- 3`, a parameter, a `for` variable), an argument name (`f(assist = 1)`), + * a member (`x$assist`, `x@assist`) and a qualified name (`pkg::assist`). + */ +function readsNewName(node: Parser.SyntaxNode, newName: string): boolean { + const parent = node.parent; + if (parent === null) return true; + switch (parent.type) { + case 'argument': + if (parent.childForFieldName('name')?.id === node.id) return false; + break; + case 'parameter': + if (parent.childForFieldName('name')?.id === node.id) return false; + break; + case 'extract_operator': + if (parent.childForFieldName('rhs')?.id === node.id) return false; + break; + case 'namespace_operator': + return false; + case 'for_statement': + if (parent.childForFieldName('variable')?.id === node.id) return false; + break; + case 'binary_operator': + if (assignmentTarget(parent)?.id === node.id) return false; + break; + } + // A parameter of an enclosing function binds the name locally, so a plain read of it never + // reaches the global. A CALL of it may: a default proves nothing about the argument, and R + // skips a non-function binding, so the call can reach the global the renamed function becomes. + return !hasEnclosingParameter(node, newName) || isCallee(node); +} + +/** + * A name R also writes between its operands: an infix function (`%+%`, `%in%`) or an operator + * (`+`, `[[`). Its uses are operator tokens, not `identifier` nodes, so a rename would edit the + * definition and leave every `a %+% b` calling a function that no longer exists. + */ +function isOperatorName(name: string): boolean { + return /^%.*%$/.test(name) || /^[^A-Za-z0-9._]+$/.test(name); +} + +/** A regular expression that matches `name` literally. */ +const escapeRegExp = (name: string): string => name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** Characters that continue an R identifier, so a match next to one is part of a longer name. */ +const NAME_CHAR = 'A-Za-z0-9._'; + +/** The source text of a rename source (hand-built test sources may carry only the tree). */ +const sourceText = (source: RenameSource): string => + source.content ?? source.tree?.rootNode.text ?? ''; + +/** + * Generated R sources (Rcpp's `RcppExports.R`, cpp11's `cpp11.R`, roxygen2 output, usethis + * standalone files) say so in their first lines. The generator rewrites them from other inputs, + * so a rename edit there is reverted on the next regeneration. + */ +const GENERATED_HEADER = /Generated by|do not edit by hand/i; + +export function isGeneratedRSource(text: string): boolean { + return GENERATED_HEADER.test(text.split('\n', 5).join('\n')); +} + +/** Whether an identifier spelled `name` occurs anywhere in the tree. */ +function mentionsName(root: Parser.SyntaxNode, name: string): boolean { + for (const node of walkNamedNodes(root)) { + if (node.type === 'identifier' && rBareName(node.text) === name) return true; + } + return false; +} + +/** Whether `inner` lies within `outer` (byte offsets). */ +const within = (inner: Parser.SyntaxNode, outer: Parser.SyntaxNode): boolean => + inner.startIndex >= outer.startIndex && inner.endIndex <= outer.endIndex; + +const sameRange = (a: Range, b: Range): boolean => + a.startLine === b.startLine && + a.startCol === b.startCol && + a.endLine === b.endLine && + a.endCol === b.endCol; + +/** + * Whether the binding `node` (an assignment or `for` loop) is made at the top level of its file: + * in the Module scope, not in a function or in a block scope (`local({})`, `test_that()`, ...). + * The scope of the assignment's own function or R6 class, which spans the assignment node itself, + * is skipped. A source without scopes (a hand-built test source) reads only the function nesting. + */ +function bindsAtTopLevel(parsed: ParsedFile | undefined, node: Parser.SyntaxNode): boolean { + if (insideFunction(node)) return false; + if (parsed?.scopes === undefined) return true; + const range = nodeRange(node); + let scopeId = innermostRenameScopeAt(parsed, range); + const own = parsed.scopes.find((scope) => scope.id === scopeId); + if (own !== undefined && sameRange(own.range, range)) scopeId = own.parent ?? undefined; + return scopeId === parsed.moduleScope; +} + +/** Whether the file binds `name` at top level (a function or any other value). */ +function definesAtTopLevel(source: RenameSource, name: string): boolean { + if (!source.tree) return false; + for (const node of walkNamedNodes(source.tree.rootNode)) { + if (rBoundName(node) === name && bindsAtTopLevel(source.parsed, node)) return true; + } + return false; +} + +/** The value side of a local `<-`, `=` or `->` assignment of `name`, or null. */ +function locallyAssignedValue(node: Parser.SyntaxNode, name: string): Parser.SyntaxNode | null { + if (node.type !== 'binary_operator' || rBoundName(node) !== name) return null; + const operator = node.childForFieldName('operator')?.text; + if (operator === '<-' || operator === '=') return node.childForFieldName('rhs'); + if (operator === '->') return node.childForFieldName('lhs'); + return null; +} + +/** The calls of `name` (an identifier callee) under `root`. */ +function callsOf(root: Parser.SyntaxNode, name: string): Parser.SyntaxNode[] { + const calls: Parser.SyntaxNode[] = []; + for (const node of walkNamedNodes(root)) { + if (node.type === 'identifier' && isCallee(node) && rBareName(node.text) === name) + calls.push(node); + } + return calls; +} + +/** Calls whose braces get a block scope of their own (the `@scope.block` patterns of query.ts). */ +const BLOCK_SCOPE_CALLS = /^(local|test_that|describe|it|with|within)$/; + +/** Whether `node` is the braces of a block scope: `local({...})`, `test_that(desc, {...})`, ... */ +function isBlockScopeBody(node: Parser.SyntaxNode): boolean { + if (node.type !== 'braced_expression' || node.parent?.type !== 'argument') return false; + if (node.parent.childForFieldName('value')?.id !== node.id) return false; + const call = node.parent.parent?.parent; + if (call?.type !== 'call') return false; + let callee = call.childForFieldName('function'); + if (callee?.type === 'namespace_operator') callee = callee.childForFieldName('rhs'); + return callee?.type === 'identifier' && BLOCK_SCOPE_CALLS.test(callee.text); +} + +/** + * Whether a function body or a block scope's braces may call `name` before its own local + * definition of it. The scope graph binds every call of the name in a body (nested functions and + * blocks included) to a function literal the body assigns to it, wherever the assignment sits; + * R binds the local only once the assignment has run. So the binding is accepted only when one + * such assignment is a direct statement of the body's braces (not under `if`, `switch()`, a + * loop, `tryCatch()` or any call) and every call of the name in the body starts after it ends (a + * call inside that literal runs after it too). + */ +export function rCallsBeforeLocalDefinition(root: Parser.SyntaxNode, name: string): boolean { + for (const fn of walkNamedNodes(root)) { + const body = + fn.type === 'function_definition' + ? fn.childForFieldName('body') + : isBlockScopeBody(fn) + ? fn + : null; + if (body === null) continue; + // Assignments made by this body itself: nested functions and blocks own theirs. + const definitions: Parser.SyntaxNode[] = []; + const stack: Parser.SyntaxNode[] = [body]; + for (let node = stack.pop(); node !== undefined; node = stack.pop()) { + if (node.type === 'function_definition' || (node.id !== body.id && isBlockScopeBody(node))) + continue; + if (locallyAssignedValue(node, name)?.type === 'function_definition') definitions.push(node); + for (const child of node.namedChildren) stack.push(child); + } + if (definitions.length === 0) continue; + const calls = callsOf(body, name); + if (calls.length === 0) continue; + const first = definitions + .filter((d) => body.type === 'braced_expression' && d.parent?.id === body.id) + .sort((a, b) => a.startIndex - b.startIndex)[0]; + if (first === undefined) return true; + if (calls.some((call) => !within(call, first) && call.startIndex < first.endIndex)) return true; + } + return false; +} + +/** + * Whether a top-level call of `name` in the file runs before the definition at `at`: top-level + * expressions run in order, so it reaches whatever `name` was before (a base or attached + * function), while the module scope binds it to the later definition. + */ +function topLevelCallBeforeDefinition(root: Parser.SyntaxNode, name: string, at: Range): boolean { + const definition = root.descendantForPosition({ row: at.startLine - 1, column: at.startCol }); + if (insideFunction(definition)) return false; + for (const call of callsOf(root, name)) { + if (insideFunction(call)) continue; + const row = call.startPosition.row + 1; + if (row < at.startLine || (row === at.startLine && call.startPosition.column < at.startCol)) + return true; + } + return false; +} + +/** Whether `call` calls `fn`, bare or `pkg::`/`pkg:::`-qualified. */ +function callsFunction(call: Parser.SyntaxNode, fn: string): boolean { + if (call.type !== 'call') return false; + const callee = call.childForFieldName('function'); + if (callee?.type === 'identifier') return callee.text === fn; + if (callee?.type === 'namespace_operator') return callee.childForFieldName('rhs')?.text === fn; + return false; +} + +/** The arguments of a call, with the name each is passed by (`undefined` when unnamed). */ +function callArguments(call: Parser.SyntaxNode): { name?: string; value: Parser.SyntaxNode }[] { + const result: { name?: string; value: Parser.SyntaxNode }[] = []; + for (const argument of call.childForFieldName('arguments')?.namedChildren ?? []) { + if (argument.type !== 'argument') continue; + const value = argument.childForFieldName('value'); + if (value === null) continue; + const written = argument.childForFieldName('name'); + result.push({ name: written ? rBareName(written.text) : undefined, value }); + } + return result; +} + +/** + * Whether a bare call of `name` in a portable R6 method is recorded as blocked by a class member + * of that name. R6 puts the members on the class scope that encloses every method, so the + * lookup from a method stops at the member, which the resolver hides, and records the site as + * `blocked` with the member as its target; the rename engine then reads it as a use of another + * declaration and leaves it. In a portable class (the default) a method sees members only as + * `self$`/`private$`, so the call really reaches the function being renamed. A non-portable + * class (`portable = FALSE`) and a RefClass method do see the member by bare name. + */ +function portableR6MemberBlocksCall(context: RenameSemanticContext, name: string): boolean { + if (context.referenceBindings === undefined) return false; + const types = new Map((context.definitions ?? []).map((d) => [d.nodeId, d.type])); + const sourceOfScope = new Map(); + for (const source of context.files.values()) + for (const scope of source.parsed.scopes ?? []) sourceOfScope.set(scope.id, source); + for (const [site, binding] of context.referenceBindings) { + if (site.kind !== 'call' || site.explicitReceiver !== undefined) continue; + if (rBareName(site.name) !== name) continue; + if (binding.status !== 'blocked' || binding.basis !== 'callable-visibility') continue; + if ( + binding.targetDefIds.length === 0 || + !binding.targetDefIds.every((id) => types.get(id) === 'Method') + ) + continue; + const root = sourceOfScope.get(site.inScope)?.tree?.rootNode; + if (root === undefined) continue; + const at = site.nameRange ?? site.atRange; + let node: Parser.SyntaxNode | null = root.descendantForPosition({ + row: at.startLine - 1, + column: at.startCol, + }); + for (; node !== null; node = node.parent) { + if (!callsFunction(node, 'R6Class')) continue; + const nonPortable = callArguments(node).some( + (a) => a.name === 'portable' && (a.value.text === 'FALSE' || a.value.text === 'F'), + ); + if (!nonPortable) return true; + break; + } + } + return false; +} + +interface RClassShape { + readonly contains: string[]; + readonly members: Set; +} + +const strings = (node: Parser.SyntaxNode): string[] => { + if (node.type === 'string') { + const value = rStringValue(node.text); + return value === null ? [] : [value]; + } + if (callsFunction(node, 'c')) return callArguments(node).flatMap((a) => strings(a.value)); + return []; +}; + +/** Member names given by `list(a = ...)`, `c(a = "x")`, `representation(...)` or `c("a", "b")`. */ +const memberNames = (node: Parser.SyntaxNode): string[] => + node.type !== 'call' + ? [] + : callArguments(node).flatMap((a) => + a.name !== undefined ? [a.name] : a.value.type === 'string' ? strings(a.value) : [], + ); + +/** + * The classes `setRefClass()` and `setClass()` define in the indexed sources: each name (the + * `Class =` argument, else the first unnamed string) with its `contains =` parents and the + * methods, fields and slots it lists. Classes defined twice are merged. + */ +function rClassShapes(files: ReadonlyMap): Map { + const shapes = new Map(); + for (const source of files.values()) { + if (!source.tree) continue; + for (const call of walkNamedNodes(source.tree.rootNode)) { + if (!callsFunction(call, 'setRefClass') && !callsFunction(call, 'setClass')) continue; + const args = callArguments(call); + const named = args.find((a) => a.name === 'Class') ?? args.find((a) => a.name === undefined); + const className = named?.value.type === 'string' ? rStringValue(named.value.text) : null; + if (className === null) continue; + const shape = shapes.get(className) ?? { contains: [], members: new Set() }; + for (const a of args) { + if (a.name === 'contains') shape.contains.push(...strings(a.value)); + if (['methods', 'fields', 'slots', 'representation'].includes(a.name ?? '')) + for (const member of memberNames(a.value)) shape.members.add(member); + } + shapes.set(className, shape); + } + } + return shapes; +} + +/** Base classes R provides itself, which add no method a reference method can call by name. */ +const BUILTIN_REFCLASS_BASES: ReadonlySet = new Set(['envRefClass', 'VIRTUAL']); + +/** + * Whether a bare call of `name` in a RefClass method may reach an inherited method or field: R + * installs the inherited methods a reference method uses, so the call reaches a parent's member + * of that name, while the scope graph does not follow `contains =` and binds the call to the + * function being renamed. Refused also when a parent is not defined in the indexed sources. + */ +function refClassInheritsName(files: ReadonlyMap, name: string): boolean { + let shapes: Map | undefined; + for (const source of files.values()) { + if (!source.tree) continue; + for (const call of callsOf(source.tree.rootNode, name)) { + let argument: Parser.SyntaxNode | null = null; + let className: string | null = null; + for (let node: Parser.SyntaxNode | null = call.parent; node !== null; node = node.parent) { + if (node.type === 'argument') argument = node; + if (!callsFunction(node, 'setRefClass')) continue; + const written = argument?.childForFieldName('name'); + if ( + argument?.parent?.parent?.id === node.id && + written && + rBareName(written.text) === 'methods' + ) { + const args = callArguments(node); + const named = + args.find((a) => a.name === 'Class') ?? args.find((a) => a.name === undefined); + className = named?.value.type === 'string' ? rStringValue(named.value.text) : null; + } + break; + } + if (className === null) continue; + shapes ??= rClassShapes(files); + const queue = [...(shapes.get(className)?.contains ?? [])]; + const seen = new Set([className]); + for (let parent = queue.shift(); parent !== undefined; parent = queue.shift()) { + if (seen.has(parent) || BUILTIN_REFCLASS_BASES.has(parent)) continue; + seen.add(parent); + const shape = shapes.get(parent); + if (shape === undefined || shape.members.has(name)) return true; + queue.push(...shape.contains); + } + } + } + return false; +} + +/** + * Whether a `$methods(...)` call (`Gen$methods(run = function() helper())`, which adds methods to + * a RefClass generator after `setRefClass()`) mentions `name`. A method added that way sees the + * class's methods by bare name, and a method of that name added there is seen by every method of + * the class; the class scope does not model these calls, so the scope graph binds such a call to + * the function being renamed. The receiver is not traced: any `x$methods(...)` counts. + */ +function refClassMethodsCallMentionsName( + files: ReadonlyMap, + name: string, +): boolean { + for (const source of files.values()) { + if (!source.tree) continue; + for (const call of walkNamedNodes(source.tree.rootNode)) { + if (call.type !== 'call') continue; + const callee = call.childForFieldName('function'); + if (callee?.type !== 'extract_operator' || callee.childForFieldName('operator')?.text !== '$') + continue; + if (rBareName(callee.childForFieldName('rhs')?.text ?? '') !== 'methods') continue; + const args = call.childForFieldName('arguments'); + if (args !== null && mentionsName(args, name)) return true; + } + } + return false; +} + +/** + * Whether a roxygen2 line of the package's `R/` files would write `name` into NAMESPACE on the + * next `document()` although it is not attached to the definition: `@rawNamespace` text that + * names it (copied verbatim), or `@export name` (an export tag that names its object, as on a + * `NULL` block). + */ +function roxygenNamesInNamespace( + files: ReadonlyMap, + file: string, + name: string, +): boolean { + const config = recordedRPackageConfig(files.get(file)?.parsed); + if (config === undefined) return false; + const packageDir = rOwningPackageDir(file, config); + const word = new RegExp(`(? ({ code: 'name_collision', message }); + +const unsupported = (message: string): RenameIssue => ({ + code: 'unsupported_occurrence', + status: 'unsupported', + message, +}); + +const ambiguous = (message: string): RenameIssue => ({ code: 'ambiguous_reference', message }); + +const GENERATED_MESSAGE = + 'A generated R file (its header says "Generated by" or "do not edit by hand") uses this name; its generator would restore the old name.'; + +// A backticked symbol is an `identifier` whose text keeps its backticks. Both spellings name the +// same symbol, so occurrences, claims and bindings are matched on the symbol. +const parseNames = createTreeSitterRenameSource({ + isName: (node) => node.type === 'identifier', + canonicalName: rBareName, + transformParsed: (parsed) => canonicalizeRenameBindings(parsed, rBareName), + skip: (node) => node.type === 'comment', + role: (node) => { + // Every parameter is claimed as a binding, whatever its default (see `isRParameterName`). + return isRParameterName(node) ? 'binding' : 'name'; + }, +}); + +/** + * Functions R itself calls by name: package load and unload hooks and the session start and end + * hooks. A rename would leave them defined under a name nothing calls. + */ +const R_HOOK_FUNCTIONS: ReadonlySet = new Set([ + '.onLoad', + '.onAttach', + '.onUnload', + '.onDetach', + '.First', + '.Last', + '.First.sys', + '.First.lib', + '.Last.lib', +]); + +/** + * The name the engine compares a graph symbol with the scope definition of. The engine reads a + * definition's simple name as the text after the last `.` of its qualified name (the rule that + * splits an owner from a member), which turns the leading-dot name `.helper` into `helper`; for a + * name that is only a leading dot and one word, the simple name is the whole name. Dotted names + * with a dot inside (`print.foo`, `.helper.foo`) are S3-style owner-qualified names and keep the + * engine's reading, so they stay refused. + */ +function rCanonicalName( + name: string, + context?: Parameters>[1], +): string { + const bare = rBareName(name); + const qualified = context?.definition?.qualifiedName; + if (qualified === undefined) return bare; + const full = rBareName(qualified); + return /^\.[^.]+$/.test(full) && bare === full.slice(1) ? full : bare; +} + +export const rRenameProvider: RenameProvider = { + canonicalName: rCanonicalName, + validateName(name) { + // `...`, `..1`, `..2`, ... refer to the arguments passed down to a function; R reserves them. + if (name === '...' || /^\.\.[0-9]+$/.test(name)) + return { + code: 'invalid_name', + message: + 'Choose a valid nonreserved R identifier: `...` and `..1`, `..2`, ... are reserved.', + }; + if (/[^\x00-\x7f]/.test(name)) + return { + code: 'invalid_name', + message: + 'Choose a valid nonreserved R identifier. New names are limited to ASCII letters, digits, `.` and `_` (conservative by design).', + }; + if (!/^(?:[A-Za-z]|\.(?![0-9]))[A-Za-z0-9._]*$/.test(name) || RESERVED.has(name)) + return { code: 'invalid_name', message: 'Choose a valid nonreserved R identifier.' }; + }, + async parseSource(input) { + const source = await parseNames(input); + // Parameters shadow same-named functions: claim them per scope so the engine + // leaves an unrelated parameter alone. + const claims = new Map(); + for (const occurrence of source.occurrences) { + if (occurrence.role !== 'binding' || !occurrence.node) continue; + const scope = innermostRenameScope(source.parsed, occurrence.node); + if (!scope) continue; + claims.set(scope, [ + ...(claims.get(scope) ?? []), + { + name: occurrence.name, + range: occurrence.range, + kind: 'binding', + hoisted: true, + purpose: 'value', + }, + ]); + } + return { + ...source, + parsed: { + ...source.parsed, + scopes: source.parsed.scopes.map((scope) => ({ + ...scope, + nameClaims: [...(scope.nameClaims ?? []), ...(claims.get(scope.id) ?? [])], + })), + }, + }; + }, + checkSafety(context) { + const { target, newName, files, graph } = context; + const name = rBareName(target.qualifiedName ?? ''); + if (target.type !== 'Function') + return unsupported('Only plain R functions can be renamed; classes and members are refused.'); + if (R_HOOK_FUNCTIONS.has(name)) + return unsupported( + 'R calls this function by name (a package or session hook); a renamed one would no longer run.', + ); + // `f(x) <- v` calls the replacement function `f<-`, so a name ending in `<-` is half of a pair + // whose other half is written differently. + if (name.endsWith('<-') || newName.endsWith('<-')) + return unsupported( + 'A replacement function (`name<-`) is called as `name(x) <- value`; those calls are not renamed.', + ); + // A name made only of non-ASCII characters (`λ`) is an identifier in a UTF-8 session, not an + // operator; the provider reads ASCII names only, so it says that rather than "operator". + if (/^[^\x00-\x7f]+$/.test(name)) + return unsupported( + 'Only ASCII identifiers are renamed; this name is not a supported identifier.', + ); + if (isOperatorName(name) || isOperatorName(newName)) + return unsupported( + 'An infix function or operator (`%op%`, `+`) is also written between its operands; those uses are not renamed.', + ); + // Rcpp/cpp11 wrappers and other generated files are rewritten from their inputs, which + // still say the old name: the definition, or any use, in such a file would be reverted. + for (const [path, source] of files) { + if (!isGeneratedRSource(sourceText(source))) continue; + if (path === target.filePath || (source.tree && mentionsName(source.tree.rootNode, name))) + return unsupported(GENERATED_MESSAGE); + } + // Only a function can be an S3 generic: an R6 or S4 class, a member or a field of the same + // spelling is not one, so a name like `Widget.foo` beside `Widget <- R6Class(...)` is free. + const defined = new Set( + [...files.values()].flatMap((source) => + source.parsed.localDefs.flatMap((d) => + d.type !== 'Function' || d.qualifiedName === undefined + ? [] + : [rBareName(d.qualifiedName)], + ), + ), + ); + if (registersS3Method(newName, defined)) + return unsupported( + 'The new name would register an S3 method of an existing function or base generic.', + ); + // R keeps one global per name. A function of the new name in another file is a second + // definition (load order decides which survives); one in this file is the engine's check. + for (const [path, source] of files) { + if (path === target.filePath) continue; + if ( + source.parsed.localDefs.some( + (d) => + d.type === 'Function' && + d.qualifiedName !== undefined && + rBareName(d.qualifiedName) === newName, + ) + ) + return collision('Another file already defines a function of the new name.'); + } + // The same holds for the old name: a second top-level definition in a file loaded with the + // target's (the same package's R/ files, or files outside every package) means load order + // decides which one callers reach. A definition inside a function or a block scope + // (`test_that()`, `local()`, ...) binds only there. + const recorded = recordedRPackageConfig(files.get(target.filePath)?.parsed); + const loadUnit = rOwningPackageDir(target.filePath, recorded); + for (const [path, source] of files) { + if (path === target.filePath || rOwningPackageDir(path, recorded) !== loadUnit) continue; + if (definesAtTopLevel(source, name)) + return ambiguous( + 'This name is defined in more than one file; load order decides which definition callers reach.', + ); + } + const callInString = new RegExp(`(?>; the global re-assignment is not renamed with it.', + }; + if (node.type !== 'string') continue; + const value = rStringValue(node.text); + // `"name<-" <- function(x, value)` defines the replacement function by string. + if ( + value !== null && + (value === name || value === newName || value === `${name}<-` || value === `${newName}<-`) + ) + return unsupported( + 'A string literal spells this name; S3, S4, R6 and dynamic lookup resolve it by string.', + ); + // glue/cli `{name(x)}`, `parse(text = "name(1)")`, `str2lang()` and formula strings are + // evaluated as R code. A plain mention of the word is prose and is not refused. + if (value !== null && callInString.test(value)) + return unsupported( + 'A string literal contains a call of this name (glue/cli interpolation, parse(text=), a formula string); it would not be renamed.', + ); + } + } + const targetSource = files.get(target.filePath); + const definedAt = target.nameRange ?? target.declarationRange; + if ( + targetSource?.tree && + definedAt !== undefined && + topLevelCallBeforeDefinition(targetSource.tree.rootNode, name, definedAt) + ) + return ambiguous( + 'A top-level call of this name runs before the definition; it reaches another function of that name.', + ); + for (const source of files.values()) { + if (source.tree && rCallsBeforeLocalDefinition(source.tree.rootNode, name)) + return ambiguous('A call may run before the local definition of this name.'); + } + if (portableR6MemberBlocksCall(context, name)) + return ambiguous( + 'A bare call inside a portable R6 method does not see class members; it reaches the function being renamed but has no proven edge.', + ); + if (refClassInheritsName(files, name)) + return ambiguous( + 'A RefClass method calls this name and a parent class (contains =) defines a member of it, or is not indexed; the call may reach the inherited member.', + ); + if (refClassMethodsCallMentionsName(files, name)) + return ambiguous( + 'A $methods() call (methods added to a RefClass generator) mentions this name; the index does not model those methods, so a call may reach a class method instead of the function being renamed.', + ); + // R files the index does not cover (skipped directories, `.Rprofile`, `R/*.S`), recorded + // by `refreshResolutionConfig` for this plan. + for (const unindexed of recorded?.unindexedRSources ?? []) { + if (unindexed.unreadable === true) + return { + code: 'incomplete_semantics', + status: 'unsupported', + message: `An R file the index does not cover (${unindexed.path}) could not be read in full, so it may call this name; it would not be updated.`, + }; + if (unindexed.words.includes(name)) + return { + code: 'incomplete_semantics', + status: 'unsupported', + message: `An R file the index does not cover (${unindexed.path}) mentions this name; it would not be updated.`, + }; + } + // NAMESPACE is not an indexed source, so an `export()`, `exportPattern()` or `S3method()` + // entry naming this function would be left behind by a rename. The persisted `isExported` + // is not proof of privacy: an edit of NAMESPACE alone does not re-process the unchanged + // `R/` files, so the flag can be stale. It is only half of the proof; the other half is the + // NAMESPACE the planner has just loaded from disk for this plan. + const file = target.filePath; + if (!/(^|\/)R\//.test(file)) return; + // roxygen2 regenerates NAMESPACE from `@export*` tags on the next `document()`, so a tag on + // the function's block would export the new name (and drop the old one) even when the + // NAMESPACE on disk says private now. + const tree = files.get(file)?.tree; + if (tree && hasRoxygenExportTag(tree.rootNode, name)) + return unsupported( + 'A roxygen2 @export tag is attached to this function; renaming it would change the exported name when NAMESPACE is next regenerated.', + ); + if (roxygenNamesInNamespace(files, file, name)) + return unsupported( + 'A roxygen2 @rawNamespace or @export line of the package names this function; the next document() would write the old name into NAMESPACE.', + ); + let exported = false; + graph.forEachNode((node) => { + if ( + node.label === 'Function' && + node.properties.filePath === file && + typeof node.properties.name === 'string' && + rBareName(node.properties.name) === name && + node.properties.isExported !== false + ) + exported = true; + }); + if (exported) + return unsupported( + 'A package function that NAMESPACE exports, or that the index cannot prove private, may have an export entry that is not renamed.', + ); + return currentNamespaceIssue(files.get(file)?.parsed, file, name, newName); + }, + // A second line of defence for generated files (see `checkSafety`): never edit one. + validateRewrite({ source, edits }) { + if (edits.length > 0 && isGeneratedRSource(sourceText(source))) + return unsupported(GENERATED_MESSAGE); + }, +}; + +/** A roxygen2 tag that adds its object to NAMESPACE: `@export`, `@exportMethod`, `@exportS3Method`, ... */ +const ROXYGEN_EXPORT_TAG = /^#'[ \t]*@export\w*(?:[ \t]|$)/; + +/** + * Whether a top-level assignment of `name` in the file carries a roxygen2 `@export*` tag: a + * `#'` line in the run of comments that directly precedes it (a plain `#` comment in between + * does not break the run, as when roxygen2 collects the block). Text only; the tags are not parsed. + */ +function hasRoxygenExportTag(root: Parser.SyntaxNode, name: string): boolean { + for (const node of walkNamedNodes(root)) { + if (rBoundName(node) !== name || insideFunction(node)) continue; + for (let sibling = node.previousSibling; sibling; sibling = sibling.previousSibling) { + if (sibling.type !== 'comment') { + if (sibling.isNamed) break; + continue; + } + if (ROXYGEN_EXPORT_TAG.test(sibling.text)) return true; + } + } + return false; +} + +/** + * Whether the NAMESPACE read from disk for this plan leaves `name` (defined in `file`) private. + * Fail-closed: anything short of a NAMESPACE that was read, parsed completely and does not export + * the name is a refusal, so a stale persisted verdict cannot make a rename that strands an export + * go through. + */ +function currentNamespaceIssue( + parsed: object | undefined, + file: string, + name: string, + newName: string, +): RenameIssue | void { + const config = recordedRPackageConfig(parsed); + if (config === undefined) + return { + code: 'incomplete_semantics', + status: 'unsupported', + message: + 'The package NAMESPACE was not read for this rename, so the function cannot be proven private.', + }; + const packageDir = rOwningPackageDir(file, config); + const info = + packageDir === undefined ? undefined : config.namespaceInfoByPackageDir.get(packageDir); + if (info === undefined || !info.hasNamespaceFile) + return { + code: 'incomplete_semantics', + status: 'unsupported', + message: + 'The package has no NAMESPACE file that could be read now (it may have been removed or be unreadable), so the function cannot be proven private.', + }; + // A registered S3 method is not exported for resolution, but it is public API: refused alike. + if (rNamespaceExports(info, name) || info.s3Registered?.has(name) === true) + return unsupported( + 'The package NAMESPACE now exports this function (export(), exportPattern() or S3method()); that entry would not be renamed. Re-run analyze to refresh the index.', + ); + // A private function renamed into an exported name changes the package API. + if (rNamespaceExports(info, newName)) + return unsupported( + 'The new name would be exported by NAMESPACE (export()/exportPattern()), changing the package API.', + ); + if (rExportPatternsIncomplete(info)) + return { + code: 'incomplete_semantics', + status: 'unsupported', + message: + 'An exportPattern() of the package NAMESPACE could not be fully applied, so the function cannot be proven private.', + }; + if (info.exportsUnreadable === true) + return { + code: 'incomplete_semantics', + status: 'unsupported', + message: + 'The package NAMESPACE has an export directive the reader cannot apply (a conditional export, a raw-string or multi-line argument, or a malformed directive), so the function cannot be proven private.', + }; +} diff --git a/gitnexus/src/core/ingestion/languages/r/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/r/scope-resolver.ts new file mode 100644 index 000000000..b8f44c6e2 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/scope-resolver.ts @@ -0,0 +1,337 @@ +/** + * R `ScopeResolver` registered in `SCOPE_RESOLVERS` and consumed by the + * generic `runScopeResolution` orchestrator. + * + * Thin wiring: import resolution reuses `resolveRImportTarget` + * (`languages/r/import-target.ts`), an adapter over the existing + * `resolveRImportInternal`, with the repo's local R packages + * threaded through `loadRPackageConfig`. Heritage (R6 `inherit=`, S4 + * `contains=`) mostly rides the generic `@reference.inherits` mechanism — + * R6/S4 base names are ordinary scope-bound class names, unlike Ruby's + * `include`/`extend`, which are method calls. The one exception is S4's + * `contains = "VIRTUAL"` sentinel (`setClass(x, contains = "VIRTUAL", ...)`, + * R's spelling for an abstract base): no fixture ever declares + * `setClass("VIRTUAL", ...)`, so the generic scope-based lookup + * (`resolveInheritanceBaseInScope` / `findClassBindingInScope`) can never + * find a binding for it — that lookup is scope-model-only and never + * consults graph nodes directly. `emitRVirtualHeritageEdges` below mints + * the placeholder `VIRTUAL` graph node and its EXTENDS edge directly, + * mirroring Ruby's `emitRubyMixinEdges` shape for the same reason: a + * heritage target that R code names but never declares. + */ + +import type { ParsedFile, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; +import { + findEnclosingClassDef, + populateClassOwnedMembers, +} from '../../scope-resolution/scope/walkers.js'; +import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { resolveDefGraphId } from '../../scope-resolution/graph-bridge/ids.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import { generateId } from '../../../../lib/utils.js'; +import { + loadRPackageConfig, + recordedRPackageConfig, + recordRPackageConfig, + rSymbolVisibility, +} from './package-config.js'; +import { rBareName } from './bare-name.js'; +import { applyRCaptureSideChannel, rResolveSelfMember } from './r6-self.js'; +import { perFileSet } from '../../utils/per-file-set.js'; +import { resolveRImportTarget } from './import-target.js'; +import { collectRUnindexedSources } from './unindexed-sources.js'; +import { + isRGlobalNameFallbackPlausible, + populateRNamespaceImports, + rFileTopLevel, +} from './namespace-imports.js'; +import { rProvider } from '../r.js'; +import { populateRQualifiedCalls, resolveRQualifiedFreeCall } from './qualified-call.js'; +import { rArityCompatibility, rMergeBindings } from './simple-hooks.js'; + +/** Literal string `"VIRTUAL"` graph id — one node shared by every S4 class + * that names it as a base, minted lazily on first use. */ +const VIRTUAL_CLASS_NAME = 'VIRTUAL'; + +/** + * Whether `def` is declared by a call that carries the class name as a string + * (`setClass("A", ...)`, `setRefClass("A", ...)`): the name starts after the declaration + * does. `A <- R6Class(...)` starts at its name. Only the call form can write the S4 + * sentinel `contains = "VIRTUAL"`; an R6 `inherit = VIRTUAL` yields an `inherits` reference + * of the same name that is an ordinary base class. + */ +function isDeclaredByNamingCall(def: SymbolDefinition): boolean { + const name = def.nameRange; + const declaration = def.declarationRange; + if (name === undefined || declaration === undefined) return false; + return ( + name.startLine > declaration.startLine || + (name.startLine === declaration.startLine && name.startCol > declaration.startCol) + ); +} + +/** + * Mint the `VIRTUAL` placeholder Class node (once) and an EXTENDS edge from + * every S4 class whose `contains=` names it (not an R6 class whose `inherit=` names a + * class called `VIRTUAL`). Scope is deliberately limited to + * the literal string `"VIRTUAL"`: other S4 base-type sentinels (`"numeric"`, + * `"list"`, …) are not treated as placeholder classes, which would be a + * separate design decision. + * + * Idempotent: re-seeds its dedup set from any `EXTENDS` edge already + * targeting a node named `VIRTUAL` before emitting, and re-uses the graph's + * existing `VIRTUAL` node instead of minting a second one when the + * orchestrator re-runs resolution. + */ +function emitRVirtualHeritageEdges( + graph: KnowledgeGraph, + _parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, + scopes?: ScopeResolutionIndexes, +): void { + if (scopes === undefined) return; + + const virtualGraphId = generateId('Class', VIRTUAL_CLASS_NAME); + const emitted = new Set(); + for (const rel of graph.iterRelationshipsByType('EXTENDS')) { + if (rel.targetId === virtualGraphId) emitted.add(rel.sourceId); + } + + let virtualNodeMinted = graph.getNode(virtualGraphId) !== undefined; + + for (const site of scopes.referenceSites) { + if (site.kind !== 'inherits' || site.name !== VIRTUAL_CLASS_NAME) continue; + const callerClass = findEnclosingClassDef(site.inScope, scopes); + if (callerClass === undefined || !isDeclaredByNamingCall(callerClass)) continue; + const callerGraphId = resolveDefGraphId(callerClass.filePath, callerClass, nodeLookup); + if (callerGraphId === undefined || emitted.has(callerGraphId)) continue; + + if (!virtualNodeMinted) { + graph.addNode({ + id: virtualGraphId, + label: 'Class', + properties: { name: VIRTUAL_CLASS_NAME, filePath: callerClass.filePath }, + }); + virtualNodeMinted = true; + } + emitted.add(callerGraphId); + graph.addRelationship({ + id: generateId('EXTENDS', `${callerGraphId}->${virtualGraphId}`), + sourceId: callerGraphId, + targetId: virtualGraphId, + type: 'EXTENDS', + confidence: 0.85, + reason: 'scope-resolution: inherits (virtual)', + }); + } +} + +/** + * `moduleScope` → its `ParsedFile`, built once per pass and keyed on the identity of the pass's + * `parsedFiles` array, which finalize passes unchanged to every `expandsWildcardTo` call. The + * lookup used to be `parsedFiles.find`, one scan of all files per `library()` edge: 450,000 edges + * over 3,000 files took 5.9 s. As in the C, C++ and Python providers, the first file wins on a + * repeated scope, as `find` did. + */ +const parsedFileByModuleScope = perFileSet( + (parsedFiles: readonly ParsedFile[]): Map => { + const index = new Map(); + for (const parsed of parsedFiles) { + if (!index.has(parsed.moduleScope)) index.set(parsed.moduleScope, parsed); + } + return index; + }, +); + +/** + * Enumerate the top-level names a `library()`/`require()` of a target module scope's file brings + * into scope (module-scope function and class bindings; functions nested in another function are + * not). `library()`/`require()` are R's wildcard imports, and R attaches only a package's + * EXPORTS: a name is kept when the target's package has no NAMESPACE, or the NAMESPACE provably + * exports it (`rSymbolVisibility` is `public`). A name the NAMESPACE does not export, or may + * export but cannot be proven to (unreadable NAMESPACE, a dropped or cut-short + * `exportPattern()`), is left out: strict, because the edge this binding yields is a precise + * 0.85 one. A bare call of such a name is not lost to the other tiers: the 0.5 unique-name guess + * skips it only when it is proven private (`isRGlobalNameFallbackPlausible`). + * + * The package config is read from the target's `ParsedFile` (`recordedRPackageConfig`); without + * one (hand-built input) nothing is filtered. The hook has no importer, so a `library(ownpkg)` + * issued from inside the package loses the package's private names as well (a documented gap). + * This reverses the earlier `load_all()` reading ("every name visible"): see + * docs/languages/r-provider.md. + * + * A bare name is dropped when the same file also defines a dotted top-level + * name with that tail (`foo` next to `print.foo`). The shared finalize indexes + * every def by the text after its last `.`, so `foo` would be bound to + * whichever of the two comes first in the file, and `print.foo` (an S3 method, + * never callable as `foo()`) can win. The dropped name resolves through the + * free-call fallback instead, where {@link rIsCallableVisibleFromCaller} + * removes the dotted def from the candidates. Order-blind, like the matching + * guard in `languages/r/import-target.ts`. + */ +export function expandRWildcardNames( + targetModuleScope: ScopeId, + parsedFiles: readonly ParsedFile[], +): readonly string[] { + const target = parsedFileByModuleScope(parsedFiles).get(targetModuleScope); + if (target === undefined) return []; + const { names: topLevel, dottedTails } = rFileTopLevel(target); + const cfg = recordedRPackageConfig(target); + const seen = new Set(); + const names: string[] = []; + for (const def of target.localDefs) { + const qn = def.qualifiedName; + if (qn === undefined || qn.length === 0 || qn.includes('.')) continue; // no dotted names + // `localDefs` is flat: it also lists functions nested in other functions, which a + // `library()` does not bring into scope. Only module-scope bindings are top-level. + // `topLevel` holds symbols and `qn` is the name as written, backticks included. + if (!topLevel.has(rBareName(qn))) continue; + if (dottedTails.has(qn)) continue; // shares its tail with an S3 method: see above (as written, as finalize keys it) + if (cfg !== undefined && rSymbolVisibility(cfg, target.filePath, rBareName(qn)) !== 'public') { + continue; // not an export of the package (or not provably one) + } + if (seen.has(qn)) continue; + seen.add(qn); + names.push(qn); + } + return names; +} + +/** + * The free-call name fallback must not bind a bare call `foo(x)` to a def whose + * name contains a `.`: `print.foo` is reached as `print.foo(...)` or through S3 + * dispatch, yet the fallback's simple-name index keys it under `foo`. Refuse + * such candidates so `foo(x)` cannot collide with `print.foo`. This is not a + * claim that a bare call never runs a dotted function: `my.helper(x)` and + * `read.config(x)` are ordinary calls. They skip this fallback and bind in the + * same file locally (0.85) or elsewhere by qualified-name lookup at 0.35 + * (`scope-resolution: call`), which `docs/languages/r-provider.md` describes. + * + * Applies to every candidate the hook sees, including R6/R5 members whose + * qualified name is `Class.member`. A bare `member(x)` never reaches an R6 + * member (it is called through `self$member()` or `obj$member()`), so the + * refusal is right for R6. RefClass methods are different: a method calls its + * siblings and inherited methods by bare name, and those calls get no edge + * here (see "Limits" in `docs/languages/r-provider.md`). + */ +export function rIsCallableVisibleFromCaller(ctx: { + readonly candidate: SymbolDefinition; +}): boolean { + return !(ctx.candidate.qualifiedName ?? '').includes('.'); +} + +export const rScopeResolver: ScopeResolver = { + language: SupportedLanguages.R, + languageProvider: rProvider, + importEdgeReason: 'r-scope: import', + + loadResolutionConfig: (repoPath: string) => loadRPackageConfig(repoPath), + + // The rename preview reloads the config uncached (as the contract requires), and also records + // the repository's R files that are not indexed sources, for the rename guard (`checkSafety` + // refuses a name such a file uses; see `collectRUnindexedSources`). + refreshResolutionConfig: async (repoPath: string, filePaths: readonly string[]) => ({ + ...(await loadRPackageConfig(repoPath)), + unindexedRSources: await collectRUnindexedSources(repoPath, filePaths), + }), + + resolveImportTarget: resolveRImportTarget, + + expandsWildcardTo: (targetModuleScope, parsedFiles) => + expandRWildcardNames(targetModuleScope, parsedFiles), + + mergeBindings: rMergeBindings, + arityCompatibility: rArityCompatibility, + + buildMro: (graph, parsedFiles, nodeLookup) => + buildMro(graph, parsedFiles, nodeLookup, defaultLinearize), + + populateOwners: (parsed: ParsedFile) => populateClassOwnedMembers(parsed), + + emitHeritageEdges: (graph, parsedFiles, nodeLookup, scopes) => + emitRVirtualHeritageEdges(graph, parsedFiles, nodeLookup, scopes), + + // A typed receiver (`self$m()`, `u <- User$new(); u$m()`) reaches the public members of the + // class and its parents; where none defines `m` the call is unresolved, not a call of a free + // function `m`. + resolveReceiverMember: rResolveSelfMember, + // An R6 or RefClass generator object (`User` in `User$save()`) is not an instance: it carries + // `new()` and the class metadata, never the class's instance methods. So a class name is never + // the receiver of a member call here. + isBareClassReceiverCandidate: () => false, + // A dotted class name (`My.Class`) is read by the shared receiver pass as a qualified one, which + // does not walk the class's parents and lets a lexically visible free function of the member's + // name win. Such a receiver gets no lookup: the call is recorded as `receiver-unresolved`, which + // also gives up the class's own and inherited methods (R-13). + suppressReceiverLookup: (typeRef) => typeRef.rawName.includes('.'), + // Restores which R6 members are private for a file reused from a worker or the parse cache. + applyCaptureSideChannel: applyRCaptureSideChannel, + + // R6's `super$method()` is not resolved: nothing binds `super` to the parent class, so there + // is no receiver type to walk from. Off, rather than answering a wrong parent. (`self` is + // bound, to the enclosing class, by `emitRScopeCaptures`; `private$` is not.) + isSuperReceiver: () => false, + + // R's dynamic-dispatch `self$field$method()` chains and R6/S4 field access + // benefit from the same heuristic Ruby/Python rely on. It does not bear on `self$m()` itself: + // that is settled by the typed `self` and `resolveReceiverMember` above. + fieldFallbackOnMethodLookup: true, + + // `source()` is a side-effect import (binds no name) — a cross-file free + // call into a `source()`d file (e.g. `HelperFunc(x)`) has no lexical or + // import binding to resolve through, so it resolves via a unique + // workspace-wide name match instead. `library()`/`require()`'s own + // wildcard-bound names still resolve through `expandsWildcardTo` above; + // this only covers what that path cannot reach. + allowGlobalFreeCallFallback: true, + + // A named import binds only to a MODULE-LEVEL definition of the target + // file. Without this, `findExportByName` indexes every `localDefs` entry by + // simple name, so an R6/R5 method named like an imported symbol could win + // the binding over the top-level function. + namedImportsBindTopLevelOnly: true, + + // NAMESPACE `importFrom()` names provided by a LOCAL package become + // synthesised `named` imports (`resolveRImportTarget`'s named branch binds + // them). Runs before finalize and over warm-cache ParsedFiles too. + // + // Followed by `populateRQualifiedCalls`: it decides each + // `pkg::name()` site by the qualifier — drops external / ambiguous ones and + // records the single definition a local one names for + // `resolveQualifiedFreeCall` below. Import synthesis is unchanged and first. + populateWorkspaceReferences: (parsedFiles, ctx) => { + // The rename guard reads the NAMESPACE this pass was configured with (see `checkSafety`), + // and looks it up by the ParsedFile objects it was handed. Both populate passes below + // replace entries of `parsedFiles` with copies (import synthesis; any dropped `pkg::fn` + // site), so the config is recorded against the objects as received, before any + // replacement, and again against the final array. + recordRPackageConfig(parsedFiles, ctx.resolutionConfig); + populateRNamespaceImports(parsedFiles, ctx); + populateRQualifiedCalls(parsedFiles, ctx); + recordRPackageConfig(parsedFiles, ctx.resolutionConfig); + }, + resolveQualifiedFreeCall: resolveRQualifiedFreeCall, + + // `populateRNamespaceImports` and `populateRQualifiedCalls` read only ParsedFile scopes, never source + // text: without this, merely declaring the hook makes the pipeline read the + // text of every R file on the main thread. + postExtractSourceTextPolicy: 'uncached-files', + + // A bare call whose name the caller package's NAMESPACE imports from a + // package other than the one the unique-name guess landed in is impossible + // in R (see `isRGlobalNameFallbackPlausible` for the exact rule, the + // qualifier-first handling of `pkg::name()` calls and the documented + // `import()` limitation). + isGlobalNameFallbackPlausible: isRGlobalNameFallbackPlausible, + + // The free-call fallback keys defs by the text after the last `.` of their + // qualified name, so an S3 method `print.foo` sits next to a bare `foo` and + // makes the bare call ambiguous (no edge). Drop those candidates from the + // name fallback; a call that spells the full dotted name binds by + // qualified-name lookup instead (see `rIsCallableVisibleFromCaller`). + isCallableVisibleFromCaller: rIsCallableVisibleFromCaller, +}; diff --git a/gitnexus/src/core/ingestion/languages/r/simple-hooks.ts b/gitnexus/src/core/ingestion/languages/r/simple-hooks.ts new file mode 100644 index 000000000..1129db419 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/simple-hooks.ts @@ -0,0 +1,121 @@ +import type { BindingRef, Callsite, Range, SymbolDefinition } from 'gitnexus-shared'; + +const TIER: Record = { + local: 0, + namespace: 1, + import: 2, + reexport: 3, + wildcard: 4, +}; + +/** `a` before `b` by the line, then the column, at which each starts. */ +const startsBefore = (a: Range, b: Range): boolean => + a.startLine !== b.startLine ? a.startLine < b.startLine : a.startCol < b.startCol; + +/** A package's binding of a name and the position of the `library()` call that first attached it. */ +interface Attachment { + readonly ref: BindingRef; + readonly at: Range; +} + +/** + * What a merge result remembers: every package that defines the name and where it was first + * attached, including the ones that result masks. The pipeline folds the bindings of a name in + * one at a time, so a package masked at one step would otherwise be forgotten, and a later + * `library()` of it would look like a fresh attachment. Keyed by the surviving bindings (not by + * the result array), so a copy of the array keeps it. A binding that has lost its entry is read as + * if nothing were masked. + */ +const attachments = new WeakMap>(); + +/** + * Local declarations shadow imports; deterministic order within a tier. + * Same shape as Zig's `zigMergeBindings`. + * + * Within the wildcard tier (`library()` / `require()` of a local package) a name that several + * attached packages define is masked as R masks it: the package attached last wins, so only the + * bindings of the `library()` call that comes last in the file are kept. A call that attaches a + * package of several files keeps all of that package's bindings for the name, because a name + * defined in two of its files is a real ambiguity the order cannot settle. The order is the + * position of the call in the file, which is the order the file attaches them in when it runs + * top to bottom; a `library()` inside a function, run later or never, is not told apart. + * + * A package counts at the call that attached it first: R leaves an attached package where it is, + * so `library(a); library(b); library(a)` keeps `b` masking `a`. When any of the bindings has no + * call position the whole tier is kept, as it was before. + */ +export function rMergeBindings( + existing: readonly BindingRef[], + incoming: readonly BindingRef[], + _scopeId: string, +): BindingRef[] { + const all = [...existing, ...incoming]; + if (all.length === 0) return []; + let bestTier = 99; + for (const b of all) { + const t = TIER[b.origin] ?? 99; + if (t < bestTier) bestTier = t; + } + const tierRefs = all.filter((b) => (TIER[b.origin] ?? 99) === bestTier); + const kept = bestTier === TIER.wildcard ? firstAttached(tierRefs) : firstPerNode(tierRefs); + // By code unit, not `localeCompare`: the order must not depend on the machine's locale. + return kept.sort((a, b) => + a.def.nodeId < b.def.nodeId ? -1 : a.def.nodeId > b.def.nodeId ? 1 : 0, + ); +} + +/** One binding per node: the first one seen. */ +function firstPerNode(bindings: readonly BindingRef[]): BindingRef[] { + const byNode = new Map(); + for (const b of bindings) if (!byNode.has(b.def.nodeId)) byNode.set(b.def.nodeId, b); + return [...byNode.values()]; +} + +/** + * The bindings that the `library()` call attached last brought in, a package counting at the call + * that attached it first. The packages that earlier merges masked take part, from their record. + */ +function firstAttached(bindings: readonly BindingRef[]): BindingRef[] { + const positioned: Array<{ ref: BindingRef; at: Range }> = []; + for (const ref of bindings) { + const at = ref.via?.atRange; + // Without a call position on every binding the order is unknown: keep the whole tier. + if (at === undefined) return firstPerNode(bindings); + positioned.push({ ref, at }); + } + const byNode = new Map(); + const offer = (a: Attachment): void => { + const seen = byNode.get(a.ref.def.nodeId); + if (seen === undefined || startsBefore(a.at, seen.at)) byNode.set(a.ref.def.nodeId, a); + }; + for (const a of positioned) { + for (const remembered of attachments.get(a.ref)?.values() ?? []) offer(remembered); + offer(a); + } + let last: Range | undefined; + for (const { at } of byNode.values()) if (last === undefined || startsBefore(last, at)) last = at; + const lastAt = last; + const kept = [...byNode.values()].filter( + ({ at }) => + lastAt !== undefined && at.startLine === lastAt.startLine && at.startCol === lastAt.startCol, + ); + for (const { ref } of kept) attachments.set(ref, byNode); + return kept.map(({ ref }) => ref); +} + +/** R declarations carry no synthesized arity metadata (no `...`/default-arg + * capture exists yet), so the comparison is always 'unknown' — a real bounds + * check that turns on if arity captures are added later. Same rationale as + * Zig's `zigArityCompatibility`. */ +export function rArityCompatibility( + callsite: Callsite, + def: SymbolDefinition, +): 'compatible' | 'unknown' | 'incompatible' { + const max = def.parameterCount; + const min = def.requiredParameterCount; + if (max === undefined && min === undefined) return 'unknown'; + if (!Number.isFinite(callsite.arity) || callsite.arity! < 0) return 'unknown'; + if (min !== undefined && callsite.arity! < min) return 'incompatible'; + if (max !== undefined && callsite.arity! > max) return 'incompatible'; + return 'compatible'; +} diff --git a/gitnexus/src/core/ingestion/languages/r/unindexed-sources.ts b/gitnexus/src/core/ingestion/languages/r/unindexed-sources.ts new file mode 100644 index 000000000..2fc2638f8 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/r/unindexed-sources.ts @@ -0,0 +1,129 @@ +/** + * R files of a repository that the index does not cover, for the rename guard. + * + * The indexer skips whole directories (`bin/`, `build/`, `tmp/`, ...), `.gitignore`d paths and + * files over its size cap, and indexes only `.R`/`.r` files, so a script there that calls a + * function would keep the old name after a rename. R itself still runs such files (`Rscript + * bin/run.R`, `.Rprofile`, the `.S`/`.q`/`.s` code files R installs from `R/`). The rename preview + * reloads its configuration (`refreshResolutionConfig`), which records each such file with the + * identifier tokens it contains (tokens, not text, to keep the retained plan small); the guard + * refuses a rename whose old name one of them uses, and the plan's resolution snapshot makes an + * edit of such a file between preview and apply a `resolution_changed`. + */ +import type { Dirent } from 'fs'; +import fs from 'fs/promises'; +import path from 'path'; +import ignore, { type Ignore } from 'ignore'; + +export interface RUnindexedSource { + /** POSIX path relative to the repository root. */ + readonly path: string; + /** The distinct identifier tokens of the file, sorted (backticks removed). */ + readonly words: readonly string[]; + /** The file is over {@link MAX_UNINDEXED_R_SOURCE_BYTES} or could not be read. */ + readonly unreadable?: true; +} + +/** Larger files are not read; the guard then fails closed. */ +export const MAX_UNINDEXED_R_SOURCE_BYTES = 8 * 1024 * 1024; + +/** A repository-relative path as graph File nodes spell it: forward slashes, no leading `./`. */ +export function rPosixRelativePath(filePath: string): string { + let normalized = filePath.replace(/\\/g, '/'); + while (normalized.startsWith('./')) normalized = normalized.slice(2); + return normalized; +} + +/** `.R`/`.r` files, `.Rprofile`, and the `.S`/`.q`/`.s` code files under an `R/` directory. */ +export function isRSourcePath(relative: string): boolean { + const base = relative.slice(relative.lastIndexOf('/') + 1); + return ( + /\.[Rr]$/.test(base) || base === '.Rprofile' || /(^|\/)R\/(.*\/)?[^/]*\.[Sqs]$/.test(relative) + ); +} + +/** + * Directories that hold no code of this repository: version control, the index store, installed + * packages (`node_modules`, `renv/library`, `packrat/lib*`) and `R CMD check` output (`*.Rcheck`, + * which holds a copy of the package's own sources). + */ +function isSkippedDirectory(relative: string): boolean { + const segments = relative.split('/'); + const name = segments[segments.length - 1]; + const parent = segments[segments.length - 2]; + return ( + name === '.git' || + name === '.gitnexus' || + name === 'node_modules' || + name.endsWith('.Rcheck') || + (parent === 'renv' && name === 'library') || + (parent === 'packrat' && name.startsWith('lib')) + ); +} + +/** Identifier tokens: plain names (`helper`, `.helper`, `to.json`) and backticked ones. */ +const TOKEN = /`([^`\n]+)`|[A-Za-z.][A-Za-z0-9._]*/g; + +function tokens(text: string): string[] { + const words = new Set(); + for (const match of text.matchAll(TOKEN)) words.add(match[1] ?? match[0]); + return [...words].sort(); +} + +/** The repository's `.gitnexusignore` rules (the user's explicit "not mine"), if any. */ +async function gitnexusIgnore(repoRoot: string): Promise { + try { + const file = path.join(repoRoot, '.gitnexusignore'); + if (!(await fs.lstat(file)).isFile()) return undefined; + return ignore().add(await fs.readFile(file, 'utf8')); + } catch { + return undefined; + } +} + +/** + * Every regular R file under `repoRoot` that is not in `indexedPaths`, with its tokens. The walk + * does not follow symbolic links, skips the directories of {@link isSkippedDirectory} and what + * `.gitnexusignore` excludes, and deliberately ignores the indexer's built-in ignore list, + * `.gitignore` and its size cap (those are what hide the files). Sorted by path. + */ +export async function collectRUnindexedSources( + repoRoot: string, + indexedPaths: readonly string[], +): Promise { + const indexed = new Set(indexedPaths.map(rPosixRelativePath)); + const ignored = await gitnexusIgnore(repoRoot); + const result: RUnindexedSource[] = []; + const pending: string[] = ['']; + for (let directory = pending.pop(); directory !== undefined; directory = pending.pop()) { + let entries: Dirent[]; + try { + entries = await fs.readdir(path.join(repoRoot, directory), { withFileTypes: true }); + } catch { + // It may hold R files: fail closed. + result.push({ path: directory, words: [], unreadable: true }); + continue; + } + for (const entry of entries) { + const relative = directory === '' ? entry.name : `${directory}/${entry.name}`; + if (entry.isDirectory()) { + if (isSkippedDirectory(relative) || ignored?.ignores(`${relative}/`)) continue; + pending.push(relative); + continue; + } + if (!entry.isFile() || !isRSourcePath(relative) || indexed.has(relative)) continue; + if (ignored?.ignores(relative)) continue; + const absolute = path.join(repoRoot, relative); + try { + if ((await fs.stat(absolute)).size > MAX_UNINDEXED_R_SOURCE_BYTES) { + result.push({ path: relative, words: [], unreadable: true }); + continue; + } + result.push({ path: relative, words: tokens(await fs.readFile(absolute, 'utf8')) }); + } catch { + result.push({ path: relative, words: [], unreadable: true }); + } + } + } + return result.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); +} diff --git a/gitnexus/src/core/ingestion/method-extractors/r.ts b/gitnexus/src/core/ingestion/method-extractors/r.ts new file mode 100644 index 000000000..37e2deea6 --- /dev/null +++ b/gitnexus/src/core/ingestion/method-extractors/r.ts @@ -0,0 +1,266 @@ +// gitnexus/src/core/ingestion/method-extractors/r.ts + +/** + * R method extractor — hand-written because R expresses classes and methods + * as function calls rather than dedicated syntax. + * + * Handles: + * - R6 methods: function entries in public/private/active list() + * - S4 setMethod(): owner hints for method implementations dispatched on signature + * - R5/setRefClass methods: methods = list(name = function(...) {...}) + */ + +import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { isRLeftAssignment } from '../languages/r/assignment.js'; +import { getRMatchedArgument } from '../languages/r/naming-argument.js'; +import { getRClassNameArgument } from '../field-extractors/r.js'; +import type { + MethodExtractor, + MethodExtractorContext, + ExtractedMethods, + MethodInfo, + MethodVisibility, + ParameterInfo, +} from '../method-types.js'; + +/** `setMethod`'s formals, in order (methods package). */ +const SET_METHOD_FORMALS: readonly string[] = [ + 'f', + 'signature', + 'definition', + 'where', + 'valueClass', + 'sealed', +]; + +export const rMethodExtractor: MethodExtractor = { + language: SupportedLanguages.R, + + isTypeDeclaration(node: SyntaxNode): boolean { + return isR6Class(node) || isRefClass(node); + }, + + extract(node: SyntaxNode, context: MethodExtractorContext): ExtractedMethods | null { + if (isR6Class(node)) return extractR6Methods(node, context); + if (isRefClass(node)) return extractRefClassMethods(node, context); + return null; + }, +}; + +/** Resolve the owning class name for a top-level R setMethod("foo", "Bar", ...) definition. */ +export function getRTopLevelMethodOwnerName(node: SyntaxNode): string | null { + if (!isS4SetMethod(node)) return null; + const args = node.childForFieldName('arguments'); + if (!args) return null; + + // The owner is the `signature` argument, as R matches it (named first, then by position). + const signature = getRMatchedArgument(args, SET_METHOD_FORMALS, 'signature'); + const value = signature?.childForFieldName('value') ?? null; + return extractQuotedString(value) ?? extractFirstSignatureClass(value); +} + +/** + * The class a `signature("A", "B")`, `signature(object = "A")` or `c("A", "B")` value dispatches + * on first: the string of its first argument, named or not. A method on several classes (multiple + * dispatch) belongs to the first one, so methods of one generic on different first classes keep + * distinct ids and an owner; two methods that differ only in a later class share an id and the + * first written wins. A first argument that is not a string (a variable, a call) names no owner. + */ +function extractFirstSignatureClass(value: SyntaxNode | null): string | null { + if (!value || value.type !== 'call') return null; + const fn = value.childForFieldName('function'); + const callee = fn?.type === 'namespace_operator' ? fn.childForFieldName('rhs') : fn; + if (callee?.type !== 'identifier' || (callee.text !== 'signature' && callee.text !== 'c')) { + return null; + } + const first = value + .childForFieldName('arguments') + ?.namedChildren.find((c) => c.type === 'argument'); + return extractQuotedString(first?.childForFieldName('value') ?? null); +} + +// --------------------------------------------------------------------------- +// R6 method extraction +// --------------------------------------------------------------------------- + +function extractR6Methods( + node: SyntaxNode, + context: MethodExtractorContext, +): ExtractedMethods | null { + const ownerName = node.childForFieldName('lhs')?.text; + if (!ownerName) return null; + + const call = node.childForFieldName('rhs'); + if (!call) return null; + + const args = call.childForFieldName('arguments'); + if (!args) return null; + + const methods: MethodInfo[] = []; + + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (!arg || arg.type !== 'argument') continue; + + const section = arg.childForFieldName('name')?.text; + if (section !== 'public' && section !== 'private' && section !== 'active') continue; + + const visibility: MethodVisibility = section === 'private' ? 'private' : 'public'; + + const listCall = arg.childForFieldName('value'); + if (!listCall || listCall.type !== 'call') continue; + + const listArgs = listCall.childForFieldName('arguments'); + if (!listArgs) continue; + + for (let j = 0; j < listArgs.namedChildCount; j++) { + const entry = listArgs.namedChild(j); + if (!entry || entry.type !== 'argument') continue; + + const nameNode = entry.childForFieldName('name'); + const valueNode = entry.childForFieldName('value'); + if (!nameNode || valueNode?.type !== 'function_definition') continue; + + methods.push({ + name: nameNode.text, + receiverType: null, + returnType: null, + parameters: extractRParameters(valueNode), + visibility, + isStatic: false, + isAbstract: false, + isFinal: false, + annotations: section === 'active' ? ['@active'] : [], + sourceFile: context.filePath, + line: entry.startPosition.row + 1, + column: entry.startPosition.column, + }); + } + } + + return methods.length > 0 ? { ownerName, methods } : null; +} + +// --------------------------------------------------------------------------- +// S4 / R5 (setRefClass) method extraction +// --------------------------------------------------------------------------- + +function extractRefClassMethods( + node: SyntaxNode, + context: MethodExtractorContext, +): ExtractedMethods | null { + const args = node.childForFieldName('arguments'); + if (!args) return null; + + // The class name is the argument that takes the `Class` formal (by name, else the + // first unnamed one), not the first string: `contains = "Base"` may precede it. + const ownerName = getRClassNameArgument(args); + if (!ownerName) return null; + + const methods: MethodInfo[] = []; + + // setRefClass(... methods = list(name = function(...) { ... })) + for (let i = 0; i < args.namedChildCount; i++) { + const arg = args.namedChild(i); + if (!arg || arg.type !== 'argument') continue; + + const paramName = arg.childForFieldName('name')?.text; + if (paramName !== 'methods') continue; + + const listCall = arg.childForFieldName('value'); + if (!listCall || listCall.type !== 'call') continue; + + const listArgs = listCall.childForFieldName('arguments'); + if (!listArgs) continue; + + for (let j = 0; j < listArgs.namedChildCount; j++) { + const entry = listArgs.namedChild(j); + if (!entry || entry.type !== 'argument') continue; + + const nameNode = entry.childForFieldName('name'); + const valueNode = entry.childForFieldName('value'); + if (!nameNode || valueNode?.type !== 'function_definition') continue; + + methods.push({ + name: nameNode.text, + receiverType: null, + returnType: null, + parameters: extractRParameters(valueNode), + visibility: 'public', + isStatic: false, + isAbstract: false, + isFinal: false, + annotations: [], + sourceFile: context.filePath, + line: entry.startPosition.row + 1, + column: entry.startPosition.column, + }); + } + } + + return methods.length > 0 ? { ownerName, methods } : null; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function isR6Class(node: SyntaxNode): boolean { + if (!isRLeftAssignment(node)) return false; + const rhs = node.childForFieldName('rhs'); + if (!rhs || rhs.type !== 'call') return false; + const fn = rhs.childForFieldName('function'); + if (!fn) return false; + if (fn.type === 'identifier' && fn.text === 'R6Class') return true; + if (fn.type === 'namespace_operator') { + return fn.childForFieldName('rhs')?.text === 'R6Class'; + } + return false; +} + +function isRefClass(node: SyntaxNode): boolean { + if (node.type !== 'call') return false; + const fn = node.childForFieldName('function'); + return fn?.type === 'identifier' && fn.text === 'setRefClass'; +} + +function isS4SetMethod(node: SyntaxNode): boolean { + if (node.type !== 'call') return false; + const fn = node.childForFieldName('function'); + return fn?.type === 'identifier' && fn.text === 'setMethod'; +} + +function extractQuotedString(node: SyntaxNode | null): string | null { + if (!node || node.type !== 'string') return null; + const content = node.namedChildren.find((c) => c.type === 'string_content'); + return content?.text ?? node.text.replace(/^["']|["']$/g, ''); +} + +/** + * Extract the formals of an R function_definition node. The grammar emits one + * `parameter` node per formal (`name` is an `identifier`, or `dots` for `...`); + * a formal with a `default` is optional, and `...` is variadic. + */ +function extractRParameters(fnNode: SyntaxNode): ParameterInfo[] { + const params: ParameterInfo[] = []; + const paramList = fnNode.childForFieldName('parameters'); + if (!paramList) return params; + + for (let i = 0; i < paramList.namedChildCount; i++) { + const param = paramList.namedChild(i); + if (!param || param.type !== 'parameter') continue; + + const nameNode = param.childForFieldName('name'); + if (!nameNode) continue; + + params.push({ + name: nameNode.text, + type: null, + isOptional: param.childForFieldName('default') !== null, + isVariadic: nameNode.type === 'dots', + }); + } + + return params; +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index a93a4b0a7..3bb4f6ec4 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -63,6 +63,7 @@ import { readFileContents } from '../filesystem-walker.js'; import { isLanguageAvailable, isGrammarRuntimeSkipped, + grammarRebuildPackage, createParserForLanguage, } from '../../tree-sitter/parser-loader.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; @@ -108,7 +109,7 @@ import { import { normalizeExtractedRoutePath } from '../route-extractors/route-path.js'; import { resolveOperands } from '../route-extractors/python-const-resolver.js'; import type { ModuleConstants } from '../route-extractors/constant-resolver.js'; -import { prepareRouteConstantsByProvider } from '../language-provider.js'; +import { prepareRouteConstantsByProvider, runPostParseHooks } from '../language-provider.js'; import { resolveInheritedSpringRoutes, type SharedSpringType, @@ -541,7 +542,7 @@ export async function runChunkedParseAndResolve( ); } else { logger.warn( - `Skipping ${count} ${lang} file(s) — ${lang} parser not available (native binding may not have built). Try: npm rebuild tree-sitter-${lang}`, + `Skipping ${count} ${lang} file(s) — ${lang} parser not available (native binding may not have built). Try: npm rebuild ${grammarRebuildPackage(lang)}`, ); } } @@ -1553,6 +1554,15 @@ export async function runChunkedParseAndResolve( } } + // Language post-parse hooks: whole-graph work a language can only do once every chunk is merged + // and before scope resolution (e.g. deferred owner resolution). Only languages with parsed files pay. + const presentLanguages = new Set(); + for (const file of parseableScanned) { + const language = languageForScannedFile(file); + if (language !== null) presentLanguages.add(language); + } + await runPostParseHooks(presentLanguages, getProvider, { graph, model, repoPath }); + // Worker-path enrichment: if exportedTypeMap is empty (e.g. the worker pool // built TypeEnv inside workers without access to SymbolTable), reconstruct // the map from graph nodes + SymbolTable here in the main thread before diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index d61865bbb..7a1ffff9d 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -1455,11 +1455,16 @@ export interface ScopeResolver { * risk LOW with 0 affected processes). * * `emitFreeCallFallback` invokes this BEFORE the implicit-`this` and - * scope-chain lookups, so an explicit path outranks a lexical shadow. + * scope-chain lookups, so an explicit path outranks a lexical shadow. That + * includes a lexical NAME CLAIM (`lexicalNames`, a parameter or a local of the + * same name): the contract left open whether such a claim blocks the call + * before the hook runs, and it does not — a defined result for a site with + * `rawQualifiedName` bypasses the block, because the written path names the + * module and a local binding of the bare tail cannot be what it means. * Returning `undefined` (unqualified call, unknown module, or no such - * member in the named module) falls through to the unchanged chain — the - * hook is strictly additive and never removes an edge the prior tiers - * would have produced. + * member in the named module) falls through to the unchanged chain, name-claim + * block included — the hook never removes an edge the prior tiers would have + * produced. * * Languages whose qualified calls carry a value/type receiver (`x.foo()`, * `Type::foo()`) are served by the receiver-bound-calls pass and leave diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 25f9748c3..a09beeee6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -285,7 +285,14 @@ export function emitFreeCallFallback( b.def.type === 'Struct' || b.def.type === 'Record', ))); - if (blocksCall) { + // A module-qualified free call (`mod::fn()`) names its module explicitly, so the path is + // evidence the lexical name claim is not: a local `fn` binds the bare tail, never `mod::fn`. + // The hook therefore runs before the block, and a defined hit bypasses it. + const qualifiedFnDef = + options.resolveQualifiedFreeCall !== undefined && site.rawQualifiedName !== undefined + ? options.resolveQualifiedFreeCall(site, parsed, scopes, workspaceIndex, allFilePaths()) + : undefined; + if (blocksCall && qualifiedFnDef === undefined) { options.recordSourceBinding?.(site, { status: 'blocked', targetDefIds: [], @@ -421,20 +428,9 @@ export function emitFreeCallFallback( // the caller's own file and emits a self-loop instead of the real // cross-module edge (#2730). Only fires when the language populated // `rawQualifiedName` AND provides the hook; `undefined` falls through to - // the unchanged chain, so this tier is strictly additive. - if ( - fnDef === undefined && - options.resolveQualifiedFreeCall !== undefined && - site.rawQualifiedName !== undefined - ) { - fnDef = options.resolveQualifiedFreeCall( - site, - parsed, - scopes, - workspaceIndex, - allFilePaths(), - ); - } + // the unchanged chain, so this tier is strictly additive. Resolved above, before the + // name-claim block, so that a hit also outranks a lexical name claim. + if (fnDef === undefined) fnDef = qualifiedFnDef; // Implicit-this overload narrowing: an unqualified call inside // a method body might be calling a sibling overload on the // enclosing class. When the workspace has multiple methods of diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts index ee98e3807..82f7c681c 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/registry.ts @@ -29,6 +29,7 @@ import { dartScopeResolver } from '../../languages/dart/scope-resolver.js'; import { vueScopeResolver } from '../../languages/vue/scope-resolver.js'; import { zigScopeResolver } from '../../languages/zig/scope-resolver.js'; import { objectiveCScopeResolver } from '../../languages/objective-c/scope-resolver.js'; +import { rScopeResolver } from '../../languages/r/scope-resolver.js'; /** Map of `SupportedLanguages` → `ScopeResolver`. The scope-resolution phase * iterates this map directly — every registered resolver runs. This is the @@ -55,4 +56,5 @@ export const SCOPE_RESOLVERS: ReadonlyMap = n [SupportedLanguages.Vue, vueScopeResolver], [SupportedLanguages.Zig, zigScopeResolver], [SupportedLanguages.ObjectiveC, objectiveCScopeResolver], + [SupportedLanguages.R, rScopeResolver], ]); diff --git a/gitnexus/src/core/ingestion/tree-sitter-queries.ts b/gitnexus/src/core/ingestion/tree-sitter-queries.ts index 2e900e642..8572ebfd9 100644 --- a/gitnexus/src/core/ingestion/tree-sitter-queries.ts +++ b/gitnexus/src/core/ingestion/tree-sitter-queries.ts @@ -3064,6 +3064,232 @@ export const ZIG_QUERIES = ` function: (field_expression member: (identifier) @call.name)) @call `; +/** + * Every named expression kind the R grammar allows as an argument value, except + * `function_definition` (an R6 entry holding a function is a method, not a field). + * Mirrors RFieldExtractor, which accepts any non-function value, so the query and + * the extractor agree on what a field is. + */ +const R6_FIELD_VALUE = `[ + (binary_operator) (braced_expression) (break) (call) (complex) (dot_dot_i) (dots) + (extract_operator) (false) (float) (for_statement) (identifier) (if_statement) (inf) + (integer) (na) (namespace_operator) (nan) (next) (null) (parenthesized_expression) + (repeat_statement) (return) (string) (subset) (subset2) (true) (unary_operator) + (while_statement) +]`; +// Every R `binary_operator` anchor below is an assignment (`operator: ["<-" "<<-" "="]`, see +// languages/r/assignment.ts): the grammar has one node for all infix operators. +export const R_QUERIES = ` +; ── Functions (name <- function(...) or name = function(...)) ──────────────── +(binary_operator + lhs: (identifier) @name + operator: ["<-" "<<-" "="] + rhs: (function_definition)) @definition.function + +; ── S4 Classes (setClass("ClassName", ...)) ────────────────────────────────── +; The S4 patterns below capture EVERY string argument of the call on purpose. +; Which argument names the definition is decided by the provider's +; \`shouldSkipDefinitionCapture\` (languages/r/naming-argument.ts): a query cannot skip an +; unbounded run of leading comments and named arguments without a repeated +; sibling group, which is quadratic in the argument count. +(call + function: (identifier) @_fn + (#match? @_fn "^(setClass|setRefClass)$") + arguments: (arguments + (argument + value: (string + content: (string_content) @name)))) @definition.class + +; ── R6 Classes via namespace (ClassName <- R6::R6Class(...)) ───────────────── +(binary_operator + lhs: (identifier) @name + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_nsfn + (#match? @_nsfn "^R6Class$")))) @definition.class + +; ── R6 Classes via bare call (ClassName <- R6Class(...)) ───────────────────── +(binary_operator + lhs: (identifier) @name + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6fn + (#match? @_r6fn "^R6Class$"))) @definition.class + +; ── R6 Methods via namespace (inside public/private/active = list(...)) ───── +(binary_operator + lhs: (identifier) @_class + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_nsfn2 + (#match? @_nsfn2 "^R6Class$")) + arguments: (arguments + (argument + name: (identifier) @_section + (#match? @_section "^(public|private|active)$") + value: (call + function: (identifier) @_listfn + (#match? @_listfn "^list$") + arguments: (arguments + (argument + name: (identifier) @name + value: (function_definition)) @definition.method)))))) + +; ── R6 Methods via bare call (inside public/private/active = list(...)) ───── +(binary_operator + lhs: (identifier) @_class2 + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6fn2 + (#match? @_r6fn2 "^R6Class$") + arguments: (arguments + (argument + name: (identifier) @_section2 + (#match? @_section2 "^(public|private|active)$") + value: (call + function: (identifier) @_listfn2 + (#match? @_listfn2 "^list$") + arguments: (arguments + (argument + name: (identifier) @name + value: (function_definition)) @definition.method)))))) + +; ── R5 Methods (inside setRefClass(... methods = list(...))) ───────────────── +; The call itself, assigned or not: \`Person <- setRefClass(...)\` and a bare +; \`setRefClass(...)\` define the same methods. +(call + function: (identifier) @_refFn + (#match? @_refFn "^setRefClass$") + arguments: (arguments + (argument + name: (identifier) @_methods + (#match? @_methods "^methods$") + value: (call + function: (identifier) @_listfn3 + (#match? @_listfn3 "^list$") + arguments: (arguments + (argument + name: (identifier) @name + value: (function_definition)) @definition.method))))) + +; ── S4 setGeneric() (generic function declaration) ─────────────────────────── +(call + function: (identifier) @_fn + (#match? @_fn "^setGeneric$") + arguments: (arguments + (argument + value: (string + content: (string_content) @name)))) @definition.function + +; ── S4 setMethod() (method implementation for a class) ────────────────────── +(call + function: (identifier) @_fn + (#match? @_fn "^setMethod$") + arguments: (arguments + (argument + value: (string + content: (string_content) @name)))) @definition.method + +; ── All function calls ─────────────────────────────────────────────────────── +(call + function: (identifier) @call.name) @call + +; ── Namespaced calls (pkg::func()) ─────────────────────────────────────────── +(call + function: (namespace_operator + lhs: (identifier) @call.namespace + rhs: (identifier) @call.name)) @call + +; ── Member calls via $ (obj$method()) ─────────────────────────────────────── +(call + function: (extract_operator + rhs: (identifier) @call.name)) @call + +; ── Imports (library/require/source) ───────────────────────────────────────── +(call + function: (identifier) @_fn + (#match? @_fn "^(library|require|source)$") + arguments: (arguments + (argument + value: [(identifier) (string)] @import.source))) @import + +; ── R6 Fields via namespace (non-function entries in public/private list()) ── +(binary_operator + lhs: (identifier) @_class + operator: ["<-" "<<-" "="] + rhs: (call + function: (namespace_operator + rhs: (identifier) @_nsfn + (#match? @_nsfn "^R6Class$")) + arguments: (arguments + (argument + name: (identifier) @_section + (#match? @_section "^(public|private)$") + value: (call + function: (identifier) @_listfn + (#match? @_listfn "^list$") + arguments: (arguments + (argument + name: (identifier) @name + value: ${R6_FIELD_VALUE} @_val) @definition.property)))))) + +; ── R6 Fields via bare call ────────────────────────────────────────────────── +(binary_operator + lhs: (identifier) @_class2 + operator: ["<-" "<<-" "="] + rhs: (call + function: (identifier) @_r6fn + (#match? @_r6fn "^R6Class$") + arguments: (arguments + (argument + name: (identifier) @_section2 + (#match? @_section2 "^(public|private)$") + value: (call + function: (identifier) @_listfn2 + (#match? @_listfn2 "^list$") + arguments: (arguments + (argument + name: (identifier) @name + value: ${R6_FIELD_VALUE} @_val2) @definition.property)))))) + +; ── S4 slots / representation ──────────────────────────────────────────────── +(call + function: (identifier) @_fn + (#match? @_fn "^(setClass|setRefClass)$") + arguments: (arguments + (argument + name: (identifier) @_slotArg + (#match? @_slotArg "^(representation|slots|fields)$") + value: (call + function: (identifier) @_listfn + (#match? @_listfn "^(list|representation|c)$") + arguments: (arguments + (argument + name: (identifier) @name) @definition.property))))) + +; ── S4 / RefClass unnamed names: slots = c("a", "b"), fields = c("a", "b") ── +; An unnamed character vector names the slots or fields (their class is ANY). Only +; c(...) reads this way: an unnamed string in representation(...) is a superclass. +; The callee is captured as @_vecOwner, not @_fn, so these string captures are not +; mistaken for the class-naming argument by isRNonNamingArgumentCapture. +(call + function: (identifier) @_vecOwner + (#match? @_vecOwner "^(setClass|setRefClass)$") + arguments: (arguments + (argument + name: (identifier) @_vecArg + (#match? @_vecArg "^(slots|fields)$") + value: (call + function: (identifier) @_vecfn + (#match? @_vecfn "^c$") + arguments: (arguments + (argument + !name + value: (string + content: (string_content) @name)) @definition.property))))) +`; import { SupportedLanguages } from 'gitnexus-shared'; @@ -3086,6 +3312,7 @@ export const LANGUAGE_QUERIES: Record = { [SupportedLanguages.Swift]: SWIFT_QUERIES, [SupportedLanguages.Dart]: DART_QUERIES, [SupportedLanguages.Vue]: TYPESCRIPT_QUERIES, // Vue ', ], + r: ['R', 'twice <- function(x) {\n return(x + x)\n}\n'], cobol: [ 'cbl', ' IDENTIFICATION DIVISION.\n PROGRAM-ID. SAMPLE.\n PROCEDURE DIVISION.\n MAIN-PARA.\n DISPLAY "HI".\n STOP RUN.\n', diff --git a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts index 0d38a1f17..7da9cb6f0 100644 --- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts @@ -417,6 +417,21 @@ const CASES: ReadonlyMap = new Map([ reachesDecoy: 'std.zig', }, ], + [ + SupportedLanguages.R, + { + files: ['scripts/main.R', 'vendor/ggplot2.py', 'pkgA/R/utils.R'], + fromFile: 'scripts/main.R', + resolutionConfig: undefined, + external: 'ggplot2', + decoy: 'vendor/ggplot2.py', + reachesDecoy: 'ggplot2.py', + parsedImport: (targetRaw) => + targetRaw === 'ggplot2' + ? { kind: 'wildcard', targetRaw } + : { kind: 'side-effect', targetRaw }, + }, + ], [ SupportedLanguages.ObjectiveC, { @@ -546,4 +561,35 @@ describe('external imports never resolve into the repository (#2953)', () => { `${language}: '${testCase.external}' names nothing in this repo, but resolved to ${JSON.stringify(resolved)}`, ).toEqual([]); }); + // R's NAMESPACE `importFrom(ggplot2, geom)` is a `named` import (the wildcard/side-effect arms + // above cover `library()`/`source()`). The bare package name must not suffix-match the + // same-named `vendor/ggplot2.py`, whether or not a local package config exists. + it.each([ + ['no package config', undefined], + [ + 'a package config that lists only other packages', + { packages: new Map([['pkgA', 'pkgA']]), namespaceInfoByPackageDir: new Map() }, + ], + ])( + 'R: a named importFrom() of an external package resolves to nothing (%s)', + (_label, config) => { + const testCase = CASES.get(SupportedLanguages.R)!; + const answer = SCOPE_RESOLVERS.get(SupportedLanguages.R)!.resolveImportTarget( + testCase.external, + testCase.fromFile, + new Set(testCase.files), + config, + { + parsedFiles: [], + parsedImport: { + kind: 'named', + localName: 'geom', + importedName: 'geom', + targetRaw: testCase.external, + }, + }, + ); + expect(filesOf(answer)).toEqual([]); + }, + ); }); diff --git a/gitnexus/test/unit/scope-resolution/free-call-qualified-over-name-claim.test.ts b/gitnexus/test/unit/scope-resolution/free-call-qualified-over-name-claim.test.ts new file mode 100644 index 000000000..63ddd790b --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/free-call-qualified-over-name-claim.test.ts @@ -0,0 +1,236 @@ +/** + * A module-qualified free call (`mod::fn()`) names its module, so a lexical name claim for the + * bare tail (a local binding or parameter called `fn`) cannot be what the call means. The + * `resolveQualifiedFreeCall` hook therefore runs BEFORE the name-claim block, and a defined hit + * bypasses it. An undefined hit, a site without a written path and a language without the hook + * keep the block, so the reorder only ever adds an edge for a site the hook resolved. + * + * The hook is a fake: this pins the ordering in the shared pass, not any language. + */ +import { describe, it, expect } from 'vitest'; +import { + buildDefIndex, + buildMethodDispatchIndex, + buildModuleScopeIndex, + buildQualifiedNameIndex, + buildScopeTree, + type NodeLabel, + type ParsedFile, + type Range, + type ReferenceSite, + type Scope, + type SymbolDefinition, +} from 'gitnexus-shared'; +import { createKnowledgeGraph } from '../../../src/core/graph/graph.js'; +import type { ScopeResolutionIndexes } from '../../../src/core/ingestion/model/scope-resolution-indexes.js'; +import { buildGraphNodeLookup } from '../../../src/core/ingestion/scope-resolution/graph-bridge/node-lookup.js'; +import { emitFreeCallFallback } from '../../../src/core/ingestion/scope-resolution/passes/free-call-fallback.js'; +import { buildWorkspaceResolutionIndex } from '../../../src/core/ingestion/scope-resolution/workspace-index.js'; +import { createSemanticModel } from '../../../src/core/ingestion/model/semantic-model.js'; +import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; + +const CALLER_FILE = 'caller.src'; +const TARGET_FILE = 'target.src'; + +const range = (line: number): Range => ({ + startLine: line, + startCol: 2, + endLine: line, + endCol: 8, +}); + +const targetDef: SymbolDefinition = { + nodeId: 'def:helper', + filePath: TARGET_FILE, + type: 'Function', + qualifiedName: 'helper', +}; +const callerDef: SymbolDefinition = { + nodeId: 'def:main', + filePath: CALLER_FILE, + type: 'Function', + qualifiedName: 'main', +}; + +/** `helper()` or `target::helper()`: the second form carries the written path. */ +const site = (qualified: boolean): ReferenceSite => ({ + name: 'helper', + atRange: range(3), + inScope: 'scope:caller-mod', + kind: 'call', + callForm: 'free', + arity: 0, + ...(qualified ? { rawQualifiedName: 'target::helper' } : {}), +}); + +function mkScope( + id: string, + filePath: string, + ownedDefs: SymbolDefinition[], + lexicalNames?: ReadonlySet, +): Scope { + return { + id, + parent: null, + kind: 'Module', + range: { startLine: 1, startCol: 0, endLine: 100, endCol: 0 }, + filePath, + bindings: new Map(), + ownedDefs, + imports: [], + typeBindings: new Map(), + ...(lexicalNames === undefined ? {} : { lexicalNames }), + }; +} + +type Hook = NonNullable; + +function run(reference: ReferenceSite, hook: Hook | undefined, claimed = true) { + const callerScope = mkScope( + 'scope:caller-mod', + CALLER_FILE, + [callerDef], + claimed ? new Set(['helper']) : undefined, + ); + const targetScope = mkScope('scope:target-mod', TARGET_FILE, [targetDef]); + const callerParsed: ParsedFile = { + filePath: CALLER_FILE, + moduleScope: 'scope:caller-mod', + scopes: [callerScope], + parsedImports: [], + localDefs: [callerDef], + referenceSites: [reference], + }; + const targetParsed: ParsedFile = { + filePath: TARGET_FILE, + moduleScope: 'scope:target-mod', + scopes: [targetScope], + parsedImports: [], + localDefs: [targetDef], + referenceSites: [], + }; + const scopes = [callerScope, targetScope]; + const allDefs = [callerDef, targetDef]; + const indexes = { + scopeTree: buildScopeTree(scopes), + defs: buildDefIndex(allDefs), + qualifiedNames: buildQualifiedNameIndex(allDefs), + moduleScopes: buildModuleScopeIndex( + scopes.map((s) => ({ filePath: s.filePath, moduleScopeId: s.id })), + ), + methodDispatch: buildMethodDispatchIndex({ + owners: [], + computeMro: () => [], + implementsOf: () => [], + }), + imports: new Map(), + bindings: new Map(), + bindingAugmentations: new Map(), + workspaceFqnBindings: new Map(), + workspaceTypeBindings: new Map(), + namespaceFqnBindings: new Map(), + namespaceTypeBindings: new Map(), + accessibleNamespacesByScope: new Map(), + referenceSites: [], + sccs: [], + stats: { + totalFiles: 2, + totalEdges: 0, + linkedEdges: 0, + unresolvedEdges: 0, + sccCount: 0, + largestSccSize: 0, + ambiguousWildcardExports: [], + }, + } as unknown as ScopeResolutionIndexes; + const graph = createKnowledgeGraph(); + for (const [id, name, filePath] of [ + ['fn:main', 'main', CALLER_FILE], + ['fn:helper', 'helper', TARGET_FILE], + ] as const) { + graph.addNode({ + id, + label: 'Function' as NodeLabel, + properties: { name, filePath, qualifiedName: name }, + }); + } + const bindings: Array<{ status: string; basis?: string }> = []; + emitFreeCallFallback( + graph, + indexes, + [callerParsed, targetParsed], + buildGraphNodeLookup(graph), + { bySourceScope: new Map() }, + new Set(), + createSemanticModel(), + buildWorkspaceResolutionIndex([callerParsed, targetParsed]), + { + recordSourceBinding: (_site, binding) => bindings.push(binding), + ...(hook === undefined ? {} : { resolveQualifiedFreeCall: hook }), + }, + ); + return { + calls: graph.relationships.filter((r) => r.type === 'CALLS'), + bindings, + }; +} + +const hitHook = (): { hook: Hook; calls: number } => { + const state = { calls: 0 }; + const hook: Hook = () => { + state.calls++; + return targetDef; + }; + return { + hook, + get calls() { + return state.calls; + }, + }; +}; + +describe('the qualified hook outranks a lexical name claim', () => { + it('control: a name claim blocks an unqualified call', () => { + const { calls, bindings } = run(site(false), undefined); + expect(calls).toHaveLength(0); + expect(bindings.map((b) => b.basis)).toContain('lexical-name-claim'); + }); + + it('a defined hit for a qualified site bypasses the block and emits the edge', () => { + const probe = hitHook(); + const { calls, bindings } = run(site(true), probe.hook); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ + targetId: 'fn:helper', + confidence: 0.85, + reason: 'import-resolved', + }); + expect(bindings.map((b) => b.basis)).not.toContain('lexical-name-claim'); + expect(probe.calls).toBe(1); + }); + + it('an undefined hit leaves the block in place', () => { + const { calls, bindings } = run(site(true), () => undefined); + expect(calls).toHaveLength(0); + expect(bindings.map((b) => b.basis)).toContain('lexical-name-claim'); + }); + + it('a site with no written path never reaches the hook and stays blocked', () => { + const probe = hitHook(); + const { calls } = run(site(false), probe.hook); + expect(calls).toHaveLength(0); + expect(probe.calls).toBe(0); + }); + + it('a language without the hook keeps the block for a qualified site', () => { + const { calls } = run(site(true), undefined); + expect(calls).toHaveLength(0); + }); + + it('without a name claim the hook still resolves the site, once', () => { + const probe = hitHook(); + const { calls } = run(site(true), probe.hook, false); + expect(calls).toHaveLength(1); + expect(probe.calls).toBe(1); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts index 658820f14..ee7adb690 100644 --- a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts +++ b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts @@ -550,6 +550,19 @@ const FIXTURES: ReadonlyMap = new Map< minimumParsedFileReads: 0, }, ], + [ + SupportedLanguages.R, + { + files: ['scripts/main.R', 'scripts/utils.R', 'pkgA/R/other.R'], + fromFile: 'scripts/main.R', + resolutionConfig: undefined, + missTarget: (i) => `ghost${i}.R`, + hitTarget: 'utils.R', + parsedImport: (targetRaw) => ({ kind: 'side-effect', targetRaw }), + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], [ SupportedLanguages.ObjectiveC, { diff --git a/gitnexus/test/unit/scope-resolution/r/__snapshots__/r-import-name-characterisation.test.ts.snap b/gitnexus/test/unit/scope-resolution/r/__snapshots__/r-import-name-characterisation.test.ts.snap new file mode 100644 index 000000000..f564e18e5 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/__snapshots__/r-import-name-characterisation.test.ts.snap @@ -0,0 +1,256 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`unbackticked R import and name lookups (characterisation) > answers exactly as before the backtick change 1`] = ` +{ + "imports": { + "analytics/R/own.R": [ + "["helper","helper","decoy"]", + "["shared","shared","corelib"]", + "["only_b","only_b","corelib"]", + "["Stack","Stack","corelib"]", + ], + "analytics/R/use.R": [ + "["helper","helper","decoy"]", + "["shared","shared","corelib"]", + "["only_b","only_b","corelib"]", + "["Stack","Stack","corelib"]", + ], + "corelib/R/a.R": [], + "corelib/R/b.R": [], + "decoy/R/d.R": [], + }, + "named": { + ""corelib"::Stack": [ + "corelib/R/a.R", + ], + ""corelib"::dup": null, + ""corelib"::filter": null, + ""corelib"::helper": [ + "corelib/R/a.R", + ], + ""corelib"::hidden": null, + ""corelib"::missing_fn": null, + ""corelib"::only_b": [ + "corelib/R/b.R", + ], + ""corelib"::print.tidy": null, + ""corelib"::shared": [ + "corelib/R/a.R", + "corelib/R/b.R", + ], + ""corelib"::tidy": null, + "corelib::Stack": [ + "corelib/R/a.R", + ], + "corelib::dup": null, + "corelib::filter": null, + "corelib::helper": [ + "corelib/R/a.R", + ], + "corelib::hidden": null, + "corelib::missing_fn": null, + "corelib::only_b": [ + "corelib/R/b.R", + ], + "corelib::print.tidy": null, + "corelib::shared": [ + "corelib/R/a.R", + "corelib/R/b.R", + ], + "corelib::tidy": null, + "decoy::Stack": null, + "decoy::dup": null, + "decoy::filter": null, + "decoy::helper": [ + "decoy/R/d.R", + ], + "decoy::hidden": null, + "decoy::missing_fn": null, + "decoy::only_b": null, + "decoy::print.tidy": null, + "decoy::shared": [ + "decoy/R/d.R", + ], + "decoy::tidy": null, + "dplyr::Stack": null, + "dplyr::dup": null, + "dplyr::filter": null, + "dplyr::helper": null, + "dplyr::hidden": null, + "dplyr::missing_fn": null, + "dplyr::only_b": null, + "dplyr::print.tidy": null, + "dplyr::shared": null, + "dplyr::tidy": null, + }, + "qualifiedOut": { + "kept": [ + "helper", + "tidy", + ], + "precise": [ + "corelib/R/a.R", + null, + "corelib/R/a.R", + null, + null, + ], + }, + "recorded": { + "analytics": [ + "filter", + "hidden", + "own_filter", + "run_all", + ], + "corelib": [ + "Stack", + "filter", + "helper", + "make", + "only_b", + "print.tidy", + "shared", + "tidy", + ], + "decoy": [ + "helper", + "shared", + ], + }, + "topLevel": { + "analytics/R/own.R": { + "dottedTails": [], + "names": [ + "filter", + "hidden", + ], + }, + "analytics/R/use.R": { + "dottedTails": [], + "names": [ + "own_filter", + "run_all", + ], + }, + "corelib/R/a.R": { + "dottedTails": [ + "tidy", + ], + "names": [ + "Stack", + "filter", + "helper", + "make", + "print.tidy", + "shared", + "tidy", + ], + }, + "corelib/R/b.R": { + "dottedTails": [], + "names": [ + "only_b", + "shared", + ], + }, + "decoy/R/d.R": { + "dottedTails": [], + "names": [ + "helper", + "shared", + ], + }, + }, + "veto": { + "filter|analytics/R/own.R|-": true, + "filter|analytics/R/own.R|corelib::filter": true, + "filter|analytics/R/own.R|decoy::filter": true, + "filter|corelib/R/a.R|-": true, + "filter|corelib/R/a.R|corelib::filter": true, + "filter|corelib/R/a.R|decoy::filter": true, + "filter|decoy/R/d.R|-": false, + "filter|decoy/R/d.R|corelib::filter": false, + "filter|decoy/R/d.R|decoy::filter": true, + "helper|analytics/R/own.R|-": true, + "helper|analytics/R/own.R|corelib::helper": true, + "helper|analytics/R/own.R|decoy::helper": true, + "helper|corelib/R/a.R|-": false, + "helper|corelib/R/a.R|corelib::helper": true, + "helper|corelib/R/a.R|decoy::helper": false, + "helper|decoy/R/d.R|-": true, + "helper|decoy/R/d.R|corelib::helper": true, + "helper|decoy/R/d.R|decoy::helper": true, + "mutate|analytics/R/own.R|-": true, + "mutate|analytics/R/own.R|corelib::mutate": true, + "mutate|analytics/R/own.R|decoy::mutate": true, + "mutate|corelib/R/a.R|-": false, + "mutate|corelib/R/a.R|corelib::mutate": true, + "mutate|corelib/R/a.R|decoy::mutate": false, + "mutate|decoy/R/d.R|-": false, + "mutate|decoy/R/d.R|corelib::mutate": false, + "mutate|decoy/R/d.R|decoy::mutate": true, + "nothing|analytics/R/own.R|-": true, + "nothing|analytics/R/own.R|corelib::nothing": true, + "nothing|analytics/R/own.R|decoy::nothing": true, + "nothing|corelib/R/a.R|-": false, + "nothing|corelib/R/a.R|corelib::nothing": true, + "nothing|corelib/R/a.R|decoy::nothing": true, + "nothing|decoy/R/d.R|-": false, + "nothing|decoy/R/d.R|corelib::nothing": true, + "nothing|decoy/R/d.R|decoy::nothing": true, + "print.tidy|analytics/R/own.R|-": true, + "print.tidy|analytics/R/own.R|corelib::print.tidy": true, + "print.tidy|analytics/R/own.R|decoy::print.tidy": true, + "print.tidy|corelib/R/a.R|-": true, + "print.tidy|corelib/R/a.R|corelib::print.tidy": true, + "print.tidy|corelib/R/a.R|decoy::print.tidy": true, + "print.tidy|decoy/R/d.R|-": false, + "print.tidy|decoy/R/d.R|corelib::print.tidy": false, + "print.tidy|decoy/R/d.R|decoy::print.tidy": true, + "shared|analytics/R/own.R|-": true, + "shared|analytics/R/own.R|corelib::shared": true, + "shared|analytics/R/own.R|decoy::shared": true, + "shared|corelib/R/a.R|-": true, + "shared|corelib/R/a.R|corelib::shared": true, + "shared|corelib/R/a.R|decoy::shared": true, + "shared|decoy/R/d.R|-": false, + "shared|decoy/R/d.R|corelib::shared": false, + "shared|decoy/R/d.R|decoy::shared": true, + "tidy|analytics/R/own.R|-": true, + "tidy|analytics/R/own.R|corelib::tidy": true, + "tidy|analytics/R/own.R|decoy::tidy": true, + "tidy|corelib/R/a.R|-": true, + "tidy|corelib/R/a.R|corelib::tidy": true, + "tidy|corelib/R/a.R|decoy::tidy": true, + "tidy|decoy/R/d.R|-": false, + "tidy|decoy/R/d.R|corelib::tidy": false, + "tidy|decoy/R/d.R|decoy::tidy": true, + }, + "wildcard": { + "analytics/R/own.R": [ + "filter", + "hidden", + ], + "analytics/R/use.R": [ + "run_all", + "own_filter", + ], + "corelib/R/a.R": [ + "helper", + "filter", + "shared", + "make", + "Stack", + ], + "corelib/R/b.R": [ + "shared", + "only_b", + ], + "decoy/R/d.R": [ + "helper", + "shared", + ], + }, +} +`; diff --git a/gitnexus/test/unit/scope-resolution/r/r-backtick-imports.test.ts b/gitnexus/test/unit/scope-resolution/r/r-backtick-imports.test.ts new file mode 100644 index 000000000..bce910118 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-backtick-imports.test.ts @@ -0,0 +1,253 @@ +/** + * Backtick-quoted names through the R import helpers. A definition keeps its backticks in the + * Module scope (`` `%||%` ``), NAMESPACE entries and `importFrom()` carry the bare symbol (`%||%`), + * and a call site spells the name as written. Each lookup that joins two of these has to compare + * the symbol, and a synthesised import has to keep the definition's own spelling so the call site + * and the target file's export both still match it. + * + * The unbackticked answers of the same helpers are pinned by `r-import-name-characterisation`. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, ParsedImport, ReferenceSite } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { populateClassOwnedMembers } from '../../../../src/core/ingestion/scope-resolution/scope/walkers.js'; +import type { + RNamespaceInfo, + RPackageConfig, +} from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + isRGlobalNameFallbackPlausible, + populateRNamespaceImports, + rFileTopLevel, +} from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; +import { + populateRQualifiedCalls, + resolveRQualifiedFreeCall, +} from '../../../../src/core/ingestion/languages/r/qualified-call.js'; +import { expandRWildcardNames } from '../../../../src/core/ingestion/languages/r/scope-resolver.js'; +import { + resolveRImportTarget, + resolveRNamedImport, +} from '../../../../src/core/ingestion/languages/r/import-target.js'; + +function parse(src: string, filePath: string): ParsedFile { + const parsed = extractParsedFile(rProvider, src, filePath); + if (parsed === undefined) throw new Error(`no ParsedFile for ${filePath}`); + populateClassOwnedMembers(parsed); + return parsed; +} + +function config( + namespaces: Record>, + packages: Record = { prov: 'prov', consumer: 'consumer', decoy: 'decoy' }, +): RPackageConfig { + const namespaceInfoByPackageDir = new Map(); + for (const [name, spec] of Object.entries(namespaces)) { + namespaceInfoByPackageDir.set(packages[name], { + hasNamespaceFile: true, + namedExports: new Set(), + exportPatterns: [], + importFrom: [], + ...spec, + }); + } + return { packages: new Map(Object.entries(packages)), namespaceInfoByPackageDir }; +} + +const importsOf = (parsed: ParsedFile): string[] => + parsed.parsedImports + .filter((i): i is Extract => i.kind === 'named') + .map((i) => `${i.localName}<-${i.targetRaw}::${i.importedName}`); + +describe('rFileTopLevel with backtick-quoted definitions', () => { + const top = rFileTopLevel( + parse( + ['`%||%` <- function(a, b) a', 'plain <- function() 1', '`odd fn` <- function() 2'].join( + '\n', + ), + 'p/R/a.R', + ), + ); + + it('lists a definition by its symbol, whichever way it is written', () => { + expect([...top.names].sort()).toEqual(['%||%', 'odd fn', 'plain']); + }); + + it('lists the spellings of the names that differ from the symbol', () => { + expect([...top.writtenAs].sort()).toEqual([ + ['%||%', ['`%||%`']], + ['odd fn', ['`odd fn`']], + ]); + }); +}); + +describe('resolveRNamedImport with backtick-quoted definitions', () => { + const cfg = config({ prov: { namedExports: new Set(['%||%', 'odd fn']) } }); + const files = [ + parse('`%||%` <- function(a, b) a\n`odd fn` <- function() 1', 'prov/R/ops.R'), + parse('hidden <- function() 1', 'prov/R/hidden.R'), + ]; + + it('finds the defining file from the NAMESPACE symbol and from the definition spelling', () => { + expect([ + resolveRNamedImport(cfg, 'prov', '%||%', files), + resolveRNamedImport(cfg, 'prov', '`%||%`', files), + resolveRNamedImport(cfg, 'prov', 'odd fn', files), + ]).toEqual([['prov/R/ops.R'], ['prov/R/ops.R'], ['prov/R/ops.R']]); + }); + + it('still refuses a name the NAMESPACE does not export', () => { + expect(resolveRNamedImport(cfg, 'prov', 'hidden', files)).toBeNull(); + }); + + it('serves a named import carrying the definition spelling through resolveRImportTarget', () => { + const all = new Set(files.map((f) => f.filePath)); + const named: ParsedImport = { + kind: 'named', + localName: '`%||%`', + importedName: '`%||%`', + targetRaw: 'prov', + }; + expect( + resolveRImportTarget('prov', 'consumer/R/a.R', all, cfg, { + parsedImport: named, + parsedFiles: files, + } as never), + ).toEqual(['prov/R/ops.R']); + }); + + it('keeps the S3 tail guard exactly where finalize would collide', () => { + // finalize keys `print.tidy` under "tidy" and a backticked `tidy` under "`tidy`", so only the + // request spelled like the dotted tail is refused. + const dotted = [parse('`tidy` <- function(x) x\nprint.tidy <- function(x) x', 'prov/R/t.R')]; + const cfgT = config({ prov: { namedExports: new Set(['tidy']) } }); + expect([ + resolveRNamedImport(cfgT, 'prov', '`tidy`', dotted), + resolveRNamedImport(cfgT, 'prov', 'tidy', dotted), + ]).toEqual([['prov/R/t.R'], null]); + }); +}); + +describe('populateRNamespaceImports with backtick-quoted definitions', () => { + const caller = () => parse('run <- function() 1', 'consumer/R/use.R'); + + it('synthesises the import with the spelling the definition and the call site use', () => { + const files = [caller(), parse('`%||%` <- function(a, b) a', 'prov/R/ops.R')]; + populateRNamespaceImports(files, { + resolutionConfig: config({ + consumer: { importFrom: [{ pkg: 'prov', name: '%||%' }] }, + prov: { namedExports: new Set(['%||%']) }, + }), + }); + expect(importsOf(files[0])).toEqual(['`%||%`<-prov::`%||%`']); + }); + + it('adds one import per spelling when the provider writes the name two ways', () => { + const files = [ + caller(), + parse('f <- function() 1', 'prov/R/a.R'), + parse('`f` <- function() 2', 'prov/R/b.R'), + ]; + populateRNamespaceImports(files, { + resolutionConfig: config({ consumer: { importFrom: [{ pkg: 'prov', name: 'f' }] } }), + }); + expect(importsOf(files[0]).sort()).toEqual(['`f`<-prov::`f`', 'f<-prov::f']); + }); + + it('lets a backtick-defined top-level function of the caller package mask its import', () => { + const files = [ + caller(), + parse('`op` <- function() 0', 'consumer/R/own.R'), + parse('`op` <- function() 1', 'prov/R/ops.R'), + ]; + populateRNamespaceImports(files, { + resolutionConfig: config({ consumer: { importFrom: [{ pkg: 'prov', name: 'op' }] } }), + }); + expect(files.map(importsOf)).toEqual([[], [], []]); + }); +}); + +describe('isRGlobalNameFallbackPlausible with backtick-quoted names', () => { + const verdict = (siteName: string, candidate: string, rawQualifiedName?: string) => { + const cfg = config({ + consumer: { importFrom: [{ pkg: 'prov', name: '%||%' }] }, + prov: {}, + }); + const files = [ + parse('run <- function() 1', 'consumer/R/use.R'), + parse('`%||%` <- function(a, b) a', 'prov/R/ops.R'), + parse('`%||%` <- function(a, b) b', 'decoy/R/ops.R'), + ]; + populateRNamespaceImports(files, { resolutionConfig: cfg }); + return isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'consumer/R/use.R' }, + candidate: { filePath: candidate }, + resolutionConfig: cfg, + site: { name: siteName, rawQualifiedName }, + }); + }; + + it('vetoes a guess in another package for a backticked call the NAMESPACE imports from prov', () => { + expect(verdict('`%||%`', 'decoy/R/ops.R')).toBe(false); + }); + + it('allows the guess inside the providing package', () => { + expect(verdict('`%||%`', 'prov/R/ops.R')).toBe(true); + }); + + it('judges a qualified call by the provider defining the symbol, however the call spells it', () => { + expect(verdict('`%||%`', 'decoy/R/ops.R', 'prov::`%||%`')).toBe(false); + }); +}); + +describe('expandRWildcardNames with backtick-quoted definitions', () => { + it('returns the definition spelling, which is the name the importing scope binds', () => { + const target = parse( + '`%||%` <- function(a, b) a\nplain <- function() 1\nnested <- function() { `inner` <- function() 2 }', + 'prov/R/ops.R', + ); + expect([...expandRWildcardNames(target.moduleScope, [target])].sort()).toEqual([ + '`%||%`', + 'nested', + 'plain', + ]); + }); +}); + +describe('populateRQualifiedCalls with backtick-quoted definitions', () => { + const run = (defSrc: string, callName: string) => { + // prov exports f: a private name is not reachable through `::` from a script. + const cfg = config({ prov: { namedExports: new Set(['f']) } }); + const site = { + kind: 'call', + name: callName, + rawQualifiedName: 'prov::f', + atRange: { startLine: 1, startCol: 0, endLine: 1, endCol: 8 }, + } as unknown as ReferenceSite; + const files = [ + parse(defSrc, 'prov/R/a.R'), + { ...parse('x <- 1', 'scripts/q.R'), referenceSites: [site] } as ParsedFile, + ]; + populateRQualifiedCalls(files, { resolutionConfig: cfg }); + return { + kept: files[1].referenceSites.length, + precise: resolveRQualifiedFreeCall(site, files[1])?.filePath ?? null, + }; + }; + + it('attributes a call written without backticks to a definition written with them', () => { + expect(run('`f` <- function() 1', 'f')).toEqual({ kept: 1, precise: 'prov/R/a.R' }); + }); + + it('attributes a call written with backticks to a definition written without them', () => { + expect(run('f <- function() 1', '`f`')).toEqual({ kept: 1, precise: 'prov/R/a.R' }); + }); + + it('drops the call when the package defines the symbol twice, once with backticks', () => { + expect(run('f <- function() 1\n`f` <- function() 2', 'f')).toEqual({ + kept: 0, + precise: null, + }); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-closure-local-veto.test.ts b/gitnexus/test/unit/scope-resolution/r/r-closure-local-veto.test.ts new file mode 100644 index 000000000..bba75764b --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-closure-local-veto.test.ts @@ -0,0 +1,75 @@ +/** + * The R unique-name fallback veto refuses a candidate bound inside another function, and only that: + * a top-level definition, a candidate it cannot inspect, and every non-function kind stay plausible. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { isRGlobalNameFallbackPlausible } from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; + +const SOURCE = [ + 'make_counter <- function() {', + ' inner_fn <- function() 2', + ' Inner <- setRefClass("Inner", fields = list(a = "numeric"))', + ' inner_fn', + '}', + 'top_fn <- function() 1', +].join('\n'); + +const parsed = (): ParsedFile => { + const file = extractParsedFile(rProvider, SOURCE, 'R/a.R'); + if (file === undefined) throw new Error('no ParsedFile'); + return file; +}; + +const defNamed = (file: ParsedFile, name: string): SymbolDefinition => { + const def = file.localDefs.find((d) => d.qualifiedName?.split('.').pop() === name); + if (def === undefined) throw new Error(`no def ${name}`); + return def; +}; + +const verdict = ( + file: ParsedFile, + candidate: Pick, + withParse = true, +): boolean => + isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'R/b.R' }, + candidate, + parsedFileOf: withParse ? (p) => (p === file.filePath ? file : undefined) : undefined, + site: { name: 'x' }, + }); + +describe('isRGlobalNameFallbackPlausible: closure-local candidates', () => { + it('refuses a function bound inside another function', () => { + const file = parsed(); + expect(verdict(file, defNamed(file, 'inner_fn'))).toBe(false); + expect(verdict(file, defNamed(file, 'Inner'))).toBe(false); + }); + + it('keeps a top-level function', () => { + const file = parsed(); + expect(verdict(file, defNamed(file, 'top_fn'))).toBe(true); + expect(verdict(file, defNamed(file, 'make_counter'))).toBe(true); + }); + + it('does not judge a candidate the scope model does not hold (a graph node id)', () => { + const file = parsed(); + expect( + verdict(file, { nodeId: 'Function:R/a.R:inner_fn', filePath: 'R/a.R', type: 'Function' }), + ).toBe(true); + }); + + it('cannot decide without the candidate parse, and allows', () => { + const file = parsed(); + expect(verdict(file, defNamed(file, 'inner_fn'), false)).toBe(true); + expect( + isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'R/b.R' }, + candidate: { filePath: 'R/a.R' }, + site: { name: 'x' }, + }), + ).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-dotted-s3-hooks.test.ts b/gitnexus/test/unit/scope-resolution/r/r-dotted-s3-hooks.test.ts new file mode 100644 index 000000000..67938d658 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-dotted-s3-hooks.test.ts @@ -0,0 +1,167 @@ +/** + * R dotted (S3) names vs bare names in the shared simple-name indexes: + * the `isCallableVisibleFromCaller` hook and the + * `expandRWildcardNames` tail guard. ParsedFiles come from the real R extractor. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { populateClassOwnedMembers } from '../../../../src/core/ingestion/scope-resolution/scope/walkers.js'; +import { + expandRWildcardNames, + rIsCallableVisibleFromCaller, + rScopeResolver, +} from '../../../../src/core/ingestion/languages/r/scope-resolver.js'; +import { countedParsedFiles } from '../../../helpers/counting-file-set.js'; + +function parse(src: string, filePath: string): ParsedFile { + const parsed = extractParsedFile(rProvider, src, filePath); + if (parsed === undefined) throw new Error(`no ParsedFile for ${filePath}`); + populateClassOwnedMembers(parsed); + return parsed; +} + +const def = (parsed: ParsedFile, qualifiedName: string): SymbolDefinition => { + const found = parsed.localDefs.find((d) => d.qualifiedName === qualifiedName); + if (found === undefined) throw new Error(`no def ${qualifiedName}`); + return found; +}; + +const visible = (callerParsed: ParsedFile, candidate: SymbolDefinition): boolean => + rIsCallableVisibleFromCaller({ callerParsed, candidate } as never); + +describe('rIsCallableVisibleFromCaller', () => { + const p = parse( + [ + 'print.foo <- function(x, ...) x', + 'foo <- function(v) v', + '`odd name` <- function(v) v', + 'Scorer <- R6::R6Class("Scorer", public = list(score = function(d) d))', + ].join('\n'), + 'pkg/R/a.R', + ); + + it('is registered on rScopeResolver', () => { + expect(rScopeResolver.isCallableVisibleFromCaller).toBe(rIsCallableVisibleFromCaller); + }); + + it('refuses a dotted S3 def for a bare call', () => { + expect(visible(p, def(p, 'print.foo'))).toBe(false); + }); + + it('allows a bare def and a backticked non-dotted def', () => { + expect(visible(p, def(p, 'foo'))).toBe(true); + expect(visible(p, def(p, '`odd name`'))).toBe(true); + }); + + it('refuses a class member whose qualified name is dotted (never callable bare)', () => { + const member = p.localDefs.find( + (d) => d.type === 'Method' && d.qualifiedName?.endsWith('score'), + ); + if (member === undefined) throw new Error('no R6 member def'); + expect(member.qualifiedName).toContain('.'); + expect(visible(p, member)).toBe(false); + }); + + it('allows a def with no qualified name (nothing to prove)', () => { + expect(visible(p, { ...def(p, 'foo'), qualifiedName: undefined })).toBe(true); + }); +}); + +describe('expandRWildcardNames', () => { + const names = (src: string): readonly string[] => { + const parsed = parse(src, 'pkg/R/a.R'); + return expandRWildcardNames(parsed.moduleScope, [parsed]); + }; + + it('drops a bare name whose file defines a dotted def with that tail (S3 method first)', () => { + expect(names('print.foo <- function(x) x\nfoo <- function(v) v\nqux <- function(v) v')).toEqual( + ['qux'], + ); + }); + + it('drops it in the mirror layout too (order-blind)', () => { + expect(names('bar <- function(v) v\nprint.bar <- function(x) x\nqux <- function(v) v')).toEqual( + ['qux'], + ); + }); + + it('keeps names that share no tail with a dotted def', () => { + expect(names('print.foo <- function(x) x\nplain_fn <- function(v) v')).toEqual(['plain_fn']); + }); + + it('keeps a bare name when the dotted def sits in a different file', () => { + const a = parse('foo <- function(v) v', 'pkg/R/a.R'); + const b = parse('print.foo <- function(x) x', 'pkg/R/b.R'); + expect(expandRWildcardNames(a.moduleScope, [a, b])).toEqual(['foo']); + }); + + it('keeps a bare name that only matches a class member (not a top-level dotted def)', () => { + expect( + names( + 'Scorer <- R6::R6Class("Scorer", public = list(tidy = function(d) d))\ntidy <- function(d) d', + ), + ).toEqual(['Scorer', 'tidy']); + }); + + it('lists only top-level definitions, not functions nested in another function', () => { + expect( + names('outer <- function() {\n helper <- function() 1\n helper()\n}\ntop <- function() 2'), + ).toEqual(['outer', 'top']); + }); + + it('does not list a function nested several levels deep or inside a control block', () => { + expect( + names( + [ + 'a <- function() {', + ' b <- function() {', + ' c <- function() 1', + ' }', + ' if (TRUE) d <- function() 2', + '}', + ].join('\n'), + ), + ).toEqual(['a']); + }); + + it('still lists top-level classes alongside top-level functions', () => { + expect( + names('Scorer <- R6::R6Class("Scorer", public = list(f = function() 1))\nrun = function() 1'), + ).toEqual(['Scorer', 'run']); + }); + + it('still lists an S4 class defined by a bare setClass() call', () => { + expect( + names('setClass("Shape", representation(n = "numeric"))\nplain <- function() 1'), + ).toEqual(['Shape', 'plain']); + }); + + it('never lists dotted names themselves', () => { + expect(names('print.foo <- function(x) x')).toEqual([]); + }); + + it('returns nothing for an unknown module scope', () => { + const a = parse('foo <- function(v) v', 'pkg/R/a.R'); + expect(expandRWildcardNames('scope:missing' as never, [a])).toEqual([]); + }); +}); + +describe('expandRWildcardNames: the module-scope lookup is built once per pass', () => { + it('reads each parsed file once however many wildcard edges ask', () => { + // `parsedFiles.find` per edge read the files up to the match every time. Counted element + // reads, not wall time: the figure depends on the file count and the number of passes only. + const paths = Array.from({ length: 200 }, (_, i) => `pkg/R/f${i}.R`); + const { parsedFiles, reads } = countedParsedFiles(paths); + for (let edge = 0; edge < 1_000; edge++) { + expandRWildcardNames(`module:${paths[(edge * 7) % paths.length]}`, parsedFiles); + } + expect(reads()).toBe(paths.length); + }); + + it('answers a scope no file owns with nothing', () => { + const { parsedFiles } = countedParsedFiles(['pkg/R/a.R']); + expect(expandRWildcardNames('module:absent' as never, parsedFiles)).toEqual([]); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-export-evidence.test.ts b/gitnexus/test/unit/scope-resolution/r/r-export-evidence.test.ts new file mode 100644 index 000000000..02ffb4b58 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-export-evidence.test.ts @@ -0,0 +1,42 @@ +/** + * R gives scope resolution no export verdict. `SymbolDefinition.isExported` is tri-state and + * `undefined` means "unknown" (finalization then keeps its prior behaviour); R's export checker + * says "public" for every graph node and the NAMESPACE refinement later flips graph nodes to + * private, but neither writes a verdict onto a scope definition. That is deliberate: a + * verdict of `false` would make finalization drop the name from wildcard providers and from + * every other lookup, with no way to tell a caller inside the package from one outside. + * Resolution instead derives a per-name visibility from the NAMESPACE (`rSymbolVisibility`, + * `package-config.ts`): `library()` is filtered by name in `expandRWildcardNames`, `pkg::name` + * and the unique-name guess consult the same verdicts (docs/languages/r-provider.md, Export + * status). Earlier, a `library()` attached every top-level name, as `devtools::load_all()` does. + */ +import { describe, expect, it } from 'vitest'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; + +describe('R scope definitions carry no export verdict', () => { + const parsed = extractParsedFile( + rProvider, + [ + 'exported_fn <- function() 1', + 'Widget <- R6Class("Widget", public = list(size = 1, grow = function() 1))', + 'setClass("Point", representation(x = "numeric"))', + 'setGeneric("area", function(shape) standardGeneric("area"))', + ].join('\n'), + 'R/a.R', + ); + + it('extracts functions, classes and members (positive control)', () => { + const types = new Set(parsed?.localDefs.map((d) => d.type)); + expect(types.has('Function')).toBe(true); + expect(types.has('Class')).toBe(true); + expect(types.has('Method')).toBe(true); + }); + + it('leaves isExported unset on every definition', () => { + expect(parsed?.localDefs.length).toBeGreaterThan(0); + for (const def of parsed?.localDefs ?? []) { + expect(def.isExported, `${def.type} ${def.qualifiedName}`).toBeUndefined(); + } + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-import-name-characterisation.test.ts b/gitnexus/test/unit/scope-resolution/r/r-import-name-characterisation.test.ts new file mode 100644 index 000000000..b6046c0c6 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-import-name-characterisation.test.ts @@ -0,0 +1,188 @@ +/** + * Characterisation of the R import and name-lookup helpers over names written WITHOUT backticks. + * + * `rFileTopLevel`, `resolveRNamedImport`, `populateRNamespaceImports`, the global-name-fallback + * veto, `expandRWildcardNames` and `populateRQualifiedCalls` all compare a NAMESPACE symbol or a + * call site's name with a definition's binding key. Reading backtick-quoted names through them + * (`rBareName`) must not change a single answer for the common case, so the whole observable + * output over a set of unbackticked layouts is pinned here. The snapshot was recorded before the + * backtick change and has not been edited since. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, ParsedImport, ReferenceSite } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { populateClassOwnedMembers } from '../../../../src/core/ingestion/scope-resolution/scope/walkers.js'; +import type { + RNamespaceInfo, + RPackageConfig, +} from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + isRGlobalNameFallbackPlausible, + populateRNamespaceImports, + rFileTopLevel, + rRecordedPackageTopLevelNames, +} from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; +import { + populateRQualifiedCalls, + resolveRQualifiedFreeCall, +} from '../../../../src/core/ingestion/languages/r/qualified-call.js'; +import { expandRWildcardNames } from '../../../../src/core/ingestion/languages/r/scope-resolver.js'; +import { resolveRNamedImport } from '../../../../src/core/ingestion/languages/r/import-target.js'; + +function parse(src: string, filePath: string): ParsedFile { + const parsed = extractParsedFile(rProvider, src, filePath); + if (parsed === undefined) throw new Error(`no ParsedFile for ${filePath}`); + populateClassOwnedMembers(parsed); + return parsed; +} + +const sorted = (values: Iterable): string[] => [...values].sort(); + +const SOURCES: Record = { + 'corelib/R/a.R': [ + 'helper <- function(x) x', + 'filter <- function(x) x', + 'print.tidy <- function(x) x', + 'tidy <- function(x) x', + 'shared <- function() 1', + 'make <- function() { inner <- function() 1; inner() }', + 'Stack <- R6::R6Class("Stack", public = list(push = function(v) v))', + ].join('\n'), + 'corelib/R/b.R': ['shared <- function() 2', 'only_b <- function() 3', 'dup <- 1'].join('\n'), + 'analytics/R/use.R': [ + 'run_all <- function() { helper(1); filter(2); tidy(3); shared(); only_b(); corelib::helper(4) }', + 'own_filter <- function() 1', + ].join('\n'), + 'analytics/R/own.R': 'filter <- function(x) x\nhidden <- function() 1', + 'decoy/R/d.R': 'helper <- function(x) x\nshared <- function() 9', +}; + +const NAMESPACES: Record = { + corelib: { + hasNamespaceFile: true, + namedExports: new Set(['helper', 'tidy', 'shared', 'only_b', 'Stack']), + exportPatterns: [], + importFrom: [], + }, + analytics: { + hasNamespaceFile: true, + namedExports: new Set(), + exportPatterns: [], + importFrom: [ + { pkg: 'corelib', name: 'helper' }, + { pkg: 'corelib', name: 'filter' }, + { pkg: 'corelib', name: 'tidy' }, + { pkg: 'corelib', name: 'shared' }, + { pkg: 'corelib', name: 'only_b' }, + { pkg: 'corelib', name: 'print.tidy' }, + { pkg: 'corelib', name: 'missing_fn' }, + { pkg: 'corelib', name: 'dup' }, + { pkg: 'corelib', name: 'Stack' }, + { pkg: 'dplyr', name: 'mutate' }, + { pkg: 'decoy', name: 'helper' }, + ], + }, +}; + +function config(): RPackageConfig { + return { + packages: new Map([ + ['corelib', 'corelib'], + ['analytics', 'analytics'], + ['decoy', 'decoy'], + ]), + namespaceInfoByPackageDir: new Map(Object.entries(NAMESPACES)), + }; +} + +function namedImports(parsed: ParsedFile): string[] { + return parsed.parsedImports + .filter((i): i is Extract => i.kind === 'named') + .map((i) => JSON.stringify([i.localName, i.importedName, i.targetRaw])); +} + +describe('unbackticked R import and name lookups (characterisation)', () => { + it('answers exactly as before the backtick change', () => { + const cfg = config(); + const files = Object.entries(SOURCES).map(([path, src]) => parse(src, path)); + + const topLevel = Object.fromEntries( + files.map((f) => { + const t = rFileTopLevel(f); + return [f.filePath, { names: sorted(t.names), dottedTails: sorted(t.dottedTails) }]; + }), + ); + + const resolve = (pkg: string, name: string) => + resolveRNamedImport(cfg, pkg, name, files)?.slice() ?? null; + const named: Record = {}; + for (const pkg of ['corelib', 'decoy', 'dplyr', '"corelib"']) { + for (const name of [ + 'helper', + 'filter', + 'tidy', + 'print.tidy', + 'shared', + 'only_b', + 'dup', + 'Stack', + 'missing_fn', + 'hidden', + ]) { + named[`${pkg}::${name}`] = resolve(pkg, name); + } + } + + populateRNamespaceImports(files, { resolutionConfig: cfg }); + const imports = Object.fromEntries(files.map((f) => [f.filePath, namedImports(f)])); + const recorded = Object.fromEntries( + [...(rRecordedPackageTopLevelNames(cfg) ?? [])].map(([pkg, names]) => [pkg, sorted(names)]), + ); + + const veto: Record = {}; + for (const name of ['helper', 'filter', 'tidy', 'shared', 'mutate', 'print.tidy', 'nothing']) { + for (const candidate of ['corelib/R/a.R', 'decoy/R/d.R', 'analytics/R/own.R']) { + for (const rawQualifiedName of [undefined, 'corelib::' + name, 'decoy::' + name]) { + veto[`${name}|${candidate}|${rawQualifiedName ?? '-'}`] = isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'analytics/R/use.R' }, + candidate: { filePath: candidate }, + resolutionConfig: cfg, + site: { name, rawQualifiedName }, + }); + } + } + } + + const wildcard = Object.fromEntries( + files.map((f) => [f.filePath, expandRWildcardNames(f.moduleScope, files).slice()]), + ); + + const qualified = files.map((f) => f); + let line = 0; + const callFor = (name: string, rawQualifiedName: string): ReferenceSite => + ({ + kind: 'call', + name, + rawQualifiedName, + atRange: { startLine: ++line, startCol: 0, endLine: line, endCol: 10 }, + }) as unknown as ReferenceSite; + const caller = parse('x <- 1', 'scripts/q.R'); + const sites = [ + callFor('helper', 'corelib::helper'), + callFor('shared', 'corelib::shared'), + callFor('tidy', 'corelib::tidy'), + callFor('mutate', 'dplyr::mutate'), + callFor('nothing', 'corelib::nothing'), + ]; + qualified.push({ ...caller, referenceSites: sites } as ParsedFile); + populateRQualifiedCalls(qualified, { resolutionConfig: cfg }); + const last = qualified[qualified.length - 1]; + const qualifiedOut = { + kept: last.referenceSites.map((s) => s.name), + precise: sites.map((s) => resolveRQualifiedFreeCall(s, last)?.filePath ?? null), + }; + + expect({ topLevel, named, imports, recorded, veto, wildcard, qualifiedOut }).toMatchSnapshot(); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-lexical-names.test.ts b/gitnexus/test/unit/scope-resolution/r/r-lexical-names.test.ts new file mode 100644 index 000000000..bdfed07bb --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-lexical-names.test.ts @@ -0,0 +1,156 @@ +/** + * The names a function binds without a definition (formals, `for` variables) are published on its + * scope as lexical names, which stop a call through them from reaching a top-level function of the + * same name. This pins what is read from the tree and what is not. + */ +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import R from '@eagleoutice/tree-sitter-r'; +import { rFunctionLexicalNames } from '../../../../src/core/ingestion/languages/r/captures.js'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; + +const parser = new Parser(); +parser.setLanguage(R as Parameters[0]); + +/** Lexical names of the first function_definition in `source`. */ +function namesOf(source: string): string[] { + const tree = parser.parse(source, undefined, { bufferSize: 1 << 24 }); + const fn = tree.rootNode.descendantsOfType('function_definition')[0]; + return rFunctionLexicalNames(fn as never).sort(); +} + +describe('rFunctionLexicalNames', () => { + it('reads the named formals, with and without defaults, and not the dots', () => { + expect(namesOf('f <- function(data, g = function(z) z, h = make(), ...) 1')).toEqual([ + 'data', + 'g', + 'h', + ]); + expect(namesOf('f <- function() 1')).toEqual([]); + }); + + it('reads every named formal, whatever its default: a default proves nothing about the argument', () => { + expect(namesOf('f <- function(a, n = 2, s = "x", b = TRUE, m = -1) 1')).toEqual([ + 'a', + 'b', + 'm', + 'n', + 's', + ]); + expect(namesOf('f <- function(a, z = NULL) 1')).toEqual(['a', 'z']); + expect(namesOf('f <- function(a, n = 2) { n <- identity; n(a) }')).toEqual(['a', 'n']); + expect(namesOf('f <- function(n = 2) for (n in 1:3) n')).toEqual(['n']); + }); + + it('keeps a backticked formal as written', () => { + expect(namesOf('f <- function(`my fn`, b) 1')).toEqual(['`my fn`', 'b']); + }); + + it('reads the variable of a for loop anywhere in the body', () => { + expect( + namesOf( + 'f <- function(a) {\n for (i in a) g(i)\n if (a) { for (j in 1:3) { for (k in j) k } }\n}', + ), + ).toEqual(['a', 'i', 'j', 'k']); + }); + + it('leaves the loop variables and formals of a nested function to that function', () => { + const source = 'f <- function(a) {\n h <- function(b) { for (z in b) z }\n a\n}'; + expect(namesOf(source)).toEqual(['a']); + }); + + it('does not read a for loop outside every function', () => { + const tree = parser.parse('for (i in 1:3) f(i)\n', undefined, { bufferSize: 1 << 24 }); + expect(tree.rootNode.descendantsOfType('function_definition')).toHaveLength(0); + }); + + it('reads a function nested 500 deep without recursing', () => { + const body = '{ for (i in 1) '.repeat(500) + '1' + ' }'.repeat(500); + expect(namesOf(`f <- function(a) ${body}`)).toEqual(['a', 'i']); + }); +}); + +describe('rFunctionLexicalNames: ordinary assignments', () => { + it('reads the target of `<-`, `=`, `->` and `->>` whose value may be a function', () => { + expect(namesOf('f <- function(a) { b <- identity; c = g(); h -> d; 1 ->> e; a }')).toEqual([ + 'a', + 'b', + 'c', + 'd', + ]); + }); + + it('reads an identifier or computed value, and any complex-target assignment, as a binding', () => { + expect(namesOf('f <- function() { p <- q; r <- s(1); t <- if (u) v else w }')).toEqual([ + 'p', + 'r', + 't', + ]); + }); + + it('does not read a function literal (a real definition) or a constant (R skips it at a call)', () => { + expect(namesOf('f <- function() { a <- function(y) 1; b <- 2; c <- "s"; d <- TRUE }')).toEqual( + [], + ); + expect(namesOf('f <- function() { a <- -1; b <- NA; c <- Inf; d <- \\(z) z }')).toEqual([]); + }); + + it('keeps NULL as a binding: it is the placeholder of an optional callback', () => { + expect(namesOf('f <- function() { a <- NULL }')).toEqual(['a']); + }); + + it('reads the base identifier of a complex target, never the called name', () => { + expect( + namesOf('f <- function(x) { names(y) <- v; z$a <- v; w[[1]] <- v; foo(x) <- 1 }'), + ).toEqual(['w', 'x', 'y', 'z']); + }); + + it('does not read a write through self as a binding of self; a plain rebinding is one', () => { + // An R6 method that sets a field (`self$count <- 1`) keeps self the instance: only `self <- v` + // replaces it. Other bases keep the rule above (`z$a <- v` binds `z`). + expect( + namesOf( + 'f <- function(v) { self$a <- v; self[["b"]] <- v; self$c[[1]] <- v; names(self$d) <- v; self@e <- v }', + ), + ).toEqual(['v']); + expect(namesOf('f <- function() { self <- other() }')).toEqual(['self']); + expect(namesOf('f <- function() { other() -> self }')).toEqual(['self']); + }); + + it('does not read `<<-` or `->>`: they assign outward', () => { + expect(namesOf('f <- function() { a <<- identity; identity ->> b }')).toEqual([]); + }); + + it('is flow-insensitive: a name is read when any of its assignments is a binding', () => { + expect(namesOf('f <- function() { a <- 2; a <- identity; b <- identity; b <- 2 }')).toEqual([ + 'a', + 'b', + ]); + }); + + it('leaves the assignments of a nested function to that function', () => { + expect(namesOf('f <- function(a) { h <- function(b) { c <- identity; b }; a }')).toEqual(['a']); + }); + + it('does not model assign(), delayedAssign(), list2env() or with()', () => { + expect( + namesOf('f <- function() { assign("a", identity); delayedAssign("b", g()); with(e, c) }'), + ).toEqual([]); + }); +}); + +describe('the function scope publishes them as lexicalNames', () => { + it('puts the formals and loop variables on the function_definition scope only', () => { + const file = extractParsedFile( + rProvider, + 'transform <- function(x) x\nrun <- function(data, transform) {\n for (cb in data) cb(1)\n transform(data)\n}\nplain <- function() 1\n', + 'R/a.R', + ); + if (file === undefined) throw new Error('no ParsedFile'); + const named = file.scopes + .filter((s) => s.kind === 'Function' && s.lexicalNames !== undefined) + .map((s) => [...(s.lexicalNames ?? [])].sort()); + expect(named).toEqual([['x'], ['cb', 'data', 'transform']]); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-merge-bindings.test.ts b/gitnexus/test/unit/scope-resolution/r/r-merge-bindings.test.ts new file mode 100644 index 000000000..c825018a4 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-merge-bindings.test.ts @@ -0,0 +1,157 @@ +/** + * `rMergeBindings` orders the bindings of one tier by node id. The order must be the same on every + * machine, so it compares code units, not `localeCompare`, which follows the locale: in an + * English collation `a` sorts before `B` and `é` before `f`; by code unit `B` and `f` come first. + */ +import { describe, expect, it } from 'vitest'; +import type { BindingRef, ImportEdge, Range, SymbolDefinition } from 'gitnexus-shared'; +import { rMergeBindings } from '../../../../src/core/ingestion/languages/r/simple-hooks.js'; + +const local = (nodeId: string): BindingRef => ({ + origin: 'local', + def: { nodeId } as SymbolDefinition, +}); + +const order = (...ids: string[]): string[] => + rMergeBindings(ids.map(local), [], 'scope:test').map((b) => b.def.nodeId); + +describe('rMergeBindings', () => { + it('orders same-tier bindings by code unit', () => { + expect(order('Function:a.R:a', 'Function:a.R:B')).toEqual(['Function:a.R:B', 'Function:a.R:a']); + expect(order('Function:a.R:é', 'Function:a.R:f')).toEqual(['Function:a.R:f', 'Function:a.R:é']); + }); + + it('gives the same order whichever way round the bindings arrive', () => { + expect(order('Function:a.R:B', 'Function:a.R:a')).toEqual(['Function:a.R:B', 'Function:a.R:a']); + }); + + it('keeps one binding per node and prefers the tightest tier', () => { + const merged = rMergeBindings( + [local('Function:a.R:x'), { origin: 'import', def: { nodeId: 'Function:b.R:x' } as never }], + [local('Function:a.R:x')], + 'scope:test', + ); + expect(merged.map((b) => b.def.nodeId)).toEqual(['Function:a.R:x']); + }); +}); + +describe('rMergeBindings: library() masking in the wildcard tier', () => { + const at = (startLine: number, startCol = 0): Range => ({ + startLine, + startCol, + endLine: startLine, + endCol: startCol + 12, + }); + /** A name brought in by the `library()` call that starts at `range` (none: no position known). */ + const attached = (nodeId: string, range?: Range): BindingRef => ({ + origin: 'wildcard', + def: { nodeId } as SymbolDefinition, + via: { atRange: range } as ImportEdge, + }); + const ids = (merged: BindingRef[]): string[] => merged.map((b) => b.def.nodeId); + + it('keeps only the package attached last, whichever way round the bindings arrive', () => { + const lp = attached('Function:lp/R/f.R:dup', at(1)); + const lp2 = attached('Function:lp2/R/f.R:dup', at(2)); + expect(ids(rMergeBindings([lp], [lp2], 's'))).toEqual(['Function:lp2/R/f.R:dup']); + expect(ids(rMergeBindings([lp2], [lp], 's'))).toEqual(['Function:lp2/R/f.R:dup']); + expect(ids(rMergeBindings([lp, lp2], [], 's'))).toEqual(['Function:lp2/R/f.R:dup']); + }); + + it('is decided by the position of the call, not by the id order of the files', () => { + const early = attached('Function:z/R/f.R:dup', at(1)); + const late = attached('Function:a/R/f.R:dup', at(2)); + expect(ids(rMergeBindings([early, late], [], 's'))).toEqual(['Function:a/R/f.R:dup']); + }); + + it('compares the column when two calls share a line', () => { + const first = attached('Function:b/R/f.R:dup', at(3, 0)); + const second = attached('Function:a/R/f.R:dup', at(3, 14)); + expect(ids(rMergeBindings([first, second], [], 's'))).toEqual(['Function:a/R/f.R:dup']); + }); + + it('keeps a package named by several calls where it was first attached, however the bindings are batched', () => { + const a1 = attached('Function:pa/R/f.R:dup', at(1)); + const b2 = attached('Function:pb/R/f.R:dup', at(2)); + const a3 = attached('Function:pa/R/f.R:dup', at(3)); + // library(pa); library(pb); library(pa): R leaves pa where it was, so pb was attached last. + expect(ids(rMergeBindings([a1, b2], [a3], 's'))).toEqual(['Function:pb/R/f.R:dup']); + expect(ids(rMergeBindings([a3], [a1, b2], 's'))).toEqual(['Function:pb/R/f.R:dup']); + expect(ids(rMergeBindings([a1, b2, a3], [], 's'))).toEqual(['Function:pb/R/f.R:dup']); + // library(pb); library(pa); library(pb): pa was attached last. + const a2 = attached('Function:pa/R/f.R:dup', at(2)); + const b1 = attached('Function:pb/R/f.R:dup', at(1)); + const b3 = attached('Function:pb/R/f.R:dup', at(3)); + expect(ids(rMergeBindings([b1, a2], [b3], 's'))).toEqual(['Function:pa/R/f.R:dup']); + }); + + it('keeps first-attachment order when the bindings arrive one at a time, as the pipeline folds them', () => { + /** Fold `bindings` through the merger one at a time, in the given order. */ + const fold = (...bindings: BindingRef[]): string[] => { + let merged: BindingRef[] = []; + for (const b of bindings) merged = rMergeBindings(merged, [b], 's'); + return ids(merged); + }; + const pa = (line: number) => attached('Function:pa/R/f.R:dup', at(line)); + const pb = (line: number) => attached('Function:pb/R/f.R:dup', at(line)); + // A/B/A -> B, B/A/B -> A, A/B/A/B -> B, A/A/B -> B, A/B/B -> B. + expect(fold(pa(1), pb(2), pa(3))).toEqual(['Function:pb/R/f.R:dup']); + expect(fold(pb(1), pa(2), pb(3))).toEqual(['Function:pa/R/f.R:dup']); + expect(fold(pa(1), pb(2), pa(3), pb(4))).toEqual(['Function:pb/R/f.R:dup']); + expect(fold(pa(1), pa(2), pb(3))).toEqual(['Function:pb/R/f.R:dup']); + expect(fold(pa(1), pb(2), pb(3))).toEqual(['Function:pb/R/f.R:dup']); + // The same shapes arriving in the opposite order give the same answer. + expect(fold(pa(3), pb(2), pa(1))).toEqual(['Function:pb/R/f.R:dup']); + expect(fold(pb(3), pa(2), pb(1))).toEqual(['Function:pa/R/f.R:dup']); + }); + + it('keeps every file of a package that is attached again, as one call attached them first', () => { + const x1 = attached('Function:px/R/x.R:dup', at(1)); + const y1 = attached('Function:px/R/y.R:dup', at(1)); + const q2 = attached('Function:pq/R/f.R:dup', at(2)); + const x3 = attached('Function:px/R/x.R:dup', at(3)); + const y3 = attached('Function:px/R/y.R:dup', at(3)); + let merged: BindingRef[] = []; + for (const b of [x1, y1, q2, x3, y3]) merged = rMergeBindings(merged, [b], 's'); + expect(ids(merged)).toEqual(['Function:pq/R/f.R:dup']); + const r1 = attached('Function:pq/R/f.R:dup', at(1)); + const s2 = attached('Function:px/R/x.R:dup', at(2)); + const t2 = attached('Function:px/R/y.R:dup', at(2)); + const r3 = attached('Function:pq/R/f.R:dup', at(3)); + merged = []; + for (const b of [r1, s2, t2, r3]) merged = rMergeBindings(merged, [b], 's'); + expect(ids(merged)).toEqual(['Function:px/R/x.R:dup', 'Function:px/R/y.R:dup']); + }); + + it('keeps every file of the one package the last call attaches: that ambiguity stays', () => { + const old = attached('Function:lp/R/f.R:dup', at(1)); + const x = attached('Function:lp2/R/x.R:dup', at(2)); + const y = attached('Function:lp2/R/y.R:dup', at(2)); + expect(ids(rMergeBindings([old, y, x], [], 's'))).toEqual([ + 'Function:lp2/R/x.R:dup', + 'Function:lp2/R/y.R:dup', + ]); + }); + + it('keeps the whole tier when a binding has no call position', () => { + const a = attached('Function:a/R/f.R:dup', at(1)); + const b = attached('Function:b/R/f.R:dup'); + expect(ids(rMergeBindings([a, b], [], 's'))).toEqual([ + 'Function:a/R/f.R:dup', + 'Function:b/R/f.R:dup', + ]); + }); + + it('leaves a name that one package defines, and the other tiers, as they were', () => { + const only = attached('Function:lp/R/f.R:only', at(1)); + expect(ids(rMergeBindings([only], [], 's'))).toEqual(['Function:lp/R/f.R:only']); + const named: BindingRef = { + origin: 'import', + def: { nodeId: 'Function:p/R/f.R:dup' } as SymbolDefinition, + via: { atRange: at(9) } as ImportEdge, + }; + const lp = attached('Function:lp/R/f.R:dup', at(1)); + const lp2 = attached('Function:lp2/R/f.R:dup', at(2)); + expect(ids(rMergeBindings([lp, lp2], [named], 's'))).toEqual(['Function:p/R/f.R:dup']); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-named-import-lookup.test.ts b/gitnexus/test/unit/scope-resolution/r/r-named-import-lookup.test.ts new file mode 100644 index 000000000..c75d545e2 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-named-import-lookup.test.ts @@ -0,0 +1,96 @@ +/** + * Looking up an `importFrom()` name that a provider's NAMESPACE does not export must cost one set + * lookup, not a pass over every export: a NAMESPACE can carry any number of `export()` entries and + * a caller any number of `importFrom()` names, so a scan per lookup is quadratic in repository text. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import type { + RNamespaceInfo, + RPackageConfig, +} from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { resolveRNamedImport } from '../../../../src/core/ingestion/languages/r/import-target.js'; + +/** A set that counts every way of walking all of its entries. */ +class CountingSet extends Set { + walks = 0; + override [Symbol.iterator](): SetIterator { + this.walks++; + return super[Symbol.iterator](); + } + override values(): SetIterator { + this.walks++; + return super.values(); + } + override entries(): SetIterator<[string, string]> { + this.walks++; + return super.entries(); + } + override forEach(...args: Parameters['forEach']>): void { + this.walks++; + super.forEach(...args); + } +} + +function setup(exportCount: number): { + cfg: RPackageConfig; + exports: CountingSet; + file: ParsedFile; +} { + const exports = new CountingSet(); + for (let i = 0; i < exportCount; i++) exports.add(`exported_${String(i)}`); + const info: RNamespaceInfo = { + hasNamespaceFile: true, + namedExports: exports, + exportPatterns: [], + importFrom: [], + }; + const cfg: RPackageConfig = { + packages: new Map([['prov', 'prov']]), + namespaceInfoByPackageDir: new Map([['prov', info]]), + }; + const file = extractParsedFile( + rProvider, + 'exported_5 <- function() 1\nprivate_fn <- function() 2', + 'prov/R/a.R', + ); + if (file === undefined) throw new Error('no ParsedFile'); + return { cfg, exports, file }; +} + +describe('resolveRNamedImport: the cost of a name the NAMESPACE does not export', () => { + it('walks the export set once for any number of lookups of names that are not exported', () => { + const { cfg, exports, file } = setup(1_000); + for (let i = 0; i < 100; i++) { + expect(resolveRNamedImport(cfg, 'prov', `private_fn_${String(i)}`, [file])).toBeNull(); + } + expect(resolveRNamedImport(cfg, 'prov', 'private_fn', [file])).toBeNull(); + // One pass builds the index of backticked entries; a scan per lookup would be 101 walks. + expect(exports.walks).toBe(1); + }); + + it('still matches a backticked entry, and reads the set again after it changed', () => { + const { cfg, exports, file } = setup(10); + expect(resolveRNamedImport(cfg, 'prov', 'private_fn', [file])).toBeNull(); + exports.add('`private_fn`'); + expect(resolveRNamedImport(cfg, 'prov', 'private_fn', [file])).toEqual(['prov/R/a.R']); + }); + + it('does not walk the export set for an exported name', () => { + const { cfg, exports, file } = setup(1_000); + expect(resolveRNamedImport(cfg, 'prov', 'exported_5', [file])).toEqual(['prov/R/a.R']); + expect(exports.walks).toBe(0); + }); + + it('answers 20,000 lookups against 20,000 exports with one walk of the export set', () => { + const { cfg, exports, file } = setup(20_000); + for (let i = 0; i < 20_000; i++) { + expect(resolveRNamedImport(cfg, 'prov', `private_fn_${String(i)}`, [file])).toBeNull(); + } + // Counted, not timed: the scan this replaced walked all 20,000 entries for every lookup + // (400 million steps, about two seconds); the index is built by a single walk. + expect(exports.walks).toBe(1); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-namespace-imports.test.ts b/gitnexus/test/unit/scope-resolution/r/r-namespace-imports.test.ts new file mode 100644 index 000000000..b244fb98d --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-namespace-imports.test.ts @@ -0,0 +1,1143 @@ +/** + * R NAMESPACE `importFrom()` binding: + * + * - `rPackageDirForFile`: which package's `R/` directory a file belongs to; + * - `populateRNamespaceImports`: synthesised `named` imports for local + * providers (last-wins, self-import drop, external skip, own-package mask); + * - `resolveRImportTarget`'s `named` branch: local-package check, dotted-name + * guards, NAMESPACE export check, top-level-definition lookup, and the + * "builds no workspace file index" property. + * + * ParsedFiles come from the real R extractor; package configs are built by hand + * (`RPackageConfig` has no other constructor than `loadRPackageConfig`). + */ +import { describe, expect, it } from 'vitest'; +import { fastestWithin, timeWork } from '../../../helpers/timing-attempts.js'; +import type { ParsedFile, ParsedImport } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { populateClassOwnedMembers } from '../../../../src/core/ingestion/scope-resolution/scope/walkers.js'; +import type { + RNamespaceInfo, + RPackageConfig, +} from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + isRGlobalNameFallbackPlausible, + populateRNamespaceImports, + rFileBelongsToDiscoveredPackage, + rFileTopLevel, + rPackageDirForFile, + rRecordedPackageTopLevelNames, +} from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; +import { + compileRExportPattern, + type RExportMatcher, +} from '../../../../src/core/ingestion/languages/r/export-pattern.js'; +import { resolveRImportTarget } from '../../../../src/core/ingestion/languages/r/import-target.js'; +import { CountingSet } from '../../../helpers/counting-file-set.js'; + +// ─── builders ─────────────────────────────────────────────────────────────── + +function parse(src: string, filePath: string): ParsedFile { + const parsed = extractParsedFile(rProvider, src, filePath); + if (parsed === undefined) throw new Error(`no ParsedFile for ${filePath}`); + populateClassOwnedMembers(parsed); + return parsed; +} + +interface NsSpec { + readonly exports?: readonly string[]; + readonly exportPatterns?: readonly RExportMatcher[]; + readonly importFrom?: readonly (readonly [pkg: string, name: string])[]; + /** Omit the NAMESPACE entry altogether (package without a NAMESPACE file). */ + readonly none?: boolean; +} + +function config( + packages: Record, + namespaces: Record = {}, +): RPackageConfig { + const namespaceInfoByPackageDir = new Map(); + for (const [name, dir] of Object.entries(packages)) { + const spec = namespaces[name]; + if (spec === undefined || spec.none === true) continue; + namespaceInfoByPackageDir.set(dir, { + hasNamespaceFile: true, + namedExports: new Set(spec.exports ?? []), + exportPatterns: [...(spec.exportPatterns ?? [])], + importFrom: (spec.importFrom ?? []).map(([pkg, name]) => ({ pkg, name })), + }); + } + return { packages: new Map(Object.entries(packages)), namespaceInfoByPackageDir }; +} + +const namedImportsOf = (parsed: ParsedFile): string[] => + parsed.parsedImports + .filter((i): i is Extract => i.kind === 'named') + .map((i) => `${i.targetRaw}::${i.importedName}`) + .sort(); + +const populate = (files: ParsedFile[], cfg: RPackageConfig | null): ParsedFile[] => { + populateRNamespaceImports(files, { resolutionConfig: cfg ?? undefined }); + return files; +}; + +// ─── rPackageDirForFile ───────────────────────────────────────────────────── + +describe('rPackageDirForFile', () => { + const cfg = config({ alpha: 'alpha', beta: 'sub/beta' }); + + it('finds the package whose R/ directory holds the file', () => { + expect(rPackageDirForFile('alpha/R/a.R', cfg)).toEqual({ name: 'alpha', dir: 'alpha' }); + }); + + it('finds a nested package directory', () => { + expect(rPackageDirForFile('sub/beta/R/b.R', cfg)).toEqual({ name: 'beta', dir: 'sub/beta' }); + }); + + it('accepts Windows separators', () => { + expect(rPackageDirForFile('alpha\\R\\a.R', cfg)?.name).toBe('alpha'); + }); + + it('returns nothing for a file outside every package R/ directory', () => { + expect( + [ + 'alpha/tests/testthat/test-a.R', + 'alpha/scripts/x.R', + 'alpha/plumber.R', + 'other/R/x.R', + 'alphaR/R/x.R', + ].map((f) => rPackageDirForFile(f, cfg)), + ).toEqual([undefined, undefined, undefined, undefined, undefined]); + }); + + it('handles a root-level package (dir "") without building a "/" prefix', () => { + const root = config({ rootpkg: '' }); + expect([ + rPackageDirForFile('R/a.R', root), + rPackageDirForFile('R/sub/a.R', root), + rPackageDirForFile('tests/R/a.R', root), + rPackageDirForFile('plumber.R', root), + ]).toEqual([{ name: 'rootpkg', dir: '' }, { name: 'rootpkg', dir: '' }, undefined, undefined]); + }); + + it('prefers the longest package directory when package directories nest', () => { + const nested = config({ outer: 'outer', inner: 'outer/inner' }); + expect(rPackageDirForFile('outer/inner/R/x.R', nested)?.name).toBe('inner'); + expect(rPackageDirForFile('outer/R/x.R', nested)?.name).toBe('outer'); + }); + + it('returns nothing without a config', () => { + expect([ + rPackageDirForFile('alpha/R/a.R', null), + rPackageDirForFile('a/R/a.R', undefined), + ]).toEqual([undefined, undefined]); + }); +}); + +// ─── rFileTopLevel ────────────────────────────────────────────────────────── + +describe('rFileBelongsToDiscoveredPackage', () => { + const cfg: RPackageConfig = { + packages: new Map([['pkg', 'pkgs/pkg']]), + namespaceInfoByPackageDir: new Map(), + packageDirs: new Set(['other']), + truncated: false, + }; + + it.each([undefined, null])('is false without a package config (%s)', (missing) => { + expect(rFileBelongsToDiscoveredPackage('R/a.R', missing)).toBe(false); + expect(rFileBelongsToDiscoveredPackage('pkgs/pkg/R/a.R', missing)).toBe(false); + }); + + it('is true for an R/ file of a registered package', () => { + expect(rFileBelongsToDiscoveredPackage('pkgs/pkg/R/a.R', cfg)).toBe(true); + }); + + it('is true for an R/ file of a directory only recorded in packageDirs', () => { + expect(rFileBelongsToDiscoveredPackage('other/R/a.R', cfg)).toBe(true); + }); + + it('is false outside every discovered package, and outside R/', () => { + expect(rFileBelongsToDiscoveredPackage('stray/R/a.R', cfg)).toBe(false); + expect(rFileBelongsToDiscoveredPackage('pkgs/pkg/tests/a.R', cfg)).toBe(false); + }); +}); + +describe('rFileTopLevel', () => { + const parsed = parse( + [ + 'x <- 1', + 'tidy <- function(a) { inner <- function(b) b; inner(a) }', + 'print.tidy <- function(x) x', + 'Scorer <- R6::R6Class("Scorer", public = list(method_only = function(z) z))', + ].join('\n'), + 'p/R/a.R', + ); + const top = rFileTopLevel(parsed); + + it('lists top-level functions and classes, dotted names included', () => { + expect([...top.names].sort()).toEqual(['Scorer', 'print.tidy', 'tidy']); + }); + + it('excludes nested functions, R6 members and plain variables', () => { + expect([...top.names].filter((n) => ['inner', 'method_only', 'x'].includes(n))).toEqual([]); + }); + + it('records the tail of every dotted name', () => { + expect([...top.dottedTails]).toEqual(['tidy']); + }); +}); + +// ─── populateRNamespaceImports ────────────────────────────────────────────── + +describe('populateRNamespaceImports', () => { + const pkgs = { analytics: 'analytics', scorelib: 'scorelib', legacyscore: 'legacyscore' }; + const provider = (path: string, names: string[]) => + parse(names.map((n) => `${n} <- function(d) d`).join('\n'), path); + const callerFile = (path: string, src = 'noop <- function() 1') => parse(src, path); + + it('synthesises a named import for a local-package importFrom() into every R/ file of the caller', () => { + const files = populate( + [ + callerFile('analytics/R/a.R'), + callerFile('analytics/R/b.R'), + callerFile('analytics/R/c.R'), + provider('scorelib/R/s.R', ['tidy_scores']), + ], + config(pkgs, { + analytics: { importFrom: [['scorelib', 'tidy_scores']] }, + scorelib: { exports: ['tidy_scores'] }, + }), + ); + expect(files.map(namedImportsOf)).toEqual([ + ['scorelib::tidy_scores'], + ['scorelib::tidy_scores'], + ['scorelib::tidy_scores'], + [], + ]); + }); + + it('shapes the import as a per-name, un-renamed named import of the package', () => { + const [caller] = populate( + [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['tidy_scores'])], + config(pkgs, { analytics: { importFrom: [['scorelib', 'tidy_scores']] } }), + ); + expect(caller.parsedImports).toEqual([ + { + kind: 'named', + localName: 'tidy_scores', + importedName: 'tidy_scores', + targetRaw: 'scorelib', + }, + ]); + }); + + it('skips an importFrom() of an external package (nothing local to bind)', () => { + const files = populate( + [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['mutate'])], + config(pkgs, { analytics: { importFrom: [['dplyr', 'mutate']] } }), + ); + expect(files.map(namedImportsOf)).toEqual([[], []]); + }); + + it('drops a self-import (the parser returns it; the package cannot import from itself)', () => { + const files = populate( + [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['f'])], + config(pkgs, { analytics: { importFrom: [['analytics', 'f']] } }), + ); + expect(files.map(namedImportsOf)).toEqual([[], []]); + }); + + it('skips a name the caller package defines at top level, even in a different file', () => { + const files = populate( + [ + callerFile('analytics/R/a.R', 'uses_it <- function() 1'), + callerFile('analytics/R/own.R', 'own_dup <- function(d) d'), + provider('scorelib/R/s.R', ['own_dup', 'other_fn']), + ], + config(pkgs, { + analytics: { + importFrom: [ + ['scorelib', 'own_dup'], + ['scorelib', 'other_fn'], + ], + }, + }), + ); + expect(files.map(namedImportsOf)).toEqual([['scorelib::other_fn'], ['scorelib::other_fn'], []]); + }); + + it('does not treat an R6 method of the caller package as a top-level definition', () => { + const files = populate( + [ + callerFile( + 'analytics/R/own.R', + 'Runner <- R6::R6Class("Runner", public = list(tidy_scores = function(d) d))', + ), + provider('scorelib/R/s.R', ['tidy_scores']), + ], + config(pkgs, { analytics: { importFrom: [['scorelib', 'tidy_scores']] } }), + ); + expect(namedImportsOf(files[0])).toEqual(['scorelib::tidy_scores']); + }); + + it('keeps only the last entry when two local packages import one name (last wins)', () => { + const files = populate( + [ + callerFile('analytics/R/a.R'), + provider('scorelib/R/s.R', ['dup']), + provider('legacyscore/R/l.R', ['dup']), + ], + config(pkgs, { + analytics: { + importFrom: [ + ['scorelib', 'dup'], + ['legacyscore', 'dup'], + ], + }, + }), + ); + expect(namedImportsOf(files[0])).toEqual(['legacyscore::dup']); + }); + + it('synthesises nothing when an external package is imported LAST for a name (no resurrection)', () => { + const files = populate( + [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['dup_ext'])], + config(pkgs, { + analytics: { + importFrom: [ + ['scorelib', 'dup_ext'], + ['dplyr', 'dup_ext'], + ], + }, + }), + ); + expect(namedImportsOf(files[0])).toEqual([]); + }); + + it('binds the local package when it is imported after an external one', () => { + const files = populate( + [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['dup_ext'])], + config(pkgs, { + analytics: { + importFrom: [ + ['dplyr', 'dup_ext'], + ['scorelib', 'dup_ext'], + ], + }, + }), + ); + expect(namedImportsOf(files[0])).toEqual(['scorelib::dup_ext']); + }); + + it('touches only R/ files: scripts/ and tests/ of the caller package get no import', () => { + const files = populate( + [ + callerFile('analytics/R/a.R'), + callerFile('analytics/scripts/x.R'), + callerFile('analytics/tests/testthat/test-x.R'), + callerFile('analytics/plumber.R'), + provider('scorelib/R/s.R', ['f']), + ], + config(pkgs, { analytics: { importFrom: [['scorelib', 'f']] } }), + ); + expect(files.map(namedImportsOf)).toEqual([['scorelib::f'], [], [], [], []]); + }); + + it('handles a root-level caller package', () => { + const files = populate( + [callerFile('R/a.R'), callerFile('scripts/x.R'), provider('scorelib/R/s.R', ['f'])], + config( + { rootpkg: '', scorelib: 'scorelib' }, + { rootpkg: { importFrom: [['scorelib', 'f']] } }, + ), + ); + expect(files.map(namedImportsOf)).toEqual([['scorelib::f'], [], []]); + }); + + it('replaces array entries with frozen copies and leaves the originals untouched', () => { + const original = callerFile('analytics/R/a.R'); + const files = [original, provider('scorelib/R/s.R', ['f'])]; + populate(files, config(pkgs, { analytics: { importFrom: [['scorelib', 'f']] } })); + expect([ + files[0] === original, + Object.isFrozen(files[0]), + original.parsedImports.length, + ]).toEqual([false, true, 0]); + }); + + it('is idempotent: a second pass does not duplicate the synthesised imports', () => { + const files = [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['f'])]; + const cfg = config(pkgs, { analytics: { importFrom: [['scorelib', 'f']] } }); + populate(files, cfg); + populate(files, cfg); + expect(namedImportsOf(files[0])).toEqual(['scorelib::f']); + }); + + it('is a no-op without a config, with no packages, or when no package imports a local one', () => { + const files = [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', ['f'])]; + const before = [...files]; + populate(files, null); + populate(files, config({})); + populate(files, config(pkgs, { analytics: { importFrom: [['dplyr', 'f']] } })); + populate(files, config(pkgs, { scorelib: { exports: ['f'] } })); + expect(files.every((f, i) => f === before[i])).toBe(true); + }); + + it("records each participating package's top-level names for the fallback veto", () => { + const cfg = config(pkgs, { analytics: { importFrom: [['scorelib', 'f']] } }); + populate( + [ + callerFile('analytics/R/a.R', 'run <- function() 1'), + provider('scorelib/R/s.R', ['f', 'g']), + provider('legacyscore/R/l.R', ['h']), + ], + cfg, + ); + const recorded = rRecordedPackageTopLevelNames(cfg); + expect([ + [...(recorded?.get('analytics') ?? [])], + [...(recorded?.get('scorelib') ?? [])].sort(), + recorded?.has('legacyscore'), + ]).toEqual([['run'], ['f', 'g'], false]); + }); + + // 75eaac75: an importFrom() that resolveRNamedImport() rejects can never bind. A synthesised + // import for it would stay unresolved yet count as an explicit name claim, suppressing the + // documented 0.5-confidence `global-name-fallback`. Only `populate*` is exercised here, so + // these fail if the skip is removed (the integration suite used to be the only pin). + describe('skips importFrom() entries that can never bind', () => { + const wholePkg = (entries: readonly string[], provided: string[], exports?: string[]) => { + const files = [callerFile('analytics/R/a.R'), provider('scorelib/R/s.R', provided)]; + populate( + files, + config(pkgs, { + analytics: { importFrom: entries.map((n) => ['scorelib', n] as const) }, + ...(exports === undefined ? {} : { scorelib: { exports } }), + }), + ); + return namedImportsOf(files[0]); + }; + + it('positive control: an exported, top-level, undotted name still binds', () => { + expect(wholePkg(['tidy_scores'], ['tidy_scores'], ['tidy_scores'])).toEqual([ + 'scorelib::tidy_scores', + ]); + }); + + it('skips a name the provider NAMESPACE does not export', () => { + expect(wholePkg(['hidden_fn'], ['hidden_fn', 'tidy_scores'], ['tidy_scores'])).toEqual([]); + }); + + it('skips a name no provider file defines at top level', () => { + expect( + wholePkg(['never_defined'], ['tidy_scores'], ['never_defined', 'tidy_scores']), + ).toEqual([]); + }); + + it('skips a dotted name, even when the provider exports and defines it', () => { + expect(wholePkg(['my.helper'], ['my.helper'], ['my.helper'])).toEqual([]); + }); + + it('skips a name defined beside a dotted S3-style def of the same tail', () => { + // finalize would index `print.tidy` under `tidy` and bind the wrong def. + expect(wholePkg(['tidy'], ['tidy', 'print.tidy'], ['tidy'])).toEqual([]); + }); + + it('skips only the unbindable entries and keeps the bindable ones', () => { + expect( + wholePkg( + ['tidy_scores', 'hidden_fn', 'never_defined', 'my.helper', 'tidy'], + ['tidy_scores', 'hidden_fn', 'my.helper', 'tidy', 'print.tidy'], + ['tidy_scores', 'never_defined', 'my.helper', 'tidy'], + ), + ).toEqual(['scorelib::tidy_scores']); + }); + + it('is decided against the files supplied, so a missing provider file drops the import', () => { + const files = [callerFile('analytics/R/a.R')]; + populate(files, config(pkgs, { analytics: { importFrom: [['scorelib', 'tidy_scores']] } })); + expect(namedImportsOf(files[0])).toEqual([]); + }); + }); + + it('records nothing for an unrelated config object', () => { + expect(rRecordedPackageTopLevelNames(config(pkgs))).toBeUndefined(); + expect(rRecordedPackageTopLevelNames(undefined)).toBeUndefined(); + }); +}); + +// ─── resolveRImportTarget: named branch ───────────────────────────────────── + +describe('resolveRImportTarget — named imports (NAMESPACE importFrom)', () => { + const namedImport = (pkg: string, name: string): ParsedImport => ({ + kind: 'named', + localName: name, + importedName: name, + targetRaw: pkg, + }); + + interface Setup { + readonly cfg: RPackageConfig; + readonly files: ParsedFile[]; + } + + /** scorelib provider files (source per path) plus a caller file. */ + function setup( + providerSources: Record, + ns: NsSpec | undefined, + dir = 'scorelib', + ): Setup { + const files = [ + parse('run <- function() 1', 'analytics/R/use.R'), + ...Object.entries(providerSources).map(([p, src]) => parse(src, p)), + ]; + return { + cfg: config( + { analytics: 'analytics', scorelib: dir }, + ns === undefined ? {} : { scorelib: ns }, + ), + files, + }; + } + + const resolve = (s: Setup, pkg: string, name: string) => + resolveRImportTarget(pkg, 'analytics/R/use.R', new Set(s.files.map((f) => f.filePath)), s.cfg, { + parsedFiles: s.files, + parsedImport: namedImport(pkg, name), + }); + + const F = 'scorelib/R/s.R'; + + it('returns nothing for an external package', () => { + const s = setup({ [F]: 'mutate <- function(d) d' }, { exports: ['mutate'] }); + expect(resolve(s, 'dplyr', 'mutate')).toBeNull(); + }); + + it('returns nothing without a package config', () => { + const s = setup({ [F]: 'f <- function(d) d' }, undefined); + expect( + resolveRImportTarget('scorelib', 'analytics/R/use.R', new Set([F]), undefined, { + parsedFiles: s.files, + parsedImport: namedImport('scorelib', 'f'), + }), + ).toBeNull(); + }); + + it('returns nothing when the local package has no definition of the name', () => { + const s = setup({ [F]: 'other <- function(d) d' }, { exports: ['f', 'other'] }); + expect(resolve(s, 'scorelib', 'f')).toBeNull(); + }); + + it('returns only the files that define the name at top level', () => { + const s = setup( + { + [F]: 'f <- function(d) d', + 'scorelib/R/t.R': 'g <- function(d) d', + 'scorelib/R/u.R': 'f <- function(d) d', + }, + { exports: ['f', 'g'] }, + ); + expect(resolve(s, 'scorelib', 'f')).toEqual([F, 'scorelib/R/u.R']); + }); + + it('ignores same-named definitions outside the package R/ directory and in other packages', () => { + const s = setup( + { + 'scorelib/scripts/x.R': 'f <- function(d) d', + 'legacyscore/R/l.R': 'f <- function(d) d', + }, + { exports: ['f'] }, + ); + expect(resolve(s, 'scorelib', 'f')).toBeNull(); + }); + + it('does not bind an R6 method, nor a nested function, as a top-level definition', () => { + const s = setup( + { + [F]: [ + 'Scorer <- R6::R6Class("Scorer", public = list(method_only = function(d) d))', + 'host <- function(d) { nested_only <- function(y) y; nested_only(d) }', + ].join('\n'), + }, + { exports: ['method_only', 'nested_only'] }, + ); + expect([resolve(s, 'scorelib', 'method_only'), resolve(s, 'scorelib', 'nested_only')]).toEqual([ + null, + null, + ]); + }); + + it('resolves an R6 class defined at top level', () => { + const s = setup( + { [F]: 'Scorer <- R6::R6Class("Scorer", public = list())' }, + { exports: ['Scorer'] }, + ); + expect(resolve(s, 'scorelib', 'Scorer')).toEqual([F]); + }); + + it('strips quotes around the package name', () => { + const s = setup({ [F]: 'f <- function(d) d' }, { exports: ['f'] }); + expect(resolve(s, '"scorelib"', 'f')).toEqual([F]); + }); + + describe('dotted-name guards', () => { + it('refuses a dotted importedName (finalize can never bind it)', () => { + const s = setup( + { [F]: 'normalise.scores <- function(d) d' }, + { exports: ['normalise.scores'] }, + ); + expect(resolve(s, 'scorelib', 'normalise.scores')).toBeNull(); + }); + + it('refuses tidy when print.tidy is defined BEFORE it in the same file', () => { + const s = setup( + { [F]: 'print.tidy <- function(x) x\ntidy <- function(d) d' }, + { exports: ['tidy'] }, + ); + expect(resolve(s, 'scorelib', 'tidy')).toBeNull(); + }); + + it('refuses tidy when print.tidy is defined AFTER it too (order-blind, conservative)', () => { + const s = setup( + { [F]: 'tidy <- function(d) d\nprint.tidy <- function(x) x' }, + { exports: ['tidy'] }, + ); + expect(resolve(s, 'scorelib', 'tidy')).toBeNull(); + }); + + it('does not fire for an underscore name in the same file as dotted defs', () => { + const s = setup( + { [F]: 'print.tidy <- function(x) x\nplain_fn <- function(d) d' }, + { exports: ['plain_fn'] }, + ); + expect(resolve(s, 'scorelib', 'plain_fn')).toEqual([F]); + }); + + it('does not fire when the dotted def sits in a different file from the definition', () => { + const s = setup( + { + [F]: 'tidy <- function(d) d', + 'scorelib/R/t.R': 'print.tidy <- function(x) x', + }, + { exports: ['tidy'] }, + ); + expect(resolve(s, 'scorelib', 'tidy')).toEqual([F]); + }); + + it('does not fire for a same-named R6 method (only Module-scope bindings count)', () => { + const s = setup( + { + [F]: [ + 'Scorer <- R6::R6Class("Scorer", public = list(tidy = function(d) d))', + 'tidy <- function(d) d', + ].join('\n'), + }, + { exports: ['tidy'] }, + ); + expect(resolve(s, 'scorelib', 'tidy')).toEqual([F]); + }); + }); + + describe('NAMESPACE export check', () => { + const src = 'f <- function(d) d\nhidden <- function(d) d\n`%+%` <- function(a, b) a'; + + it('resolves a name listed in export()', () => { + const s = setup({ [F]: src }, { exports: ['f'] }); + expect(resolve(s, 'scorelib', 'f')).toEqual([F]); + }); + + it('refuses a defined name the provider NAMESPACE does not export', () => { + const s = setup({ [F]: src }, { exports: ['f'] }); + expect(resolve(s, 'scorelib', 'hidden')).toBeNull(); + }); + + it('resolves a name matched by an exportPattern()', () => { + const s = setup({ [F]: src }, { exportPatterns: [/^hid/] }); + expect(resolve(s, 'scorelib', 'hidden')).toEqual([F]); + }); + + it('matches a backticked export() entry against the unquoted name (backticks stripped)', () => { + const s = setup({ [F]: src }, { exports: ['`%+%`'] }); + // Export check passes, and the def is found by its symbol although it keeps its backticks. + expect(resolve(s, 'scorelib', '%+%')).toEqual([F]); + const t = setup({ [F]: 'g <- function(d) d' }, { exports: ['`g`'] }); + expect(resolve(t, 'scorelib', 'g')).toEqual([F]); + }); + + it('looks a name up among export entries that hold a long run of backticks in linear time', () => { + // `/^`+|`+$/g` took ~12 s per entry on this shape; the bound leaves a ten-fold margin. + const hostile = `x${'`'.repeat(200_000)}y`; + // Each attempt sets up afresh, so a memo built by an earlier one cannot hide the cost. + const { ms, value: answers } = fastestWithin(2_000, () => + timeWork(() => { + const s = setup({ [F]: src }, { exports: [hostile, 'f'] }); + return () => [resolve(s, 'scorelib', 'hidden'), resolve(s, 'scorelib', 'f')]; + }), + ); + expect(ms).toBeLessThan(2_000); + expect(answers).toEqual([null, [F]]); + }); + + it('treats a package without a NAMESPACE file as exporting everything', () => { + const s = setup({ [F]: src }, undefined); + expect(resolve(s, 'scorelib', 'hidden')).toEqual([F]); + }); + + it('treats a package whose NAMESPACE entry is absent as exporting everything', () => { + const s = setup({ [F]: src }, { none: true }); + expect(resolve(s, 'scorelib', 'hidden')).toEqual([F]); + }); + + it('checks a root-level provider package (dir "")', () => { + const files = [ + parse('run <- function() 1', 'analytics/R/use.R'), + parse('f <- function(d) d\nhidden <- function(d) d', 'R/s.R'), + ]; + const cfg = config({ analytics: 'analytics', rootpkg: '' }, { rootpkg: { exports: ['f'] } }); + const ask = (name: string) => + resolveRImportTarget('rootpkg', 'analytics/R/use.R', new Set(), cfg, { + parsedFiles: files, + parsedImport: namedImport('rootpkg', name), + }); + expect([ask('f'), ask('hidden')]).toEqual([['R/s.R'], null]); + }); + + const compiled = (pattern: string): RExportMatcher => { + const re = compileRExportPattern(pattern); + if (!re) throw new Error(`exportPattern did not compile: ${pattern}`); + return re; + }; + + it('resolves a name matched by a POSIX-class exportPattern (the RStudio default)', () => { + // `exportPattern("^[[:alpha:]]+")` used to compile to a JS RegExp that matched nothing, so + // every name was judged non-exported and the import degraded to the baseline name-guess. + // Deliberate change: POSIX classes are translated by `compileRExportPattern`. + const s = setup( + { [F]: 'tidy_scores <- function(d) d' }, + { + exportPatterns: [compiled('^[[:alpha:]]+')], + }, + ); + expect(resolve(s, 'scorelib', 'tidy_scores')).toEqual([F]); + }); + + it('still refuses a name the POSIX-class exportPattern does not match', () => { + const s = setup( + { [F]: 'hidden <- function(d) d' }, + { + exportPatterns: [compiled('^[[:upper:]]+$')], + }, + ); + expect(resolve(s, 'scorelib', 'hidden')).toBeNull(); + // Control: the same definition resolves when the package exports everything. + const t = setup({ [F]: 'hidden <- function(d) d' }, undefined); + expect(resolve(t, 'scorelib', 'hidden')).toEqual([F]); + }); + }); + + describe('workspace file index', () => { + it('builds no file index for named imports (the file Set is never traversed)', () => { + const s = setup({ [F]: 'f <- function(d) d' }, { exports: ['f'] }); + const paths = new CountingSet(s.files.map((f) => f.filePath)); + for (let i = 0; i < 25; i++) { + resolveRImportTarget('scorelib', 'analytics/R/use.R', paths, s.cfg, { + parsedFiles: s.files, + parsedImport: namedImport('scorelib', 'f'), + }); + resolveRImportTarget('dplyr', 'analytics/R/use.R', paths, s.cfg, { + parsedFiles: s.files, + parsedImport: namedImport('dplyr', 'f'), + }); + } + expect(paths.scans).toBe(0); + }); + + it('still builds one memoised index for wildcard imports', () => { + const s = setup({ [F]: 'f <- function(d) d' }, { exports: ['f'] }); + const paths = new CountingSet(s.files.map((f) => f.filePath)); + for (let i = 0; i < 25; i++) { + resolveRImportTarget('scorelib', 'analytics/R/use.R', paths, s.cfg, { + parsedFiles: s.files, + parsedImport: { kind: 'wildcard', targetRaw: 'scorelib' }, + }); + } + expect(paths.scans).toBe(1); + }); + + it('indexes the parsed files once for many named imports', () => { + const s = setup({ [F]: 'f <- function(d) d\ng <- function(d) d' }, { exports: ['f', 'g'] }); + const first = resolve(s, 'scorelib', 'f'); + const second = resolve(s, 'scorelib', 'f'); + expect(second).toBe(first); + }); + }); + + describe('non-named kinds and context-less calls', () => { + it('leaves a wildcard import of an external package unresolved', () => { + const s = setup({ [F]: 'f <- function(d) d' }, { exports: ['f'] }); + expect( + resolveRImportTarget('dplyr', 'analytics/R/use.R', new Set([F]), s.cfg, { + parsedFiles: s.files, + parsedImport: { kind: 'wildcard', targetRaw: 'dplyr' }, + }), + ).toBeNull(); + }); + + it('resolves a wildcard import of a local package to all its R/ files, unchanged', () => { + const s = setup( + { [F]: 'f <- function(d) d', 'scorelib/R/t.R': 'g <- function(d) d' }, + { exports: ['f'] }, + ); + expect( + resolveRImportTarget( + 'scorelib', + 'analytics/R/use.R', + new Set(s.files.map((f) => f.filePath)), + s.cfg, + { parsedFiles: s.files, parsedImport: { kind: 'wildcard', targetRaw: 'scorelib' } }, + ), + ).toEqual([F, 'scorelib/R/t.R']); + }); + + it('a context-less call is treated as before (suffix resolution, no named branch)', () => { + const s = setup({ [F]: 'f <- function(d) d' }, { exports: ['f'] }); + expect( + resolveRImportTarget( + 'scorelib', + 'analytics/R/use.R', + new Set(s.files.map((f) => f.filePath)), + s.cfg, + ), + ).toEqual([F]); + }); + }); +}); + +// ─── isRGlobalNameFallbackPlausible (truth table T1-T11) ──────────────────── + +describe('isRGlobalNameFallbackPlausible — global-name guess veto', () => { + const pkgs = { + analytics: 'analytics', + scorelib: 'scorelib', + legacyscore: 'legacyscore', + corelib: 'corelib', + }; + const def = (path: string, names: string[]) => + parse(names.map((n) => `${n} <- function(d) d`).join('\n'), path); + + /** + * Runs the real `populateRNamespaceImports` (which records the per-package top-level names the + * veto reads) over `files`, then asks the veto about a call to `name` in `callerFile` whose + * unique-name guess landed in `candidateFile`. + */ + const verdict = (opts: { + importFrom: readonly (readonly [string, string])[]; + files: readonly ParsedFile[]; + callerFile?: string; + candidateFile: string; + name: string; + packages?: Record; + rawQualifiedName?: string; + skipPopulate?: boolean; + }): boolean => { + const cfg = config(opts.packages ?? pkgs, { + analytics: { importFrom: opts.importFrom }, + scorelib: { exports: ['f', 'tidy_scores', 'reexp_fn'] }, + legacyscore: { exports: ['f', 'tidy_scores', 'mutate'] }, + corelib: { exports: ['f', 'reexp_fn'] }, + }); + const files = [...opts.files]; + if (opts.skipPopulate !== true) populate(files, cfg); + return isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: opts.callerFile ?? 'analytics/R/use.R' }, + candidate: { filePath: opts.candidateFile }, + resolutionConfig: cfg, + site: { name: opts.name, rawQualifiedName: opts.rawQualifiedName }, + }); + }; + + const caller = () => def('analytics/R/use.R', ['run_all']); + + it('T1: allows a candidate in the caller own package (own namespace masks; stays a fallback edge)', () => { + expect( + verdict({ + importFrom: [['dplyr', 'filter']], + files: [caller(), def('analytics/R/own.R', ['filter'])], + candidateFile: 'analytics/R/own.R', + name: 'filter', + }), + ).toBe(true); + }); + + it('T2: vetoes a decoy in another local package when the name is imported from an external package', () => { + expect( + verdict({ + importFrom: [['dplyr', 'mutate']], + files: [caller(), def('legacyscore/R/l.R', ['mutate'])], + candidateFile: 'legacyscore/R/l.R', + name: 'mutate', + }), + ).toBe(false); + }); + + it('T3a: vetoes a candidate in a third local package when the provider package defines the name', () => { + const files = [ + caller(), + def('scorelib/R/s.R', ['tidy_scores']), + def('legacyscore/R/l.R', ['tidy_scores']), + ]; + expect( + verdict({ + importFrom: [['scorelib', 'tidy_scores']], + files, + candidateFile: 'legacyscore/R/l.R', + name: 'tidy_scores', + }), + ).toBe(false); + }); + + it('T3b: allows the candidate that lives in the imported-from package itself', () => { + const files = [ + caller(), + def('scorelib/R/s.R', ['tidy_scores']), + def('legacyscore/R/l.R', ['tidy_scores']), + ]; + expect( + verdict({ + importFrom: [['scorelib', 'tidy_scores']], + files, + candidateFile: 'scorelib/R/s.R', + name: 'tidy_scores', + }), + ).toBe(true); + }); + + it('T4: allows the origin package when a local provider re-exports the name (it defines none)', () => { + expect( + verdict({ + importFrom: [['scorelib', 'reexp_fn']], + files: [caller(), def('scorelib/R/s.R', ['other_fn']), def('corelib/R/c.R', ['reexp_fn'])], + candidateFile: 'corelib/R/c.R', + name: 'reexp_fn', + }), + ).toBe(true); + }); + + it('T4 (undecidable): allows when no per-package names were recorded for the provider', () => { + expect( + verdict({ + importFrom: [['scorelib', 'f']], + files: [caller(), def('scorelib/R/s.R', ['f']), def('legacyscore/R/l.R', ['f'])], + candidateFile: 'legacyscore/R/l.R', + name: 'f', + skipPopulate: true, + }), + ).toBe(true); + }); + + describe('duplicate importFrom() entries: only the LAST entry counts', () => { + const files = () => [caller(), def('scorelib/R/s.R', ['f']), def('legacyscore/R/l.R', ['f'])]; + + it('T5a: external package last (scorelib, then dplyr) vetoes the candidate in scorelib', () => { + expect( + verdict({ + importFrom: [ + ['scorelib', 'f'], + ['dplyr', 'f'], + ], + files: files(), + candidateFile: 'scorelib/R/s.R', + name: 'f', + }), + ).toBe(false); + }); + + it('T5b: local package last (dplyr, then scorelib) vetoes a candidate in a third package', () => { + expect( + verdict({ + importFrom: [ + ['dplyr', 'f'], + ['scorelib', 'f'], + ], + files: files(), + candidateFile: 'legacyscore/R/l.R', + name: 'f', + }), + ).toBe(false); + }); + + it('T5c: local package last allows the candidate in that package', () => { + expect( + verdict({ + importFrom: [ + ['dplyr', 'f'], + ['scorelib', 'f'], + ], + files: files(), + candidateFile: 'scorelib/R/s.R', + name: 'f', + }), + ).toBe(true); + }); + }); + + it('T6a: allows a candidate outside every package R/ directory (scripts/)', () => { + expect( + verdict({ + importFrom: [['dplyr', 'mutate']], + files: [caller(), def('analytics/scripts/x.R', ['mutate'])], + candidateFile: 'analytics/scripts/x.R', + name: 'mutate', + }), + ).toBe(true); + }); + + it('T6b: allows any candidate when the caller file is outside the package R/ directory', () => { + for (const callerFile of [ + 'analytics/tests/testthat/test-x.R', + 'analytics/scripts/run.R', + 'plumber.R', + ]) { + expect( + verdict({ + importFrom: [['dplyr', 'mutate']], + files: [def('legacyscore/R/l.R', ['mutate'])], + callerFile, + candidateFile: 'legacyscore/R/l.R', + name: 'mutate', + }), + ).toBe(true); + } + }); + + it('T7 (known limitation): a later import(corelib) is not tokenised, so the candidate in corelib is still vetoed', () => { + // NAMESPACE: importFrom(scorelib, f) then import(corelib). R would take the later binding; + // `import()` is dropped by the parser, so only the importFrom entry exists. + expect( + verdict({ + importFrom: [['scorelib', 'f']], + files: [caller(), def('scorelib/R/s.R', ['f']), def('corelib/R/c.R', ['f'])], + candidateFile: 'corelib/R/c.R', + name: 'f', + }), + ).toBe(false); + }); + + it('T8: allows everything for a self-import (nothing is imported)', () => { + expect( + verdict({ + importFrom: [['analytics', 'f']], + files: [caller(), def('legacyscore/R/l.R', ['f'])], + candidateFile: 'legacyscore/R/l.R', + name: 'f', + }), + ).toBe(true); + }); + + it('T9: refuses a call with no importFrom() entry into a package the caller declares no link to (rule A)', () => { + expect( + verdict({ + importFrom: [['scorelib', 'f']], + files: [caller(), def('scorelib/R/s.R', ['f']), def('legacyscore/R/l.R', ['g'])], + candidateFile: 'legacyscore/R/l.R', + name: 'g', + }), + ).toBe(false); + }); + + describe('T9b: allows it when the caller package declares a link, or its imports are not fully read', () => { + const plausible = ( + extra: Partial, + callerExtra: { parsedImports?: readonly { kind: string; targetRaw: string }[] } = {}, + ): boolean => { + const cfg = config(pkgs, { + analytics: { importFrom: [['scorelib', 'f']] }, + legacyscore: { exports: ['g'] }, + }); + const info = cfg.namespaceInfoByPackageDir.get(pkgs.analytics); + if (info === undefined) throw new Error('no analytics NAMESPACE'); + Object.assign(info, extra); + return isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'analytics/R/use.R', ...callerExtra }, + candidate: { filePath: 'legacyscore/R/l.R' }, + resolutionConfig: cfg, + site: { name: 'g' }, + }); + }; + it('import(legacyscore)', () => { + expect(plausible({ importPackages: new Set(['legacyscore']) })).toBe(true); + }); + it('Depends: legacyscore', () => { + expect(plausible({ depends: new Set(['legacyscore']) })).toBe(true); + }); + it('library(legacyscore) in the caller file', () => { + expect( + plausible({}, { parsedImports: [{ kind: 'wildcard', targetRaw: '"legacyscore"' }] }), + ).toBe(true); + }); + it('an import the NAMESPACE reader could not read', () => { + expect(plausible({ importsUnreadable: true })).toBe(true); + }); + it('control: an import of another package', () => { + expect(plausible({ importPackages: new Set(['scorelib']) })).toBe(false); + }); + }); + + describe('T10: cannot decide', () => { + const ctx = (resolutionConfig: unknown) => ({ + callerParsed: { filePath: 'analytics/R/use.R' }, + candidate: { filePath: 'legacyscore/R/l.R' }, + resolutionConfig, + site: { name: 'mutate' }, + }); + + it('allows without a resolution config, and for a config of an unexpected shape', () => { + expect([ + isRGlobalNameFallbackPlausible(ctx(undefined)), + isRGlobalNameFallbackPlausible(ctx(null)), + isRGlobalNameFallbackPlausible(ctx({})), + isRGlobalNameFallbackPlausible(ctx('nope')), + ]).toEqual([true, true, true, true]); + }); + + it('allows when the caller package has no NAMESPACE entry', () => { + expect( + isRGlobalNameFallbackPlausible(ctx(config(pkgs, { legacyscore: { exports: ['mutate'] } }))), + ).toBe(true); + }); + + it('allows a caller file that belongs to no package', () => { + const cfg = config(pkgs, { analytics: { importFrom: [['dplyr', 'mutate']] } }); + expect( + isRGlobalNameFallbackPlausible({ ...ctx(cfg), callerParsed: { filePath: 'other/R/x.R' } }), + ).toBe(true); + }); + }); + + it('T11 (qualifier flip): allows the correct legacyscore::mutate() edge — the qualifier names the candidate package', () => { + expect( + verdict({ + importFrom: [['dplyr', 'mutate']], + files: [caller(), def('legacyscore/R/l.R', ['mutate'])], + candidateFile: 'legacyscore/R/l.R', + name: 'mutate', + rawQualifiedName: 'legacyscore::mutate', + }), + ).toBe(true); + }); + + it('handles a root-level caller package (dir "")', () => { + const packages = { rootpkg: '', legacyscore: 'legacyscore' }; + const cfg = config(packages, { rootpkg: { importFrom: [['dplyr', 'mutate']] } }); + const ask = (callerFile: string) => + isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: callerFile }, + candidate: { filePath: 'legacyscore/R/l.R' }, + resolutionConfig: cfg, + site: { name: 'mutate' }, + }); + expect([ask('R/use.R'), ask('tests/testthat/test-x.R')]).toEqual([false, true]); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-qualified-call-populate.test.ts b/gitnexus/test/unit/scope-resolution/r/r-qualified-call-populate.test.ts new file mode 100644 index 000000000..d1d41b402 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-qualified-call-populate.test.ts @@ -0,0 +1,107 @@ +/** + * `populateRQualifiedCalls` attributes a local package's top-level definitions to the files of + * that package. A package known only from a `/R/` directory owns the files under it, but + * never a file a discovered package owns, whatever that directory is called. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, SymbolDefinition } from 'gitnexus-shared'; +import type { RPackageConfig } from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { rQualifiedDefinitionCount } from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; +import { + populateRQualifiedCalls, + resolveRQualifiedFreeCall, +} from '../../../../src/core/ingestion/languages/r/qualified-call.js'; + +const callSite = { + kind: 'call', + name: 'helper', + rawQualifiedName: 'foo::helper', + atRange: { startLine: 3, startCol: 2, endLine: 3, endCol: 16 }, +}; + +function definition(filePath: string): SymbolDefinition { + return { nodeId: `Function:${filePath}:helper`, filePath, type: 'Function' } as SymbolDefinition; +} + +/** A file whose Module scope binds `helper` as a local top-level function. */ +function definingFile(filePath: string): ParsedFile { + const def = definition(filePath); + return { + filePath, + moduleScope: 'module', + scopes: [{ id: 'module', bindings: new Map([['helper', [{ origin: 'local', def }]]]) }], + referenceSites: [], + } as unknown as ParsedFile; +} + +function callerFile(filePath: string): ParsedFile { + return { + filePath, + moduleScope: 'module', + scopes: [{ id: 'module', bindings: new Map() }], + referenceSites: [callSite], + } as unknown as ParsedFile; +} + +describe('populateRQualifiedCalls: /R/ ownership', () => { + it('binds `foo::helper` to the undiscovered foo/R/ package, not to a discovered package in a directory named foo', () => { + // Discovery found `Package: bar` in pkgs/foo; x/foo/R/ has no DESCRIPTION, so `foo` is a + // package only through its directory name. + const cfg: RPackageConfig = { + packages: new Map([['bar', 'pkgs/foo']]), + namespaceInfoByPackageDir: new Map(), + truncated: false, + }; + const parsedFiles = [ + definingFile('pkgs/foo/R/a.R'), + definingFile('x/foo/R/b.R'), + callerFile('scripts/use.R'), + ]; + populateRQualifiedCalls(parsedFiles, { resolutionConfig: cfg }); + + const caller = parsedFiles[2]; + expect(caller.referenceSites).toHaveLength(1); // not dropped as a duplicate definition + expect(resolveRQualifiedFreeCall(callSite, caller)?.filePath).toBe('x/foo/R/b.R'); + }); +}); + +describe('populateRQualifiedCalls: module state', () => { + const config = (): RPackageConfig => ({ + packages: new Map([['foo', 'pkgs/foo']]), + namespaceInfoByPackageDir: new Map(), + truncated: false, + }); + + it.each([ + ['an empty context', {}], + ['a context whose resolutionConfig is undefined', { resolutionConfig: undefined }], + ['a context whose resolutionConfig is null', { resolutionConfig: null }], + ])('runs without a package config (%s)', (_name, ctx) => { + const parsedFiles = [definingFile('pkgs/foo/R/a.R'), callerFile('scripts/use.R')]; + expect(() => populateRQualifiedCalls(parsedFiles, ctx)).not.toThrow(); + // No config, so nothing proves `foo` external: the site stays and nothing is recorded. + expect(parsedFiles[1].referenceSites).toHaveLength(1); + }); + + it('forgets the previous run: a run with no qualified call records no definition count', () => { + const cfg = config(); + populateRQualifiedCalls([definingFile('pkgs/foo/R/a.R'), callerFile('scripts/use.R')], { + resolutionConfig: cfg, + }); + expect(rQualifiedDefinitionCount(cfg, 'foo', 'helper')).toBe(1); + + populateRQualifiedCalls([definingFile('pkgs/foo/R/a.R')], { resolutionConfig: cfg }); + expect(rQualifiedDefinitionCount(cfg, 'foo', 'helper')).toBeUndefined(); + }); + + it('forgets the previous run precise targets: a later run does not resolve its site', () => { + const cfg = config(); + const first = [definingFile('pkgs/foo/R/a.R'), callerFile('scripts/use.R')]; + populateRQualifiedCalls(first, { resolutionConfig: cfg }); + expect(resolveRQualifiedFreeCall(callSite, first[1])?.filePath).toBe('pkgs/foo/R/a.R'); + + const second = [callerFile('scripts/use.R')]; + populateRQualifiedCalls(second, { resolutionConfig: cfg }); + expect(resolveRQualifiedFreeCall(callSite, second[0])).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-qualified-call.test.ts b/gitnexus/test/unit/scope-resolution/r/r-qualified-call.test.ts new file mode 100644 index 000000000..b833a643a --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-qualified-call.test.ts @@ -0,0 +1,535 @@ +/** + * R qualified-call helpers: + * + * - `parseRQualifier(raw)`: the package named by a `@reference.qualified-name` text; + * - `rQualifierLocality(pkg, cfg, filePaths)`: `local` | `external` | `unknown`; + * - `loadRPackageConfig` sets `RPackageConfig.truncated` only when a subtree its depth-3 / + * 200-directory walk skipped may hide an undiscovered package (or a directory of the walk + * is unreadable); skipped subtrees without a package do not truncate; + * - the global-name-fallback veto's qualifier rule (`isRGlobalNameFallbackPlausible`). + */ +import { describe, expect, it, vi } from 'vitest'; +import { cleanupTempDirSync } from '../../../helpers/test-db.js'; +import { fastestWithin, timeWork } from '../../../helpers/timing-attempts.js'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { RPackageConfig } from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + hidesUndiscoveredPackage, + loadRPackageConfig, +} from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + parseRQualifier, + rQualifierLocality, + type RQualifierLocality, +} from '../../../../src/core/ingestion/languages/r/qualified-call.js'; +import { + isRGlobalNameFallbackPlausible, + rRecordQualifiedDefinitionCounts, +} from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; + +type Locality = RQualifierLocality; + +const FIXTURES = path.resolve(__dirname, '..', '..', '..', 'fixtures', 'lang-resolution'); + +function config(packages: Record, truncated?: boolean): RPackageConfig { + const cfg: RPackageConfig = { + packages: new Map(Object.entries(packages)), + namespaceInfoByPackageDir: new Map(), + }; + return truncated === undefined ? cfg : { ...cfg, truncated }; +} + +describe('parseRQualifier', () => { + const cases: readonly (readonly [raw: string, expected: string | undefined])[] = [ + ['pkg::f', 'pkg'], + ['pkg:::f', 'pkg'], + ['pkg ::: f', 'pkg'], + ['pkg :: f', 'pkg'], + ['"pkg"::f', 'pkg'], + ["'pkg'::f", 'pkg'], + ['`pkg`::f', 'pkg'], + ['my.pkg::f', 'my.pkg'], + ['not a qualifier', undefined], + ['::f', undefined], + ]; + + for (const [raw, expected] of cases) { + it(`${JSON.stringify(raw)} -> ${JSON.stringify(expected)}`, () => { + expect(parseRQualifier(raw)).toBe(expected); + }); + } + + it('keeps a package name with inner blanks, and uses the first separator only', () => { + expect(parseRQualifier('"my pkg"::f')).toBe('my pkg'); + expect(parseRQualifier('a::b::c')).toBe('a'); + expect(parseRQualifier(' \t pkg \n::f')).toBe('pkg'); + expect(parseRQualifier('a : :b')).toBeUndefined(); + }); + + it('finishes promptly on a very long run of blanks with no separator', () => { + // The lazy `[\s\S]*?` it used to be took ~0.4 s per 32,000 blanks and quadratically more + // beyond; 20 of them took 7 s. The linear version takes milliseconds. + const raw = '"' + ' '.repeat(32_000) + 'x"'; + const { ms } = fastestWithin(2_000, () => + timeWork(() => () => { + for (let i = 0; i < 20; i++) expect(parseRQualifier(raw)).toBeUndefined(); + }), + ); + expect(ms).toBeLessThan(2_000); + }); + + it('still reads the package after a very long quoted name', () => { + const name = ' '.repeat(16_000) + 'x'.repeat(16_000); + expect(parseRQualifier(`"${name}"::f`)).toBe(name); + }); +}); + +describe('rQualifierLocality', () => { + const paths = (...p: string[]): ReadonlySet => new Set(p); + + const cases: readonly { + readonly title: string; + readonly pkg: string; + readonly cfg: RPackageConfig | null; + readonly files: ReadonlySet; + readonly expected: Locality; + }[] = [ + { + title: 'a package in the config is local', + pkg: 'renamed', + cfg: config({ renamed: 'renamed_impl' }), + files: paths('renamed_impl/R/a.R'), + expected: 'local', + }, + { + title: 'a package in the config is local even when its discovery was truncated', + pkg: 'pkg', + cfg: config({ pkg: 'pkg' }, true), + files: paths('pkg/R/a.R'), + expected: 'local', + }, + { + title: 'a root-level package is local through the config', + pkg: 'rootpkg', + cfg: config({ rootpkg: '' }), + files: paths('R/a.R'), + expected: 'local', + }, + { + title: 'not in the config but a parsed file sits under /R/ (no DESCRIPTION)', + pkg: 'pkgthird', + cfg: config({ pkgmain: 'pkgmain' }), + files: paths('pkgmain/R/a.R', 'pkgthird/R/t.R'), + expected: 'local', + }, + { + title: 'the /R/ segment may be nested below the repo root', + pkg: 'pathpkg', + cfg: config({ caller: 'caller' }, true), + files: paths('caller/R/a.R', 'deep/a/b/c/pathpkg/R/p.R'), + expected: 'local', + }, + { + title: 'a path that merely contains the name is not a package directory', + pkg: 'path', + cfg: config({ caller: 'caller' }), + files: paths('caller/R/a.R', 'deep/pathpkg/R/p.R', 'x/path/scripts/s.R'), + expected: 'external', + }, + { + title: 'not local and discovery completed: certainly external', + pkg: 'dplyr', + cfg: config({ caller: 'caller' }, false), + files: paths('caller/R/a.R'), + expected: 'external', + }, + { + title: 'not local and the truncated flag is absent (complete discovery): external', + pkg: 'dplyr', + cfg: config({ caller: 'caller' }), + files: paths('caller/R/a.R'), + expected: 'external', + }, + { + title: 'not local but discovery was truncated: unknown', + pkg: 'invisiblepkg', + cfg: config({ caller: 'caller' }, true), + files: paths('caller/R/a.R', 'deep/a/b/c/impl/R/x.R'), + expected: 'unknown', + }, + { + title: 'no package config could be built: unknown', + pkg: 'dplyr', + cfg: null, + files: paths('R/a.R'), + expected: 'unknown', + }, + ]; + + for (const c of cases) { + it(c.title, () => { + expect(rQualifierLocality(c.pkg, c.cfg, c.files)).toBe(c.expected); + }); + } +}); + +describe('loadRPackageConfig: discovery completeness on the qualified-call fixtures', () => { + const load2 = (fixture: string) => loadRPackageConfig(path.join(FIXTURES, fixture)); + + // Permanent geometry checks: the fixtures really are shaped as the integration tests assume. + it('finds every package of r-qualified-calls, including a directory named differently', async () => { + const cfg = await load2('r-qualified-calls'); + expect([...(cfg?.packages ?? [])].sort()).toEqual([ + ['altlib', 'altlib'], + ['caller', 'caller'], + ['duplib', 'duplib'], + ['locallib', 'locallib'], + ['provlib', 'provlib'], + ['renamed', 'renamed_impl'], + ]); + }); + + it('does not find the packages nested below depth 3 in r-qualified-calls-truncated', async () => { + const cfg = await load2('r-qualified-calls-truncated'); + expect([...(cfg?.packages ?? [])]).toEqual([['caller', 'caller']]); + }); + + it('truncated is false when the walk completed', async () => { + expect((await load2('r-qualified-calls'))?.truncated).toBe(false); + }); + + it('truncated is true when a skipped directory hides an undiscovered package', async () => { + // deep/a/b/c/impl holds `Package: invisiblepkg`, below the depth limit. + expect((await load2('r-qualified-calls-truncated'))?.truncated).toBe(true); + }); + + it('truncated is false when the skipped directories (tests/testthat/fixtures) hold no package', async () => { + const cfg = await load2('r-qualified-calls-deep-fixtures'); + expect([...(cfg?.packages ?? [])]).toEqual([['deepcaller', 'caller']]); + expect(cfg?.truncated).toBe(false); + }); +}); + +describe('loadRPackageConfig: zero packages found', () => { + async function withTree(build: (root: string) => Promise) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'r-no-description-')); + try { + await build(root); + return await loadRPackageConfig(root); + } finally { + cleanupTempDirSync(root); + } + } + + it('returns a complete empty config when discovery finds no package', async () => { + const cfg = await loadRPackageConfig(path.join(FIXTURES, 'r-no-description')); + expect(cfg).not.toBeNull(); + expect(cfg?.packages.size).toBe(0); + expect(cfg?.truncated).toBe(false); + }); + + it('keeps truncated set when no package is found but a skipped subtree hides one', async () => { + const cfg = await loadRPackageConfig(path.join(FIXTURES, 'r-no-description-truncated')); + expect(cfg?.packages.size).toBe(0); + expect(cfg?.truncated).toBe(true); + }); + + it('returns a complete empty config for an R-only temporary tree without DESCRIPTION', async () => { + const cfg = await withTree(async (root) => { + await fs.mkdir(path.join(root, 'R')); + await fs.writeFile(path.join(root, 'R', 'a.R'), 'f <- function() 1\n'); + }); + expect(cfg?.truncated).toBe(false); + }); + + it('rQualifierLocality: complete empty config proves external; truncated stays unknown', () => { + const empty = config({}, false); + expect(rQualifierLocality('dplyr', empty, new Set(['R/a.R']))).toBe('external'); + expect(rQualifierLocality('dplyr', config({}, true), new Set(['R/a.R']))).toBe('unknown'); + expect(rQualifierLocality('dplyr', null, new Set(['R/a.R']))).toBe('unknown'); + // a /R/ path still makes the package local without a DESCRIPTION + expect(rQualifierLocality('pathpkg', empty, new Set(['pathpkg/R/a.R']))).toBe('local'); + }); +}); + +describe('loadRPackageConfig: packageDirs', () => { + it('lists every directory with a Package: DESCRIPTION, including same-name copies and packages without a NAMESPACE', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'r-package-dirs-')); + try { + const write = async (rel: string, text: string): Promise => { + await fs.mkdir(path.dirname(path.join(root, rel)), { recursive: true }); + await fs.writeFile(path.join(root, rel), text); + }; + await write('DESCRIPTION', 'Package: top\nVersion: 1.0\n'); + await write('NAMESPACE', 'export(f)\n'); + await write('a/dup/DESCRIPTION', 'Package: dup\nVersion: 1.0\n'); // wins the name, no NAMESPACE + await write('b/dup/DESCRIPTION', 'Package: dup\nVersion: 1.0\n'); // same-name copy that loses + await write('c/nodesc/README', 'not a package\n'); + await write('d/nopkg/DESCRIPTION', 'Version: 1.0\n'); // no Package: line + const cfg = await loadRPackageConfig(root); + expect([...(cfg.packageDirs ?? [])].sort()).toEqual(['', 'a/dup', 'b/dup']); + expect([...cfg.packages].sort()).toEqual([ + ['dup', 'a/dup'], + ['top', ''], + ]); + } finally { + cleanupTempDirSync(root); + } + }); +}); + +describe('rQualifierLocality: a /R/ directory is not evidence for a file a discovered package owns', () => { + it('does not make `foo` local from pkgs/foo/R/ when that directory is the package `bar`', () => { + const cfg = config({ bar: 'pkgs/foo' }, false); + expect(rQualifierLocality('foo', cfg, new Set(['pkgs/foo/R/x.R']))).toBe('external'); + expect(rQualifierLocality('bar', cfg, new Set(['pkgs/foo/R/x.R']))).toBe('local'); + }); + + it('applies to a same-name copy of a package that discovery did not register by name', () => { + const cfg: RPackageConfig = { + ...config({ bar: 'a/bar' }, false), + packageDirs: new Set(['a/bar', 'b/foo']), + }; + expect(rQualifierLocality('foo', cfg, new Set(['b/foo/R/x.R']))).toBe('external'); + }); + + it('still treats an undiscovered /R/ directory as local', () => { + const cfg = config({ bar: 'pkgs/bar' }, false); + expect(rQualifierLocality('foo', cfg, new Set(['pkgs/foo/R/x.R', 'pkgs/bar/R/y.R']))).toBe( + 'local', + ); + expect(rQualifierLocality('foo', cfg, new Set(['pkgs\\foo\\R\\x.R']))).toBe('local'); + }); +}); + +describe('loadRPackageConfig: truncation only when a skipped subtree hides an undiscovered package', () => { + // Layout: root/DESCRIPTION (rootpkg). `skipDir` is the first directory below the depth-3 + // limit: root(0) > a(1) > b(2) > c(3) lists d, which is skipped. + const SKIP = ['a', 'b', 'c', 'd']; + + async function withRepo( + build: (root: string) => Promise, + ): Promise>> { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'r-truncated-hidden-')); + try { + await fs.writeFile(path.join(root, 'DESCRIPTION'), 'Package: rootpkg\n'); + await build(root); + return await loadRPackageConfig(root); + } finally { + cleanupTempDirSync(root); + } + } + + const writePkg = async (dir: string, name: string): Promise => { + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'DESCRIPTION'), `Package: ${name}\n`); + }; + + it('a skipped directory with no package inside is not truncated', async () => { + const cfg = await withRepo(async (root) => { + await fs.mkdir(path.join(root, ...SKIP, 'data', 'raw'), { recursive: true }); + await fs.writeFile(path.join(root, ...SKIP, 'data', 'raw', 'x.csv'), 'a,b\n'); + }); + expect(cfg?.truncated).toBe(false); + }); + + it('a skipped subtree with a real package of an undiscovered name is truncated', async () => { + const cfg = await withRepo((root) => writePkg(path.join(root, ...SKIP, 'realpkg'), 'realpkg')); + expect(cfg?.truncated).toBe(true); + }); + + it('a skipped subtree with only a same-name copy of a discovered package is not truncated', async () => { + const cfg = await withRepo((root) => writePkg(path.join(root, ...SKIP, 'copy'), 'rootpkg')); + expect(cfg?.truncated).toBe(false); + }); + + it('a package at depth 5 (hidden below the skipped directory itself) is truncated', async () => { + const cfg = await withRepo((root) => + writePkg(path.join(root, ...SKIP, 'e', 'deeppkg'), 'deeppkg'), + ); + expect(cfg?.truncated).toBe(true); + }); + + it('a DESCRIPTION under node_modules, .git or .Rproj.user of a skipped subtree is ignored', async () => { + const cfg = await withRepo(async (root) => { + await writePkg(path.join(root, ...SKIP, 'node_modules', 'jspkg'), 'jspkg'); + await writePkg(path.join(root, ...SKIP, '.git', 'gitpkg'), 'gitpkg'); + await writePkg(path.join(root, ...SKIP, '.Rproj.user', 'rprojpkg'), 'rprojpkg'); + }); + expect(cfg?.truncated).toBe(false); + }); + + it('a package left in the queue when the 200-directory cap stops the walk is truncated', async () => { + const cfg = await withRepo(async (root) => { + for (let i = 0; i < 210; i++) await fs.mkdir(path.join(root, `filler${String(i)}`)); + // Sorts last, so it is still queued when the cap is hit. + await writePkg(path.join(root, 'zz_hidden'), 'zzhidden'); + }); + expect(cfg?.truncated).toBe(true); + }); + + it('a skipped subtree larger than the 5000-directory budget is truncated', async () => { + const cfg = await withRepo(async (root) => { + const big = path.join(root, ...SKIP); + await fs.mkdir(big, { recursive: true }); + await Promise.all( + Array.from({ length: 5001 }, (_, i) => fs.mkdir(path.join(big, `d${String(i)}`))), + ); + }); + expect(cfg?.truncated).toBe(true); + }, 30000); + + // The read failure is injected rather than provoked with chmod(0o000): permission bits are a + // POSIX concept (Windows ignores them, root bypasses them), whereas the behaviour under test is + // "an I/O error while walking means the package set is incomplete". + function eacces(target: string): NodeJS.ErrnoException { + return Object.assign(new Error(`EACCES: permission denied, '${target}'`), { code: 'EACCES' }); + } + + it('an unreadable skipped directory is truncated', async () => { + const realReaddir = fs.readdir.bind(fs) as (...a: unknown[]) => Promise; + const spy = vi + .spyOn(fs, 'readdir') + .mockImplementation(((dir: unknown, ...rest: unknown[]) => + path.basename(String(dir)) === 'locked' + ? Promise.reject(eacces(String(dir))) + : realReaddir(dir, ...rest)) as unknown as typeof fs.readdir); + try { + const cfg = await withRepo(async (root) => { + await fs.mkdir(path.join(root, ...SKIP, 'locked'), { recursive: true }); + }); + expect(spy.mock.calls.some(([d]) => path.basename(String(d)) === 'locked')).toBe(true); + expect(cfg?.truncated).toBe(true); + } finally { + spy.mockRestore(); + } + }); + + it('an unreadable DESCRIPTION is truncated, so its qualifier stays unknown', async () => { + // Manifests are read through one opened handle, so the open is where a read can fail. + const realOpen = fs.open.bind(fs) as (...a: unknown[]) => Promise; + const spy = vi + .spyOn(fs, 'open') + .mockImplementation(((file: unknown, ...rest: unknown[]) => + path.basename(String(file)) === 'DESCRIPTION' && + path.basename(path.dirname(String(file))) === 'sub' + ? Promise.reject(eacces(String(file))) + : realOpen(file, ...rest)) as unknown as typeof fs.open); + try { + const cfg = await withRepo(async (root) => { + await writePkg(path.join(root, 'sub'), 'hiddenpkg'); + }); + expect(spy.mock.calls.some(([f]) => path.basename(String(f)) === 'DESCRIPTION')).toBe(true); + expect(cfg?.packages.has('hiddenpkg')).toBe(false); + expect(cfg?.truncated).toBe(true); + expect(rQualifierLocality('hiddenpkg', cfg as RPackageConfig, new Set())).toBe('unknown'); + } finally { + spy.mockRestore(); + } + }); +}); + +describe('hidesUndiscoveredPackage', () => { + it('shares one budget across all roots (exceeded on the second root)', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'r-hidden-budget-')); + try { + const a = path.join(root, 'a'); + const b = path.join(root, 'b'); + for (const d of [a, b]) { + await fs.mkdir(path.join(d, 'x'), { recursive: true }); + } + // Each root costs 2 directories (itself + x): 4 in total. + expect(await hidesUndiscoveredPackage([a, b], new Set(), 4)).toBe(false); + expect(await hidesUndiscoveredPackage([a, b], new Set(), 3)).toBe(true); + } finally { + cleanupTempDirSync(root); + } + }); + + it('a nonexistent root cannot be read: true', async () => { + expect( + await hidesUndiscoveredPackage([path.join(os.tmpdir(), 'r-no-such-dir-x9')], new Set()), + ).toBe(true); + }); + + it('no roots: false', async () => { + expect(await hidesUndiscoveredPackage([], new Set())).toBe(false); + }); +}); + +describe('loadRPackageConfig: directory cap', () => { + it('truncated is false when the directories left queued by the 200-directory cap hold no package', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'r-cap-empty-')); + try { + await fs.writeFile(path.join(root, 'DESCRIPTION'), 'Package: rootpkg\n'); + for (let i = 0; i < 210; i++) await fs.mkdir(path.join(root, `filler${String(i)}`)); + const cfg = await loadRPackageConfig(root); + expect([...(cfg?.packages ?? [])]).toEqual([['rootpkg', '']]); + expect(cfg?.truncated).toBe(false); + } finally { + cleanupTempDirSync(root); + } + }); +}); + +describe('isRGlobalNameFallbackPlausible: qualifier rule', () => { + const cfg: RPackageConfig = { + packages: new Map([ + ['caller', 'caller'], + ['named', 'named'], + ['other', 'other'], + ]), + namespaceInfoByPackageDir: new Map([ + [ + 'caller', + { + hasNamespaceFile: true, + namedExports: new Set(), + exportPatterns: [], + importFrom: [{ pkg: 'ext', name: 'mutate' }], + }, + ], + ]), + }; + const verdict = (qualified: string | undefined, candidate: string, config = cfg): boolean => + isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'caller/R/a.R' }, + candidate: { filePath: candidate }, + resolutionConfig: config, + site: { name: 'mutate', rawQualifiedName: qualified }, + }); + + it('allows a candidate inside the named local package (whatever importFrom says)', () => { + expect(verdict('named::mutate', 'named/R/m.R')).toBe(true); + }); + + it('vetoes a candidate elsewhere when the named package defines the name', () => { + rRecordQualifiedDefinitionCounts(cfg, new Map([['named', new Map([['mutate', 1]])]])); + expect(verdict('named::mutate', 'other/R/o.R')).toBe(false); + }); + + it('falls through to the NAMESPACE rules when the named package defines none (re-export)', () => { + rRecordQualifiedDefinitionCounts(cfg, new Map([['named', new Map()]])); + // importFrom(ext, mutate) + a candidate in a third package: today's importFrom veto. + expect(verdict('named::mutate', 'other/R/o.R')).toBe(false); + const noImportFrom: RPackageConfig = { ...cfg, namespaceInfoByPackageDir: new Map() }; + rRecordQualifiedDefinitionCounts(noImportFrom, new Map([['named', new Map()]])); + expect(verdict('named::mutate', 'other/R/o.R', noImportFrom)).toBe(true); + }); + + it('falls through when no counts were recorded for the config', () => { + const fresh: RPackageConfig = { ...cfg }; + expect(verdict('named::mutate', 'other/R/o.R', fresh)).toBe(false); // importFrom rule, as before + }); + + it('ignores a qualifier naming a package outside the config', () => { + expect(verdict('elsewhere::mutate', 'other/R/o.R')).toBe(false); // importFrom rule, as before + }); + + it('a site without a qualifier is judged by the NAMESPACE rules only', () => { + expect(verdict(undefined, 'other/R/o.R')).toBe(false); + expect(verdict(undefined, 'caller/R/c.R')).toBe(true); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-qualified-visibility.test.ts b/gitnexus/test/unit/scope-resolution/r/r-qualified-visibility.test.ts new file mode 100644 index 000000000..aa2887c11 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-qualified-visibility.test.ts @@ -0,0 +1,392 @@ +/** + * Export visibility (`public` / `private` / `unknown`) of R package definitions and the rules built + * on it: `populateRQualifiedCalls` (no qualified site survives without a precise target), + * the `library()` expansion (strict), the unique-name guess veto (proven private only), the + * package-tree predicate and the re-export reader. The end-to-end cells are in + * `integration/resolvers/r-qualified-exports.test.ts`. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { + compileRExportPatternDetailed, + type RExportMatcher, +} from '../../../../src/core/ingestion/languages/r/export-pattern.js'; +import { SharedWorkBudget } from '../../../../src/core/ingestion/languages/r/linear-regex.js'; +import { + isRGlobalNameFallbackPlausible, + parseRQualifierOperator, + rEnclosingPackageDir, + rReexportOrigin, + rSymbolVisibility, +} from '../../../../src/core/ingestion/languages/r/namespace-imports.js'; +import { + rBuildSymbolVisibilityMemo, + recordRPackageConfig, + type RNamespaceInfo, + type RPackageConfig, +} from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + populateRQualifiedCalls, + resolveRQualifiedFreeCall, +} from '../../../../src/core/ingestion/languages/r/qualified-call.js'; +import { expandRWildcardNames } from '../../../../src/core/ingestion/languages/r/scope-resolver.js'; + +function parse(src: string, filePath: string): ParsedFile { + const parsed = extractParsedFile(rProvider, src, filePath); + if (parsed === undefined) throw new Error(`no ParsedFile for ${filePath}`); + return parsed; +} + +function ns(spec: Partial): RNamespaceInfo { + return { + hasNamespaceFile: true, + namedExports: new Set(), + exportPatterns: [], + importFrom: [], + ...spec, + }; +} + +function config( + packages: Record, + namespaces: Record>, + extra: Partial = {}, +): RPackageConfig { + return { + packages: new Map(Object.entries(packages)), + namespaceInfoByPackageDir: new Map( + Object.entries(namespaces).map(([dir, spec]) => [dir, ns(spec)]), + ), + truncated: false, + ...extra, + }; +} + +const qualifiedSites = (parsed: ParsedFile) => + parsed.referenceSites.filter((s) => s.kind === 'call' && s.rawQualifiedName !== undefined); + +describe('rSymbolVisibility', () => { + const files = [ + parse( + 'shown <- function() 1\nhidden <- function() 2\nnested <- function() { inner <- function() 3 }', + 'p/R/a.R', + ), + parse('open <- function() 1', 'q/R/a.R'), + parse('u1 <- function() 1', 'u/R/a.R'), + ]; + const cfg = config( + { p: 'p', q: 'q', u: 'u' }, + { p: { namedExports: new Set(['shown']) }, u: { exportsUnreadable: true } }, + ); + rBuildSymbolVisibilityMemo(files, cfg); + + it('is public for an exported name, private for a readable complete NAMESPACE that omits it', () => { + expect(rSymbolVisibility(cfg, 'p/R/a.R', 'shown')).toBe('public'); + expect(rSymbolVisibility(cfg, 'p/R/a.R', 'hidden')).toBe('private'); + }); + + it('is public for a package without a NAMESPACE file', () => { + expect(rSymbolVisibility(cfg, 'q/R/a.R', 'open')).toBe('public'); + }); + + it('is unknown when the NAMESPACE is unreadable, and for a name the memo does not hold', () => { + expect(rSymbolVisibility(cfg, 'u/R/a.R', 'u1')).toBe('unknown'); + expect(rSymbolVisibility(cfg, 'p/R/a.R', 'inner')).toBe('unknown'); + }); + + it('judges only definitions under a package R/ directory', () => { + expect(rSymbolVisibility(cfg, 'p/tests/x.R', 'hidden')).toBe('public'); + expect(rSymbolVisibility(cfg, 'scripts/x.R', 'hidden')).toBe('public'); + expect(rSymbolVisibility(null, 'p/R/a.R', 'hidden')).toBe('public'); + }); + + it('is unknown for every name of a NAMESPACE package when no memo was built for the config', () => { + expect(rSymbolVisibility(config({ p: 'p' }, { p: {} }), 'p/R/a.R', 'hidden')).toBe('unknown'); + }); +}); + +describe('rSymbolVisibility under a dropped or cut-short exportPattern()', () => { + it('is unknown (not private) for a name no pattern matched when a pattern was dropped', () => { + const files = [parse('abc <- function() 1', 'p/R/a.R')]; + const cfg = config( + { p: 'p' }, + { p: { droppedExportPatterns: [{ pattern: '\\ { + const names = ['pub_a', 'pub_b', 'pub_c', 'pub_d', 'priv_a', 'priv_b', 'priv_c', 'priv_d']; + const defFiles = (): ParsedFile[] => + names.map((n, i) => parse(`${n} <- function() ${i}`, `p/R/${n}.R`)); + const callerSource = [...names, 'missing_reexport'] + .map((n) => `c_${n} <- function() p::${n}()`) + .join('\n'); + + /** A fresh config whose package budget lasts for only a few matcher steps. */ + const freshConfig = (): RPackageConfig => { + const budget = new SharedWorkBudget(60, 1_000_000); + const compiled = compileRExportPatternDetailed('^pub', budget); + if ('reason' in compiled || compiled.matcher === null) throw new Error('pattern not compiled'); + return config( + { p: 'p' }, + { p: { exportPatterns: [compiled.matcher as RExportMatcher], exportPatternBudget: budget } }, + ); + }; + + const outcome = (order: readonly number[], reverseSites: boolean) => { + const cfg = freshConfig(); + const defs = defFiles(); + let caller = parse(callerSource, 'scripts/use.R'); + if (reverseSites) { + caller = { ...caller, referenceSites: [...caller.referenceSites].reverse() }; + } + const files = [...order.map((i) => defs[i]), caller]; + populateRQualifiedCalls(files, { resolutionConfig: cfg }); + const after = files[files.length - 1]; + return { + visibility: names.map((n) => rSymbolVisibility(cfg, `p/R/${n}.R`, n)), + bound: qualifiedSites(after) + .map((s) => s.name) + .sort(), + }; + }; + + it('gives the same verdicts for permuted and reversed files and sites, across runs', () => { + const base = outcome([0, 1, 2, 3, 4, 5, 6, 7], false); + // Guard: the budget really ran out (some name is `unknown`), so order could have mattered. + expect(base.visibility).toContain('unknown'); + expect(base.visibility).toContain('public'); + for (const order of [ + [7, 6, 5, 4, 3, 2, 1, 0], + [3, 1, 7, 0, 5, 2, 6, 4], + [4, 5, 6, 7, 0, 1, 2, 3], + ]) { + for (const reverse of [false, true]) { + expect(outcome(order, reverse)).toEqual(base); + } + } + }); +}); + +describe('populateRQualifiedCalls: no qualified site survives without a precise target', () => { + const cfgFor = (extra: Partial = {}): RPackageConfig => + config( + { exp: 'exp', oth: 'oth', dup: 'dup' }, + { + exp: { + namedExports: new Set(['shown', 'reexp']), + importFrom: [{ pkg: 'oth', name: 'reexp' }], + }, + oth: { namedExports: new Set(['reexp']) }, + }, + extra, + ); + const sources: Record = { + 'exp/R/a.R': 'shown <- function() 1\nhidden <- function() 2', + 'oth/R/o.R': 'reexp <- function() 3', + 'dup/R/d1.R': 'twice <- function() 1', + 'dup/R/d2.R': 'twice <- function() 2', + }; + const caller = [ + 'ghost <- function() 0', + 'k1 <- function() exp::shown()', + 'k2 <- function() exp::hidden()', + 'k3 <- function() exp:::hidden()', + 'k4 <- function() exp::ghost()', + 'k5 <- function() exp::reexp()', + 'k6 <- function() dup::twice()', + 'k7 <- function() ext::anything()', + 'k8 <- function() exp:::reexp()', + ].join('\n'); + + const run = (cfg: RPackageConfig | null) => { + const files = [ + ...Object.entries(sources).map(([p, s]) => parse(s, p)), + parse(caller, 'scripts/use.R'), + ]; + populateRQualifiedCalls(files, cfg === null ? {} : { resolutionConfig: cfg }); + return files[files.length - 1]; + }; + + it('keeps exactly the sites that have a precise target', () => { + const after = run(cfgFor()); + const kept = qualifiedSites(after); + for (const site of kept) + expect(resolveRQualifiedFreeCall(site, after), site.name).toBeDefined(); + // shown (public), hidden via ::: , reexp via exp (re-export, one origin definition) + expect(kept.map((s) => `${s.rawQualifiedName}`).sort()).toEqual([ + 'exp:::hidden', + 'exp::reexp', + 'exp::shown', + ]); + const reexport = kept.find((s) => s.name === 'reexp'); + expect(reexport).toBeDefined(); + expect(resolveRQualifiedFreeCall(reexport as object, after)?.filePath).toBe('oth/R/o.R'); + }); + + it('drops every qualified site when locality cannot be decided (no config, truncated discovery)', () => { + for (const cfg of [null, cfgFor({ truncated: true })]) { + const after = run(cfg); + for (const site of qualifiedSites(after)) { + expect(resolveRQualifiedFreeCall(site, after), site.name).toBeDefined(); + } + } + // truncated: `ext` is unknown, `exp`/`dup` are still local through the config. + expect(qualifiedSites(run(cfgFor({ truncated: true }))).map((s) => s.name)).not.toContain( + 'anything', + ); + }); + + it('a caller inside the package tree reaches a private name through ::, one outside does not', () => { + const cfg = cfgFor(); + const files = [ + ...Object.entries(sources).map(([p, s]) => parse(s, p)), + parse('inside <- function() exp::hidden()', 'exp/tests/testthat/test-a.R'), + parse('outside <- function() exp::hidden()', 'scripts/use.R'), + ]; + populateRQualifiedCalls(files, { resolutionConfig: cfg }); + expect(qualifiedSites(files[files.length - 2])).toHaveLength(1); + expect(qualifiedSites(files[files.length - 1])).toHaveLength(0); + }); + + it('does not resolve a re-export whose origin is an external package', () => { + const cfg = cfgFor(); + cfg.namespaceInfoByPackageDir.set( + 'exp', + ns({ namedExports: new Set(['reexp']), importFrom: [{ pkg: 'ext', name: 'reexp' }] }), + ); + const files = [ + ...Object.entries(sources).map(([p, s]) => parse(s, p)), + parse('k <- function() exp::reexp()', 'scripts/use.R'), + ]; + populateRQualifiedCalls(files, { resolutionConfig: cfg }); + expect(qualifiedSites(files[files.length - 1])).toHaveLength(0); + }); +}); + +describe('isRGlobalNameFallbackPlausible: proven-private veto', () => { + const files = [ + parse('shown <- function() 1\nhidden <- function() 2', 'exp/R/a.R'), + parse('u1 <- function() 1', 'unr/R/a.R'), + ]; + const cfg = config( + { exp: 'exp', unr: 'unr' }, + { exp: { namedExports: new Set(['shown']) }, unr: { exportsUnreadable: true } }, + ); + rBuildSymbolVisibilityMemo(files, cfg); + const verdict = (name: string, candidateFile: string, callerFile: string): boolean => + isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: callerFile }, + candidate: { filePath: candidateFile, type: 'Function' }, + resolutionConfig: cfg, + site: { name }, + }); + + it('vetoes a private candidate for a caller outside the package tree', () => { + expect(verdict('hidden', 'exp/R/a.R', 'scripts/s.R')).toBe(false); + expect(verdict('hidden', 'exp/R/a.R', 'other/R/b.R')).toBe(false); + }); + + it('allows a private candidate for callers inside the tree, including tests/ and inst/', () => { + expect(verdict('hidden', 'exp/R/a.R', 'exp/R/b.R')).toBe(true); + expect(verdict('hidden', 'exp/R/a.R', 'exp/tests/testthat/test-a.R')).toBe(true); + expect(verdict('hidden', 'exp/R/a.R', 'exp/inst/x.R')).toBe(true); + }); + + it('allows an exported name and an unknown-visibility name (a guess is vetoed only when impossible)', () => { + expect(verdict('shown', 'exp/R/a.R', 'scripts/s.R')).toBe(true); + expect(verdict('u1', 'unr/R/a.R', 'scripts/s.R')).toBe(true); + }); + + it('does not judge a candidate that is not a function or class', () => { + expect( + isRGlobalNameFallbackPlausible({ + callerParsed: { filePath: 'scripts/s.R' }, + candidate: { filePath: 'exp/R/a.R', type: 'Method' }, + resolutionConfig: cfg, + site: { name: 'hidden' }, + }), + ).toBe(true); + }); +}); + +describe('expandRWildcardNames: library() brings in exports only', () => { + const target = parse('shown <- function() 1\nhidden <- function() 2', 'exp/R/a.R'); + const open = parse('a <- function() 1\nb <- function() 2', 'nons/R/a.R'); + const cfg = config({ exp: 'exp', nons: 'nons' }, { exp: { namedExports: new Set(['shown']) } }); + rBuildSymbolVisibilityMemo([target, open], cfg); + recordRPackageConfig([target, open], cfg); + + it('keeps exported names, drops the rest', () => { + expect([...expandRWildcardNames(target.moduleScope, [target])]).toEqual(['shown']); + }); + + it('keeps every name of a package without a NAMESPACE', () => { + expect([...expandRWildcardNames(open.moduleScope, [open])].sort()).toEqual(['a', 'b']); + }); + + it('filters nothing when no config was recorded for the file (hand-built input)', () => { + const bare = parse('shown <- function() 1\nhidden <- function() 2', 'exp/R/a.R'); + expect([...expandRWildcardNames(bare.moduleScope, [bare])].sort()).toEqual(['hidden', 'shown']); + }); +}); + +describe('rEnclosingPackageDir', () => { + it('is the longest package directory that is a path prefix, so tests/ and inst/ are inside', () => { + const cfg = config({ outer: 'outer', inner: 'outer/inner' }, { outer: {}, 'outer/inner': {} }); + expect(rEnclosingPackageDir('outer/R/a.R', cfg)).toBe('outer'); + expect(rEnclosingPackageDir('outer/tests/testthat/t.R', cfg)).toBe('outer'); + expect(rEnclosingPackageDir('outer/inst/x.R', cfg)).toBe('outer'); + expect(rEnclosingPackageDir('outer/inner/R/a.R', cfg)).toBe('outer/inner'); + expect(rEnclosingPackageDir('outer/inner/tests/t.R', cfg)).toBe('outer/inner'); + expect(rEnclosingPackageDir('outerx/R/a.R', cfg)).toBeUndefined(); + expect(rEnclosingPackageDir('scripts/s.R', cfg)).toBeUndefined(); + }); + + it('claims every file of a root-level package that no nested package owns', () => { + const cfg = config({ root: '', sub: 'sub' }, { '': {} }); + expect(rEnclosingPackageDir('scripts/s.R', cfg)).toBe(''); + expect(rEnclosingPackageDir('R/a.R', cfg)).toBe(''); + expect(rEnclosingPackageDir('sub/R/a.R', cfg)).toBe('sub'); + }); + + it('is undefined without a config', () => { + expect(rEnclosingPackageDir('R/a.R', null)).toBeUndefined(); + }); +}); + +describe('qualifier operator and re-export origin', () => { + it.each([ + ['pkg::f', '::'], + ['pkg:::f', ':::'], + ['pkg ::: f', ':::'], + ['"pkg"::f', '::'], + ['f', undefined], + ])('reads the operator of %s', (raw, op) => { + expect(parseRQualifierOperator(raw)).toBe(op); + }); + + it('takes the last importFrom entry for the name, and ignores a self-import', () => { + const cfg = config( + { a: 'a' }, + { + a: { + importFrom: [ + { pkg: 'x', name: 'f' }, + { pkg: 'y', name: 'f' }, + { pkg: 'a', name: 'g' }, + ], + }, + }, + ); + expect(rReexportOrigin(cfg, 'a', 'a', 'f')).toBe('y'); + expect(rReexportOrigin(cfg, 'a', 'a', 'g')).toBeUndefined(); + expect(rReexportOrigin(cfg, 'a', 'a', 'h')).toBeUndefined(); + expect(rReexportOrigin(cfg, 'nodir', 'a', 'f')).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-qualifier-locality-index.test.ts b/gitnexus/test/unit/scope-resolution/r/r-qualifier-locality-index.test.ts new file mode 100644 index 000000000..423990826 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-qualifier-locality-index.test.ts @@ -0,0 +1,86 @@ +/** + * `rQualifierLocality` is asked about every distinct `pkg::` qualifier of a run. Answering each + * from a scan of every file path is qualifiers x files (4,000 x 5,000 took 0.84 s); an index of the + * `/R/` directory names, built once, makes it linear. The index must give the same answers. + */ +import { describe, expect, it } from 'vitest'; +import type { RPackageConfig } from '../../../../src/core/ingestion/languages/r/package-config.js'; +import { + rQualifierLocality, + rUnownedPackageDirNames, +} from '../../../../src/core/ingestion/languages/r/qualified-call.js'; + +const config = (packages: Record, truncated = false): RPackageConfig => ({ + packages: new Map(Object.entries(packages)), + namespaceInfoByPackageDir: new Map(), + truncated, +}); + +/** A set of paths that counts every way of walking all of them. */ +class CountingSet extends Set { + walks = 0; + override [Symbol.iterator](): SetIterator { + this.walks++; + return super[Symbol.iterator](); + } + override values(): SetIterator { + this.walks++; + return super.values(); + } +} + +describe('rUnownedPackageDirNames', () => { + const cfg = config({ owner: 'pkgs/owner' }); + + it('lists the directories that precede R/ in a path no discovered package owns', () => { + expect( + [ + ...rUnownedPackageDirNames( + ['foo/R/a.R', 'deep/x/bar/R/b.R', 'scripts/a.R', 'R/top.R', 'baz\\R\\c.R', 'foo/R'], + cfg, + ), + ].sort(), + ).toEqual(['bar', 'baz', 'foo']); + }); + + it('leaves out the directory of a file a discovered package owns', () => { + expect([...rUnownedPackageDirNames(['pkgs/owner/R/a.R', 'other/R/b.R'], cfg)]).toEqual([ + 'other', + ]); + }); +}); + +describe('rQualifierLocality with the index', () => { + const cfg = config({ known: 'pkgs/known', owner: 'pkgs/owner' }); + const paths = new Set([ + 'pkgs/known/R/a.R', + 'pkgs/owner/R/b.R', + 'extra/R/c.R', + 'deep/x/nested/R/d.R', + 'scripts/e.R', + ]); + + it.each(['known', 'owner', 'extra', 'nested', 'x', 'scripts', 'R', 'missing', 'pkgs', 'deep'])( + 'answers %s as the scan does', + (pkg) => { + for (const c of [cfg, config({}, true), null]) { + const index = rUnownedPackageDirNames(paths, c); + expect(rQualifierLocality(pkg, c, paths, index)).toBe(rQualifierLocality(pkg, c, paths)); + } + }, + ); + + it('never walks the file paths once the index is given', () => { + const counted = new CountingSet(paths); + const index = rUnownedPackageDirNames(paths, cfg); + for (let i = 0; i < 1_000; i++) rQualifierLocality(`pkg${String(i)}`, cfg, counted, index); + expect(rQualifierLocality('extra', cfg, counted, index)).toBe('local'); + expect(counted.walks).toBe(0); + }); + + it('walks them for every qualifier without it (the cost the index removes)', () => { + const counted = new CountingSet(paths); + for (let i = 0; i < 10; i++) rQualifierLocality(`pkg${String(i)}`, cfg, counted); + expect(counted.walks).toBe(10); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-r6-private-members.test.ts b/gitnexus/test/unit/scope-resolution/r/r-r6-private-members.test.ts new file mode 100644 index 000000000..4270e4f1a --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-r6-private-members.test.ts @@ -0,0 +1,81 @@ +/** + * `self$m()` reaches only the public members of an R6 class, so the capture pass records where + * the private members are. The definitions carry no section, so the record is keyed by the + * position their definition id encodes, and it crosses the worker boundary (and the parse cache) + * as the file's capture side channel. This pins the keys against the real definition ids and the + * round trip through `collectCaptureSideChannel` / `applyCaptureSideChannel`. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { rScopeResolver } from '../../../../src/core/ingestion/languages/r/scope-resolver.js'; +import { definitionIdPosition } from '../../../../src/core/ingestion/scope-resolution/utils/definition-id.js'; + +const SOURCE = [ + 'Foo <- R6::R6Class("Foo",', + ' public = list(', + ' pub = function() 1', + ' ),', + ' private = list(', + ' hidden = function() 2,', + ' count = 0', + ' ),', + ' active = list(', + ' size = function(value) 3', + ' )', + ')', + 'free <- function() 4', + '', +].join('\n'); + +const collect = (filePath: string) => rProvider.collectCaptureSideChannel?.(filePath); + +function parse(source: string, filePath: string): ParsedFile { + const file = extractParsedFile(rProvider, source, filePath); + if (file === undefined) throw new Error('no ParsedFile'); + return file; +} + +describe('R6 private members: the capture side channel', () => { + it('records exactly the private methods, at the position their definition id encodes', () => { + const file = parse(SOURCE, 'R/privacy.R'); + const channel = collect('R/privacy.R') as { kind: string; r6PrivateMembers: string[] }; + expect(channel.kind).toBe('r'); + const keyOf = (name: string): string => { + const def = file.localDefs.find((d) => d.nodeId.endsWith(`:Method:${name}`)); + const position = definitionIdPosition(def?.nodeId, 'R/privacy.R'); + if (position === undefined) throw new Error(`no Method ${name}`); + return `${position.line}:${position.column}`; + }; + expect(channel.r6PrivateMembers).toEqual([keyOf('hidden')]); + expect(channel.r6PrivateMembers).not.toContain(keyOf('pub')); + expect(channel.r6PrivateMembers).not.toContain(keyOf('size')); + }); + + it('is plain data that survives a structured clone', () => { + parse(SOURCE, 'R/privacy.R'); + const channel = collect('R/privacy.R'); + expect(structuredClone(channel)).toEqual(channel); + }); + + it('records nothing for a file without private members, and a re-parse replaces the record', () => { + parse(SOURCE, 'R/reparsed.R'); + expect(collect('R/reparsed.R')).toBeDefined(); + parse('free <- function() 4\n', 'R/reparsed.R'); + expect(collect('R/reparsed.R')).toBeUndefined(); + }); + + it('a reused ParsedFile restores its record, and one without a channel clears a stale one', () => { + const file = parse(SOURCE, 'R/reused.R'); + const channel = collect('R/reused.R'); + parse('free <- function() 4\n', 'R/reused.R'); + expect(collect('R/reused.R')).toBeUndefined(); + + rScopeResolver.applyCaptureSideChannel?.({ ...file, captureSideChannel: channel }); + expect(collect('R/reused.R')).toEqual(channel); + + rScopeResolver.applyCaptureSideChannel?.({ ...file, captureSideChannel: undefined }); + expect(collect('R/reused.R')).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-receiver-chain.test.ts b/gitnexus/test/unit/scope-resolution/r/r-receiver-chain.test.ts new file mode 100644 index 000000000..943327d1c --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-receiver-chain.test.ts @@ -0,0 +1,137 @@ +/** + * R chained receivers (`a$b$c()`). + * + * `emitRScopeCaptures` mints the chain R-locally, with the shared codec's `truncated` marker: + * R never types through a chain (R6 fields and results have no declared type), so the marker's + * only job is to make the shared fold refuse the chain and leave the site unresolved, with a + * `receiver-unresolved` outcome. The shared comment on the fold calls the marker defence in + * depth and says the producer never mints one; the last block pins that reliance, so an + * upstream change that folds or refuses a truncated chain fails an R test. + * + * A chain with no variable base (`get_user()$save()`, `pkg::Cls$new()$m()`, `x[[1]]$f$m()`) + * mints no site at all. + */ +import { describe, expect, it } from 'vitest'; +import type { CaptureMatch } from 'gitnexus-shared'; +import { emitRScopeCaptures } from '../../../../src/core/ingestion/languages/r/captures.js'; +import { + decodeReceiverChain, + encodeReceiverChain, +} from '../../../../src/core/ingestion/utils/receiver-chain-codec.js'; +import { foldReceiverChain } from '../../../../src/core/ingestion/scope-resolution/passes/compound-receiver.js'; + +/** The member-call matches of `source`, as `{receiver, name, chain}`. */ +const memberCalls = (source: string) => + emitRScopeCaptures(source, 'a.R') + .filter((m: CaptureMatch) => m['@reference.call.member'] !== undefined) + .map((m) => ({ + receiver: m['@reference.receiver']?.text, + name: m['@reference.name']?.text, + chain: decodeReceiverChain(m['@reference.receiver-chain']?.text), + })); + +describe('a chain with a variable base is minted truncated', () => { + it.each([ + ['a$b$c()', 'a$b', 'c', 'a', [{ kind: 'field', name: 'b' }]], + ['Cls$new()$m()', 'Cls$new()', 'm', 'Cls', [{ kind: 'call', name: 'new' }]], + [ + 'a$b$c$d$m()', + 'a$b$c$d', + 'm', + 'a', + [ + { kind: 'field', name: 'b' }, + { kind: 'field', name: 'c' }, + { kind: 'field', name: 'd' }, + ], + ], + [ + 'a$b()$c$m()', + 'a$b()$c', + 'm', + 'a', + [ + { kind: 'call', name: 'b' }, + { kind: 'field', name: 'c' }, + ], + ], + ['self$f$m()', 'self$f', 'm', 'self', [{ kind: 'field', name: 'f' }]], + ['a@b$m()', 'a@b', 'm', 'a', [{ kind: 'field', name: 'b' }]], + ])('%s', (text, receiver, name, base, steps) => { + // The last member call is the outer one (`Cls$new()$m()` also holds the inner `Cls$new()`). + const outer = memberCalls(`f <- function(a) {\n ${text}\n}\n`).filter((c) => c.name === name); + expect(outer).toHaveLength(1); + expect(outer[0].receiver).toBe(receiver); + expect(outer[0].chain).toEqual({ baseReceiverName: base, steps, truncated: true }); + }); + + it('a plain receiver keeps its single site and gets no chain', () => { + const calls = memberCalls('f <- function(u) u$save()\n'); + expect(calls).toEqual([{ receiver: 'u', name: 'save', chain: undefined }]); + }); + + it('one member-call site per call: the inner field step is not a call of its own', () => { + expect(memberCalls('f <- function(a) a$b$c()\n').map((c) => c.name)).toEqual(['c']); + }); + + it('an inner call step keeps its own plain site (a$b() receiver a, no chain)', () => { + const calls = memberCalls('f <- function(a) a$b()$m()\n'); + expect(calls.map((c) => [c.name, c.receiver, c.chain === undefined])).toEqual([ + ['b', 'a', true], + ['m', 'a$b()', false], + ]); + }); +}); + +describe('a chain with no variable base mints no site', () => { + it.each([ + 'get_user()$save()', + 'pkg::Cls$new()$m()', + 'pkg:::Cls$new()$m()', + 'x[[1]]$f$m()', + 'x[1]$f$m()', + '(x)$f$m()', + 'g(x)$f$m()', + 'g(x)$m()', + 'a$b[[1]]$m()', + 'a$"b"$m()', + ])('%s', (text) => { + expect(memberCalls(`f <- function(x) {\n ${text}\n}\n`)).toEqual([]); + }); + + it('a chain longer than the codec accepts mints no site', () => { + const long = Array.from({ length: 40 }, (_, i) => `s${i}`).join('$'); + expect(memberCalls(`f <- function(a) a$${long}$m()\n`)).toEqual([]); + }); + + it('an unencodable name mints no site', () => { + expect(memberCalls('f <- function(a) a$`b c`$m()\n')).toEqual([]); + }); +}); + +describe('the shared fold refuses a truncated chain (the reliance behind the marker)', () => { + const wire = (truncated: boolean) => { + const encoded = encodeReceiverChain( + 'a', + [{ kind: 'field', name: 'b' }], + truncated ? { truncated } : undefined, + ); + const decoded = decodeReceiverChain(encoded); + if (decoded === undefined) throw new Error(`chain did not round-trip: ${String(encoded)}`); + return decoded; + }; + + it('decodes the marker', () => { + expect(wire(true).truncated).toBe(true); + expect(wire(false).truncated).toBe(false); + }); + + it('foldReceiverChain returns undefined for a truncated chain before reading any index', () => { + // A complete chain reaches into the indexes and throws on these empty stand-ins; the + // truncated one is refused first. If upstream stops refusing a truncated chain, this fails. + const scopes = {} as never; + const index = {} as never; + expect(foldReceiverChain(wire(true), 'scope' as never, scopes, index)).toBeUndefined(); + expect(() => foldReceiverChain(wire(false), 'scope' as never, scopes, index)).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-replacement-scaling.test.ts b/gitnexus/test/unit/scope-resolution/r/r-replacement-scaling.test.ts new file mode 100644 index 000000000..b9ff800f4 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-replacement-scaling.test.ts @@ -0,0 +1,264 @@ +/** + * R replacement-call detection scales linearly with nesting depth and argument count. + * + * `emitRScopeCaptures` asks, for every free call, whether it lies in the target chain of an + * assignment (`foo(x) <- v` runs `` `foo<-` ``). Asking that per call by walking up the tree + * costs O(depth) per call (and tree-sitter's `.parent` re-descends from the root, so more in + * real time), and finding the first argument by materialising the whole argument list costs + * O(args) per call: a deep `nn(nn(...))` or a wide `c(f(1), ..., f(n))` then stalls the parse + * worker, the file is dropped, and a large enough one aborts the whole analyse. + * + * The work is counted, not timed: every node read made through a counting wrapper is one + * visit, and `.parent` is served from a precomputed map at one visit, so the count is the + * same on every machine and gate load cannot flip it. Doubling the input may at most + * (about) double the visits of the path `emitRScopeCaptures` uses. A control shows the + * per-call oracle is super-linear on the same inputs, so the inputs do exercise the problem. + * `rReplacementCallIds` must give exactly the oracle's answer on every R fixture. + */ +import { describe, expect, it } from 'vitest'; +import type { CaptureMatch } from 'gitnexus-shared'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { SyntaxNode } from '../../../../src/core/ingestion/utils/ast-helpers.js'; +import { getRParser, getRScopeQuery } from '../../../../src/core/ingestion/languages/r/query.js'; +import { + isRReplacementCall, + rReplacementCallIds, + rReplacementCallTest, +} from '../../../../src/core/ingestion/languages/r/assignment.js'; +import { emitRScopeCaptures } from '../../../../src/core/ingestion/languages/r/captures.js'; + +const FIXTURES = fileURLToPath(new URL('../../../fixtures', import.meta.url)); + +const parse = (source: string) => + getRParser().parse(source, undefined, { bufferSize: 4 * source.length + 65536 }); + +/** Every node of the tree, top-down, without recursion. */ +function allNodes(root: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + const stack = [root]; + for (let node = stack.pop(); node !== undefined; node = stack.pop()) { + out.push(node); + for (const child of node.namedChildren) stack.push(child); + } + return out; +} + +/** + * A view of the tree that counts node reads: every property read on a wrapped node is one + * visit, and so is every node an array result hands back. `.parent` is answered from a map + * built once, at one visit, so the count does not depend on how the binding finds a parent. + */ +function countingView(root: SyntaxNode) { + const counter = { visits: 0 }; + const parentOf = new Map(); + for (const node of allNodes(root)) { + for (const child of node.children) parentOf.set(child.id, node); + } + const proto = Object.getPrototypeOf(root); + const wrapped = new Map(); + const isNode = (value: unknown): value is SyntaxNode => + typeof value === 'object' && value !== null && Object.getPrototypeOf(value) === proto; + const wrapValue = (value: unknown): unknown => { + if (isNode(value)) return wrap(value); + if (Array.isArray(value)) { + counter.visits += value.length; + return value.map(wrapValue); + } + return value; + }; + const wrap = (node: SyntaxNode): SyntaxNode => { + const known = wrapped.get(node.id); + if (known !== undefined) return known; + const proxy = new Proxy(node, { + get(target, prop) { + counter.visits++; + if (prop === 'parent') { + const parent = parentOf.get(target.id); + return parent === undefined ? null : wrap(parent); + } + const value = Reflect.get(target, prop, target); + if (typeof value === 'function') { + return (...args: unknown[]) => wrapValue(value.apply(target, args)); + } + return wrapValue(value); + }, + }); + wrapped.set(node.id, proxy); + return proxy; + }; + return { counter, wrap }; +} + +/** The free calls the scope query captures, as `emitRScopeCaptures` sees them. */ +function freeCalls(root: SyntaxNode): SyntaxNode[] { + const calls: SyntaxNode[] = []; + for (const match of getRScopeQuery().matches(root)) { + for (const capture of match.captures) { + if (capture.name === 'reference.call.free') calls.push(capture.node); + } + } + return calls; +} + +/** Visits the capture path spends answering "is this free call a replacement call?" for every free call. */ +function capturePathVisits(source: string): number { + const root = parse(source).rootNode; + const calls = freeCalls(root); + const { counter, wrap } = countingView(root); + const isReplacement = rReplacementCallTest(wrap(root)); + for (const call of calls) isReplacement(wrap(call)); + return counter.visits; +} + +/** Visits the per-call oracle spends on every call node (the control). */ +function oracleVisits(source: string): number { + const root = parse(source).rootNode; + const calls = allNodes(root).filter((n) => n.type === 'call'); + const { counter, wrap } = countingView(root); + for (const call of calls) isRReplacementCall(wrap(call)); + return counter.visits; +} + +/** A shape both as an expression and as an assignment target, so the target chain is walked too. */ +const both = (shape: string) => `${shape}\n${shape} <- v\n`; + +const nested = (open: string, close: string, depth: number, leaf: string) => + open.repeat(depth) + leaf + close.repeat(depth); + +const DEPTH_AXES: Array<[string, (depth: number) => string]> = [ + ['nn(nn(...))', (d) => both(nested('nn(', ')', d, 'x'))], + ['list(a = list(...))', (d) => both(nested('list(a = ', ')', d, 'x'))], + ['g(a$g(a$...))', (d) => both('g(' + nested('a$g(', ')', d - 1, 'x') + ')')], +]; + +const wide = (n: number) => both(`c(${Array.from({ length: n }, (_, i) => `f(${i})`).join(', ')})`); + +const AXES: Array<[string, (n: number) => string, number[]]> = [ + ...DEPTH_AXES.map(([name, gen]): [string, (n: number) => string, number[]] => [ + `depth ${name}`, + gen, + [100, 200, 400], + ]), + ['width c(f(1), ..., f(n))', wide, [1000, 2000, 4000]], +]; + +describe('replacement-position detection on the capture path is linear', () => { + it.each(AXES)('%s: doubling the input at most doubles the visits', (_axis, gen, sizes) => { + const visits = sizes.map((n) => capturePathVisits(gen(n))); + for (let i = 1; i < sizes.length; i++) { + expect( + visits[i], + `visits ${sizes[i - 1]} -> ${sizes[i]}: ${visits.join(', ')}`, + ).toBeLessThanOrEqual(2.2 * visits[i - 1]); + } + }); + + it.each(AXES)( + 'control, %s: the per-call oracle is super-linear on the same inputs', + (_axis, gen, sizes) => { + // Two pairs are enough to show the growth, and keep the control cheap. + const visits = sizes.slice(0, 2).map((n) => oracleVisits(gen(n))); + expect(visits[1], `oracle visits: ${visits.join(', ')}`).toBeGreaterThan(3 * visits[0]); + }, + ); +}); + +/** Whether some capture of `matches` has exactly the text `text`. */ +const capturesText = (matches: readonly CaptureMatch[], text: string): boolean => + matches.some((m) => Object.values(m).some((c) => c?.text === text)); + +describe('a pathological file finishes emitRScopeCaptures quickly', () => { + // Generous absolute bounds: the old per-call walk took minutes on these inputs. + it('first-argument nesting 1,500 deep', () => { + const source = nested('nn(', ')', 1500, 'x') + '\nsentinel <- function() 1\n'; + const started = performance.now(); + const matches = emitRScopeCaptures(source, 'deep.R'); + expect(performance.now() - started).toBeLessThan(5000); + expect(capturesText(matches, 'sentinel')).toBe(true); + }, 30_000); + + it('10,000 call-valued arguments', () => { + const args = Array.from({ length: 10_000 }, (_, i) => `k${i} = fn(${i})`).join(', '); + const source = `x <- c(${args})\nsentinel <- function() 1\n`; + const started = performance.now(); + const matches = emitRScopeCaptures(source, 'wide.R'); + expect(performance.now() - started).toBeLessThan(5000); + expect(capturesText(matches, 'sentinel')).toBe(true); + }, 30_000); +}); + +/** Every R source file under the test fixtures. */ +function fixtureRFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...fixtureRFiles(full)); + else if (entry.isFile() && /\.[Rr]$/.test(entry.name)) out.push(full); + } + return out; +} + +describe('rReplacementCallIds gives the per-call answer exactly', () => { + const files = fixtureRFiles(FIXTURES); + + it('finds the R fixtures', () => { + expect(files.length).toBeGreaterThan(300); + }); + + it('agrees with isRReplacementCall on every call of every R fixture', () => { + let replacementCalls = 0; + const mismatches: string[] = []; + for (const file of files) { + const root = parse(fs.readFileSync(file, 'utf-8')).rootNode; + const expected = allNodes(root) + .filter((n) => n.type === 'call' && isRReplacementCall(n)) + .map((n) => n.id) + .sort((a, b) => a - b); + const actual = [...rReplacementCallIds(root)].sort((a, b) => a - b); + replacementCalls += expected.length; + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + mismatches.push(path.relative(FIXTURES, file)); + } + } + expect(mismatches).toEqual([]); + // The fixtures do contain replacement calls, so the agreement is not vacuous. + expect(replacementCalls).toBeGreaterThan(20); + }); + + it.each([ + 'foo(x) <- 1', + 'foo(bar(x)) <- 1', + 'foo(x, bar(1)) <- v', + 'foo(x)[bar(1)] <- v', + 'foo(x)$a <- 1', + 'names(foo(x)$a) <- v', + 'foo(x)[[i]] <- v', + 'v -> foo(x)', + 'v ->> foo(bar(x))', + 'foo(x) <<- v', + 'foo(x) = v', + 'pkg::foo(x) <- v', + 'foo(a = bar(x)) <- v', + 'foo(bar(x))@slot <- v', + 'y <- foo(bar(x))', + '(foo(x)) <- 1', + 'foo(x <- bar(y))', + 'f(g(h(x))) <- k(1)', + 'a$g(b(x)) <- 1', + 'x[f(1)](g(2)) <- 3', + ])('agrees on %s', (source) => { + const root = parse(source).rootNode; + const expected = allNodes(root) + .filter((n) => n.type === 'call' && isRReplacementCall(n)) + .map((n) => n.text) + .sort(); + const ids = rReplacementCallIds(root); + const actual = allNodes(root) + .filter((n) => ids.has(n.id)) + .map((n) => n.text) + .sort(); + expect(actual).toEqual(expected); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/r-source-import-lookup.test.ts b/gitnexus/test/unit/scope-resolution/r/r-source-import-lookup.test.ts new file mode 100644 index 000000000..42f21f7cc --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/r-source-import-lookup.test.ts @@ -0,0 +1,117 @@ +/** + * R `source("file.R")` exact-path lookup. The live resolver answers it from the per-pass + * `normToRaw` map instead of `indexOf` over the whole file list for every `source()` call. + * The map must pick exactly the file `indexOf` picked (first in list order), so these cases + * compare both paths on the same corpus, including the duplicate-spelling and missing-file + * cases where an index could plausibly diverge, and pin that the hot path no longer scans. + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + resolveRImportInternal, + resolveRImportTarget, +} from '../../../../src/core/ingestion/languages/r/import-target.js'; +import { getWorkspaceFileIndex } from '../../../../src/core/ingestion/utils/workspace-file-index.js'; + +// Raw spellings: the Windows-style duplicate of 'pkg/R/dup.R' comes FIRST, so first-wins must +// return the backslash spelling for both lookups. +const FILES = [ + 'scripts\\win.R', + 'pkg\\R\\dup.R', + 'pkg/R/dup.R', + 'pkg/R/a.R', + 'pkg/R/b.R', + 'scripts/run.R', + 'scripts/helpers/util.R', + 'util.R', + 'vendor/other/util.R', + '/abs/root.R', +]; + +interface Case { + readonly title: string; + readonly from: string; + readonly raw: string; +} + +const CASES: readonly Case[] = [ + { title: 'relative, found beside the caller', from: 'pkg/R/a.R', raw: '"b.R"' }, + { title: 'relative into a subdirectory', from: 'scripts/run.R', raw: '"helpers/util.R"' }, + { title: 'relative with ../', from: 'scripts/helpers/x.R', raw: '"../run.R"' }, + { title: 'duplicate spellings: first wins', from: 'pkg/R/a.R', raw: '"dup.R"' }, + { title: 'backslash spelling of the file', from: 'scripts/x.R', raw: '"win.R"' }, + { title: 'absolute, present', from: 'pkg/R/a.R', raw: '"/abs/root.R"' }, + { title: 'absolute, absent', from: 'pkg/R/a.R', raw: '"/abs/missing.R"' }, + { title: 'relative, absent: suffix fallback', from: 'pkg/R/a.R', raw: '"util.R"' }, + { title: 'relative, absent everywhere', from: 'pkg/R/a.R', raw: '"nowhere.R"' }, + { title: 'lowercase extension', from: 'pkg/R/a.R', raw: '"a.r"' }, +]; + +const run = (c: Case, withMap: boolean): ReturnType => { + const { normalized, all, index, normToRaw } = getWorkspaceFileIndex(new Set(FILES)); + return resolveRImportInternal( + c.from, + c.raw, + normalized, + all, + null, + index, + withMap ? normToRaw : undefined, + ); +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('resolveRImportInternal: source() exact-path lookup', () => { + it.each(CASES)('$title: the index agrees with indexOf', (c) => { + expect(run(c, true)).toEqual(run(c, false)); + }); + + it('positive controls: the corpus really exercises found, duplicate and missing files', () => { + const byTitle = (title: string) => run(CASES.find((c) => c.title === title)!, true); + expect(byTitle('relative, found beside the caller')).toBe('pkg/R/b.R'); + expect(byTitle('duplicate spellings: first wins')).toBe('pkg\\R\\dup.R'); + expect(byTitle('backslash spelling of the file')).toBe('scripts\\win.R'); + expect(byTitle('absolute, present')).toBe('/abs/root.R'); + expect(byTitle('absolute, absent')).toBeNull(); + expect(byTitle('relative, absent everywhere')).toBeNull(); + }); + + it('does not scan the file list for an exact hit when the index is supplied', () => { + const { normalized, all, index, normToRaw } = getWorkspaceFileIndex(new Set(FILES)); + const indexOf = vi.spyOn(normalized, 'indexOf'); + expect( + resolveRImportInternal('pkg/R/a.R', '"b.R"', normalized, all, null, index, normToRaw), + ).toBe('pkg/R/b.R'); + expect( + resolveRImportInternal('pkg/R/a.R', '"/abs/root.R"', normalized, all, null, index, normToRaw), + ).toBe('/abs/root.R'); + expect(indexOf).not.toHaveBeenCalled(); + }); + + it('still scans (and agrees) when called without the index', () => { + const { normalized, all, index } = getWorkspaceFileIndex(new Set(FILES)); + const indexOf = vi.spyOn(normalized, 'indexOf'); + expect(resolveRImportInternal('pkg/R/a.R', '"b.R"', normalized, all, null, index)).toBe( + 'pkg/R/b.R', + ); + expect(indexOf).toHaveBeenCalled(); + }); +}); + +describe('resolveRImportTarget: source() through the live adapter', () => { + // `source()` carries no wildcard/named context, so the adapter takes the file-path branch. + it.each(CASES)('$title: matches the indexOf result', (c) => { + const viaAdapter = resolveRImportTarget(c.raw, c.from, new Set(FILES)); + expect(viaAdapter).toEqual(run(c, false)); + }); + + it('does not scan the file list on a repeat call for the same file set', () => { + const files = new Set(FILES); + resolveRImportTarget('"b.R"', 'pkg/R/a.R', files); // builds the per-set index + const indexOf = vi.spyOn(Array.prototype, 'indexOf'); + expect(resolveRImportTarget('"b.R"', 'pkg/R/a.R', files)).toBe('pkg/R/b.R'); + expect(indexOf.mock.calls.filter(([needle]) => needle === 'pkg/R/b.R')).toEqual([]); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/r/scope-function-anchors.test.ts b/gitnexus/test/unit/scope-resolution/r/scope-function-anchors.test.ts new file mode 100644 index 000000000..22ed656af --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/r/scope-function-anchors.test.ts @@ -0,0 +1,429 @@ +/** + * R caller attribution: the R scope query must anchor + * `@scope.function` on the SAME AST node as `@declaration.function` (the whole + * `name <- function(...)` assignment) and `@declaration.method` (the + * `name = function(...)` argument), so a def is owned by its OWN Function scope. + * If the two anchors differ, the def is owned by the enclosing + * Module / outer-function / Class scope, and `resolveCallerGraphId` credits every + * call in a body to the FIRST callable that scope owns. + * + * Every `it` holds ONE assertion that compares a value, and the + * lookups never dereference a possibly-missing scope/def, so a failure is + * always a wrong VALUE, never a thrown TypeError. + * + * Positions: lines are 1-based, columns 0-based (the scope `Range` convention). + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, Range, Scope, SymbolDefinition } from 'gitnexus-shared'; +import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js'; +import { rProvider } from '../../../../src/core/ingestion/languages/r.js'; +import { populateClassOwnedMembers } from '../../../../src/core/ingestion/scope-resolution/scope/walkers.js'; + +const FILE = 'R/x.R'; + +/** Parse + run R's `populateOwners` (the generic `populateClassOwnedMembers`). */ +function parse(src: string): ParsedFile | undefined { + const parsed = extractParsedFile(rProvider, src, FILE); + if (parsed !== undefined) populateClassOwnedMembers(parsed); + return parsed; +} + +const scopesOf = (src: string): readonly Scope[] => parse(src)?.scopes ?? []; + +const fmt = (r: Range): string => `${r.startLine}:${r.startCol}-${r.endLine}:${r.endCol}`; + +/** + * The range string of the single occurrence of `snippet` in `src`. Throws at + * describe time (never inside a test body) when the snippet is missing or + * ambiguous, so a typo cannot masquerade as an expected failure. + */ +function spanOf(src: string, snippet: string): string { + const start = src.indexOf(snippet); + if (start < 0 || src.indexOf(snippet, start + 1) >= 0) { + throw new Error(`snippet must occur exactly once: ${snippet}`); + } + const at = (offset: number): string => { + const before = src.slice(0, offset); + const line = before.split('\n').length; + const col = offset - (before.lastIndexOf('\n') + 1); + return `${line}:${col}`; + }; + return `${at(start)}-${at(start + snippet.length)}`; +} + +const allDefs = (scopes: readonly Scope[]): SymbolDefinition[] => + scopes.flatMap((s) => [...s.ownedDefs]); + +const findDef = ( + scopes: readonly Scope[], + type: string, + name: string, +): SymbolDefinition | undefined => + allDefs(scopes).find((d) => d.nodeId.endsWith(`:${type}:${name}`)); + +const scopeByRange = ( + scopes: readonly Scope[], + kind: Scope['kind'], + range: string, +): Scope | undefined => scopes.find((s) => s.kind === kind && fmt(s.range) === range); + +const moduleOf = (scopes: readonly Scope[]): Scope | undefined => + scopes.find((s) => s.kind === 'Module'); + +/** Who owns the def, and where that scope sits. Always an object, so a wrong owner is a value diff. */ +function ownerInfo( + scopes: readonly Scope[], + type: string, + name: string, +): { kind: string; parentKind: string | undefined; range: string } | undefined { + const owner = scopes.find((s) => s.ownedDefs.some((d) => d.nodeId.endsWith(`:${type}:${name}`))); + if (owner === undefined) return undefined; + return { + kind: owner.kind, + parentKind: scopes.find((s) => s.id === owner.parent)?.kind, + range: fmt(owner.range), + }; +} + +/** Ranges of every ancestor scope, nearest first. */ +function ancestorRanges(scopes: readonly Scope[], scope: Scope | undefined): string[] { + const out: string[] = []; + let cur = scope === undefined ? undefined : scopes.find((s) => s.id === scope.parent); + while (cur !== undefined) { + out.push(fmt(cur.range)); + const parentId = cur.parent; + cur = scopes.find((s) => s.id === parentId); + } + return out; +} + +const defNames = (defs: readonly SymbolDefinition[]): string[] => + defs.map((d) => d.nodeId.split(':').slice(-1)[0]); + +// ─── Sources ──────────────────────────────────────────────────────────────── + +const FORMS_SRC = [ + 'f <- function(x) g(x)', + 'g = function(x) g2(x)', + 'h <<- function(x) g3(x)', + 'k <- \\(x) g4(x)', + 'dotted.name <- function(x) g5(x)', + '`weird name` <- function(x) g6(x)', + '', +].join('\n'); + +const FORMS = [ + { label: '<-', name: 'f', line: 'f <- function(x) g(x)', row: 1 }, + { label: '=', name: 'g', line: 'g = function(x) g2(x)', row: 2 }, + { label: '<<-', name: 'h', line: 'h <<- function(x) g3(x)', row: 3 }, + { label: '\\(x) lambda', name: 'k', line: 'k <- \\(x) g4(x)', row: 4 }, + { label: 'dotted name', name: 'dotted.name', line: 'dotted.name <- function(x) g5(x)', row: 5 }, + { + label: 'backtick name', + name: '`weird name`', + line: '`weird name` <- function(x) g6(x)', + row: 6, + }, +] as const; + +const NESTED_SRC = [ + 'outer <- function(x) {', + ' inner <- function(y) g(y)', + ' inner(x)', + ' lapply(x, function(z) h(z))', + ' sapply(x, FUN = function(z) h2(z))', + ' tryCatch(e(), error = function(e) h3(e))', + '}', + '', +].join('\n'); +const OUTER_FN = spanOf(NESTED_SRC, NESTED_SRC.slice(NESTED_SRC.indexOf('function(x) {'), -1)); +const INNER_ASSIGN = spanOf(NESTED_SRC, 'inner <- function(y) g(y)'); + +const R6_SRC = [ + 'Acct <- R6::R6Class("Acct",', + ' public = list(', + ' cache = build_cache(),', + ' deposit = function(x) 1,', + ' run = function() {', + ' helper <- function() 1', + ' helper()', + ' }', + ' ),', + ' private = list(', + ' audit = function() 2', + ' ),', + ' active = list(', + ' balance = function() 3', + ' )', + ')', + '', +].join('\n'); + +const BARE_R6_SRC = 'Bare <- R6Class("Bare", public = list(get_value = function() 1))\n'; +const R5_SRC = + 'Rc <- setRefClass("Rc", methods = list(process = function() 1, other = function() 2))\n'; +const FIELD_SRC = 'A <- R6Class("A", public = list(cache = build_cache(), m = function() 1))\n'; +const SOLO_SRC = 'solo <- function(x) g(x)'; +const MEMBER_SRC = 'obj$f <- function(x) g(x)\nx[["k"]] <- function(y) g(y)\n'; + +describe('R scope function anchors', () => { + // ─── Sanity ─────────────────────────────────────────────────────────────── + it('parses every source in this file', () => { + const parsed = [ + FORMS_SRC, + NESTED_SRC, + R6_SRC, + BARE_R6_SRC, + R5_SRC, + FIELD_SRC, + SOLO_SRC, + MEMBER_SRC, + ].map((src) => parse(src) !== undefined); + expect(parsed).toEqual([true, true, true, true, true, true, true, true]); + }); + + // ─── named forms ───────────────────────────────────── + describe('named assignment forms', () => { + const scopes = scopesOf(FORMS_SRC); + + // Assignment-anchored ownership: the def is owned by a Function scope whose range is + // the assignment (the def's anchor) and whose parent is the Module. + for (const form of FORMS) { + it(`${form.label}: the def is owned by its own assignment-anchored Function scope`, () => { + expect(ownerInfo(scopes, 'Function', form.name)).toEqual({ + kind: 'Function', + parentKind: 'Module', + range: spanOf(FORMS_SRC, form.line), + }); + }); + } + + // Assignment-anchored ownership: the Module scope no longer owns any callable. + it('the Module scope owns no callable def', () => { + const callables = (moduleOf(scopes)?.ownedDefs ?? []).filter((d) => d.type === 'Function'); + expect(defNames(callables)).toEqual([]); + }); + + // Behaviour the anchoring leaves unchanged: the binding of each name is visible from the Module. + for (const form of FORMS) { + it(`binds ${form.label} name in the Module scope with origin local`, () => { + const refs = moduleOf(scopes)?.bindings.get(form.name) ?? []; + expect(refs.map((r) => r.origin)).toEqual(['local']); + }); + } + + // Behaviour the anchoring leaves unchanged: scope ids stay unique per file. + it('keeps scope ids unique', () => { + expect(new Set(scopes.map((s) => s.id)).size).toBe(scopes.length); + }); + + // Behaviour the anchoring leaves unchanged: the def id still encodes the assignment start. + it('keeps the def node ids anchored at the assignment start', () => { + const expected = FORMS.map((f) => `def:${FILE}#${f.row}:0:Function:${f.name}`).sort(); + expect( + allDefs(scopes) + .map((d) => d.nodeId) + .sort(), + ).toEqual(expected); + }); + }); + + // ─── nested functions ──────────────────────────────────── + describe('nested functions', () => { + const scopes = scopesOf(NESTED_SRC); + + // Assignment-anchored ownership. + it("the inner def is owned by inner's own assignment-anchored Function scope", () => { + expect(ownerInfo(scopes, 'Function', 'inner')).toEqual({ + kind: 'Function', + parentKind: 'Function', + range: INNER_ASSIGN, + }); + }); + + it("outer's function_definition scope owns nothing", () => { + expect( + defNames( + scopeByRange(scopes, 'Function', OUTER_FN)?.ownedDefs ?? [ + { nodeId: 'missing-scope' } as SymbolDefinition, + ], + ), + ).toEqual([]); + }); + + // Behaviour the anchoring leaves unchanged: the inner binding lives in outer's function_definition scope. + it("binds the inner name in outer's function_definition scope", () => { + const refs = scopeByRange(scopes, 'Function', OUTER_FN)?.bindings.get('inner') ?? []; + expect(refs.map((r) => r.origin)).toEqual(['local']); + }); + + // Behaviour the anchoring leaves unchanged: anonymous / named-argument lambdas own nothing + // and hang (transitively) under outer's function_definition scope. + for (const snippet of ['function(z) h(z)', 'function(z) h2(z)', 'function(e) h3(e)']) { + const anonymous = scopeByRange(scopes, 'Function', spanOf(NESTED_SRC, snippet)); + it(`anonymous "${snippet}" owns no def`, () => { + expect( + defNames(anonymous?.ownedDefs ?? [{ nodeId: 'missing-scope' } as SymbolDefinition]), + ).toEqual([]); + }); + it(`anonymous "${snippet}" sits under outer's function_definition scope`, () => { + expect(ancestorRanges(scopes, anonymous).includes(OUTER_FN)).toBe(true); + }); + } + }); + + // ─── R6 and R5 methods ─────────────────────────────── + describe('R6 / R5 methods', () => { + const r6 = scopesOf(R6_SRC); + const bare = scopesOf(BARE_R6_SRC); + const r5 = scopesOf(R5_SRC); + const classOwns = (scopes: readonly Scope[]): string[] => + (scopes.find((s) => s.kind === 'Class')?.ownedDefs ?? []).map((d) => d.type); + + // Assignment-anchored ownership: one test each. + const cases = [ + { + label: 'R6 public', + scopes: r6, + src: R6_SRC, + name: 'deposit', + snippet: 'deposit = function(x) 1', + }, + { + label: 'R6 private', + scopes: r6, + src: R6_SRC, + name: 'audit', + snippet: 'audit = function() 2', + }, + { + label: 'R6 active', + scopes: r6, + src: R6_SRC, + name: 'balance', + snippet: 'balance = function() 3', + }, + { + label: 'bare R6Class', + scopes: bare, + src: BARE_R6_SRC, + name: 'get_value', + snippet: 'get_value = function() 1', + }, + { + label: 'R5 methods', + scopes: r5, + src: R5_SRC, + name: 'process', + snippet: 'process = function() 1', + }, + ]; + for (const c of cases) { + it(`${c.label}: the method is owned by a Function scope under the Class, which owns only the class def`, () => { + expect({ + owner: ownerInfo(c.scopes, 'Method', c.name), + classOwns: classOwns(c.scopes), + }).toEqual({ + owner: { kind: 'Function', parentKind: 'Class', range: spanOf(c.src, c.snippet) }, + classOwns: ['Class'], + }); + }); + } + + // Behaviour the anchoring leaves unchanged: ownerId + qualifiedName after populateClassOwnedMembers. + // HEAD reaches this through branch 2 (defs owned by the Class scope); after + // the fix through branch 1 (Function scope under Class): the invariant must + // survive the mechanism swap. + it('stamps a method ownerId with the class def id', () => { + const classDef = findDef(r6, 'Class', 'Acct'); + expect(findDef(r6, 'Method', 'deposit')?.ownerId).toBe(classDef?.nodeId); + }); + it('qualifies a method as Class.method', () => { + expect(findDef(r6, 'Method', 'deposit')?.qualifiedName).toBe('Acct.deposit'); + }); + it('gives the class def a resolvable id (guards the two ownerId/qualifiedName assertions above)', () => { + expect(findDef(r6, 'Class', 'Acct')?.nodeId).toBe(`def:${FILE}#1:0:Class:Acct`); + }); + + // ─── nested named helper inside an R6 method ──────────── + // HEAD stamps ownerId = class and qualifiedName `Acct.helper` on it (branch 1 + // matches its method's Function scope, whose parent is the Class). After the + // fix its scope parent is a Function scope and nothing is stamped + // (walkers.ts depth invariant). Losing the class attribution for such a + // helper is an accepted trade-off: it is a local function inside a method, + // not a class member, and keeps a bare qualifiedName. + it('a nested helper in an R6 method gets no ownerId', () => { + expect(findDef(r6, 'Function', 'helper')?.ownerId).toBeUndefined(); + }); + it('a nested helper in an R6 method keeps a bare qualifiedName', () => { + expect(findDef(r6, 'Function', 'helper')?.qualifiedName).toBe('helper'); + }); + it('the method itself keeps ownerId = class and Class.method', () => { + const run = findDef(r6, 'Method', 'run'); + expect({ ownerId: run?.ownerId, qualifiedName: run?.qualifiedName }).toEqual({ + ownerId: `def:${FILE}#1:0:Class:Acct`, + qualifiedName: 'Acct.run', + }); + }); + }); + + // ─── class body outside any method ─────────────────────── + describe('class-body field default', () => { + const scopes = scopesOf(FIELD_SRC); + + // Assignment-anchored ownership: the Class scope owns ONLY the class def. + it('the Class scope owns only the class def', () => { + const owned = scopes.find((s) => s.kind === 'Class')?.ownedDefs ?? []; + expect(owned.map((d) => d.type)).toEqual(['Class']); + }); + + // Behaviour the anchoring leaves unchanged: no Function scope contains the field-default call, so the + // caller walk from it ends at the Class scope (the source is the Class node). + it('has no Function scope around the field-default call', () => { + const call = spanOf(FIELD_SRC, 'build_cache()'); + const [start] = call.split('-'); + const [line, col] = start.split(':').map(Number); + const containing = scopes.filter( + (s) => + s.kind === 'Function' && + (s.range.startLine < line || (s.range.startLine === line && s.range.startCol <= col)) && + (s.range.endLine > line || (s.range.endLine === line && s.range.endCol >= col)), + ); + expect(containing.map((s) => fmt(s.range))).toEqual([]); + }); + }); + + // ─── single-function file, no trailing newline ─────────── + describe('single-function file without a trailing newline', () => { + const scopes = scopesOf(SOLO_SRC); + + it('keeps scope ids unique when the assignment range equals the program range', () => { + expect(new Set(scopes.map((s) => s.id)).size).toBe(scopes.length); + }); + + // Assignment-anchored ownership. + it('the def is owned by a Function scope whose parent is the Module', () => { + expect(ownerInfo(scopes, 'Function', 'solo')).toEqual({ + kind: 'Function', + parentKind: 'Module', + range: spanOf(SOLO_SRC, SOLO_SRC), + }); + }); + }); + + // ─── Member-assigned functions ──────────────────────────────── + describe('member-assigned functions', () => { + const scopes = scopesOf(MEMBER_SRC); + + it('declares no def for obj$f <- function / x[["k"]] <- function', () => { + expect(defNames(allDefs(scopes))).toEqual([]); + }); + + it('creates only the function_definition scopes (no assignment-anchored scope)', () => { + const functionScopes = scopes.filter((s) => s.kind === 'Function').map((s) => fmt(s.range)); + expect(functionScopes.sort()).toEqual( + [spanOf(MEMBER_SRC, 'function(x) g(x)'), spanOf(MEMBER_SRC, 'function(y) g(y)')].sort(), + ); + }); + }); +}); diff --git a/gitnexus/test/unit/type-env-r.test.ts b/gitnexus/test/unit/type-env-r.test.ts new file mode 100644 index 000000000..45fb10629 --- /dev/null +++ b/gitnexus/test/unit/type-env-r.test.ts @@ -0,0 +1,660 @@ +import { describe, it, expect } from 'vitest'; +import { buildTypeEnv } from '../../src/core/ingestion/type-env.js'; +import { fastestWithin, timeWork } from '../helpers/timing-attempts.js'; +import Parser from 'tree-sitter'; +import R from '@eagleoutice/tree-sitter-r'; +import { SupportedLanguages } from 'gitnexus-shared'; + +const parser = new Parser(); + +const parse = (code: string) => { + parser.setLanguage(R); + return parser.parse(code); +}; + +function flatGet(typeEnv: ReturnType, varName: string): string | undefined { + for (const [, scopeMap] of typeEnv.allScopes()) { + const val = scopeMap.get(varName); + if (val) return val; + } + return undefined; +} + +function flatSize(typeEnv: ReturnType): number { + let count = 0; + for (const [, scopeMap] of typeEnv.allScopes()) count += scopeMap.size; + return count; +} + +/** + * Identifiers named `varName` in the body of the function assigned to `fnName` + * (not in its formals, and not inside a function nested in it): the nodes a + * receiver lookup starts from. + */ +function bodyIdentifiers(tree: ReturnType, fnName: string, varName: string) { + const fnNode = tree.rootNode + .descendantsOfType('binary_operator') + .find((n) => n.childForFieldName('lhs')?.text === fnName) + ?.childForFieldName('rhs'); + if (fnNode?.type !== 'function_definition') throw new Error(`no function ${fnName}`); + return fnNode.descendantsOfType('identifier').filter((id) => { + if (id.text !== varName) return false; + let current = id.parent; + while (current && current.id !== fnNode.id) { + if (current.type === 'parameters' || current.type === 'function_definition') return false; + current = current.parent; + } + return true; + }); +} + +function lookupIn( + tree: ReturnType, + typeEnv: ReturnType, + fnName: string, + varName: string, +): string | undefined { + const [node] = bodyIdentifiers(tree, fnName, varName); + if (!node) throw new Error(`no ${varName} in ${fnName}`); + return typeEnv.lookup(varName, node); +} + +describe('buildTypeEnv', () => { + describe('R roxygen2 annotations', () => { + it('extracts @param type bindings from roxygen2 comments', () => { + const tree = parse(` +DataFrame <- R6::R6Class("DataFrame", public = list(rows = function() 1)) + +#' @param data DataFrame +#' @param outliers DataFrame +AddOutlierStatuses <- function(data, outliers) { + data +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'data')).toBe('DataFrame'); + expect(flatGet(typeEnv, 'outliers')).toBe('DataFrame'); + }); + + it('binds a formal written in backticks to the type its @param tag gives the symbol', () => { + const tree = parse(` +DataFrame <- R6::R6Class("DataFrame", public = list(rows = function() 1)) + +#' @param data DataFrame +Summarise <- function(\`data\`) { + \`data\`$rows() +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, '`data`')).toBe('DataFrame'); + }); + + it('binds nothing for a type the file does not define (primitive or capitalised)', () => { + const tree = parse(` +#' @param x numeric +#' @param name Character +process <- function(x, name) { + x +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'x')).toBeUndefined(); + expect(flatGet(typeEnv, 'name')).toBeUndefined(); + }); + + it('extracts no types when no roxygen2 comments present', () => { + const tree = parse(` +process <- function(x, y) { + x + y +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatSize(typeEnv)).toBe(0); + }); + + it.each([ + ['<-', 'helper <- function(repo) {\n repo\n}'], + ['=', 'helper = function(repo) {\n repo\n}'], + ['<<-', 'helper <<- function(repo) {\n repo\n}'], + ])('extracts types from %s function definitions', (_op, definition) => { + const tree = parse(` +UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1)) + +#' @param repo UserRepo +${definition} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'repo')).toBe('UserRepo'); + expect(lookupIn(tree, typeEnv, 'helper', 'repo')).toBe('UserRepo'); + expect(typeEnv.fileScope().get('repo')).toBeUndefined(); + }); + + it('returns constructor binding for R6 obj <- ClassName$new()', () => { + const tree = parse(` +rs <- ResultSet$new(items) +`); + const { constructorBindings } = buildTypeEnv(tree, SupportedLanguages.R); + const binding = constructorBindings.find((b) => b.varName === 'rs'); + expect(binding).toBeDefined(); + expect(binding!.calleeName).toBe('ResultSet'); + }); + + it('returns constructor binding for S4 obj <- new("ClassName")', () => { + const tree = parse(` +model <- new("DataModel", name = "test") +`); + const { constructorBindings } = buildTypeEnv(tree, SupportedLanguages.R); + const binding = constructorBindings.find((b) => b.varName === 'model'); + expect(binding).toBeDefined(); + expect(binding!.calleeName).toBe('DataModel'); + }); + + it.each([ + ['Class= after another argument', 'model <- new(name = "test", Class = "DataModel")'], + ['Class= first', 'model <- new(Class = "DataModel", name = "test")'], + ['a leading named argument', 'model <- new(name = "test", "DataModel")'], + ])('returns the S4 constructor binding for new() with %s', (_form, code) => { + const { constructorBindings } = buildTypeEnv(parse(code), SupportedLanguages.R); + expect(constructorBindings.find((b) => b.varName === 'model')?.calleeName).toBe('DataModel'); + }); + + it('does not take a later string as the class of new() when Class= is another value', () => { + const { constructorBindings } = buildTypeEnv( + parse('model <- new(Class = base_class, "Other")'), + SupportedLanguages.R, + ); + expect(constructorBindings.find((b) => b.varName === 'model')).toBeUndefined(); + }); + + it('extracts @param types when @examples block is present', () => { + const tree = parse(` +DataFrame <- R6::R6Class("DataFrame", public = list(rows = function() 1)) +Config <- R6::R6Class("Config", public = list(get = function() 1)) + +#' @param data DataFrame +#' @param config {Config} configuration object +#' @return Result +#' @examples +#' result <- process(my_data, my_config) +#' print(result) +process <- function(data, config) { + data +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'data')).toBe('DataFrame'); + expect(flatGet(typeEnv, 'config')).toBe('Config'); + }); + + it('extracts @param types when regular comments appear before function', () => { + const tree = parse(` +DataFrame <- R6::R6Class("DataFrame", public = list(rows = function() 1)) + +#' @param x DataFrame +# TODO: refactor this later +compute <- function(x) { x } +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'x')).toBe('DataFrame'); + }); + + it('binds a parameter only when the tag names a class the file defines', () => { + const tree = parse(` +#' @param data DataFrame +#' @param name Character +process <- function(data, name) { + data +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'data')).toBeUndefined(); + expect(flatGet(typeEnv, 'name')).toBeUndefined(); + }); + + it('does not bind a lowercase word even when the file defines a class of that name', () => { + const tree = parse(` +setClass("repo", representation(name = "character")) + +#' @param r repo +save <- function(r) { r } +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'r')).toBeUndefined(); + }); + + describe('description prose is not a type', () => { + it.each([ + ['R6', 'Data', 'df', 'Data frame of observations', 'Data <- R6::R6Class("Data")'], + ['S4', 'Data', 'df', 'Data frame of observations', 'setClass("Data", representation())'], + ['R6', 'Config', 'cfg', 'Config object to read from', 'Config <- R6::R6Class("Config")'], + [ + 'S4', + 'Config', + 'cfg', + 'Config object to read from', + 'setClass("Config", representation())', + ], + ])( + 'does not bind the first word of a description that names a %s class %s', + (_kind, _className, param, description, definition) => { + const tree = parse(` +${definition} + +#' @param ${param} ${description} +run <- function(${param}) { + ${param} +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, param)).toBeUndefined(); + }, + ); + + it('does not bind an article or determiner that happens to be a class name', () => { + const tree = parse(` +A <- R6::R6Class("A", public = list(go = function() 1)) + +#' @param x A numeric vector +f <- function(x) { x } +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'x')).toBeUndefined(); + }); + + it('does not bind from the first line of a description that continues on the next line', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param df Data +#' frame of observations +run <- function(df) { + df$fit() +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'df')).toBeUndefined(); + }); + + it('binds a braced type whose prose continues on the next line', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param df {Data} the observations, +#' one row per case +run <- function(df) { + df$fit() +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'df')).toBe('Data'); + }); + }); + + it('reads a tag only at the start of a roxygen line', () => { + const tree = parse(` +UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1)) + +#' Write the tag as #' @param repo UserRepo in your own comments. +save <- function(repo) { + repo +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'repo')).toBeUndefined(); + }); + + it('does not read a tag written mid-line, even when it names a defined class', () => { + const tree = parse(` +UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1)) + +#' Write the tag as @param repo UserRepo +save <- function(repo) { + repo +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'repo')).toBeUndefined(); + }); + + it('does not read a longer tag name that starts with param', () => { + const tree = parse(` +UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1)) + +#' @paramfoo repo UserRepo +save <- function(repo) { + repo +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'repo')).toBeUndefined(); + }); + + describe('the binding belongs to the documented function', () => { + const userRepo = 'UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1))'; + + it('does not bind a name that is not a formal of the documented function', () => { + const tree = parse(` +${userRepo} + +#' @param repo UserRepo +f <- function(x) { x } + +g <- function(repo) { repo$save() } + +h <- function() { + repo <- 1 + repo$save() +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'f', 'x')).toBeUndefined(); + expect(lookupIn(tree, typeEnv, 'g', 'repo')).toBeUndefined(); + expect(lookupIn(tree, typeEnv, 'h', 'repo')).toBeUndefined(); + expect(typeEnv.fileScope().get('repo')).toBeUndefined(); + expect(flatGet(typeEnv, 'repo')).toBeUndefined(); + }); + + it('binds a documented formal inside its function only', () => { + const tree = parse(` +${userRepo} + +#' @param repo UserRepo +f <- function(repo) { repo$save() } + +g <- function(repo) { repo$save() } + +h <- function() { + repo <- 1 + repo$save() +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'f', 'repo')).toBe('UserRepo'); + expect(lookupIn(tree, typeEnv, 'g', 'repo')).toBeUndefined(); + expect(lookupIn(tree, typeEnv, 'h', 'repo')).toBeUndefined(); + expect(typeEnv.fileScope().get('repo')).toBeUndefined(); + }); + + it('binds a formal that has a default value', () => { + const tree = parse(` +${userRepo} + +#' @param repo UserRepo +f <- function(x, repo = NULL) { repo$save() } + +g <- function(repo) { repo$save() } +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'f', 'repo')).toBe('UserRepo'); + expect(lookupIn(tree, typeEnv, 'g', 'repo')).toBeUndefined(); + expect(typeEnv.fileScope().get('repo')).toBeUndefined(); + }); + + it('skips the dots formal and still binds a named formal beside it', () => { + const tree = parse(` +${userRepo} + +#' @param ... UserRepo +#' @param repo UserRepo +f <- function(..., repo) { repo$save() } +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'f', 'repo')).toBe('UserRepo'); + expect(flatGet(typeEnv, '...')).toBeUndefined(); + expect(typeEnv.fileScope().get('repo')).toBeUndefined(); + }); + + it.each([ + ['a plain value', 'repo <- 1'], + ['a call', 'setup()'], + ])('binds nothing when the block is followed by %s (guard)', (_label, statement) => { + const tree = parse(` +${userRepo} + +#' @param repo UserRepo +${statement} + +g <- function(repo) { repo$save() } +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'g', 'repo')).toBeUndefined(); + expect(typeEnv.fileScope().get('repo')).toBeUndefined(); + }); + + it('does not carry an outer function documentation into a function nested in it', () => { + const tree = parse(` +${userRepo} + +#' @param repo UserRepo +outer <- function(repo) { + inner <- function(repo) { repo$save() } + repo$save() +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'outer', 'repo')).toBe('UserRepo'); + expect(lookupIn(tree, typeEnv, 'inner', 'repo')).toBeUndefined(); + }); + + it('binds a documented nested function only inside itself', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +outer <- function(x) { + #' @param y Data + inner <- function(y) { y$fit() } + y +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(lookupIn(tree, typeEnv, 'inner', 'y')).toBe('Data'); + expect(lookupIn(tree, typeEnv, 'outer', 'y')).toBeUndefined(); + expect(typeEnv.fileScope().get('y')).toBeUndefined(); + }); + }); + + describe('class definition forms', () => { + it.each([ + ['bare R6Class', 'UserRepo <- R6Class("UserRepo", public = list(save = function() 1))'], + [ + 'namespaced R6::R6Class', + 'UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1))', + ], + ['setClass', 'setClass("UserRepo", representation(name = "character"))'], + ['setRefClass', 'setRefClass("UserRepo", fields = list(name = "character"))'], + ])('binds a type defined with %s', (_form, definition) => { + const tree = parse(` +#' @param repo UserRepo +save <- function(repo) { + repo +} + +${definition} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'repo')).toBe('UserRepo'); + }); + + it('binds a backticked R6 class name, as it does a backticked formal', () => { + const tree = parse(` +#' @param repo UserRepo +save <- function(repo) { + repo +} + +\`UserRepo\` <- R6::R6Class("UserRepo", public = list(save = function() 1)) +`); + expect(flatGet(buildTypeEnv(tree, SupportedLanguages.R), 'repo')).toBe('UserRepo'); + }); + + it.each([ + ['setClass', 'setClass(contains = "Base", Class = "Derived")'], + ['setRefClass', 'setRefClass(fields = list(n = "numeric"), Class = "Derived")'], + ['a backticked Class', 'setClass(contains = "Base", `Class` = "Derived")'], + ])('names the class from Class= when %s puts it after another argument', (_form, call) => { + const tree = parse(` +#' @param a Derived +#' @param b Base +save <- function(a, b) { + a +} + +${call} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'a')).toBe('Derived'); + // `Base` is only a parent named by contains=; the file does not define it. + expect(flatGet(typeEnv, 'b')).toBeUndefined(); + }); + + it('still names the class from the first unnamed argument', () => { + const tree = parse(` +#' @param a Derived +save <- function(a) { + a +} + +setClass(contains = "Base", "Derived") +`); + expect(flatGet(buildTypeEnv(tree, SupportedLanguages.R), 'a')).toBe('Derived'); + }); + + it('binds the braced form, with or without prose after it', () => { + const tree = parse(` +UserRepo <- R6::R6Class("UserRepo", public = list(save = function() 1)) + +#' @param repo {UserRepo} the repository +#' @param backup {UserRepo} +save <- function(repo, backup) { + repo +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'repo')).toBe('UserRepo'); + expect(flatGet(typeEnv, 'backup')).toBe('UserRepo'); + }); + }); + + describe('dotted formal names', () => { + it.each([ + ['na.rm', 'na.rm'], + ['a leading dot', '.data'], + ['dots and underscores', 'max_n.rows'], + ])('binds the documented type of %s', (_label, formal) => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param ${formal} Data +#' @param other Data +run <- function(${formal}, other) { + ${formal} +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, formal)).toBe('Data'); + expect(flatGet(typeEnv, 'other')).toBe('Data'); + }); + + it('keeps the braced form and the continuation lines of a dotted name', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param na.rm {Data} +#' second line of prose +run <- function(na.rm) na.rm +`); + expect(flatGet(buildTypeEnv(tree, SupportedLanguages.R), 'na.rm')).toBe('Data'); + }); + + it('binds nothing for a dotted name the function does not have', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param na.rm Data +run <- function(x) x +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + expect(flatGet(typeEnv, 'na.rm')).toBeUndefined(); + expect(flatGet(typeEnv, 'x')).toBeUndefined(); + }); + }); + + describe('adversarial long lines', () => { + // A roxygen line is repository-controlled text. The name pattern used to retry + // quadratically on a long name followed by punctuation: 20 lines of 32,000 + // characters took over ten seconds then and take milliseconds now, so the bound + // has a wide margin on a slow machine. + const LONG = 32_000; + const LINES = 20; + const BOUND_MS = 2_000; + + // Each attempt parses afresh, so a per-tree cache warmed by an earlier one cannot hide the cost. + const timedEnv = (parsed: ReturnType) => + fastestWithin(BOUND_MS, () => + timeWork(() => { + const tree = parse(parsed.rootNode.text); + return () => buildTypeEnv(tree, SupportedLanguages.R); + }), + ); + + it('finishes promptly on a long name followed by punctuation', () => { + const line = "#' @param " + 'a'.repeat(LONG) + '!'; + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +${Array(LINES).fill(line).join('\n')} +#' @param ok Data +run <- function(ok) ok +`); + const { value: typeEnv, ms } = timedEnv(tree); + expect(ms).toBeLessThan(BOUND_MS); + // The malformed lines are skipped; the well-formed tag after them still binds. + expect(flatGet(typeEnv, 'ok')).toBe('Data'); + }); + + it('binds a dotted name that is followed by a very long description', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param na.rm {Data} ${'word '.repeat(LONG / 5)} +run <- function(na.rm) na.rm +`); + const { value: typeEnv, ms } = timedEnv(tree); + expect(ms).toBeLessThan(BOUND_MS); + expect(flatGet(typeEnv, 'na.rm')).toBe('Data'); + }); + + it('does not bind a name made only of dots, however long', () => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +#' @param ${'.'.repeat(LONG)} Data +run <- function(x) x +`); + const { value: typeEnv, ms } = timedEnv(tree); + expect(ms).toBeLessThan(BOUND_MS); + expect(flatGet(typeEnv, 'x')).toBeUndefined(); + }); + }); + + describe('forms that are not bound (guards)', () => { + it.each([ + ['\\code{Data} markup', "#' @param df \\code{Data}", ['df']], + ['\\linkS4class{Data} markup', "#' @param df \\linkS4class{Data}", ['df']], + ['a comma-separated name list', "#' @param x,y Data", ['x', 'y']], + ['the dots parameter', "#' @param ... Data", ['...']], + ])('does not bind %s', (_label, tag, names) => { + const tree = parse(` +Data <- R6::R6Class("Data", public = list(fit = function() 1)) + +${tag} +run <- function(x, y, df, ..., na.rm = FALSE) { + x +} +`); + const typeEnv = buildTypeEnv(tree, SupportedLanguages.R); + for (const name of names) expect(flatGet(typeEnv, name)).toBeUndefined(); + }); + }); + }); +}); diff --git a/type-resolution-system.md b/type-resolution-system.md index dc54c83c4..9442339dc 100644 --- a/type-resolution-system.md +++ b/type-resolution-system.md @@ -82,6 +82,15 @@ The receiver benchmark has its own language and shape inventory. Read its including deliberate untyped controls, unsupported shapes and N/A entries. Optional grammar availability must be recorded separately from resolver results. +R (experimental) is registered too. It has no inline type annotations: parameter +types come only from roxygen2 `@param` tags and S4 slot types from +`representation(...)` string literals. A receiver is typed in the scope only by a +constructor call in the same function (`x <- Cls$new()`, R6 and RefClass); the S4 +`new("Cls")` binding and the roxygen types live in the per-file type environment, which no +R call edge reads, so S4 receiver typing is not scope-bound and types no receiver. Coverage, limits and the roxygen2 rules are in +[docs/languages/r-provider.md](docs/languages/r-provider.md); wrong, lost and deferred +results are in [docs/languages/r-ingestion-gaps.md](docs/languages/r-ingestion-gaps.md). + ## Design Constraints - Prefer an unresolved reference over an edge to an unrelated declaration.