From c417ccfc81574462a11c00780ae6ee9458bd0967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Mon, 26 Oct 2026 15:13:48 +0000 Subject: [PATCH] perf(ingestion): reduce indexing overhead and declaration memory (#3726) * refactor(ingestion): share callable declaration facts Reuse pure parameter and signature facts between structure and scope extraction, with tree-owned caches and read-only parameter contracts. Preserve provider-specific wrappers and consumer projections. Add characterization tests, real-worker parity checks, and reproducible performance measurements. Keep the overall indexing effect explicit. Refs #3723 * fix(bench): create result files exclusively Address CodeQL review feedback on #3726. Use a private default directory and an exclusive file handle so existing files and symlinks cannot be overwritten. * perf: speed analyzer identity guard probes * perf(ingestion): reuse callable type and overload facts * ci: retain callable time and memory benchmarks * chore: use established CI performance measurements * fix(bench): check callable baseline in CI * perf(ingestion): guard callable scaling and reuse export checks * fix(ci): budget for callable scaling matrix * refactor(ingestion): simplify callable handling --------- Co-authored-by: Gergo Magyar --- .github/workflows/ci-tests.yml | 16 +- ARCHITECTURE.md | 11 + .../bench/method-declaration-facts/README.md | 155 ++++ .../bench/method-declaration-facts/RESULTS.md | 93 ++ .../method-declaration-facts/baseline.json | 541 ++++++++++++ .../method-declaration-facts/measure.mjs | 799 ++++++++++++++++++ .../bench/method-declaration-facts/memory.mjs | 105 +++ .../method-declaration-facts/scaling.mjs | 28 + .../trace-parameters.mjs | 47 ++ gitnexus/src/core/analyzer-identity.ts | 47 +- .../src/core/ingestion/export-detection.ts | 30 +- .../languages/csharp/arity-metadata.ts | 6 +- .../languages/dart/arity-metadata.ts | 4 +- .../languages/java/arity-metadata.ts | 6 +- .../languages/kotlin/arity-metadata.ts | 6 +- .../ingestion/languages/php/arity-metadata.ts | 6 +- .../languages/python/arity-metadata.ts | 4 +- .../languages/swift/arity-metadata.ts | 11 +- .../languages/typescript/arity-metadata.ts | 16 +- .../method-extractors/configs/csharp.ts | 3 +- .../method-extractors/configs/dart.ts | 7 +- .../method-extractors/configs/jvm.ts | 5 +- .../method-extractors/configs/php.ts | 3 +- .../method-extractors/configs/python.ts | 3 +- .../method-extractors/configs/swift.ts | 7 +- .../configs/typescript-javascript.ts | 3 +- .../method-extractors/declaration-facts.ts | 61 ++ gitnexus/src/core/ingestion/method-types.ts | 4 +- .../src/core/ingestion/utils/method-props.ts | 4 +- .../core/ingestion/workers/parse-worker.ts | 20 +- gitnexus/test/integration/parsing.test.ts | 48 +- .../unit/analyzer-identity-guard-io.test.ts | 189 +++++ .../unit/declaration-parameter-types.test.ts | 86 ++ .../unit/method-declaration-facts.test.ts | 296 +++++++ .../unit/method-declaration-scaling.test.ts | 70 ++ gitnexus/test/unit/method-props.test.ts | 4 +- 36 files changed, 2676 insertions(+), 68 deletions(-) create mode 100644 gitnexus/bench/method-declaration-facts/README.md create mode 100644 gitnexus/bench/method-declaration-facts/RESULTS.md create mode 100644 gitnexus/bench/method-declaration-facts/baseline.json create mode 100644 gitnexus/bench/method-declaration-facts/measure.mjs create mode 100644 gitnexus/bench/method-declaration-facts/memory.mjs create mode 100644 gitnexus/bench/method-declaration-facts/scaling.mjs create mode 100644 gitnexus/bench/method-declaration-facts/trace-parameters.mjs create mode 100644 gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts create mode 100644 gitnexus/test/unit/analyzer-identity-guard-io.test.ts create mode 100644 gitnexus/test/unit/declaration-parameter-types.test.ts create mode 100644 gitnexus/test/unit/method-declaration-facts.test.ts create mode 100644 gitnexus/test/unit/method-declaration-scaling.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1da1c4662..c9e0a8d41 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -691,7 +691,8 @@ jobs: benchmarks: name: benchmarks (GITNEXUS_BENCH) runs-on: ubuntu-latest - timeout-minutes: 25 + # The callable size matrix runs 63 full analyses and 18 heap probes. + timeout-minutes: 30 steps: # persist-credentials: false — this job only runs npm + vitest benchmarks # and never pushes; the default-persisted token in .git/config would be at @@ -710,6 +711,19 @@ jobs: node --import tsx bench/python-scope/import-target-fingerprint.mjs --check working-directory: gitnexus + - name: Callable declaration facts / time / memory baseline + if: ${{ !cancelled() }} + run: node --import tsx bench/method-declaration-facts/measure.mjs --check --output method-declaration-facts.json + working-directory: gitnexus + + - name: Upload callable declaration measurements + if: always() + uses: ./.github/actions/upload-test-artifact + with: + name: method-declaration-facts-benchmark + path: gitnexus/method-declaration-facts.json + retention-days: 5 + - name: Java wildcard-static route constant guards (#3110) if: ${{ !cancelled() }} # Build-free: named-import control vs wildcard materialization; diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 26bc36874..4635dbff8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -356,6 +356,17 @@ accesses, value uses and inheritance edges. Register a `ScopeResolver` in `SCOPE_RESOLVERS` to participate; the old per-language migration flag and parallel call-resolution DAG were removed in RING4-1 #942. +Method extraction still supplies graph properties and overload identity; scope +extraction builds the declaration metadata used by resolution. Their repeated +node-only signature callbacks share facts through `memoizeDeclarationFact` +(`ingestion/method-extractors/declaration-facts.ts`). Each callback owns a WeakMap +from the immutable parse tree to native node IDs, so separate traversal wrappers +reuse one result without retaining old trees. Parameter facts are read-only; +providers can map syntax wrappers to the same declaration identity. Each consumer +keeps its own arity/type normalization. Owner-dependent flags and file-dependent +receiver rules remain separate. The cache stays inside parser isolates; serialized +graph/ParsedFile payloads and the persistent parse-cache format remain unchanged. + A provider can explicitly restrict its edge mode. COBOL currently uses `callable-flow-only`: its established processor retains structural CALLS/IMPORTS, and scope resolution contributes procedure-pointer calls. Registration alone is diff --git a/gitnexus/bench/method-declaration-facts/README.md b/gitnexus/bench/method-declaration-facts/README.md new file mode 100644 index 000000000..16bf844ea --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/README.md @@ -0,0 +1,155 @@ +# Shared callable declaration facts + +## CI measurements + +The existing **benchmarks (GITNEXUS_BENCH)** job in `ci-tests.yml` runs: + +```sh +node --import tsx bench/method-declaration-facts/measure.mjs --check --output method-declaration-facts.json +``` + +`--check` compares the current build with committed `baseline.json`; no second +checkout is required. It checks all eight providers' semantic fingerprints and +shared parameter/type-array counts, cold/warm/changed graph and cache counts at +each workload size, and tree collection. Three fresh-process +samples report median wall/CPU time, endpoint heap, peak RSS, and retained heap, +with percentage differences from the recorded measurements. Timing and heap/RSS +are machine-dependent references, not absolute CI failure thresholds. + +Like the C# pipeline benchmark, this check increases input size and compares cost +growth with input growth. It uses the stricter **1.5** normalized growth budget +already used by the Python scope benchmark: + +```text +normalized growth = (large cost / small cost) / (large input / small input) +``` + +| Workload | Sizes | Gated costs | +| --- | --- | --- | +| Java + TypeScript, spread across files | 8 / 32 / 128 files per language, 40 methods per class | Cold/warm/changed wall time, CPU, endpoint heap, sampled peak RSS and process maximum RSS | +| Java + TypeScript, concentrated in one class per language | 320 / 1,280 / 5,120 methods per class | Same full-indexing measurements | +| Retained declaration metadata, separately for Java and TypeScript | 2,500 / 10,000 / 40,000 declarations | Extraction time, extraction retained heap, emitted retained heap | + +Both indexing shapes contain 640 / 2,560 / 10,240 generated methods, plus fixed +call and edit probes. The harness verifies the emitted method and parsed/cache-hit +file counts, so silently dropping work cannot improve a scaling result. Size order +alternates between repetitions. Every adjacent pair **and the full size span** must +stay within the budget, using per-size medians from the same run. Missing, zero, +negative or non-finite measurements fail. `--record` also enforces these guards. +Unit tests demonstrate that quadratic time or memory growth fails the check. + +The artifact includes raw samples, median costs and normalized ratios. These +bounded workloads guard against scaling regressions; they do not prove linear +complexity for every repository. Fixed startup costs can mask growth at small +sizes. Released heap deltas remain diagnostic because they can be zero or negative +after collection. + +CI uploads raw samples and comparisons as `method-declaration-facts-benchmark`, +using the same five-day retention as the other benchmark artifacts. The separate +all-language [scope-resolution benchmark](../scope-resolution/README.md) remains +unchanged. + +After reviewing an intentional fixture, semantic, or allocation change, run the +same command with `--record` instead of `--check`, then inspect `baseline.json`. +CI never records or accepts a new baseline automatically. Output files must not +already exist. The two-checkout commands below remain available for diagnostics. + +## Detailed local measurements + +Build the pre-change revision and candidate with `npm run build` in each package, +then run from the candidate's `gitnexus/` directory: + +```sh +node bench/method-declaration-facts/measure.mjs \ + --baseline-root /path/to/baseline/gitnexus \ + --samples 3 --output /tmp/method-declaration-facts.json +``` + +The output file must not exist; existing files and symlinks are rejected before +measurements start. Without `--output`, the harness creates a private temporary +directory and prints the result path. Results are written after all pairs finish. + +Use `--candidate-root` to measure a built candidate snapshot elsewhere. Put both +runtimes and their dependencies on the same filesystem. The source corpus always +comes from the checkout containing the script, and its content hash must match +between revisions. Each pair alternates revision order and uses fresh processes. + +The harness measures three forced analysis runs per process: a cold index, warm +parse-cache replay, and a changed method signature. It uses one real parse worker, +disables FTS and embeddings, and materializes graph edges for comparison. It checks +all structural nodes and edges, excluding Community/Process nodes and their membership edges, +between revisions; warm replay must match cold output, and the edit must change +the graph. Cache hit/reparse counts accompany every measurement. + +Two corpora exercise different costs: 64 generated Java/TypeScript class files +with 40 methods each, and the first 96 tracked ingestion provider sources plus +tracked same-arity, primary-constructor, record-component, and variadic fixtures. +Each includes a small file whose signature changes. These are bounded workloads, +not a claim about every repository or language distribution. + +Pass `--providers-only` for a separate nine-file parity corpus covering all eight +migrated providers plus the changed-file probe. It includes overload declarations, +a Java record, a C# primary constructor, a Kotlin companion, Python decoration, +PHP variadics, and Swift/Dart constructors. Every provider must emit its sample +callable and a resolved call to it. This mode uses the same repeated cold, warm, +and changed-file graph comparisons; all eight grammars must be installed. + +Counters separately run real scope and MethodInfo consumers for all eight +migrated languages. They report distinct parameter arrays and declaration-node +AST accesses, with semantic output hashes. These are instrumentation runs, outside +the timed analysis. A forced-GC probe reports collection of 100 discarded trees; +it does not add GC assertions to the unit suite. +Separate parameter-only timings call the callback twice for each of 1,000 +declarations in a fresh tree, across nine trials per provider. Parsing, forced GC, +and the counter instrumentation are outside those timed regions. + +Wall time includes analysis and persistence. CPU and RSS include worker threads. +RSS sampling uses 20 ms intervals; cumulative maximum RSS is the process high-water +mark, so later scenarios may inherit an earlier peak. GC duration covers only the +main isolate. Progress-phase times use callback intervals, not exclusive CPU +profiles. Compare distributions and report noise; reduced extraction counts alone +do not establish an end-to-end speedup. + +See [the recorded results](RESULTS.md) for the initial measurements and their limits. + +## Retained memory + +Run the same probe against both built revisions, alternating their order across +at least five fresh-process pairs for each language: + +```sh +node --expose-gc bench/method-declaration-facts/memory.mjs \ + --root /path/to/runtime/gitnexus --language java --declarations 10000 +node --expose-gc bench/method-declaration-facts/memory.mjs \ + --root /path/to/runtime/gitnexus --language typescript --declarations 10000 +``` + +The probe holds the parsed tree, runs scope arity and structure extraction, then +releases method records, the tree, and emitted metadata in stages. It reports +forced-GC JavaScript heap at each stage, a semantic digest, and whether the native +tree became collectible. `extractionRetainedBytes` subtracts the tree-only heap; +`emittedRetainedBytes` subtracts the initialized empty heap after releasing the +tree. `releasedDeltaBytes` includes remaining cache capacity and runtime warmup. +Compare those deltas rather than absolute process heap. + +This probe measures one isolate and a small declaration shape. Its RSS and +extraction timing are **not** end-to-end indexing measurements. Use `measure.mjs` +for complete cold/warm/changed indexing, including worker and native-memory RSS. +Keep both runtimes on the same filesystem with the same dependencies and give +every end-to-end child a fresh private `XDG_RUNTIME_DIR` so analyzer identity +caches start equally cold. The CI check includes this retained-heap probe in +the same `method-declaration-facts-benchmark` report as the indexing measurements. + +To count requests and distinct parameter arrays in real workers, use the optional +preload below from the candidate package. Repeat with the candidate runtime and a +different, nonexistent output file. The preload also runs inside parser workers; +group records by thread, tree, provider, and node, and count unique `(thread, array)` +pairs. These runs are instrumented: discard their timing and memory measurements. + +```sh +FACT_RUNTIME_ROOT=/path/to/baseline/gitnexus \ +FACT_COUNTER_LOG=/tmp/baseline-parameters.jsonl \ +NODE_OPTIONS='--import=/absolute/path/to/gitnexus/bench/method-declaration-facts/trace-parameters.mjs' \ +node --expose-gc bench/method-declaration-facts/measure.mjs \ + --child analysis --root /path/to/baseline/gitnexus --corpus repository +``` diff --git a/gitnexus/bench/method-declaration-facts/RESULTS.md b/gitnexus/bench/method-declaration-facts/RESULTS.md new file mode 100644 index 000000000..95ef6065f --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/RESULTS.md @@ -0,0 +1,93 @@ +# Measurement results — 2026-10-25 + +The measurements below are the original extraction-cache experiment, before +the later analyzer-identity and shared parameter-type optimizations. Ongoing CI +measurements use the existing [scope-resolution benchmark](../scope-resolution/README.md). +The later isolated experiments remain available in [GitHub Actions](https://github.com/abhigyanpatwari/GitNexus/actions/runs/40289711096). + +Baseline: `ee7eff97f7f4eb7d68f7090cba448f7cc8285db2`. Candidate: this refactor before commit. The SHA-256 of sorted changed production paths and file bytes, each separated by NUL, is `b6e82236a38c3a0cbd073f2ef162885613869a32c6da9c2d906ecc85a6ff5ea6`. Both revisions were built independently and used the same installed dependencies. + +Runtime: Node v22.23.3, Linux, 12th Gen Intel(R) Core(TM) i9-12900K, one parse worker. Runtimes, dependencies, and temporary corpora were on the same native filesystem. This was a shared development host, without CPU isolation. Full tests and graph indexing were started only after these measurements. + +## Extraction work + +For all eight providers, scope and structure consumers produced identical semantic hashes. Distinct parameter arrays fell from **2 to 1**. Separate uninstrumented trials called parameter extraction twice on each of 1,000 declarations in a fresh tree; parsing and forced GC were outside the timed region. The table reports medians of nine trials per provider. These small fixtures measure extraction cost, not overall indexing. + +| Provider | Baseline wall ms | Candidate wall ms | Baseline CPU ms | Candidate CPU ms | Root AST accesses before → after | +| ---------- | ---------------: | ----------------: | --------------: | ---------------: | -------------------------------: | +| java | 32.60 | 15.46 | 60.19 | 32.73 | 63 → 62 | +| typescript | 37.75 | 19.20 | 64.63 | 36.29 | 59 → 58 | +| kotlin | 47.40 | 28.13 | 75.01 | 49.54 | 105 → 101 | +| csharp | 60.80 | 30.37 | 92.36 | 52.97 | 181 → 180 | +| python | 89.85 | 44.75 | 152.75 | 84.62 | 67 → 57 | +| php | 27.56 | 17.37 | 43.57 | 27.17 | 63 → 62 | +| swift | 85.46 | 48.28 | 98.17 | 64.23 | 269 → 246 | +| dart | 77.63 | 34.35 | 102.06 | 55.27 | 50 → 66 | + +The Dart wrapper-to-inner-declaration lookup adds root-node accesses while avoiding the second parameter traversal. Root access counts do not count descendant reads, `tree`, or `id`; they are not total AST traversal counts. Scope-specific transformations still run independently. + +A separate real-worker trace of the repository corpus, aggregated across cold, +warm, and changed-file analysis, recorded **1,391 parameter requests** in both +revisions. They produced **1,391 distinct arrays before and 784 after** (43.6% +fewer). Of 784 declarations, 607 were requested twice and 177 arrow functions were +requested once. Those single requests still incur cache bookkeeping. The observed +migrated callbacks in this corpus were TypeScript; cold analysis parsed 98 of its +142 manifest files. Graph hashes still matched. These trace runs overlapped the +test suite, so their timing and memory values are excluded from the tables below. +The optional preload in [README.md](README.md) reproduces these counters. + +## End-to-end analysis + +Three paired trials per corpus alternated revision order. Each process ran cold analysis, warm parse-cache replay, and a changed-file analysis. Wall time includes persistence. All six pairs matched structural graph hashes and node/edge counts in every scenario. Cold/warm graphs matched; changed signatures changed the graph. Cold and changed runs used real parser workers; warm runs replayed cache entries without workers. + +| Corpus | Files | Content manifest SHA-256 | +| ------------ | ----: | ------------------------------------------------------------------ | +| declarations | 65 | `a8f8aa6206812f6bdcd24fce9d89dcf25d14eaf7b1c86ba49aea1ca64f729e40` | +| repository | 142 | `ccafcd91d22aaccd886e978717499148823ca804132d64956f7f1ca9e9df1277` | + +Wall times are median [minimum–maximum], milliseconds. + +| Corpus / scenario | Baseline | Candidate | Median change | +| ---------------------- | ------------------: | ------------------: | ------------: | +| declarations / cold | 6621 [6485–7814] | 7616 [6773–8238] | +15.0% | +| declarations / warm | 2939 [2754–3693] | 3027 [2828–3125] | +3.0% | +| declarations / changed | 3565 [3564–4160] | 3721 [3343–4708] | +4.4% | +| repository / cold | 13737 [11731–15283] | 16039 [10623–18116] | +16.8% | +| repository / warm | 6829 [5838–7794] | 7580 [5162–10662] | +11.0% | +| repository / changed | 7556 [6348–7656] | 8035 [5832–9154] | +6.3% | + +Additional medians, shown baseline → candidate. RSS is the sampled peak, CPU includes workers, and GC measures only the main isolate. Parsing is a progress-callback interval rather than an exclusive profile. + +| Corpus / scenario | CPU ms | RSS MiB | Main GC ms | Parsing interval ms | +| ---------------------- | ----------------: | ------------: | ------------: | ------------------: | +| declarations / cold | 7367.8 → 8149.3 | 530.5 → 558.7 | 131.4 → 170.2 | 3286.3 → 3211.9 | +| declarations / warm | 2931.8 → 2876.1 | 636.6 → 651.3 | 195.1 → 198.4 | 175.6 → 154.4 | +| declarations / changed | 3397.3 → 3522.6 | 708.9 → 713.0 | 121.3 → 120.9 | 1008.0 → 1060.6 | +| repository / cold | 17199.8 → 20591.9 | 759.5 → 753.4 | 229.3 → 306.4 | 6954.7 → 8205.3 | +| repository / warm | 7137.5 → 7288.9 | 883.5 → 873.4 | 248.3 → 268.6 | 299.7 → 315.3 | +| repository / changed | 7698.1 → 8652.0 | 951.2 → 954.2 | 253.2 → 261.4 | 1202.5 → 1242.8 | + +The final end-to-end medians were slower. An earlier complete six-pair run, before the narrow Python/Dart wrapper identity correction, showed cold median changes of +1.7% for declarations and −4.4% for repository sources. The final run also shifted warm-cache timings, where the extraction cache is not used. These observations and the wide overlapping ranges leave the overall indexing effect unresolved; they do **not** establish a speedup or rule out a regression. Repeat on an isolated host and larger representative corpora before claiming an end-to-end gain. + +The isolated measurement does establish lower repeated parameter-extraction cost. It does not account for workload-dependent cache bookkeeping across every declaration shape. Retention remained bounded in the forced-GC probe: **100/100 discarded trees were collected in both revisions**. Post-GC main heap was 6.87 MiB before and 6.88 MiB after. Warm reparse counts were zero; changed-file runs reparsed one file. The RSS sample and small lifetime probe cannot establish a general peak-memory improvement. + +Reproduce with the command and limitations in [README.md](README.md). The harness emits per-trial JSON, including all phase intervals, CPU, GC, sampled/cumulative RSS, cache counts, corpus manifests, graph hashes, and parameter-only timings. + +## Mixed-provider worker parity + +Three additional paired runs used `--providers-only`: eight provider files plus +the changed-file probe. Every run asserted that each provider emitted its sample +callable and a resolved call to it. The fixtures include Java records, C# primary +constructors, Kotlin companions, overloads, Python decorators, PHP variadics, and +Swift/Dart constructors. Swift's existing `Function` classification is preserved. + +All pairs matched **47 structural nodes and 46 edges** in every scenario. Cold +runs parsed nine files; warm runs parsed zero; changed runs parsed one. The cold +and warm graph hash was +`131f1043fb43b9635228595748bd1a15d853b10f86ed40902303612918b15169`; +the changed graph hash was +`93734f2c8ea9ba0d8841e269467dc49cd2289be83aa18b13a8cebe871a6106ab`. +The corpus manifest was +`0a75726a1e88b189f63675a9e12476ca3d928d1a25ed4abf9648efbd0c9b339f`. +These extra runs overlapped the full test suite and provide correctness evidence; +their timing and memory measurements are excluded from the performance tables. diff --git a/gitnexus/bench/method-declaration-facts/baseline.json b/gitnexus/bench/method-declaration-facts/baseline.json new file mode 100644 index 000000000..5f6e0f671 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/baseline.json @@ -0,0 +1,541 @@ +{ + "_note": "Reviewed semantic and allocation baseline. --check compares fingerprints, cache counts and shared-array counts and gates same-run normalized time/memory growth at 1.5. Absolute timing and heap/RSS medians are references. CI never records a baseline.", + "environment": { + "node": "v22.23.3", + "platform": "linux", + "cpu": "12th Gen Intel(R) Core(TM) i9-12900K", + "workers": 1 + }, + "correctness": { + "counts": [ + { + "language": "java", + "facts": "f1acdfa027c072521b69ca5b9b694ebb3a528e45f5383edffe0b9d4adfbe70d0", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "typescript", + "facts": "51290686f9885e3ecc18c6de969b53a825d8677b3ffaeb82e752e1d996b3d56e", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "kotlin", + "facts": "32250ca227cb267e75efd4856184dd96e17ec08c23cdbdc61b8301b602efe071", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "csharp", + "facts": "e7158eb25c8b100ec6d835b32ca28df16246dbc32ac9a202e202e4bfc865ef30", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "python", + "facts": "78ba377afae0198395d638d5ff9aa3a7c44504666184003b099e2709fe544a4d", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "php", + "facts": "d23a4178521e676a4d267ff365dd6de11d9b5df8b0880ee380acadec5933158b", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "swift", + "facts": "39ef708f6d9c3515fa0c95ce79b719e2c3ef7638fa5a9c59a1254a8766568c02", + "parameterArrays": 1, + "parameterTypeArrays": 1 + }, + { + "language": "dart", + "facts": "2379fc6fb6ce0ba092304790eebc1e36030e939c16bfe51f0aabca6112590f07", + "parameterArrays": 1, + "parameterTypeArrays": 1 + } + ], + "analysis": { + "declarations-8": { + "manifest": { + "files": 17, + "manifest": "353a6948d690eaba364c8d31df4937d1b969e855ccb7bed1ded1c95b6a528847" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 691, + "edges": 690, + "hash": "c2918b76eea971e5d925afd2e912c53ce372da6abfd38ba4adcccade22c07ef9" + }, + "reparsed": 17, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 691, + "edges": 690, + "hash": "c2918b76eea971e5d925afd2e912c53ce372da6abfd38ba4adcccade22c07ef9" + }, + "reparsed": 0, + "hits": 17 + }, + { + "scenario": "changed", + "facts": { + "nodes": 691, + "edges": 690, + "hash": "e220495c5c40fba7820c452b44c6ced2302981e60a7428eb4bb73ec628d7daa8" + }, + "reparsed": 1, + "hits": 16 + } + ] + }, + "declarations": { + "manifest": { + "files": 65, + "manifest": "a8f8aa6206812f6bdcd24fce9d89dcf25d14eaf7b1c86ba49aea1ca64f729e40" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 2755, + "edges": 2754, + "hash": "f9f0363efa933fcd0fdfbb64076e43bd7eb361f880251fda2de6dac1f716a0fa" + }, + "reparsed": 65, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 2755, + "edges": 2754, + "hash": "f9f0363efa933fcd0fdfbb64076e43bd7eb361f880251fda2de6dac1f716a0fa" + }, + "reparsed": 0, + "hits": 65 + }, + { + "scenario": "changed", + "facts": { + "nodes": 2755, + "edges": 2754, + "hash": "58969c80be830cff4ba91511587aec077106936e7abe80d95f77ad284be2fcd4" + }, + "reparsed": 1, + "hits": 64 + } + ] + }, + "declarations-128": { + "manifest": { + "files": 257, + "manifest": "b9625a8c2fe98276a16847c4d77f9e4c46cd696a61d555440d7840faf0107780" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 11011, + "edges": 11010, + "hash": "11fbfb9fdf42fb9ac78c11e359fb9d7a6aea448887a65650b9d54e039f3e73a2" + }, + "reparsed": 257, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 11011, + "edges": 11010, + "hash": "11fbfb9fdf42fb9ac78c11e359fb9d7a6aea448887a65650b9d54e039f3e73a2" + }, + "reparsed": 0, + "hits": 257 + }, + { + "scenario": "changed", + "facts": { + "nodes": 11011, + "edges": 11010, + "hash": "88a8baaa628aa037c02c83b6d912ef9ca206fd63c035a284dc55a9c3f1ed4903" + }, + "reparsed": 1, + "hits": 256 + } + ] + }, + "concentrated-320": { + "manifest": { + "files": 3, + "manifest": "b2bcb40fa14120ac4ada96e78569ffa2413250ea3009dbf39b7d4d4c1bb533da" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 649, + "edges": 648, + "hash": "4606ff07912fef5a01f2b057e6b9d9bc5df3cc3a40295fd558fca8d45df9527e" + }, + "reparsed": 3, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 649, + "edges": 648, + "hash": "4606ff07912fef5a01f2b057e6b9d9bc5df3cc3a40295fd558fca8d45df9527e" + }, + "reparsed": 0, + "hits": 3 + }, + { + "scenario": "changed", + "facts": { + "nodes": 649, + "edges": 648, + "hash": "562dc4337b5840e0a64c1b3f4d4f90cc1f2fd75a49ccb23a50b7c6adcbf3fd07" + }, + "reparsed": 1, + "hits": 2 + } + ] + }, + "concentrated-1280": { + "manifest": { + "files": 3, + "manifest": "59d63e1f0a4b92388b4b9c9008e90338d50839ebc7c8779c86169f1adac57a06" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 2569, + "edges": 2568, + "hash": "ad17f7f24a6948d140ccfdca0230f9562a3e44717a5931a29ae62af504899041" + }, + "reparsed": 3, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 2569, + "edges": 2568, + "hash": "ad17f7f24a6948d140ccfdca0230f9562a3e44717a5931a29ae62af504899041" + }, + "reparsed": 0, + "hits": 3 + }, + { + "scenario": "changed", + "facts": { + "nodes": 2569, + "edges": 2568, + "hash": "90625066c2e3ef78881ef820b573f5a1201cb5b50b96a3a5d9bb4a6572dbeb5a" + }, + "reparsed": 1, + "hits": 2 + } + ] + }, + "concentrated-5120": { + "manifest": { + "files": 3, + "manifest": "8b2fb11e9b2a4ba1c124c1efdb22e55db91b8595e1a519f55da75e94af0a5ff1" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 10249, + "edges": 10248, + "hash": "f1f7ef339c2ba2f15f1c38fc9c7caffc0d5d92a222059cc21352a17fb30a2879" + }, + "reparsed": 3, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 10249, + "edges": 10248, + "hash": "f1f7ef339c2ba2f15f1c38fc9c7caffc0d5d92a222059cc21352a17fb30a2879" + }, + "reparsed": 0, + "hits": 3 + }, + { + "scenario": "changed", + "facts": { + "nodes": 10249, + "edges": 10248, + "hash": "4b521b01aef30d967ea1ce113f8021a1f2c30319ef9102f303986c77390b4c27" + }, + "reparsed": 1, + "hits": 2 + } + ] + }, + "providers": { + "manifest": { + "files": 9, + "manifest": "0a75726a1e88b189f63675a9e12476ca3d928d1a25ed4abf9648efbd0c9b339f" + }, + "runs": [ + { + "scenario": "cold", + "facts": { + "nodes": 47, + "edges": 46, + "hash": "131f1043fb43b9635228595748bd1a15d853b10f86ed40902303612918b15169" + }, + "reparsed": 9, + "hits": 0 + }, + { + "scenario": "warm", + "facts": { + "nodes": 47, + "edges": 46, + "hash": "131f1043fb43b9635228595748bd1a15d853b10f86ed40902303612918b15169" + }, + "reparsed": 0, + "hits": 9 + }, + { + "scenario": "changed", + "facts": { + "nodes": 47, + "edges": 46, + "hash": "93734f2c8ea9ba0d8841e269467dc49cd2289be83aa18b13a8cebe871a6106ab" + }, + "reparsed": 1, + "hits": 8 + } + ] + } + }, + "memory": { + "java-2500": { + "digest": "5d72a942d521e860c36563aceb57cc32d17857273a18d0d428b4266bd617e06f", + "declarations": 2500 + }, + "java": { + "digest": "44e5fc1e221e02a9644075284d18b3936d3afc36b1ff696341f7ab5a4a5b15b0", + "declarations": 10000 + }, + "java-40000": { + "digest": "bb0f0b9a3281108e1aa910478e8f9171f0e375e96bf527e7020f46bb5e04cba3", + "declarations": 40000 + }, + "typescript-2500": { + "digest": "14e9b13d31e8874849cfd64f2367716b09ed9d6577912f75318f358bdc8c741d", + "declarations": 2500 + }, + "typescript": { + "digest": "9e1ae4a2165a63a3f89aa97d474e799ffc18d09cb47578b5a112a7ed75c54e40", + "declarations": 10000 + }, + "typescript-40000": { + "digest": "632331d244854ab84e4974e4ea7ef669460e3acc5e97e933cb822ca0e39844c2", + "declarations": 40000 + } + }, + "collectedTrees": 100 + }, + "measured": { + "declarations-8/cold": { + "wallMs": 4598.0965830000005, + "cpuMs": 5028.881, + "heapUsedMiB": 69.43971252441406, + "sampledPeakRssMiB": 552.4375, + "cumulativeMaxRssMiB": 552.4375 + }, + "declarations-8/warm": { + "wallMs": 1469.5753610000002, + "cpuMs": 1310.647, + "heapUsedMiB": 100.86992645263672, + "sampledPeakRssMiB": 639.1484375, + "cumulativeMaxRssMiB": 639.1484375 + }, + "declarations-8/changed": { + "wallMs": 2224.902616000001, + "cpuMs": 2158.139, + "heapUsedMiB": 70.97301483154297, + "sampledPeakRssMiB": 691.4453125, + "cumulativeMaxRssMiB": 691.4453125 + }, + "declarations/cold": { + "wallMs": 6287.4714890000005, + "cpuMs": 7338.656, + "heapUsedMiB": 102.16807556152344, + "sampledPeakRssMiB": 568.0625, + "cumulativeMaxRssMiB": 574.22265625 + }, + "declarations/warm": { + "wallMs": 1898.1291850000007, + "cpuMs": 2115.823, + "heapUsedMiB": 107.16117858886719, + "sampledPeakRssMiB": 672.890625, + "cumulativeMaxRssMiB": 679.06640625 + }, + "declarations/changed": { + "wallMs": 2573.0706090000003, + "cpuMs": 2741.011, + "heapUsedMiB": 101.39332580566406, + "sampledPeakRssMiB": 741.0625, + "cumulativeMaxRssMiB": 747.2890625 + }, + "declarations-128/cold": { + "wallMs": 11950.934368, + "cpuMs": 15088.994, + "heapUsedMiB": 96.21513366699219, + "sampledPeakRssMiB": 642.6796875, + "cumulativeMaxRssMiB": 642.8671875 + }, + "declarations-128/warm": { + "wallMs": 3071.7164410000005, + "cpuMs": 3799.132, + "heapUsedMiB": 151.96864318847656, + "sampledPeakRssMiB": 752.5546875, + "cumulativeMaxRssMiB": 758.421875 + }, + "declarations-128/changed": { + "wallMs": 3785.085203999999, + "cpuMs": 4623.536, + "heapUsedMiB": 165.27832794189453, + "sampledPeakRssMiB": 825.6484375, + "cumulativeMaxRssMiB": 832.015625 + }, + "concentrated-320/cold": { + "wallMs": 4694.445621, + "cpuMs": 4989.052, + "heapUsedMiB": 67.74372100830078, + "sampledPeakRssMiB": 543.921875, + "cumulativeMaxRssMiB": 543.921875 + }, + "concentrated-320/warm": { + "wallMs": 1478.7918250000002, + "cpuMs": 1216.909, + "heapUsedMiB": 98.2383804321289, + "sampledPeakRssMiB": 634.32421875, + "cumulativeMaxRssMiB": 634.32421875 + }, + "concentrated-320/changed": { + "wallMs": 2217.1457689999997, + "cpuMs": 2067.268, + "heapUsedMiB": 69.39957427978516, + "sampledPeakRssMiB": 703.64453125, + "cumulativeMaxRssMiB": 703.64453125 + }, + "concentrated-1280/cold": { + "wallMs": 8088.226249, + "cpuMs": 9146.62, + "heapUsedMiB": 110.2262191772461, + "sampledPeakRssMiB": 601.5078125, + "cumulativeMaxRssMiB": 602.08203125 + }, + "concentrated-1280/warm": { + "wallMs": 1780.5142610000003, + "cpuMs": 1871.551, + "heapUsedMiB": 116.76529693603516, + "sampledPeakRssMiB": 714.1328125, + "cumulativeMaxRssMiB": 714.75390625 + }, + "concentrated-1280/changed": { + "wallMs": 2454.9453699999995, + "cpuMs": 2596.301, + "heapUsedMiB": 117.36157989501953, + "sampledPeakRssMiB": 772.80078125, + "cumulativeMaxRssMiB": 772.80078125 + }, + "concentrated-5120/cold": { + "wallMs": 19038.470589, + "cpuMs": 21988.894, + "heapUsedMiB": 87.05318450927734, + "sampledPeakRssMiB": 806.7421875, + "cumulativeMaxRssMiB": 807.34765625 + }, + "concentrated-5120/warm": { + "wallMs": 2599.4626009999993, + "cpuMs": 3164.342, + "heapUsedMiB": 196.42921447753906, + "sampledPeakRssMiB": 931.328125, + "cumulativeMaxRssMiB": 931.328125 + }, + "concentrated-5120/changed": { + "wallMs": 3300.2307550000005, + "cpuMs": 3837.783, + "heapUsedMiB": 198.03103637695312, + "sampledPeakRssMiB": 1009.9921875, + "cumulativeMaxRssMiB": 1009.9921875 + }, + "providers/cold": { + "wallMs": 5234.342168, + "cpuMs": 5416.471, + "heapUsedMiB": 64.36933135986328, + "sampledPeakRssMiB": 581.26953125, + "cumulativeMaxRssMiB": 587.6796875 + }, + "providers/warm": { + "wallMs": 1649.076336, + "cpuMs": 1537.857, + "heapUsedMiB": 80.01580810546875, + "sampledPeakRssMiB": 682.64453125, + "cumulativeMaxRssMiB": 688.83984375 + }, + "providers/changed": { + "wallMs": 2417.3744080000006, + "cpuMs": 2417.786, + "heapUsedMiB": 65.51468658447266, + "sampledPeakRssMiB": 764.375, + "cumulativeMaxRssMiB": 770.55859375 + }, + "java-2500/retained": { + "extractionMs": 97.82188899999998, + "extractionRetainedBytes": 3176480, + "emittedRetainedBytes": 1596952, + "releasedDeltaBytes": 332792 + }, + "java/retained": { + "extractionMs": 378.996999, + "extractionRetainedBytes": 12057136, + "emittedRetainedBytes": 5701312, + "releasedDeltaBytes": 660560 + }, + "java-40000/retained": { + "extractionMs": 1499.86337, + "extractionRetainedBytes": 47273448, + "emittedRetainedBytes": 21694640, + "releasedDeltaBytes": 1447192 + }, + "typescript-2500/retained": { + "extractionMs": 116.37295999999999, + "extractionRetainedBytes": 3198712, + "emittedRetainedBytes": 1619784, + "releasedDeltaBytes": 355624 + }, + "typescript/retained": { + "extractionMs": 472.583266, + "extractionRetainedBytes": 12017024, + "emittedRetainedBytes": 5662008, + "releasedDeltaBytes": 621256 + }, + "typescript-40000/retained": { + "extractionMs": 1857.2782389999998, + "extractionRetainedBytes": 47241792, + "emittedRetainedBytes": 21663784, + "releasedDeltaBytes": 1416336 + } + } +} diff --git a/gitnexus/bench/method-declaration-facts/measure.mjs b/gitnexus/bench/method-declaration-facts/measure.mjs new file mode 100644 index 000000000..efae55061 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/measure.mjs @@ -0,0 +1,799 @@ +#!/usr/bin/env node +// Build first, then use --check; paired diagnostics accept --baseline-root. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { cp, mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { performance, PerformanceObserver } from 'node:perf_hooks'; +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { evaluateScaling } from './scaling.mjs'; + +const here = fileURLToPath(import.meta.url); +const candidate = path.resolve(path.dirname(here), '../..'); +const args = process.argv.slice(2); +const option = (name, fallback) => (args.includes(name) ? args[args.indexOf(name) + 1] : fallback); +const root = path.resolve(option('--root', candidate)); +const load = (relative) => import(pathToFileURL(path.join(root, 'dist', relative))); +const digest = (value) => createHash('sha256').update(value).digest('hex'); +const stable = (value) => + JSON.stringify(value, (_, v) => + v && !Array.isArray(v) && typeof v === 'object' + ? Object.fromEntries( + Object.keys(v) + .sort() + .map((key) => [key, v[key]]), + ) + : v, + ); + +const providerFixtures = { + 'J.java': + 'record J(int value) { int f(int x) { return x; } String f(String x) { return x; } int call() { return f(1); } int fromValue() { return f(value); } }', + 'T.ts': + 'export class T { f(x: number): number; f(x: string): string; f(x: number | string) { return x; } call() { return this.f(1); } }', + 'K.kt': + 'class K(val value: Int) {\n fun f(x: Int): Int { return x }\n fun call(): Int { return f(value) }\n companion object { fun build(): K { return K(1) } }\n}', + 'C.cs': + 'public class C(int value) { public int f(int x) => x; public string f(string x) => x; public int Call() => f(1); public int FromValue() => f(value); }', + 'P.py': + 'class P:\n @staticmethod\n def f(x: int) -> int:\n return x\n def call(self) -> int:\n return P.f(1)\n', + 'Ph.php': + 'f(1, "a"); } }', + 'S.swift': + 'class S { init() {}\n func f(_ x: Int, y: String = "") -> Int { return x }\n func call() -> Int { return f(1) }\n}', + 'D.dart': + "class D { D(int value) {} int f(int x, [String y = '']) => x; int call() => f(1); int get value => 1; }", +}; + +async function countFacts() { + const require = createRequire(path.join(root, 'package.json')); + const Parser = require('tree-sitter'); + const { getLanguageGrammar } = await load('core/tree-sitter/parser-loader.js'); + const { createMethodExtractor } = await load('core/ingestion/method-extractors/generic.js'); + const { buildMethodProps } = await load('core/ingestion/utils/method-props.js'); + const specs = [ + [ + 'java', + 'jvm', + 'java', + 'method_declaration', + 'class C { int f(int x, String y) { return x; } }', + ], + [ + 'typescript', + 'typescript-javascript', + 'typescript', + 'method_definition', + 'class C { f(x: number, y: string): number { return x; } }', + ], + [ + 'kotlin', + 'jvm', + 'kotlin', + 'function_declaration', + 'class C {\n fun f(x: Int, y: String): Int { return x }\n}', + ], + [ + 'csharp', + 'csharp', + 'csharp', + 'method_declaration', + 'class C { int f(int x, string y) { return x; } }', + ], + [ + 'python', + 'python', + 'python', + 'function_definition', + 'class C:\n def f(self, x: int, y: str) -> int:\n return x\n', + ], + [ + 'php', + 'php', + 'php', + 'method_declaration', + ' Int { return x } }', + ], + [ + 'dart', + 'dart', + 'dart', + 'method_signature', + 'class C { int f(int x, String y) { return x; } }', + ], + ]; + const result = []; + for (const [language, module, prefix, nodeType, source] of specs) { + const grammar = getLanguageGrammar(language); + if (!grammar) { + result.push({ language, skipped: 'grammar unavailable' }); + continue; + } + const config = (await load(`core/ingestion/method-extractors/configs/${module}.js`))[ + `${prefix}MethodConfig` + ]; + const arityModule = await load(`core/ingestion/languages/${language}/arity-metadata.js`); + const arity = Object.values(arityModule).find((value) => typeof value === 'function'); + const parser = new Parser(); + parser.setLanguage(grammar); + const node = parser.parse(source).rootNode.descendantsOfType(nodeType)[0]; + let nativeReads = 0; + const counted = new Proxy(node, { + get(target, property) { + if (property !== 'tree' && property !== 'id') nativeReads++; + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const parameterResults = new Set(); + const original = config.extractParameters; + config.extractParameters = (...values) => { + const parameters = original(...values); + parameterResults.add(parameters); + return parameters; + }; + try { + const scope = arity(counted); + const method = createMethodExtractor(config).extractFromNode(counted, { + language, + filePath: 'fixture', + }); + const graph = buildMethodProps(method); + result.push({ + language, + nativeReads, + parameterArrays: parameterResults.size, + parameterTypeArrays: new Set([scope.parameterTypes, graph.parameterTypes]).size, + facts: digest(stable({ scope, method })), + }); + } finally { + config.extractParameters = original; + } + const timingSource = + language === 'php' + ? ' { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar('java')); + return Array.from({ length: 100 }, () => { + const tree = parser.parse('class C { int f(int x) { return x; } }'); + const node = tree.rootNode.descendantsOfType('method_declaration')[0]; + javaMethodConfig.extractParameters(node); + javaMethodConfig.extractParameters(node); + assert.equal(javaExportChecker(node.childForFieldName('name'), 'f'), false); + return new WeakRef(tree); + }); + })(); + // WeakRef targets survive the current job; collect only after yielding it. + for (let i = 0; i < 5; i++) { + await new Promise((resolve) => setImmediate(resolve)); + global.gc(); + } + return { + created: refs.length, + collected: refs.filter((ref) => ref.deref() === undefined).length, + heapAfterGcMiB: process.memoryUsage().heapUsed / 1048576, + }; +} + +async function makeCorpus(repo, corpus) { + await mkdir(repo, { recursive: true }); + await writeFile(path.join(repo, '.gitignore'), '.gitnexus/\n'); + if (corpus === 'providers') { + for (const [file, source] of Object.entries(providerFixtures)) { + await writeFile(path.join(repo, file), source + '\n'); + } + } else if (corpus === 'declarations') { + const files = Number(option('--files', '32')); + const methods = Number(option('--methods', '40')); + assert(Number.isSafeInteger(files) && files > 0); + assert(Number.isSafeInteger(methods) && methods > 0); + const ts = Array.from( + { length: methods }, + (_, i) => `m${i}(a: number, b: string, c: Array, d = 0): number { return a + d; }`, + ).join('\n'); + const java = Array.from( + { length: methods }, + (_, i) => `int m${i}(int a, String b, List c, int d) { return a + d; }`, + ).join('\n'); + for (let file = 0; file < files; file++) { + await writeFile( + path.join(repo, `T${file}.ts`), + `export class T${file} {\n${ts}\n call() { return this.m0(1, "", [], 0); }\n}\n`, + ); + await writeFile( + path.join(repo, `J${file}.java`), + `class J${file} {\n${java}\n int call() { return m0(1, "", null, 0); }\n}\n`, + ); + } + } else { + // The same tracked source snapshot feeds both revisions. + const paths = execFileSync( + 'git', + [ + 'ls-files', + 'src/core/ingestion/method-extractors', + 'src/core/ingestion/languages', + 'test/fixtures/lang-resolution', + ], + { cwd: candidate, encoding: 'utf8' }, + ) + .trim() + .split('\n'); + const sources = paths + .filter((p) => p.startsWith('src/') && /\.ts$/.test(p)) + .sort() + .slice(0, 96); + const fixtures = paths.filter((p) => + /(?:same-arity|primary-constructor|record-component|variadic)/.test(p), + ); + for (const file of [...sources, ...fixtures]) { + await mkdir(path.dirname(path.join(repo, file)), { recursive: true }); + await cp(path.join(candidate, file), path.join(repo, file)); + } + } + await writeFile( + path.join(repo, 'change.ts'), + 'export class Changed { method(value: number): number { return value; } }\n', + ); + const git = (...values) => execFileSync('git', values, { cwd: repo, stdio: 'pipe' }); + git('init', '-q'); + git('add', '-A'); + git( + '-c', + 'user.name=benchmark', + '-c', + 'user.email=benchmark@example.invalid', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'fixture', + ); + const files = git('ls-files').toString().trim().split('\n'); + const contents = await Promise.all( + files.map(async (f) => [f, digest(await readFile(path.join(repo, f)))]), + ); + return { files: files.length - 1, manifest: digest(stable(contents)) }; +} + +function graphFacts(graph) { + const nodes = []; + graph.forEachNode((node) => { + if (!['Community', 'Process'].includes(node.label)) nodes.push(stable(node)); + }); + const edges = [...graph.relationships] + .filter((edge) => !['MEMBER_OF', 'STEP_IN_PROCESS'].includes(edge.type)) + .map(stable); + return { + nodes: nodes.length, + edges: edges.length, + hash: digest(stable([nodes.sort(), edges.sort()])), + }; +} + +async function measureAnalysis(corpus) { + const temp = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-callable-facts-')); + Object.assign(process.env, { + GITNEXUS_HOME: path.join(temp, 'home'), + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_LBUG_BUFFER_POOL_SIZE: '134217728', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + GITNEXUS_MEMORY: 'off', + }); + const repo = path.join(temp, 'repo'); + try { + const manifest = await makeCorpus(repo, corpus); + const { runFullAnalysis } = await load('core/run-analyze.js'); + const runs = []; + for (const scenario of ['cold', 'warm', 'changed']) { + if (scenario === 'changed') + await writeFile( + path.join(repo, 'change.ts'), + 'export class Changed { method(value: string): string { return value; } }\n', + ); + global.gc?.(); + let peakRss = process.memoryUsage().rss; + let gcMs = 0; + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) gcMs += entry.duration; + }); + observer.observe({ entryTypes: ['gc'] }); + const sampler = setInterval(() => { + peakRss = Math.max(peakRss, process.memoryUsage().rss); + }, 20); + const phases = {}; + let previousPhase = 'startup', + previousTime = performance.now(); + const onProgress = (phase) => { + if (phase !== previousPhase) console.error(`${scenario}: ${phase}`); + const now = performance.now(); + phases[previousPhase] = (phases[previousPhase] ?? 0) + now - previousTime; + previousPhase = phase; + previousTime = now; + }; + const cpu = process.cpuUsage(), + start = performance.now(); + let analysis; + try { + analysis = await runFullAnalysis( + repo, + { + force: true, + skipAgentsMd: true, + skipSkills: true, + skipFts: true, + streamGraphEmit: false, + workerPoolSize: 1, + }, + { onProgress, onLog: (message) => console.error(message) }, + ); + } finally { + clearInterval(sampler); + } + const wallMs = performance.now() - start, + used = process.cpuUsage(cpu); + onProgress('done'); + await new Promise((resolve) => setImmediate(resolve)); + observer.disconnect(); + const pipeline = analysis.pipelineResult; + assert(pipeline, 'analysis must return its graph'); + if (scenario !== 'warm') + assert(pipeline.usedWorkerPool, 'cache misses must use real parser workers'); + if (scenario === 'warm') assert.equal(pipeline.reparsedFileCount, 0); + if (scenario === 'changed') assert(pipeline.reparsedFileCount > 0); + if (corpus === 'declarations') { + const files = Number(option('--files', '32')); + const methods = Number(option('--methods', '40')); + let declarations = 0; + pipeline.graph.forEachNode((node) => { + if (node.label === 'Method' && /^m\d+$/.test(node.properties.name)) declarations++; + }); + assert.equal(declarations, 2 * files * methods, 'Scaling workload lost declarations'); + assert.equal(pipeline.reparsedFileCount + pipeline.parseCacheHitFileCount, 2 * files + 1); + } + if (corpus === 'providers') { + if (scenario === 'cold') { + assert.equal(pipeline.reparsedFileCount, Object.keys(providerFixtures).length + 1); + } + const methods = new Set(); + pipeline.graph.forEachNode((node) => { + if (['Method', 'Function'].includes(node.label) && node.properties.name === 'f') { + methods.add(node.properties.filePath); + } + }); + assert.deepEqual([...methods].sort(), Object.keys(providerFixtures).sort()); + const calledFiles = new Set(); + for (const edge of pipeline.graph.iterRelationshipsByType('CALLS')) { + const target = pipeline.graph.getNode(edge.targetId); + if (target?.properties.name === 'f') calledFiles.add(target.properties.filePath); + } + assert.deepEqual([...calledFiles].sort(), Object.keys(providerFixtures).sort()); + } + const facts = graphFacts(pipeline.graph); + if (scenario === 'warm') + assert.equal(facts.hash, runs[0].facts.hash, 'warm replay changed graph facts'); + if (scenario === 'changed') + assert.notEqual(facts.hash, runs[0].facts.hash, 'edited signature retained stale facts'); + runs.push({ + scenario, + wallMs, + cpuMs: (used.user + used.system) / 1000, + phases, + gcMs, + sampledPeakRssMiB: Math.max(peakRss, process.memoryUsage().rss) / 1048576, + cumulativeMaxRssMiB: process.resourceUsage().maxRSS / 1024, + heapUsedMiB: process.memoryUsage().heapUsed / 1048576, + reparsed: pipeline.reparsedFileCount, + hits: pipeline.parseCacheHitFileCount, + facts, + }); + } + return { manifest, runs }; + } finally { + await rm(temp, { recursive: true, force: true }); + } +} + +async function child(revision, mode, corpus, extraArgs = []) { + const cache = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-callable-identity-')); + try { + const memory = mode === 'memory'; + const output = execFileSync( + process.execPath, + memory + ? [ + '--expose-gc', + fileURLToPath(new URL('./memory.mjs', import.meta.url)), + '--root', + revision, + '--language', + corpus, + ...extraArgs, + ] + : [ + '--expose-gc', + here, + '--child', + mode, + '--root', + revision, + '--corpus', + corpus ?? '', + ...extraArgs, + ], + { + encoding: 'utf8', + timeout: 600_000, + maxBuffer: 32 * 1024 * 1024, + env: { ...process.env, GITNEXUS_MEMORY: 'off', XDG_RUNTIME_DIR: cache }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + if (memory) return JSON.parse(output); + const line = output.split('\n').find((line) => line.startsWith('BENCH_RESULT=')); + assert(line, 'child did not return measurements'); + return JSON.parse(line.slice('BENCH_RESULT='.length)); + } finally { + await rm(cache, { recursive: true, force: true }); + } +} + +async function checkBenchmark() { + const record = args.includes('--record'); + assert(!(record && args.includes('--check')), '--check cannot record a baseline'); + assert( + !['--baseline-root', '--candidate-root', '--providers-only'].some((flag) => + args.includes(flag), + ), + 'CI modes use the fixed declaration and provider corpora', + ); + const baselinePath = new URL('./baseline.json', import.meta.url); + const baseline = record ? undefined : JSON.parse(await readFile(baselinePath, 'utf8')); + const samples = Number(option('--samples', '3')); + assert(Number.isInteger(samples) && samples >= 3, '--samples must be at least 3'); + const output = option('--output'); + assert( + !output || path.resolve(output) !== fileURLToPath(baselinePath), + 'Output must not overwrite baseline.json', + ); + const outputFile = output ? await open(output, 'wx', 0o600) : undefined; + const report = { + passed: false, + environment: { + node: process.version, + platform: process.platform, + cpu: os.cpus()[0]?.model, + workers: 1, + }, + samples, + counts: [], + lifetime: {}, + trials: {}, + memory: {}, + measured: {}, + scaling: {}, + }; + const median = (rows, key) => { + assert( + rows.every( + (row) => Number.isFinite(row[key]) && (key === 'releasedDeltaBytes' || row[key] > 0), + ), + `Invalid ${key} sample`, + ); + return rows.map((row) => row[key]).sort((a, b) => a - b)[Math.floor(rows.length / 2)]; + }; + const summarize = (rows, keys) => Object.fromEntries(keys.map((key) => [key, median(rows, key)])); + try { + report.counts = await child(root, 'counts'); + assert.equal(report.counts.length, 8); + assert( + report.counts.every((row) => !row.skipped), + 'All eight provider grammars are required', + ); + report.lifetime = await child(root, 'lifetime'); + assert.equal( + report.lifetime.collected, + report.lifetime.created, + 'declaration cache retained trees', + ); + const correctness = { + counts: report.counts.map(({ language, facts, parameterArrays, parameterTypeArrays }) => ({ + language, + facts, + parameterArrays, + parameterTypeArrays, + })), + analysis: {}, + memory: {}, + collectedTrees: report.lifetime.collected, + }; + const workloads = [ + ...[8, 32, 128].map((files) => ({ + name: files === 32 ? 'declarations' : `declarations-${files}`, + shape: 'spread', + files, + methods: 40, + corpus: 'declarations', + })), + ...[320, 1280, 5120].map((methods) => ({ + name: `concentrated-${methods}`, + shape: 'concentrated', + files: 1, + methods, + corpus: 'declarations', + })), + { name: 'providers', corpus: 'providers' }, + ]; + report.workloads = workloads; + for (const { name } of workloads) report.trials[name] = []; + for (let trial = 0; trial < samples; trial++) { + for (const { name, corpus, files, methods } of trial % 2 + ? [...workloads].reverse() + : workloads) { + console.log(`${name} ${trial + 1}/${samples}`); + const extraArgs = + corpus === 'declarations' ? ['--files', String(files), '--methods', String(methods)] : []; + report.trials[name].push(await child(root, 'analysis', corpus, extraArgs)); + } + } + const analysisMetrics = [ + 'wallMs', + 'cpuMs', + 'heapUsedMiB', + 'sampledPeakRssMiB', + 'cumulativeMaxRssMiB', + ]; + for (const { name } of workloads) { + const trials = report.trials[name]; + const facts = (trial) => ({ + manifest: trial.manifest, + runs: trial.runs.map(({ scenario, facts, reparsed, hits }) => ({ + scenario, + facts, + reparsed, + hits, + })), + }); + correctness.analysis[name] = facts(trials[0]); + for (const trial of trials) { + assert.deepEqual( + facts(trial), + correctness.analysis[name], + `${name}: nondeterministic graph or cache replay`, + ); + assert.equal(trial.runs[2].reparsed, 1); + } + for (const scenario of ['cold', 'warm', 'changed']) { + report.measured[`${name}/${scenario}`] = summarize( + trials.map((trial) => trial.runs.find((run) => run.scenario === scenario)), + analysisMetrics, + ); + } + } + for (const shape of ['spread', 'concentrated']) { + for (const scenario of ['cold', 'warm', 'changed']) { + report.scaling[`${shape}/${scenario}`] = evaluateScaling( + workloads + .filter((row) => row.shape === shape) + .map(({ name, files, methods }) => ({ + size: 2 * files * methods, + ...report.measured[`${name}/${scenario}`], + })), + analysisMetrics, + ); + } + } + const memorySizes = [2500, 10000, 40000]; + const memoryMetrics = ['extractionMs', 'extractionRetainedBytes', 'emittedRetainedBytes']; + for (const language of ['java', 'typescript']) { + const nameFor = (size) => (size === 10000 ? language : `${language}-${size}`); + for (const size of memorySizes) report.memory[nameFor(size)] = []; + for (let trial = 0; trial < samples; trial++) { + for (const size of trial % 2 ? [...memorySizes].reverse() : memorySizes) { + console.log(`${language} retained ${size} ${trial + 1}/${samples}`); + report.memory[nameFor(size)].push( + await child(root, 'memory', language, ['--declarations', String(size)]), + ); + } + } + for (const size of memorySizes) { + const name = nameFor(size); + const trials = report.memory[name]; + assert( + trials.every((trial) => trial.treeCollected), + `${language}: heap probe retained trees`, + ); + assert.equal( + new Set(trials.map((trial) => trial.digest)).size, + 1, + `${language}: nondeterministic metadata`, + ); + assert( + trials.every((trial) => trial.declarations === size), + `${name}: wrong declaration count`, + ); + correctness.memory[name] = { + digest: trials[0].digest, + declarations: trials[0].declarations, + }; + report.measured[`${name}/retained`] = summarize(trials, [ + ...memoryMetrics, + 'releasedDeltaBytes', + ]); + } + report.scaling[`${language}/retained`] = evaluateScaling( + memorySizes.map((size) => ({ size, ...report.measured[`${nameFor(size)}/retained`] })), + memoryMetrics, + ); + } + report.correctness = correctness; + console.log(JSON.stringify({ scaling: report.scaling }, null, 2)); + assert( + Object.values(report.scaling).every((row) => row.passed), + 'Callable time or memory growth exceeds the 1.5 normalized scaling budget', + ); + if (baseline) + assert.deepEqual( + correctness, + baseline.correctness, + 'callable declaration facts or allocation counts differ from baseline.json', + ); + report.baselineMeasured = baseline?.measured; + report.changesPercent = baseline + ? Object.fromEntries( + Object.entries(report.measured).map(([name, metrics]) => [ + name, + Object.fromEntries( + Object.entries(metrics).map(([key, value]) => [ + key, + 100 * (value / baseline.measured[name][key] - 1), + ]), + ), + ]), + ) + : {}; + console.log( + JSON.stringify({ measured: report.measured, changesPercent: report.changesPercent }, null, 2), + ); + if (record) + await writeFile( + baselinePath, + JSON.stringify( + { + _note: + 'Reviewed semantic and allocation baseline. --check compares fingerprints, cache counts and shared-array counts and gates same-run normalized time/memory growth at 1.5. Absolute timing and heap/RSS medians are references. CI never records a baseline.', + environment: report.environment, + correctness, + measured: report.measured, + }, + null, + 2, + ) + '\n', + ); + report.passed = true; + console.log(`[method-declaration-facts ${record ? '--record' : '--check'}] PASS`); + } finally { + if (outputFile) { + await outputFile.writeFile(JSON.stringify(report, null, 2) + '\n'); + await outputFile.close(); + } + } +} + +if (args.includes('--child')) { + const mode = option('--child'); + const result = + mode === 'counts' + ? await countFacts() + : mode === 'lifetime' + ? await retainedTrees() + : await measureAnalysis(option('--corpus')); + console.log('BENCH_RESULT=' + JSON.stringify(result)); +} else if (args.includes('--check') || args.includes('--record')) { + await checkBenchmark(); +} else { + const baseline = option('--baseline-root'); + const candidateRuntime = path.resolve(option('--candidate-root', candidate)); + assert(baseline, 'Supply --baseline-root pointing at the built pre-change package'); + const samples = Number(option('--samples', '3')); + assert(Number.isInteger(samples) && samples >= 3, '--samples must be at least 3'); + const outputPath = + option('--output') ?? + path.join(await mkdtemp(path.join(os.tmpdir(), 'gitnexus-callable-results-')), 'results.json'); + const outputFile = await open(outputPath, 'wx', 0o600); + try { + const results = { + measuredAt: new Date().toISOString(), + environment: { + node: process.version, + platform: process.platform, + cpu: os.cpus()[0]?.model, + workers: 1, + }, + counts: {}, + lifetime: {}, + samples: [], + }; + for (const [name, revision] of [ + ['baseline', baseline], + ['candidate', candidateRuntime], + ]) { + results.counts[name] = await child(revision, 'counts'); + results.lifetime[name] = await child(revision, 'lifetime'); + } + assert.deepEqual( + results.counts.baseline.map((r) => [r.language, r.facts]), + results.counts.candidate.map((r) => [r.language, r.facts]), + ); + for (const corpus of args.includes('--providers-only') + ? ['providers'] + : ['declarations', 'repository']) { + for (let sample = 0; sample < samples; sample++) { + const pair = {}; + for (const name of sample % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) { + console.log(`${corpus} pair ${sample + 1}/${samples}: ${name}`); + pair[name] = await child( + name === 'baseline' ? baseline : candidateRuntime, + 'analysis', + corpus, + ); + } + assert.deepEqual(pair.baseline.manifest, pair.candidate.manifest); + assert.deepEqual( + pair.baseline.runs.map((r) => r.facts), + pair.candidate.runs.map((r) => r.facts), + 'before/after graph facts differ', + ); + results.samples.push({ corpus, sample, ...pair }); + } + } + await outputFile.writeFile(JSON.stringify(results, null, 2) + '\n'); + console.log(`Measurements saved to ${outputPath}`); + } finally { + await outputFile.close(); + } +} diff --git a/gitnexus/bench/method-declaration-facts/memory.mjs b/gitnexus/bench/method-declaration-facts/memory.mjs new file mode 100644 index 000000000..c766b2058 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/memory.mjs @@ -0,0 +1,105 @@ +// Isolate retained JS heap from whole-process RSS. Run with --expose-gc. +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const args = process.argv.slice(2); +const option = (name, fallback) => (args.includes(name) ? args[args.indexOf(name) + 1] : fallback); +const root = path.resolve(option('--root', '.')); +const language = option('--language', 'java'); +const count = Number(option('--declarations', '10000')); +assert(global.gc, 'Run node with --expose-gc'); +assert(['java', 'typescript'].includes(language)); +assert(Number.isSafeInteger(count) && count > 0); +const load = (name) => import(pathToFileURL(path.join(root, 'dist', name))); +const Parser = createRequire(path.join(root, 'package.json'))('tree-sitter'); +const { getLanguageGrammar } = await load('core/tree-sitter/parser-loader.js'); +const { createMethodExtractor } = await load('core/ingestion/method-extractors/generic.js'); +const { buildMethodProps } = await load('core/ingestion/utils/method-props.js'); +const java = language === 'java'; +const configModule = await load( + `core/ingestion/method-extractors/configs/${java ? 'jvm' : 'typescript-javascript'}.js`, +); +const config = configModule[`${language}MethodConfig`]; +const arityModule = await load(`core/ingestion/languages/${language}/arity-metadata.js`); +const arity = arityModule[java ? 'computeJavaArityMetadata' : 'computeTsArityMetadata']; +const extractor = createMethodExtractor(config); +const parser = new Parser(); +parser.setLanguage(getLanguageGrammar(language)); +const source = `class C {\n${Array.from({ length: count }, (_, i) => + java + ? `int f${i}(int x, String y) { return x; }` + : `f${i}(x: number, y: string): number { return x; }`, +).join('\n')}\n}`; +const nodeType = java ? 'method_declaration' : 'method_definition'; +const context = { filePath: `fixture.${java ? 'java' : 'ts'}`, language }; + +async function heap() { + // WeakRef targets and pending native finalizers need a new event-loop turn. + for (let i = 0; i < 4; i++) { + await new Promise((resolve) => setImmediate(resolve)); + global.gc(); + } + return process.memoryUsage().heapUsed; +} + +// Warm module/class initialization separately from the measured tree. +for (let i = 0; i < 20; i++) { + const warm = parser.parse( + java + ? 'class W { int f(int x, String y) { return x; } }' + : 'class W { f(x: number, y: string): number { return x; } }', + ); + const node = warm.rootNode.descendantsOfType(nodeType)[0]; + arity(node); + buildMethodProps(extractor.extractFromNode(node, context)); +} +const emptyHeap = await heap(); +let tree = parser.parse(source); +assert(!tree.rootNode.hasError); +let nodes = tree.rootNode.descendantsOfType(nodeType); +assert.equal(nodes.length, count); +const treeHeap = await heap(); +function extract(declarations) { + const methods = []; + const emitted = []; + const start = performance.now(); + for (const node of declarations) { + const scope = arity(node); + const method = extractor.extractFromNode(node, context); + assert(method); + methods.push(method); + emitted.push({ scope, graph: buildMethodProps(method) }); + } + return { methods, emitted, extractionMs: performance.now() - start }; +} +let { methods, emitted, extractionMs } = extract(nodes); +const liveHeap = await heap(); +assert.equal(methods.length, count); // Keep the method records live through the heap sample. +const digest = createHash('sha256').update(JSON.stringify(emitted)).digest('hex'); +methods = null; +const emittedWithTreeHeap = await heap(); +const weakTree = new WeakRef(tree); +nodes = null; +tree = null; +const emittedHeap = await heap(); +const treeCollected = weakTree.deref() === undefined; +emitted = null; +const releasedHeap = await heap(); +console.log( + JSON.stringify({ + language, + declarations: count, + digest, + extractionMs, + treeCollected, + heapBytes: { emptyHeap, treeHeap, liveHeap, emittedWithTreeHeap, emittedHeap, releasedHeap }, + extractionRetainedBytes: liveHeap - treeHeap, + emittedRetainedBytes: emittedHeap - emptyHeap, + releasedDeltaBytes: releasedHeap - emptyHeap, + processMaxRssMiB: process.resourceUsage().maxRSS / 1024, + note: 'Fresh process; forced-GC JS heap, not native or worker heap. Extraction time excludes parse/GC. Peak RSS is for this probe only. Compare repeated same-runtime runs.', + }), +); diff --git a/gitnexus/bench/method-declaration-facts/scaling.mjs b/gitnexus/bench/method-declaration-facts/scaling.mjs new file mode 100644 index 000000000..6afbf77fc --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/scaling.mjs @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; + +export function evaluateScaling(rows, metrics) { + const budget = 1.5; + assert(rows.length >= 2 && metrics.length > 0, 'Scaling needs sizes and metrics'); + for (const [index, row] of rows.entries()) { + assert(Number.isFinite(row.size) && row.size > 0, 'Invalid workload size'); + if (index) assert(row.size > rows[index - 1].size, 'Sizes must increase'); + for (const metric of metrics) + assert(Number.isFinite(row[metric]) && row[metric] > 0, `Invalid ${metric}`); + } + const pairs = rows.slice(1).map((row, index) => [rows[index], row]); + if (rows.length > 2) pairs.push([rows[0], rows.at(-1)]); + const comparisons = pairs.map(([small, large]) => { + const inputRatio = large.size / small.size; + const ratios = Object.fromEntries( + metrics.map((metric) => [metric, large[metric] / small[metric] / inputRatio]), + ); + return { + from: small.size, + to: large.size, + inputRatio, + ratios, + passed: Object.values(ratios).every((ratio) => ratio <= budget), + }; + }); + return { budget, comparisons, passed: comparisons.every((row) => row.passed) }; +} diff --git a/gitnexus/bench/method-declaration-facts/trace-parameters.mjs b/gitnexus/bench/method-declaration-facts/trace-parameters.mjs new file mode 100644 index 000000000..2d62abb07 --- /dev/null +++ b/gitnexus/bench/method-declaration-facts/trace-parameters.mjs @@ -0,0 +1,47 @@ +// Optional instrumentation preload; discard timings from traced runs. +import assert from 'node:assert/strict'; +import { appendFileSync, writeFileSync } from 'node:fs'; +import { threadId } from 'node:worker_threads'; +import { pathToFileURL } from 'node:url'; +const root = process.env.FACT_RUNTIME_ROOT; +const target = process.env.FACT_COUNTER_LOG; +assert(root && target, 'Set FACT_RUNTIME_ROOT and FACT_COUNTER_LOG for this preload'); +if (threadId === 0) writeFileSync(target, '', { flag: 'wx' }); +const trees = new WeakMap(), + arrays = new WeakMap(); +let treeSerial = 0, + arraySerial = 0; +for (const [module, prefixes] of [ + ['jvm', ['java', 'kotlin']], + ['typescript-javascript', ['typescript', 'javascript']], + ['csharp', ['csharp']], + ['python', ['python']], + ['php', ['php']], + ['swift', ['swift']], + ['dart', ['dart']], +]) { + const exports = await import( + pathToFileURL(root + '/dist/core/ingestion/method-extractors/configs/' + module + '.js') + ); + for (const prefix of prefixes) { + const config = exports[prefix + 'MethodConfig']; + const original = config.extractParameters; + config.extractParameters = function (node, ...args) { + const result = original(node, ...args); + if (node.tree && !trees.has(node.tree)) trees.set(node.tree, ++treeSerial); + if (!arrays.has(result)) arrays.set(result, ++arraySerial); + appendFileSync( + target, + JSON.stringify({ + thread: threadId, + provider: prefix, + tree: trees.get(node.tree), + node: node.id, + kind: node.type, + array: arrays.get(result), + }) + '\n', + ); + return result; + }; + } +} diff --git a/gitnexus/src/core/analyzer-identity.ts b/gitnexus/src/core/analyzer-identity.ts index 5782a5724..9bd3902a1 100644 --- a/gitnexus/src/core/analyzer-identity.ts +++ b/gitnexus/src/core/analyzer-identity.ts @@ -2241,8 +2241,13 @@ function loadIdentityCache( } const CACHE_GUARD_PROBE_SCRIPT = String.raw` -const fs = require('node:fs/promises'); +const fs = require('node:fs'); +const { performance } = require('node:perf_hooks'); const crypto = require('node:crypto'); +const syncFs = { + lstat: fs.lstatSync, stat: fs.statSync, + readlink: fs.readlinkSync, readdir: fs.readdirSync, +}; const state = (value) => ({ dev: String(value.dev), ino: String(value.ino), mode: String(value.mode), nlink: String(value.nlink), size: String(value.size), @@ -2272,30 +2277,30 @@ const inventoryDigest = (entries) => { for (const entry of normalized) frame(hash, [entry.name, entry.kind]); return 'sha256:' + hash.digest('hex'); }; -const probe = async (request) => { +const probe = async (request, io) => { try { - const link = await fs.lstat(request.absolutePath, { bigint: true }); + const link = await io.lstat(request.absolutePath, { bigint: true }); const type = link.isDirectory() ? 'directory' : link.isFile() ? 'file' : link.isSymbolicLink() ? 'symlink' : 'other'; if (request.mode === 'link') return { type, state: state(link), - ...(type === 'symlink' ? { symlinkTarget: await fs.readlink(request.absolutePath) } : {}), + ...(type === 'symlink' ? { symlinkTarget: await io.readlink(request.absolutePath) } : {}), }; if (request.mode === 'directory-inventory') { if (!link.isDirectory() || link.isSymbolicLink()) return null; const entriesDigest = inventoryDigest( - await fs.readdir(request.absolutePath, { withFileTypes: true }), + await io.readdir(request.absolutePath, { withFileTypes: true }), ); - const after = await fs.lstat(request.absolutePath, { bigint: true }); + const after = await io.lstat(request.absolutePath, { bigint: true }); if (JSON.stringify(state(link)) !== JSON.stringify(state(after))) return null; return { type: 'directory-inventory', state: state(after), entriesDigest }; } - const target = link.isFile() ? link : await fs.stat(request.absolutePath, { bigint: true }); + const target = link.isFile() ? link : await io.stat(request.absolutePath, { bigint: true }); if (!target.isFile()) return null; const result = { type: 'readable-file', state: { link: state(link), target: state(target) } }; - if (link.isSymbolicLink()) result.state.symlinkTarget = await fs.readlink(request.absolutePath); + if (link.isSymbolicLink()) result.state.symlinkTarget = await io.readlink(request.absolutePath); return result; } catch { return null; } }; @@ -2305,9 +2310,29 @@ process.stdin.on('data', (chunk) => { input += chunk; }); process.stdin.on('end', async () => { try { const requests = JSON.parse(input); - const results = []; - for (let offset = 0; offset < requests.length; offset += 512) { - results.push(...await Promise.all(requests.slice(offset, offset + 512).map(probe))); + const results = new Array(requests.length); + // Local stats are cheaper without thread-pool dispatch. Sample across the + // install tree so slow dependency mounts still use concurrent async I/O. + const sampleCount = Math.min(16, requests.length); + const started = performance.now(); + for (let sample = 0; sample < sampleCount; sample += 1) { + const index = Math.floor(sample * requests.length / sampleCount); + results[index] = await probe(requests[index], syncFs); + if (performance.now() - started >= 8) break; + } + const useSync = performance.now() - started < 8; + if (useSync) { + for (let index = 0; index < requests.length; index += 1) { + if (results[index] === undefined) results[index] = await probe(requests[index], syncFs); + } + } else { + for (let offset = 0; offset < requests.length; offset += 512) { + await Promise.all(requests.slice(offset, offset + 512).map(async (request, index) => { + if (results[offset + index] === undefined) { + results[offset + index] = await probe(request, fs.promises); + } + })); + } } process.stdout.write(JSON.stringify(results)); } catch { process.exitCode = 1; } diff --git a/gitnexus/src/core/ingestion/export-detection.ts b/gitnexus/src/core/ingestion/export-detection.ts index 143358c6f..c436e5d23 100644 --- a/gitnexus/src/core/ingestion/export-detection.ts +++ b/gitnexus/src/core/ingestion/export-detection.ts @@ -41,25 +41,33 @@ export const tsExportChecker: ExportChecker = (node, _name) => { /** Python: public if no leading underscore (convention). */ export const pythonExportChecker: ExportChecker = (_node, name) => !name.startsWith('_'); +// Parser trees are immutable during extraction; weak keys do not retain discarded trees. +const javaPublicModifierCache = new WeakMap(); + /** Java: check for 'public' modifier — modifiers are siblings of the name node, not parents. */ export const javaExportChecker: ExportChecker = (node, _name) => { let current: SyntaxNode | null = node; while (current) { - if (current.parent) { - const parent = current.parent; - for (let i = 0; i < parent.childCount; i++) { - const child = parent.child(i); - if (child?.type === 'modifiers' && child.text?.includes('public')) { - return true; + const parent: SyntaxNode | null = current.parent; + if (parent) { + let isPublic = javaPublicModifierCache.get(parent); + if (isPublic === undefined) { + isPublic = false; + for (let i = 0; i < parent.childCount; i++) { + const child = parent.child(i); + if (child?.type === 'modifiers' && child.text?.includes('public')) { + isPublic = true; + break; + } } - } - if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') { - if (parent.text?.trimStart().startsWith('public')) { - return true; + if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') { + isPublic ||= parent.text?.trimStart().startsWith('public') ?? false; } + javaPublicModifierCache.set(parent, isPublic); } + if (isPublic) return true; } - current = current.parent; + current = parent; } return false; }; diff --git a/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts index 92b377352..20a63b375 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/arity-metadata.ts @@ -19,6 +19,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { csharpMethodConfig } from '../../method-extractors/configs/csharp.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface CsharpArityMetadata { readonly parameterCount: number | undefined; @@ -31,13 +32,12 @@ export function computeCsharpArityMetadata(fnNode: SyntaxNode): CsharpArityMetad let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + let types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) hasVariadic = true; else if (p.isOptional) optionalCount++; - if (p.type !== null) types.push(p.type); } - if (hasVariadic) types.push('params'); + if (hasVariadic) types = [...types, 'params']; const total = params.length; // `params int[] args` declares one formal param but accepts any arg diff --git a/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts index 28ab666c7..6e569f6a8 100644 --- a/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/dart/arity-metadata.ts @@ -12,6 +12,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { dartMethodConfig } from '../../method-extractors/configs/dart.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface DartArityMetadata { parameterCount: number | undefined; @@ -29,10 +30,9 @@ export function computeDartArityMetadata(fnNode: SyntaxNode): DartArityMetadata const params = dartMethodConfig.extractParameters?.(fnNode) ?? []; let optionalCount = 0; - const types: string[] = []; + const types = declarationParameterTypes(params); for (const p of params) { if (p.isOptional) optionalCount++; - if (p.type !== null) types.push(p.type); } const total = params.length; diff --git a/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts index 47cccbff9..75a92a48a 100644 --- a/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/java/arity-metadata.ts @@ -12,6 +12,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { javaMethodConfig } from '../../method-extractors/configs/jvm.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; export interface JavaArityMetadata { readonly parameterCount: number | undefined; @@ -23,12 +24,11 @@ export function computeJavaArityMetadata(fnNode: SyntaxNode): JavaArityMetadata const params = javaMethodConfig.extractParameters?.(fnNode) ?? []; let hasVariadic = false; - const types: string[] = []; + let types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) hasVariadic = true; - if (p.type !== null) types.push(p.type); } - if (hasVariadic) types.push('varargs'); + if (hasVariadic) types = [...types, 'varargs']; const total = params.length; // For varargs methods, `parameterCount` (max) is unknown — any number of diff --git a/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts index 5cdf76cf6..6f54ef673 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/arity-metadata.ts @@ -1,5 +1,6 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { kotlinMethodConfig } from '../../method-extractors/configs/jvm.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; export interface KotlinArityMetadata { readonly parameterCount: number | undefined; @@ -10,12 +11,11 @@ export interface KotlinArityMetadata { export function computeKotlinArityMetadata(fnNode: SyntaxNode): KotlinArityMetadata { const params = kotlinMethodConfig.extractParameters?.(fnNode) ?? []; let hasVararg = false; - const parameterTypes: string[] = []; + let parameterTypes = declarationParameterTypes(params); for (const param of params) { if (param.isVariadic) hasVararg = true; - if (param.type !== null) parameterTypes.push(param.type); } - if (hasVararg) parameterTypes.push('vararg'); + if (hasVararg) parameterTypes = [...parameterTypes, 'vararg']; const required = params.filter((p) => !p.isOptional && !p.isVariadic).length; return { diff --git a/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts index 40662bd61..95506884b 100644 --- a/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/php/arity-metadata.ts @@ -21,6 +21,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { phpMethodConfig } from '../../method-extractors/configs/php.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface PhpArityMetadata { readonly parameterCount: number | undefined; @@ -33,7 +34,7 @@ export function computePhpArityMetadata(fnNode: SyntaxNode): PhpArityMetadata { let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + let types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) { @@ -41,7 +42,6 @@ export function computePhpArityMetadata(fnNode: SyntaxNode): PhpArityMetadata { } else if (p.isOptional) { optionalCount++; } - if (p.type !== null) types.push(p.type); } // PHP variadic marker convention: append the literal '...' string to // `parameterTypes`. This is intentionally DIFFERENT from C#, which uses @@ -54,7 +54,7 @@ export function computePhpArityMetadata(fnNode: SyntaxNode): PhpArityMetadata { // arity logic lives in `phpArityCompatibility` (arity.ts) and now // also in `phpEmitUnresolvedReceiverEdges` (scope-resolver.ts), both // of which check `'...'`. Finding 9 of PR #1497 adversarial review. - if (hasVariadic) types.push('...'); + if (hasVariadic) types = [...types, '...']; const total = params.length; // Variadic methods accept any arg count ≥ required — leave `parameterCount` diff --git a/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts index 6d5e38af2..5168a537f 100644 --- a/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/python/arity-metadata.ts @@ -18,6 +18,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { pythonMethodConfig } from '../../method-extractors/configs/python.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface PythonArityMetadata { readonly parameterCount: number | undefined; @@ -31,11 +32,10 @@ export function computePythonArityMetadata(fnNode: SyntaxNode): PythonArityMetad let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + const types = declarationParameterTypes(params); for (const p of params) { if (p.isVariadic) hasVariadic = true; else if (p.isOptional) optionalCount++; - if (p.type !== null) types.push(p.type); } const total = params.length; diff --git a/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts index da3efaa61..527b3b7a0 100644 --- a/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/swift/arity-metadata.ts @@ -18,6 +18,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { swiftMethodConfig } from '../../method-extractors/configs/swift.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface SwiftArityMetadata { readonly parameterCount: number | undefined; @@ -30,13 +31,17 @@ export function computeSwiftArityMetadata(fnNode: SyntaxNode): SwiftArityMetadat let hasVariadic = false; let optionalCount = 0; - const types: string[] = []; + const sharedTypes = declarationParameterTypes(params); + // Scope signatures keep unknown positional slots; graph metadata omits them. + let types = + sharedTypes.length === params.length + ? sharedTypes + : params.map((parameter) => parameter.type ?? ''); for (const p of params) { if (p.isVariadic) hasVariadic = true; else if (p.isOptional) optionalCount++; - types.push(p.type ?? ''); } - if (hasVariadic) types.push('variadic'); + if (hasVariadic) types = [...types, 'variadic']; const total = params.length; const parameterCount = hasVariadic ? undefined : total; diff --git a/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts index 97ac7a0c9..3340efb1c 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/arity-metadata.ts @@ -52,6 +52,7 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { typescriptMethodConfig } from '../../method-extractors/configs/typescript-javascript.js'; +import { declarationParameterTypes } from '../../method-extractors/declaration-facts.js'; interface TsArityMetadata { readonly parameterCount: number | undefined; @@ -64,14 +65,21 @@ export function computeTsArityMetadata(fnNode: SyntaxNode): TsArityMetadata { let hasRest = false; let optionalCount = 0; - const types: string[] = []; + const sharedTypes = declarationParameterTypes(params); + let normalized: string[] | undefined; + let index = 0; for (const p of params) { if (p.isVariadic) hasRest = true; else if (p.isOptional) optionalCount++; const t = p.type !== null && p.type !== undefined ? stripGenericsAndArraySuffix(p.type) : ''; - types.push(t); + if (t !== sharedTypes[index]) { + normalized ??= sharedTypes.slice(); + normalized[index] = t; + } + index++; } - if (hasRest) types.push('params'); + const types = normalized ?? sharedTypes; + const markedTypes = hasRest ? [...types, 'params'] : types; const total = params.length; const parameterCount = hasRest ? undefined : total; @@ -82,7 +90,7 @@ export function computeTsArityMetadata(fnNode: SyntaxNode): TsArityMetadata { // registry without aiding narrowing — callers treat absence as // "types unknown". const hasAnyType = types.some((t) => t !== '' && t !== 'params'); - const parameterTypes = hasAnyType || hasRest ? (types.length > 0 ? types : undefined) : undefined; + const parameterTypes = hasAnyType || hasRest ? markedTypes : undefined; return { parameterCount, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts index c6550050f..b3a29b255 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/csharp.ts @@ -16,6 +16,7 @@ import { collectModifierTexts, } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; // --------------------------------------------------------------------------- @@ -234,7 +235,7 @@ export const csharpMethodConfig: MethodExtractionConfig = { return undefined; }, - extractParameters: extractCSharpParameters, + extractParameters: memoizeDeclarationFact(extractCSharpParameters), extractVisibility(node) { // Detect compound C# visibilities: protected internal, private protected diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts index af71c24bd..6afad2fc2 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/dart.ts @@ -9,6 +9,7 @@ import type { } from '../../method-types.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Dart helpers @@ -415,9 +416,9 @@ export const dartMethodConfig: MethodExtractionConfig = { methodNodeTypes: ['method_signature', 'declaration'], bodyNodeTypes: ['class_body', 'extension_body'], - extractName: extractDartName, - extractReturnType: extractDartReturnType, - extractParameters: extractDartParameters, + extractName: memoizeDeclarationFact(extractDartName, getInnerSignature), + extractReturnType: memoizeDeclarationFact(extractDartReturnType, getInnerSignature), + extractParameters: memoizeDeclarationFact(extractDartParameters, getInnerSignature), extractVisibility: extractDartVisibility, isStatic: isDartStatic, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts index 44d65f4b3..080996f52 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/jvm.ts @@ -13,6 +13,7 @@ import { } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Shared JVM helpers @@ -131,7 +132,7 @@ export const javaMethodConfig: MethodExtractionConfig = { extractReturnType: extractReturnTypeFromField, - extractParameters: extractJavaParameters, + extractParameters: memoizeDeclarationFact(extractJavaParameters), extractVisibility(node) { return findVisibility(node, JAVA_VIS, 'package', 'modifiers'); @@ -289,7 +290,7 @@ export const kotlinMethodConfig: MethodExtractionConfig = { extractReturnType: extractKotlinReturnType, - extractParameters: extractKotlinParameters, + extractParameters: memoizeDeclarationFact(extractKotlinParameters), extractVisibility(node) { return findVisibility(node, KOTLIN_VIS, 'public', 'modifiers'); diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/php.ts b/gitnexus/src/core/ingestion/method-extractors/configs/php.ts index 2d04a6bb3..69f3de29b 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/php.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/php.ts @@ -9,6 +9,7 @@ import type { } from '../../method-types.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // PHP helpers @@ -282,7 +283,7 @@ export const phpMethodConfig: MethodExtractionConfig = { extractReturnType: extractPhpReturnType, - extractParameters: extractPhpParameters, + extractParameters: memoizeDeclarationFact(extractPhpParameters), extractVisibility: findPhpVisibility, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/python.ts b/gitnexus/src/core/ingestion/method-extractors/configs/python.ts index 0540956b1..2a6ace071 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/python.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/python.ts @@ -15,6 +15,7 @@ import { } from '../../languages/python/receiver-binding.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Python helpers @@ -300,7 +301,7 @@ export const pythonMethodConfig: MethodExtractionConfig = { }, extractReturnType: extractPythonReturnType, - extractParameters: extractPythonParameters, + extractParameters: memoizeDeclarationFact(extractPythonParameters, unwrapDecorated), extractVisibility: extractPythonVisibility, isStatic(node) { diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts index 1ee630b6a..12d783ad9 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/swift.ts @@ -10,6 +10,7 @@ import type { import { findVisibility, hasKeyword, hasModifier } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // Swift helpers @@ -282,9 +283,9 @@ export const swiftMethodConfig: MethodExtractionConfig = { // dev-mode "body field type not in bodyNodeTypes" warning. bodyNodeTypes: ['class_body', 'protocol_body', 'enum_class_body'], - extractName: extractSwiftName, - extractReturnType: extractSwiftReturnType, - extractParameters: extractSwiftParameters, + extractName: memoizeDeclarationFact(extractSwiftName), + extractReturnType: memoizeDeclarationFact(extractSwiftReturnType), + extractParameters: memoizeDeclarationFact(extractSwiftParameters), extractVisibility(node) { return findVisibility(node, SWIFT_VIS, 'internal', 'modifiers'); diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts index b57ad110a..b80f68613 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/typescript-javascript.ts @@ -10,6 +10,7 @@ import type { import { hasKeyword } from '../../field-extractors/configs/helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { memoizeDeclarationFact } from '../declaration-facts.js'; // --------------------------------------------------------------------------- // TS/JS helpers @@ -308,7 +309,7 @@ const shared: Omit = { }, extractReturnType: extractTsJsReturnType, - extractParameters: extractTsJsParameters, + extractParameters: memoizeDeclarationFact(extractTsJsParameters), extractVisibility: extractTsJsVisibility, isStatic(node) { diff --git a/gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts b/gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts new file mode 100644 index 000000000..393dd0ea7 --- /dev/null +++ b/gitnexus/src/core/ingestion/method-extractors/declaration-facts.ts @@ -0,0 +1,61 @@ +import type { ParameterInfo } from '../method-types.js'; +import type { SyntaxNode } from '../utils/ast-helpers.js'; + +type DeclarationFact = string | undefined | readonly Readonly[]; + +const parameterTypes = new WeakMap[], readonly string[]>(); +const noTypes: readonly string[] = []; + +/** The common type projection of immutable declaration parameters. Consumers + * with positional unknowns, normalized types, or variadic markers copy only + * when their language policy differs. Neither keys nor values retain an AST. + */ +export function declarationParameterTypes( + parameters: readonly Readonly[], +): readonly string[] { + if (parameters.length === 0) return noTypes; + const cached = parameterTypes.get(parameters); + if (cached) return cached; + const types: string[] = []; + for (const parameter of parameters) { + if (parameter.type !== null) types.push(parameter.type); + } + if (types.length === 0) return noTypes; + parameterTypes.set(parameters, types); + return types; +} + +/** Share a pure, node-only fact across structure and scope extraction. + * Trees are immutable during ingestion. Native node IDs survive different JS + * wrappers; tree ownership keeps IDs from colliding across files or reparses. + * Values must contain no AST nodes. File/owner-dependent callbacks stay uncached. + * Providers may identify an equivalent declaration inside the same tree when + * one consumer sees a syntax wrapper and another sees its inner declaration. + */ +export function memoizeDeclarationFact( + extract: (node: SyntaxNode) => T, + declaration?: (node: SyntaxNode) => SyntaxNode | null, +): (node: SyntaxNode) => T { + const trees = new WeakMap>(); + return (node) => { + const tree = node.tree; + const id = node.id; + if (!tree || typeof id !== 'number') return extract(node); + let facts = trees.get(tree); + if (facts?.has(id)) return facts.get(id)!; + const declarationId = declaration?.(node)?.id ?? id; + if (declarationId !== id && facts?.has(declarationId)) { + const fact = facts.get(declarationId)!; + facts.set(id, fact); + return fact; + } + const fact = extract(node); + if (!facts) { + facts = new Map(); + trees.set(tree, facts); + } + facts.set(id, fact); + if (declarationId !== id) facts.set(declarationId, fact); + return fact; + }; +} diff --git a/gitnexus/src/core/ingestion/method-types.ts b/gitnexus/src/core/ingestion/method-types.ts index fb7c61f1f..3e9edf2e9 100644 --- a/gitnexus/src/core/ingestion/method-types.ts +++ b/gitnexus/src/core/ingestion/method-types.ts @@ -23,7 +23,7 @@ export interface MethodInfo { name: string; receiverType: string | null; returnType: string | null; - parameters: ParameterInfo[]; + parameters: readonly Readonly[]; visibility: MethodVisibility; isStatic: boolean; isAbstract: boolean; @@ -100,7 +100,7 @@ export interface MethodExtractionConfig { * name is the file stem, so `fn incr(counter: *Counter)` in `Counter.zig` * is a method only when the file is known. Same optional-trailing-argument * shape as `extractOwnerName`; every other config ignores it. */ - extractParameters: (node: SyntaxNode, filePath?: string) => ParameterInfo[]; + extractParameters: (node: SyntaxNode, filePath?: string) => readonly Readonly[]; extractVisibility: (node: SyntaxNode) => MethodVisibility; isStatic: (node: SyntaxNode, filePath?: string) => boolean; isAbstract: (node: SyntaxNode, ownerNode: SyntaxNode) => boolean; diff --git a/gitnexus/src/core/ingestion/utils/method-props.ts b/gitnexus/src/core/ingestion/utils/method-props.ts index 4ade74c8a..fcf913059 100644 --- a/gitnexus/src/core/ingestion/utils/method-props.ts +++ b/gitnexus/src/core/ingestion/utils/method-props.ts @@ -1,4 +1,5 @@ import type { MethodInfo } from '../method-types.js'; +import { declarationParameterTypes } from '../method-extractors/declaration-facts.js'; import { SupportedLanguages, type ParameterTypeClass } from 'gitnexus-shared'; /** Languages where class overload signatures are declaration-only contracts @@ -210,12 +211,11 @@ export function parameterShapeIdTag( /** Convert MethodInfo from methodExtractor into flat properties for a graph node. */ export function buildMethodProps(info: MethodInfo): Record { - const types: string[] = []; + const types = declarationParameterTypes(info.parameters); const typeClasses: ParameterTypeClass[] = []; let optionalCount = 0; let hasVariadic = false; for (const p of info.parameters) { - if (p.type !== null) types.push(p.type); if (p.typeClass !== undefined) typeClasses.push(p.typeClass); if (p.isOptional) optionalCount++; if (p.isVariadic) hasVariadic = true; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 5d999ed24..fb0a43b2b 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -871,6 +871,20 @@ function selectFieldInfo( // ============================================================================ const methodInfoCache = new Map>(); +const methodCollisionGroups = new WeakMap, Map>(); + +/** Method maps are complete before publication and immutable for this file. + * Reuse their overload groups in both definition and enclosing-callable IDs. + * Weak ownership releases the groups with the per-file method cache. + */ +function getMethodCollisionGroups(methods: Map): Map { + let groups = methodCollisionGroups.get(methods); + if (!groups) { + groups = buildCollisionGroups(methods); + methodCollisionGroups.set(methods, groups); + } + return groups; +} /** * Get (or extract and cache) method info for a class node. @@ -1183,7 +1197,7 @@ const findEnclosingFunctionId = ( ? undefined : info.parameters.length; if (methodMap && arity !== undefined) { - const g = buildCollisionGroups(methodMap); + const g = getMethodCollisionGroups(methodMap); encTypeTag = typeTagForId(methodMap, funcName, arity, info, encLang, g) + constTagForId(methodMap, funcName, arity, info, g); @@ -1251,7 +1265,7 @@ const findEnclosingFunctionId = ( ? undefined : info2.parameters.length; if (methodMap2 && arity2 !== undefined) { - const g2 = buildCollisionGroups(methodMap2); + const g2 = getMethodCollisionGroups(methodMap2); encTypeTag2 = typeTagForId(methodMap2, customResult.funcName, arity2, info2, encLang2, g2) + constTagForId(methodMap2, customResult.funcName, arity2, info2, g2); @@ -2861,7 +2875,7 @@ const processFileGroup = ( const parameterTypes = methodProps.parameterTypes as string[] | undefined; if (parameterTypes?.length) arityTag += `~${parameterTypes.join(',')}`; } else if (arityTag && defMethodMap && defMethodInfo) { - const groups = buildCollisionGroups(defMethodMap); + const groups = getMethodCollisionGroups(defMethodMap); arityTag += typeTagForId( defMethodMap, nodeName, diff --git a/gitnexus/test/integration/parsing.test.ts b/gitnexus/test/integration/parsing.test.ts index f7295b759..c1d55a345 100644 --- a/gitnexus/test/integration/parsing.test.ts +++ b/gitnexus/test/integration/parsing.test.ts @@ -6,7 +6,7 @@ * PHP export detection (#20), symbol ID with startLine (#19), * definition node range (#22). */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, assert } from 'vitest'; import fs from 'fs/promises'; import path from 'path'; import { loadParser, loadLanguage } from '../../src/core/tree-sitter/parser-loader.js'; @@ -93,6 +93,52 @@ describe('parsing', () => { }); }); + describe('java', () => { + it('keeps sibling visibility independent when a parent check is reused', async () => { + const parser = await loadParser(); + await loadLanguage(SupportedLanguages.Java); + const tree = parser.parse('class C { private void a() {} public void b() {} void c() {} }'); + const methods = tree.rootNode.descendantsOfType('method_declaration'); + for (const order of [methods, [...methods].reverse()]) { + expect( + order.map((method) => { + const name = method.childForFieldName('name'); + assert(name); + return isNodeExported(name, name.text, 'java'); + }), + ).toEqual([false, true, false]); + } + }); + + it('preserves public-ancestor behavior across repeated checks and separate trees', async () => { + const parser = await loadParser(); + await loadLanguage(SupportedLanguages.Java); + for (const modifier of ['', 'public ', '']) { + const tree = parser.parse( + `${modifier}class C { private void a() {} class Inner { void b() {} } }`, + ); + const methods = tree.rootNode.descendantsOfType('method_declaration'); + expect(methods).toHaveLength(2); + for (const method of methods) { + const name = method.childForFieldName('name'); + assert(name); + expect(isNodeExported(name, name.text, 'java')).toBe(modifier === 'public '); + expect(isNodeExported(name, name.text, 'java')).toBe(modifier === 'public '); + } + } + }); + + it.each(['method_declaration', 'constructor_declaration'])( + 'preserves the %s text fallback', + (type) => { + const name = mockNode('identifier', 'C'); + mockNode(type, ' public C() {}', undefined, [name]); + expect(isNodeExported(name, 'C', 'java')).toBe(true); + expect(isNodeExported(name, 'C', 'java')).toBe(true); + }, + ); + }); + // Go describe('go', () => { it('uppercase first letter is exported', () => { diff --git a/gitnexus/test/unit/analyzer-identity-guard-io.test.ts b/gitnexus/test/unit/analyzer-identity-guard-io.test.ts new file mode 100644 index 000000000..f89622f5e --- /dev/null +++ b/gitnexus/test/unit/analyzer-identity-guard-io.test.ts @@ -0,0 +1,189 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { mkdir, symlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; + +const captured = vi.hoisted(() => ({ script: '' })); +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + spawnSync: (...args: Parameters) => { + const argv = args[1]; + if (Array.isArray(argv) && argv.includes('--input-type=commonjs')) { + captured.script = argv[argv.indexOf('-e') + 1]; + // Let identity resolution fall back to the real direct validator. + return { status: 1, stdout: '', stderr: '' }; + } + return actual.spawnSync(...args); + }, + }; +}); + +import { + _clearAnalyzerIdentityProcessCacheForTests, + resolveAnalyzerRunnerIdentity, +} from '../../src/core/analyzer-identity.js'; + +type Request = { absolutePath: string; mode: 'link' | 'readable-file' | 'directory-inventory' }; +type Result = { + type: string; + state: unknown; + entriesDigest?: string; + symlinkTarget?: string; +} | null; + +// Exercise the actual subprocess script captured through the public resolver. +// A deterministic clock chooses the I/O route without timing-sensitive tests. +async function runProbe(requests: Request[], slow: boolean, mutateDirectory?: string) { + const { spawnSync } = + await vi.importActual('node:child_process'); + const instrumentation = ` + const probeFs = require('node:fs'); + let ticks = 0; + Object.defineProperty(require('node:perf_hooks').performance, 'now', { + value: () => ticks++ * ${slow ? 100 : 0}, + }); + const counts = { sync: 0, async: 0 }; + const mutationDirectory = ${JSON.stringify(mutateDirectory ?? null)}; + let mutated = false; + const afterRead = (file) => { + if (!mutated && file === mutationDirectory) { + mutated = true; + probeFs.writeFileSync(require('node:path').join(file, 'added'), 'changed'); + } + }; + for (const name of ['lstat', 'stat', 'readlink', 'readdir']) { + const originalSync = probeFs[name + 'Sync']; + probeFs[name + 'Sync'] = (...args) => { + counts.sync++; + const result = originalSync(...args); + if (name === 'readdir') afterRead(args[0]); + return result; + }; + const originalAsync = probeFs.promises[name]; + probeFs.promises[name] = async (...args) => { + counts.async++; + const result = await originalAsync(...args); + if (name === 'readdir') afterRead(args[0]); + return result; + }; + } + process.on('exit', () => process.stderr.write(JSON.stringify(counts))); + `; + const child = spawnSync( + process.execPath, + ['--input-type=commonjs', '-e', instrumentation + captured.script], + { + input: JSON.stringify(requests), + encoding: 'utf8', + timeout: 10_000, + }, + ); + expect(child.error).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + return { + results: JSON.parse(child.stdout) as Result[], + counts: JSON.parse(child.stderr) as { sync: number; async: number }, + }; +} + +describe('identity guard subprocess I/O', () => { + let fixture: TestDBHandle; + let file: string; + let directory: string; + + beforeAll(async () => { + fixture = await createTempDir(); + directory = path.join(fixture.dbPath, 'src'); + await mkdir(directory); + await writeFile( + path.join(fixture.dbPath, 'package.json'), + '{"name":"guard-io","version":"1.0.0"}', + ); + for (let index = 0; index < 140; index++) { + await writeFile(path.join(directory, `file-${index}.ts`), `export const value = ${index};`); + } + file = path.join(directory, 'file-0.ts'); + vi.stubEnv('GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS', '0'); + _clearAnalyzerIdentityProcessCacheForTests(); + resolveAnalyzerRunnerIdentity(pathToFileURL(file).href, { + cacheDirectory: path.join(fixture.dbPath, 'cache'), + }); + expect(captured.script).not.toBe(''); + }); + + afterAll(async () => { + vi.unstubAllEnvs(); + _clearAnalyzerIdentityProcessCacheForTests(); + await fixture?.cleanup(); + }); + + it('returns identical ordered snapshots and failures across batch boundaries on both routes', async () => { + const cases: Request[] = [ + { absolutePath: file, mode: 'link' }, + { absolutePath: directory, mode: 'directory-inventory' }, + { absolutePath: file, mode: 'readable-file' }, + { absolutePath: path.join(fixture.dbPath, 'missing'), mode: 'link' }, + { absolutePath: directory, mode: 'readable-file' }, + { absolutePath: file, mode: 'directory-inventory' }, + ]; + const requests = Array.from({ length: 529 }, (_, index) => cases[index % cases.length]); + const fast = await runProbe(requests, false); + const slow = await runProbe(requests, true); + expect(fast.counts.async).toBe(0); + expect(fast.counts.sync).toBeGreaterThan(requests.length); + // A slow first probe exhausts the sample budget; the rest stay concurrent. + expect(slow.counts.sync).toBe(1); + expect(slow.counts.async).toBeGreaterThan(512); + expect(slow.results).toEqual(fast.results); + expect(fast.results).toHaveLength(requests.length); + for (let index = 0; index < requests.length; index++) { + const result = fast.results[index]; + if (index % cases.length >= 3) expect(result).toBeNull(); + else + expect(result?.type).toBe( + ['file', 'directory-inventory', 'readable-file'][index % cases.length], + ); + } + expect(fast.results[1]?.entriesDigest).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + + it.each([false, true])('rejects an inventory changed during a probe (slow=%s)', async (slow) => { + const changing = path.join(fixture.dbPath, `changing-${slow}`); + await mkdir(changing); + const requests: Request[] = Array.from({ length: 529 }, () => ({ + absolutePath: file, + mode: 'link', + })); + // Index 1 is outside the distributed sample, exercising the selected route. + requests[1] = { absolutePath: changing, mode: 'directory-inventory' }; + const result = await runProbe(requests, slow, changing); + expect(result.results[1]).toBeNull(); + expect(result.results[0]?.type).toBe('file'); + expect(result.results[528]?.type).toBe('file'); + }); + + it.skipIf(process.platform === 'win32')( + 'preserves symlink targets and dangling-link failures on both routes', + async () => { + const link = path.join(fixture.dbPath, 'linked.ts'); + const dangling = path.join(fixture.dbPath, 'dangling.ts'); + await symlink(file, link); + await symlink(path.join(fixture.dbPath, 'absent.ts'), dangling); + const cases: Request[] = [ + { absolutePath: link, mode: 'link' }, + { absolutePath: link, mode: 'readable-file' }, + { absolutePath: dangling, mode: 'readable-file' }, + ]; + const requests = Array.from({ length: 140 }, (_, index) => cases[index % cases.length]); + const fast = await runProbe(requests, false); + const slow = await runProbe(requests, true); + expect(slow.results).toEqual(fast.results); + expect(fast.results[0]?.symlinkTarget).toBe(file); + expect(fast.results[1]?.state).toMatchObject({ symlinkTarget: file }); + expect(fast.results[2]).toBeNull(); + }, + ); +}); diff --git a/gitnexus/test/unit/declaration-parameter-types.test.ts b/gitnexus/test/unit/declaration-parameter-types.test.ts new file mode 100644 index 000000000..04bc08440 --- /dev/null +++ b/gitnexus/test/unit/declaration-parameter-types.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import TypeScript from 'tree-sitter-typescript'; +import { declarationParameterTypes } from '../../src/core/ingestion/method-extractors/declaration-facts.js'; +import { createMethodExtractor } from '../../src/core/ingestion/method-extractors/generic.js'; +import { javaMethodConfig } from '../../src/core/ingestion/method-extractors/configs/jvm.js'; +import { typescriptMethodConfig } from '../../src/core/ingestion/method-extractors/configs/typescript-javascript.js'; +import { computeJavaArityMetadata } from '../../src/core/ingestion/languages/java/arity-metadata.js'; +import { computeTsArityMetadata } from '../../src/core/ingestion/languages/typescript/arity-metadata.js'; +import { buildMethodProps } from '../../src/core/ingestion/utils/method-props.js'; + +describe('shared parameter type projection', () => { + const parameter = (type: string | null) => + Object.freeze({ name: 'p', type, isOptional: false, isVariadic: false }); + + it('shares a projection without changing immutable parameters or merging declarations', () => { + const parameters = Object.freeze([parameter('A'), parameter(null), parameter('B')]); + const types = declarationParameterTypes(parameters); + expect(types).toEqual(['A', 'B']); + expect(declarationParameterTypes(parameters)).toBe(types); + expect(declarationParameterTypes([...parameters])).not.toBe(types); + expect(parameters.map((p) => p.type)).toEqual(['A', null, 'B']); + expect(declarationParameterTypes([])).toEqual([]); + expect(declarationParameterTypes([parameter(null)])).toEqual([]); + }); + + for (const fixture of [ + { java: true, signature: 'int f(int x, String y)', scope: ['int', 'String'], shared: true }, + { + java: true, + signature: 'int f(int x, String... y)', + scope: ['int', 'String', 'varargs'], + shared: false, + }, + { + java: false, + signature: 'f(x: number, y: string): number', + scope: ['number', 'string'], + shared: true, + }, + { java: false, signature: 'f(x, y: string): number', scope: ['', 'string'], shared: false }, + { + java: false, + signature: 'f(x: string, y, z: number): number', + scope: ['string', '', 'number'], + shared: false, + }, + { + java: false, + signature: 'f(x: Box[], ...y: string[]): number', + scope: ['Box', 'string', 'params'], + shared: false, + }, + ]) { + for (const scopeFirst of [true, false]) { + it(`${fixture.signature}: preserves provider projection, scopeFirst=${scopeFirst}`, () => { + const parser = new Parser(); + parser.setLanguage(fixture.java ? Java : TypeScript.typescript); + const tree = parser.parse(`class C { ${fixture.signature} { return 0; } }`); + expect(tree.rootNode.hasError).toBe(false); + const node = tree.rootNode.descendantsOfType( + fixture.java ? 'method_declaration' : 'method_definition', + )[0]!; + const config = fixture.java ? javaMethodConfig : typescriptMethodConfig; + const arity = fixture.java ? computeJavaArityMetadata : computeTsArityMetadata; + const extractor = createMethodExtractor(config); + let scope = scopeFirst ? arity(node) : undefined; + const info = extractor.extractFromNode!(node, { + language: config.language, + filePath: 'fixture', + })!; + const graph = buildMethodProps(info); + scope ??= arity(node); + expect(scope.parameterTypes).toEqual(fixture.scope); + expect(graph.parameterTypes).toEqual( + info.parameters.flatMap((p) => (p.type === null ? [] : [p.type])), + ); + if (fixture.shared) expect(scope.parameterTypes).toBe(graph.parameterTypes); + else expect(scope.parameterTypes).not.toBe(graph.parameterTypes); + expect(arity(node)).toEqual(scope); + expect(buildMethodProps(info)).toEqual(graph); + }); + } + } +}); diff --git a/gitnexus/test/unit/method-declaration-facts.test.ts b/gitnexus/test/unit/method-declaration-facts.test.ts new file mode 100644 index 000000000..3b6576c20 --- /dev/null +++ b/gitnexus/test/unit/method-declaration-facts.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it, vi } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import TypeScript from 'tree-sitter-typescript'; +import { createMethodExtractor } from '../../src/core/ingestion/method-extractors/generic.js'; +import { + javaMethodConfig, + kotlinMethodConfig, +} from '../../src/core/ingestion/method-extractors/configs/jvm.js'; +import { + typescriptMethodConfig, + javascriptMethodConfig, +} from '../../src/core/ingestion/method-extractors/configs/typescript-javascript.js'; +import { csharpMethodConfig } from '../../src/core/ingestion/method-extractors/configs/csharp.js'; +import { pythonMethodConfig } from '../../src/core/ingestion/method-extractors/configs/python.js'; +import { phpMethodConfig } from '../../src/core/ingestion/method-extractors/configs/php.js'; +import { swiftMethodConfig } from '../../src/core/ingestion/method-extractors/configs/swift.js'; +import { dartMethodConfig } from '../../src/core/ingestion/method-extractors/configs/dart.js'; +import { computeJavaArityMetadata } from '../../src/core/ingestion/languages/java/arity-metadata.js'; +import { computeTsArityMetadata } from '../../src/core/ingestion/languages/typescript/arity-metadata.js'; +import { memoizeDeclarationFact } from '../../src/core/ingestion/method-extractors/declaration-facts.js'; +import { computeKotlinArityMetadata } from '../../src/core/ingestion/languages/kotlin/arity-metadata.js'; +import { computeCsharpArityMetadata } from '../../src/core/ingestion/languages/csharp/arity-metadata.js'; +import { computePythonArityMetadata } from '../../src/core/ingestion/languages/python/arity-metadata.js'; +import { computePhpArityMetadata } from '../../src/core/ingestion/languages/php/arity-metadata.js'; +import { computeSwiftArityMetadata } from '../../src/core/ingestion/languages/swift/arity-metadata.js'; +import { computeDartArityMetadata } from '../../src/core/ingestion/languages/dart/arity-metadata.js'; +import { + getLanguageGrammar, + isLanguageAvailable, +} from '../../src/core/tree-sitter/parser-loader.js'; +import { synthesizeSwiftSignatureBindings } from '../../src/core/ingestion/languages/swift/signature-bindings.js'; +import { synthesizeDartSignatureBindings } from '../../src/core/ingestion/languages/dart/signature-bindings.js'; + +describe('shared callable declaration facts', () => { + for (const fixture of [ + { + config: pythonMethodConfig, + code: 'class C:\n @staticmethod\n def f(x: int) -> int:\n return x\n', + owner: 'class_definition', + declaration: 'function_definition', + arity: computePythonArityMetadata, + }, + { + config: dartMethodConfig, + code: 'class C { C(int x) {} }', + owner: 'class_definition', + declaration: 'constructor_signature', + arity: computeDartArityMetadata, + }, + ]) { + for (const scopeFirst of [true, false]) { + it.skipIf(!isLanguageAvailable(fixture.config.language))( + `${fixture.config.language}: shares facts across declaration wrappers, scopeFirst=${scopeFirst}`, + () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(fixture.config.language)!); + const tree = parser.parse(fixture.code); + expect(tree.rootNode.hasError).toBe(false); + const owner = tree.rootNode.descendantsOfType(fixture.owner)[0]!; + const declaration = owner.descendantsOfType(fixture.declaration)[0]!; + const spy = vi.spyOn(fixture.config, 'extractParameters'); + try { + if (scopeFirst) fixture.arity(declaration); + const result = createMethodExtractor(fixture.config).extract(owner, { + language: fixture.config.language, + filePath: 'fixture', + })!; + if (!scopeFirst) fixture.arity(declaration); + expect(result.methods).toHaveLength(1); + expect(result.methods[0]!.parameters.map((p) => p.type)).toEqual(['int']); + expect(spy).toHaveBeenCalledTimes(2); + expect(spy.mock.results[0]!.value).toBe(spy.mock.results[1]!.value); + } finally { + spy.mockRestore(); + } + }, + ); + } + } + for (const fixture of [ + { + language: Java, + config: javaMethodConfig, + code: 'class C { int f(int x, String... rest) { return x; } }', + nodeType: 'method_declaration', + arity: computeJavaArityMetadata, + expectedTypes: ['int', 'String'], + }, + { + language: TypeScript.typescript, + config: typescriptMethodConfig, + code: 'class C { f(x: number, ...rest: string[]): number { return x; } }', + nodeType: 'method_definition', + arity: computeTsArityMetadata, + expectedTypes: ['number', 'string[]'], + }, + ]) { + for (const scopeFirst of [true, false]) { + it(`${fixture.config.language}: traverses parameters once, scopeFirst=${scopeFirst}`, () => { + const parser = new Parser(); + parser.setLanguage(fixture.language); + const tree = parser.parse(fixture.code); + const node = tree.rootNode.descendantsOfType(fixture.nodeType)[0]!; + const read = vi.spyOn(node, 'childForFieldName'); + const extractor = createMethodExtractor(fixture.config); + const structure = () => + extractor.extractFromNode!(node, { + filePath: 'fixture', + language: fixture.config.language, + }); + if (scopeFirst) fixture.arity(node); + const info = structure()!; + if (!scopeFirst) fixture.arity(node); + expect(info.name).toBe('f'); + expect(info.parameters.map((p) => p.rawType ?? p.type)).toEqual(fixture.expectedTypes); + expect(info.parameters.map((p) => p.isVariadic)).toEqual([false, true]); + expect(read.mock.calls.filter(([field]) => field === 'parameters')).toHaveLength(1); + read.mockRestore(); + }); + } + } + + for (const fixture of [ + { + config: kotlinMethodConfig, + code: 'class C {\n fun f(x: Int, y: String): Int { return x }\n}', + nodeType: 'function_declaration', + arity: computeKotlinArityMetadata, + types: ['Int', 'String'], + }, + { + config: csharpMethodConfig, + code: 'class C { public int F(int x, string y = "") { return x; } }', + nodeType: 'method_declaration', + arity: computeCsharpArityMetadata, + types: ['int', 'string'], + }, + { + config: pythonMethodConfig, + code: 'class C:\n def f(self, x: int, y: str = "") -> int:\n return x\n', + nodeType: 'function_definition', + arity: computePythonArityMetadata, + types: ['int', 'str'], + }, + { + config: phpMethodConfig, + code: ' Int { return x } }', + nodeType: 'function_declaration', + arity: computeSwiftArityMetadata, + types: ['Int', 'String'], + }, + { + config: dartMethodConfig, + code: "class C { int f(int x, [String y = '']) { return x; } }", + nodeType: 'method_signature', + arity: computeDartArityMetadata, + types: ['int', 'String'], + }, + ]) { + it.skipIf(!isLanguageAvailable(fixture.config.language))( + `${fixture.config.language}: scope and structure reuse parameter facts`, + () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(fixture.config.language)!); + const tree = parser.parse(fixture.code); + expect(tree.rootNode.hasError).toBe(false); + const node = tree.rootNode.descendantsOfType(fixture.nodeType)[0]!; + const spy = vi.spyOn(fixture.config, 'extractParameters'); + try { + const scope = fixture.arity(node); + const info = createMethodExtractor(fixture.config).extractFromNode!(node, { + language: fixture.config.language, + filePath: 'fixture', + })!; + expect(info.parameters.map((p) => p.type)).toEqual(fixture.types); + expect(scope.parameterTypes?.slice(0, 2)).toEqual(fixture.types); + expect(spy).toHaveBeenCalledTimes(2); + expect(info.parameters).toBe(spy.mock.results[0]!.value); + } finally { + spy.mockRestore(); + } + }, + ); + } + + it('shares the parameter callback through the JavaScript config copy', () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(javascriptMethodConfig.language)!); + const node = parser + .parse('class C { f(x, ...rest) {} }') + .rootNode.descendantsOfType('method_definition')[0]!; + expect(javascriptMethodConfig.extractParameters).toBe(typescriptMethodConfig.extractParameters); + expect(javascriptMethodConfig.extractParameters(node)).toBe( + typescriptMethodConfig.extractParameters(node), + ); + }); + + for (const fixture of [ + { + config: swiftMethodConfig, + code: 'class C { func f(_ x: Int) -> Int { return x } }', + nodeType: 'function_declaration', + bindings: synthesizeSwiftSignatureBindings, + }, + { + config: dartMethodConfig, + code: 'class C { int f(int x) { return x; } }', + nodeType: 'method_signature', + bindings: (node: Parser.SyntaxNode) => + synthesizeDartSignatureBindings(node, node.nextNamedSibling), + }, + ]) { + it.skipIf(!isLanguageAvailable(fixture.config.language))( + `${fixture.config.language}: signature bindings share name and return facts`, + () => { + const parser = new Parser(); + parser.setLanguage(getLanguageGrammar(fixture.config.language)!); + const node = parser.parse(fixture.code).rootNode.descendantsOfType(fixture.nodeType)[0]!; + let reads = 0; + const counted = new Proxy(node, { + get(target, property) { + if (property !== 'tree' && property !== 'id') reads++; + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + expect(fixture.bindings(counted).length).toBeGreaterThan(0); + const before = reads; + expect(before).toBeGreaterThan(0); + expect(fixture.config.extractName(counted)).toBe('f'); + expect(fixture.config.extractReturnType(counted)).toMatch(/^[Ii]nt$/); + expect(reads).toBe(before); + }, + ); + } + + it('separates same-line declarations and reparsed trees', () => { + const parser = new Parser(); + parser.setLanguage(Java); + const first = parser.parse('class C { void f(int x) {} void f(String s) {} }'); + const second = parser.parse('class C { void f(long x) {} }'); + const [intNode, stringNode] = first.rootNode.descendantsOfType('method_declaration'); + const longNode = second.rootNode.descendantsOfType('method_declaration')[0]!; + expect(javaMethodConfig.extractParameters(intNode!)[0]!.type).toBe('int'); + expect(javaMethodConfig.extractParameters(stringNode!)[0]!.type).toBe('String'); + expect(javaMethodConfig.extractParameters(longNode)[0]!.type).toBe('long'); + }); + + it('reuses empty and absent facts across wrappers with the same native identity', () => { + const parser = new Parser(); + parser.setLanguage(Java); + const tree = parser.parse('class C { void f() {} }'); + const node = tree.rootNode.descendantsOfType('method_declaration')[0]!; + // A fresh traversal may already reuse a JS wrapper. This wrapper deliberately + // differs while preserving the actual native tree/node identity. + const wrapper = new Proxy(node, {}); + expect(wrapper).not.toBe(node); + const absent = vi.fn(() => undefined); + const empty = vi.fn(() => []); + for (const extract of [absent, empty]) { + const shared = memoizeDeclarationFact(extract); + const first = shared(node); + expect(shared(wrapper)).toBe(first); + expect(extract).toHaveBeenCalledTimes(1); + } + }); + + it('retries failures and bypasses sharing when tree identity is unavailable', () => { + const parser = new Parser(); + parser.setLanguage(Java); + const node = parser.parse('class C {}').rootNode; + const extract = vi + .fn<() => string>() + .mockImplementationOnce(() => { + throw new Error('incomplete'); + }) + .mockReturnValue('complete'); + const shared = memoizeDeclarationFact(extract); + expect(() => shared(node)).toThrow('incomplete'); + expect(shared(node)).toBe('complete'); + expect(shared(node)).toBe('complete'); + expect(extract).toHaveBeenCalledTimes(2); + const partial = {} as Parser.SyntaxNode; + shared(partial); + shared(partial); + expect(extract).toHaveBeenCalledTimes(4); + }); +}); diff --git a/gitnexus/test/unit/method-declaration-scaling.test.ts b/gitnexus/test/unit/method-declaration-scaling.test.ts new file mode 100644 index 000000000..0989cd3d2 --- /dev/null +++ b/gitnexus/test/unit/method-declaration-scaling.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +const { evaluateScaling } = await import( + new URL('../../bench/method-declaration-facts/scaling.mjs', import.meta.url).href +); + +const sizes = [640, 2560, 10240]; +const series = (cost: (size: number) => number) => + sizes.map((size) => ({ size, time: cost(size), heap: size * 1000 })); + +describe('callable benchmark scaling guard', () => { + it('accepts linear growth with fixed startup overhead', () => { + expect( + evaluateScaling( + series((n) => 500 + n), + ['time', 'heap'], + ).passed, + ).toBe(true); + }); + + it('rejects quadratic time despite linear memory', () => { + expect( + evaluateScaling( + series((n) => n * n), + ['time', 'heap'], + ).passed, + ).toBe(false); + }); + + it('rejects quadratic memory despite linear time', () => { + const rows = series((n) => n).map((row) => ({ ...row, heap: row.size ** 2 })); + expect(evaluateScaling(rows, ['time', 'heap']).passed).toBe(false); + }); + + it('checks the whole size range as well as adjacent sizes', () => { + // Each 4x step grows cost by 5.28x, but the 16x span grows it by 27.86x. + expect( + evaluateScaling( + series((n) => n ** 1.2), + ['time'], + ).passed, + ).toBe(false); + }); + + it('rejects a regression confined to the largest workload', () => { + const rows = series((n) => n); + rows[2].time *= 7; + expect(evaluateScaling(rows, ['time']).passed).toBe(false); + }); + + it.each([0, -1, NaN, Infinity, undefined])('rejects an invalid reading: %s', (time) => { + const rows = series((n) => n); + expect(() => evaluateScaling([{ ...rows[0], time }, ...rows.slice(1)], ['time'])).toThrow(); + }); + + it('rejects missing sizes or metrics', () => { + expect(() => evaluateScaling(series((n) => n).slice(0, 1), ['time'])).toThrow(); + expect(() => + evaluateScaling( + series((n) => n), + [], + ), + ).toThrow(); + }); + + it.each([0, -1, NaN, Infinity, 640])('rejects invalid or repeated sizes: %s', (size) => { + const rows = series((n) => n); + expect(() => evaluateScaling([rows[0], { ...rows[1], size }, rows[2]], ['time'])).toThrow(); + }); +}); diff --git a/gitnexus/test/unit/method-props.test.ts b/gitnexus/test/unit/method-props.test.ts index cd1be05e0..468580ff0 100644 --- a/gitnexus/test/unit/method-props.test.ts +++ b/gitnexus/test/unit/method-props.test.ts @@ -119,7 +119,7 @@ describe('typeTagForId', () => { it('returns empty string for variadic method (arity undefined)', () => { const m = makeMethodInfo('log', [{ name: 'args', type: 'String' }]); - m.parameters[0].isVariadic = true; + m.parameters = m.parameters.map((parameter) => ({ ...parameter, isVariadic: true })); const map = buildMethodMap([m]); expect(typeTagForId(map, 'log', undefined, m)).toBe(''); @@ -204,7 +204,7 @@ describe('arityForIdFromInfo', () => { it('returns undefined for variadic methods', () => { const m = makeMethodInfo('log', [{ name: 'args', type: 'String' }]); - m.parameters[0].isVariadic = true; + m.parameters = m.parameters.map((parameter) => ({ ...parameter, isVariadic: true })); expect(arityForIdFromInfo(m)).toBeUndefined(); });