mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-15 09:11:33 +00:00
Merge b5cea365a3 into e3e50be954
This commit is contained in:
commit
7abc034f2a
35 changed files with 49268 additions and 98 deletions
78
.github/workflows/ci-tests.yml
vendored
78
.github/workflows/ci-tests.yml
vendored
|
|
@ -7,6 +7,84 @@ permissions:
|
|||
contents: read
|
||||
|
||||
jobs:
|
||||
lexical-lookup-benchmark:
|
||||
name: lexical lookup / separate and repeated processes
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GITNEXUS_LEXICAL_BENCH_WORKERS: '2'
|
||||
GITNEXUS_LEXICAL_BENCH_HEAP_MIB: '4096'
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
- name: Pin comparison revision
|
||||
id: baseline
|
||||
shell: bash
|
||||
run: printf 'sha=%s\n' "$(git rev-parse HEAD^1)" >> "$GITHUB_OUTPUT"
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
with:
|
||||
build: 'true'
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ steps.baseline.outputs.sha }}
|
||||
path: baseline
|
||||
persist-credentials: false
|
||||
- name: Build baseline runtime and install search extensions
|
||||
working-directory: baseline/gitnexus
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
npx tsx scripts/ensure-fts.ts
|
||||
- name: Install candidate search extensions
|
||||
working-directory: gitnexus
|
||||
run: npx tsx scripts/ensure-fts.ts
|
||||
- name: Prepare identical source corpora
|
||||
shell: bash
|
||||
env:
|
||||
BASE_SHA: ${{ steps.baseline.outputs.sha }}
|
||||
run: |
|
||||
mkdir "$RUNNER_TEMP/lexical-lookup"
|
||||
for variant in baseline candidate; do
|
||||
corpus="$RUNNER_TEMP/lexical-lookup/corpus-$variant"
|
||||
git clone --quiet --no-checkout --no-hardlinks "$GITHUB_WORKSPACE/baseline" "$corpus"
|
||||
git -C "$corpus" sparse-checkout set gitnexus/src/core/ingestion gitnexus-shared/src/scope-resolution
|
||||
git -C "$corpus" checkout --quiet --detach "$BASE_SHA"
|
||||
done
|
||||
- name: Compare lexical lookup semantics
|
||||
run: node gitnexus/bench/lexical-lookup/differential.mjs "$GITHUB_WORKSPACE/baseline" "$GITHUB_WORKSPACE" | tee "$RUNNER_TEMP/lexical-lookup/differential.log"
|
||||
shell: bash
|
||||
- name: Measure separate processes
|
||||
shell: bash
|
||||
run: >-
|
||||
node gitnexus/bench/lexical-lookup/measure.mjs
|
||||
"$GITHUB_WORKSPACE/baseline" "$GITHUB_WORKSPACE"
|
||||
"$RUNNER_TEMP/lexical-lookup/corpus-baseline" "$RUNNER_TEMP/lexical-lookup/corpus-candidate"
|
||||
"$RUNNER_TEMP/lexical-lookup/fresh" fresh
|
||||
2>&1 | tee "$RUNNER_TEMP/lexical-lookup/fresh.log"
|
||||
- name: Measure repeated processes
|
||||
if: ${{ !cancelled() }}
|
||||
shell: bash
|
||||
run: >-
|
||||
node gitnexus/bench/lexical-lookup/measure.mjs
|
||||
"$GITHUB_WORKSPACE/baseline" "$GITHUB_WORKSPACE"
|
||||
"$RUNNER_TEMP/lexical-lookup/corpus-baseline" "$RUNNER_TEMP/lexical-lookup/corpus-candidate"
|
||||
"$RUNNER_TEMP/lexical-lookup/persistent" persistent
|
||||
2>&1 | tee "$RUNNER_TEMP/lexical-lookup/persistent.log"
|
||||
- name: Retain lexical lookup measurements
|
||||
if: always()
|
||||
uses: ./.github/actions/upload-test-artifact
|
||||
with:
|
||||
name: lexical-lookup-performance
|
||||
path: |
|
||||
${{ runner.temp }}/lexical-lookup/fresh/results.json
|
||||
${{ runner.temp }}/lexical-lookup/persistent/results.json
|
||||
${{ runner.temp }}/lexical-lookup/*.log
|
||||
retention-days: 30
|
||||
|
||||
mcp-query-benchmark:
|
||||
name: MCP query performance
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
91
docs/optimize/identity-memory-scaling-20261103.csv
Normal file
91
docs/optimize/identity-memory-scaling-20261103.csv
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
category,reader,repetition,count,extra_width_bytes,identity_bytes,force_gc,requested_calls,verified_calls,status,peak_rss_bytes,peak_rss_is_lower_bound,final_gc_heap_bytes,final_gc_rss_bytes
|
||||
scale,original prepared async,1,0,0,0,true,3,3,complete,247959552,false,9354912,247320576
|
||||
scale,cooperative prepared,1,0,0,0,true,3,3,complete,248381440,false,9348896,247791616
|
||||
scale,cooperative prepared,1,16384,0,770048,true,3,3,complete,300908544,false,16922008,297074688
|
||||
scale,original prepared async,1,16384,0,770048,true,3,3,complete,303788032,false,16924952,299225088
|
||||
scale,original prepared async,1,32768,0,1540096,true,3,3,complete,322269184,false,24041632,318803968
|
||||
scale,cooperative prepared,1,32768,0,1540096,true,3,3,complete,319725568,false,24046312,315506688
|
||||
scale,cooperative prepared,1,65536,0,3080192,true,3,3,complete,354664448,false,37827224,349306880
|
||||
scale,original prepared async,1,65536,0,3080192,true,3,3,complete,356151296,false,37828664,351764480
|
||||
scale,original prepared async,1,131072,0,6160384,true,3,3,complete,418729984,false,64490336,410820608
|
||||
scale,cooperative prepared,1,131072,0,6160384,true,3,3,complete,424943616,false,65540424,412405760
|
||||
scale,cooperative prepared,1,262144,0,12320768,true,3,3,complete,558362624,false,121669088,528625664
|
||||
scale,original prepared async,1,262144,0,12320768,true,3,3,complete,559734784,false,120741000,528830464
|
||||
scale,original prepared async,1,524288,0,24641536,true,3,3,complete,818147328,false,232378984,764395520
|
||||
scale,cooperative prepared,1,524288,0,24641536,true,3,3,complete,828125184,false,232392608,766697472
|
||||
scale,cooperative prepared,1,1048576,0,49283072,true,3,3,complete,1357352960,false,455643288,1239048192
|
||||
scale,original prepared async,1,1048576,0,49283072,true,3,3,complete,1362251776,false,454759344,1241427968
|
||||
scale,cooperative prepared,2,1048576,0,49283072,true,3,3,complete,1356464128,false,454559560,1239228416
|
||||
scale,original prepared async,2,1048576,0,49283072,true,3,3,complete,1373642752,false,455630704,1242320896
|
||||
scale,original prepared async,2,524288,0,24641536,true,3,3,complete,830435328,false,232388752,770142208
|
||||
scale,cooperative prepared,2,524288,0,24641536,true,3,3,complete,814604288,false,232378264,757653504
|
||||
scale,cooperative prepared,2,262144,0,12320768,true,3,3,complete,555560960,false,121656056,528904192
|
||||
scale,original prepared async,2,262144,0,12320768,true,3,3,complete,562339840,false,121638904,530608128
|
||||
scale,original prepared async,2,131072,0,6160384,true,3,3,complete,418287616,false,64485216,410038272
|
||||
scale,cooperative prepared,2,131072,0,6160384,true,3,3,complete,423247872,false,65547064,410828800
|
||||
scale,cooperative prepared,2,65536,0,3080192,true,3,3,complete,355983360,false,37841384,351195136
|
||||
scale,original prepared async,2,65536,0,3080192,true,3,3,complete,355446784,false,37824832,351186944
|
||||
scale,original prepared async,2,32768,0,1540096,true,3,3,complete,322183168,false,24041472,317911040
|
||||
scale,cooperative prepared,2,32768,0,1540096,true,3,3,complete,321650688,false,24042880,316248064
|
||||
scale,cooperative prepared,2,16384,0,770048,true,3,3,complete,299732992,false,16924824,297156608
|
||||
scale,original prepared async,2,16384,0,770048,true,3,3,complete,304021504,false,16916120,299577344
|
||||
scale,original prepared async,2,0,0,0,true,3,3,complete,248582144,false,9354448,247795712
|
||||
scale,cooperative prepared,2,0,0,0,true,3,3,complete,248983552,false,9348520,248360960
|
||||
scale,original prepared async,3,0,0,0,true,3,3,complete,250396672,false,9354520,249204736
|
||||
scale,cooperative prepared,3,0,0,0,true,3,3,complete,248913920,false,9348664,248324096
|
||||
scale,cooperative prepared,3,262144,0,12320768,true,3,3,complete,557928448,false,121659672,530026496
|
||||
scale,original prepared async,3,262144,0,12320768,true,3,3,complete,559079424,false,121618872,531521536
|
||||
scale,original prepared async,3,32768,0,1540096,true,3,3,complete,320778240,false,24045264,316944384
|
||||
scale,cooperative prepared,3,32768,0,1540096,true,3,3,complete,319979520,false,24038744,315535360
|
||||
scale,cooperative prepared,3,1048576,0,49283072,true,3,3,complete,1356611584,false,455707776,1245691904
|
||||
scale,original prepared async,3,1048576,0,49283072,true,3,3,complete,1358626816,false,454530440,1232388096
|
||||
scale,original prepared async,3,65536,0,3080192,true,3,3,complete,356339712,false,37829552,351657984
|
||||
scale,cooperative prepared,3,65536,0,3080192,true,3,3,complete,354988032,false,37827936,349990912
|
||||
scale,cooperative prepared,3,524288,0,24641536,true,3,3,complete,824152064,false,232397976,769536000
|
||||
scale,original prepared async,3,524288,0,24641536,true,3,3,complete,818388992,false,232381400,759271424
|
||||
scale,original prepared async,3,16384,0,770048,true,3,3,complete,303210496,false,16928848,298729472
|
||||
scale,cooperative prepared,3,16384,0,770048,true,3,3,complete,301977600,false,16920680,297541632
|
||||
scale,cooperative prepared,3,131072,0,6160384,true,3,3,complete,426950656,false,65528616,412585984
|
||||
scale,original prepared async,3,131072,0,6160384,true,3,3,complete,418197504,false,64493728,409317376
|
||||
width,original prepared async,1,65536,128,28246016,true,3,3,complete,486682624,false,61827680,482648064
|
||||
width,cooperative prepared,1,65536,128,28246016,true,3,3,complete,487243776,false,62853184,482398208
|
||||
width,original prepared async,1,65536,512,103743488,true,3,3,complete,797528064,false,137084992,793079808
|
||||
width,cooperative prepared,1,65536,512,103743488,true,3,3,complete,796639232,false,138111840,791740416
|
||||
width,cooperative prepared,2,65536,128,28246016,true,3,3,complete,484167680,false,62862640,479428608
|
||||
width,original prepared async,2,65536,128,28246016,true,3,3,complete,485789696,false,61833368,480915456
|
||||
width,cooperative prepared,2,65536,512,103743488,true,3,3,complete,801923072,false,138116000,796282880
|
||||
width,original prepared async,2,65536,512,103743488,true,3,3,complete,797220864,false,137084336,791736320
|
||||
width,original prepared async,3,65536,128,28246016,true,3,3,complete,486744064,false,61823968,481890304
|
||||
width,cooperative prepared,3,65536,128,28246016,true,3,3,complete,486674432,false,62855800,481607680
|
||||
width,original prepared async,3,65536,512,103743488,true,3,3,complete,798748672,false,137085264,786731008
|
||||
width,cooperative prepared,3,65536,512,103743488,true,3,3,complete,797835264,false,137993424,792526848
|
||||
soak,original prepared async,,65536,0,3080192,true,40,40,complete,360906752,false,36883984,355561472
|
||||
soak,cooperative prepared,,65536,0,3080192,true,40,40,complete,366809088,false,37905184,359317504
|
||||
soak,cooperative prepared,,1048576,0,49283072,false,40,8,failed,2668290048,true,,
|
||||
natural control,original prepared async,,1048576,0,49283072,false,12,12,complete,2331799552,false,453534800,1473449984
|
||||
failure reproduction,cooperative prepared instrumented,,1048576,0,49283072,false,12,8,failed,2661085184,true,,
|
||||
natural soak,cooperative direct,,1048576,0,49283072,false,40,40,complete,3041075200,false,454811184,2352312320
|
||||
direct scale,cooperative direct,1,16384,0,770048,false,5,5,complete,316907520,false,16978936,312582144
|
||||
direct scale,cooperative direct,1,65536,0,3080192,false,5,5,complete,430456832,false,37997512,406716416
|
||||
direct scale,cooperative direct,1,131072,0,6160384,false,5,5,complete,583294976,false,65674696,522543104
|
||||
direct scale,cooperative direct,1,262144,0,12320768,false,5,5,complete,846909440,false,121794512,725827584
|
||||
direct scale,cooperative direct,1,524288,0,24641536,false,5,5,complete,1346248704,false,232843552,1101787136
|
||||
direct scale,cooperative direct,1,1048576,0,49283072,false,5,5,complete,2418225152,false,454894384,1884545024
|
||||
direct width,cooperative direct,1,65536,128,28246016,false,5,5,complete,642912256,false,62943728,601165824
|
||||
direct width,cooperative direct,1,65536,512,103743488,false,5,5,complete,1121271808,false,138047664,1095659520
|
||||
direct width,cooperative direct,2,65536,512,103743488,false,5,5,complete,1124130816,false,138171416,1099202560
|
||||
direct width,cooperative direct,2,65536,128,28246016,false,5,5,complete,642789376,false,62951440,603402240
|
||||
direct scale,cooperative direct,2,1048576,0,49283072,false,5,5,complete,2416967680,false,456022600,1898213376
|
||||
direct scale,cooperative direct,2,524288,0,24641536,false,5,5,complete,1343852544,false,232557824,1100718080
|
||||
direct scale,cooperative direct,2,262144,0,12320768,false,5,5,complete,845365248,false,121794224,724008960
|
||||
direct scale,cooperative direct,2,131072,0,6160384,false,5,5,complete,583536640,false,65666832,521920512
|
||||
direct scale,cooperative direct,2,65536,0,3080192,false,5,5,complete,430624768,false,38004072,407015424
|
||||
direct scale,cooperative direct,2,16384,0,770048,false,5,5,complete,315084800,false,16971856,310923264
|
||||
direct scale,cooperative direct,3,16384,0,770048,false,5,5,complete,313884672,false,16972968,309682176
|
||||
direct scale,cooperative direct,3,65536,0,3080192,false,5,5,complete,430358528,false,37998152,406716416
|
||||
direct scale,cooperative direct,3,131072,0,6160384,false,5,5,complete,583319552,false,65683272,522694656
|
||||
direct scale,cooperative direct,3,262144,0,12320768,false,5,5,complete,844750848,false,121800720,723677184
|
||||
direct scale,cooperative direct,3,524288,0,24641536,false,5,5,complete,1352495104,false,232659648,1105154048
|
||||
direct scale,cooperative direct,3,1048576,0,49283072,false,5,5,complete,2419154944,false,456088936,1902014464
|
||||
direct width,cooperative direct,3,65536,128,28246016,false,5,5,complete,644427776,false,62948408,604971008
|
||||
direct width,cooperative direct,3,65536,512,103743488,false,5,5,complete,1108217856,false,138046296,1082867712
|
||||
|
20977
docs/optimize/identity-memory-scaling-20261103.json
Normal file
20977
docs/optimize/identity-memory-scaling-20261103.json
Normal file
File diff suppressed because one or more lines are too long
156
docs/optimize/identity-memory-scaling-20261103.md
Normal file
156
docs/optimize/identity-memory-scaling-20261103.md
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
# Identity-reader memory scaling: evidence and limits
|
||||
|
||||
These measurements precede the production streaming implementation. See the
|
||||
[follow-up fix and validation](identity-reader-fix-20261103.md) for its results.
|
||||
|
||||
The source-level live-memory bound is linear in graph nodes, stored identity
|
||||
rows, and identity-string bytes. The measured process-memory results support
|
||||
that growth rate over the tested range, but exposed a native allocation failure
|
||||
in the first cooperative-reader probe. **That probe must not be retained.**
|
||||
|
||||
A separate direct-query routing probe passed the natural-GC soak and its own
|
||||
size sweep. That array-based probe remained an isolated experiment: at the end
|
||||
of this study no production adapter change had been made, and memory neutrality
|
||||
had not been established.
|
||||
|
||||
## Main findings
|
||||
|
||||
| Check | Result |
|
||||
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| Cooperative reader using prepared execution, forced GC | Approximately linear RSS; R² 0.999875; held-out 1M result within 0.56% of prediction |
|
||||
| Same reader, normal GC, 1,048,576 nodes | Native 512 MiB buffer pool exhausted on read 9; reproduced in a fresh process |
|
||||
| Unchanged reader, same fixture/pool/heap, normal GC | All 12 requested reads passed |
|
||||
| Cooperative reader using direct-query execution, normal GC | All 40 requested million-node reads passed |
|
||||
| Direct-query probe, independent natural-GC size sweep | Approximately linear RSS; R² 0.998393; held-out 1M result 0.18% above prediction |
|
||||
|
||||
The failed reproduction recorded 249 cursor-close calls with no close errors.
|
||||
Changing execution routing while retaining the SQL and row conversion avoided
|
||||
the failure. This localizes the problem to the prepared-execution path in these
|
||||
probes; it does not establish the exact native ownership or allocator bug.
|
||||
|
||||
## Normal-GC scaling of the direct-query probe
|
||||
|
||||
Each point below is the median of three fresh processes. Each process performs
|
||||
five complete verifications, with no forced collection between them. The heap
|
||||
limit is 4 GiB and the database buffer pool is fixed at 512 MiB. Collection runs
|
||||
before measurement and once at the end to measure retained heap.
|
||||
|
||||
| Nodes | Peak RSS, MiB | Range, MiB |
|
||||
| --------: | ------------: | --------------: |
|
||||
| 16,384 | 300.5 | 299.3–302.2 |
|
||||
| 65,536 | 410.5 | 410.4–410.7 |
|
||||
| 131,072 | 556.3 | 556.3–556.5 |
|
||||
| 262,144 | 806.2 | 805.6–807.7 |
|
||||
| 524,288 | 1,283.9 | 1,281.6–1,289.8 |
|
||||
| 1,048,576 | 2,306.2 | 2,305.0–2,307.1 |
|
||||
|
||||
Fitting only the points through 524,288 nodes gives:
|
||||
|
||||
`peak RSS ≈ 287.4 MiB + 2,014.7 bytes × node count`.
|
||||
|
||||
The held-out million-node prediction is 2,302.1 MiB; the observed median is
|
||||
2,306.2 MiB, a 0.18% difference. Retained JavaScript heap also scales linearly:
|
||||
about 424.7 bytes/node plus 9.7 MiB, R² 0.999995. The million-node retained heap
|
||||
is 434.9 MiB, within 0.12% of that model's held-out prediction.
|
||||
|
||||
At 65,536 nodes, increasing total identity bytes from 2.94 to 26.94 to 98.94 MiB
|
||||
increased median peak RSS from 410.5 to 613.1 to 1,069.3 MiB. The three-point
|
||||
byte-size fit has R² 0.9961. It supports a byte-sensitive bound; it is not a
|
||||
universal extrapolation to arbitrary string sizes or encodings.
|
||||
|
||||
## Repeated reads still have a cost
|
||||
|
||||
The direct-query probe's 40-read natural-GC run reached 2,900.2 MiB peak RSS.
|
||||
Its first 12 reads reached 2,416.7 MiB, versus 2,223.8 MiB for the unchanged
|
||||
reader's 12-read control: an observed **8.7% increase** in this single matched
|
||||
comparison. This is not a replicated estimate of a general memory regression.
|
||||
|
||||
Median RSS rose from 2,427.0 MiB in reads 11–20 to 2,532.2 MiB in reads 31–40.
|
||||
Final collection reduced live JavaScript heap to 433.7 MiB, but RSS remained
|
||||
2,243.3 MiB. These results do not prove a permanent RSS plateau or leak freedom.
|
||||
They establish that 40 reads completed without the prepared-path pool failure.
|
||||
All 1,240 observed cursor closes succeeded; all 1,240 diagnostic prepared-object
|
||||
finalizers had run by completion.
|
||||
|
||||
The smaller forced-GC 65K soaks were more stable: median retained RSS changed
|
||||
by approximately 0.3 MiB for the original reader and 0.4 MiB for the cooperative
|
||||
prepared reader between the same windows. That result did not predict the
|
||||
larger normal-GC failure, which is why forced-GC evidence alone is insufficient.
|
||||
|
||||
## What the source proves
|
||||
|
||||
Let `N_e` be expected graph nodes, `N_d` stored rows scanned, and `S` their
|
||||
combined identity-string size. These counts differ when the database contains
|
||||
unexpected rows. The verifier can materialize those rows before rejecting them.
|
||||
|
||||
- Expected-node maps hold at most `N_e` entries, referencing existing nodes.
|
||||
Normalized IDs add at most linear string storage.
|
||||
- Each query returns one label's identity fields, with at most five scalar
|
||||
fields per row. It does not project source content or relationships.
|
||||
- The cooperative reader appends each row once to one array. It retains one
|
||||
awaited continuation and no collection of previous arrays or promises.
|
||||
- Validation keeps no additional graph-sized result structure. Maps and row
|
||||
arrays belong to one invocation; the adapter closes cursors in `finally`.
|
||||
|
||||
Under the usual linear-storage assumption for V8 maps and dense arrays, the
|
||||
bound on reachable JavaScript entries and payload is **O(N_e + N_d + S)**.
|
||||
For a matching graph and fixed schema this is **O(N + S)**; fixed-width
|
||||
identities reduce it to O(N). The existing input graph is additional and can
|
||||
hold edges, source text, and other properties outside this claim.
|
||||
|
||||
The 1,024-row chunk controls yielding. It does **not** bound memory to 1,024
|
||||
rows: the complete table result is retained. Reachable-object bounds do not
|
||||
prove when V8 collects garbage, when native allocations are released, or how
|
||||
much memory the OS reports as resident. The observed failure is a direct reason
|
||||
not to equate the source-level bound with a total-process safety guarantee.
|
||||
|
||||
## Protocol and evidence
|
||||
|
||||
All reader probes used the unchanged PR runtime
|
||||
`6aeeace8e96603fb062dcc709e834042d5a6df74`, Node 22.23.3, and LadybugDB 0.22.0.
|
||||
Fixtures contain one Function table with unique fixed-width ASCII identities;
|
||||
this exercises the largest possible table result for a given node count.
|
||||
They contain no relationships or source content. Every completed verification
|
||||
checks all identity fields against the in-memory graph, not just counts.
|
||||
|
||||
The initial controlled sweep covered zero nodes and seven doubling sizes from
|
||||
16K through 1M, three processes per reader/size, and three reads per process.
|
||||
Separate string-width cases and soak tests followed. The direct-query variant
|
||||
then received its own prospectively recorded six-size/three-repeat natural-GC
|
||||
protocol, wider-string cases, and five reads per process. Samples from the two
|
||||
execution routes are kept separate.
|
||||
|
||||
The direct-query diagnostic still creates prepared objects for instrumentation
|
||||
but executes the SQL through `Connection.query` instead of executing those
|
||||
prepared objects. A production implementation has not been built or measured.
|
||||
Prepared-object finalizer counts are observations, not proof of native ownership.
|
||||
|
||||
An initial 256 MiB-pool bulk load failed before any reader measurement. Setup
|
||||
was then performed in separate 1 GiB-pool processes; every reported reader
|
||||
measurement uses the same 512 MiB pool. Setup peaks are excluded. A planned
|
||||
additional natural-GC sweep of the prepared path was cancelled after its soak
|
||||
failed; its frozen plan is retained and is not counted as executed work.
|
||||
|
||||
The evidence comprises **90 reader processes: 88 completed and two failed on
|
||||
read 9**, with **448 completed full verifications** across those processes.
|
||||
These are repeated synthetic checks, not 448 distinct test cases. No native
|
||||
worker was terminated. No application dependency or production source changed.
|
||||
Matplotlib 3.11.2 was used in an isolated tool environment to render the figures.
|
||||
|
||||
- [Figure](identity-memory-scaling-20261103.png) · [Vector PDF](identity-memory-scaling-20261103.pdf)
|
||||
- [Raw samples and exact probe sources](identity-memory-scaling-20261103.json)
|
||||
- [Process-level CSV](identity-memory-scaling-20261103.csv)
|
||||
|
||||
The JSON includes source/driver hashes, protocols, all completed and failed
|
||||
samples, failure messages, and original probe scripts. Each worker accepts a
|
||||
JSON config containing `runtime`, `storage`, `output`, `count`, `width`, `mode`,
|
||||
`variant`, `iterations`, and `forceGc`; it runs with
|
||||
`node --expose-gc --max-old-space-size=4096 worker.mjs config.json`.
|
||||
Use a disposable fixture directory and the pinned, built runtime. Run `seed`
|
||||
in a separate process before `measure`; never target a production index.
|
||||
|
||||
The conclusion is limited to these fixtures and ranges. It is not a
|
||||
full-indexing, watch-mode, arbitrary-input, or unlimited-repetition guarantee.
|
||||
The original cooperative prepared path is rejected. These probes left the
|
||||
direct-query route as a measured next candidate, with a memory tradeoff to
|
||||
resolve before retention. The follow-up report above records that work.
|
||||
BIN
docs/optimize/identity-memory-scaling-20261103.pdf
Normal file
BIN
docs/optimize/identity-memory-scaling-20261103.pdf
Normal file
Binary file not shown.
BIN
docs/optimize/identity-memory-scaling-20261103.png
Normal file
BIN
docs/optimize/identity-memory-scaling-20261103.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 349 KiB |
27
docs/optimize/identity-reader-fix-20261103.csv
Normal file
27
docs/optimize/identity-reader-fix-20261103.csv
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
case,reader,nodes,extra_bytes_per_string,reads,identity_bytes,peak_rss_mib,initial_heap_mib,retained_heap_mib,median_read_ms
|
||||
soak-stream-1m-40,stream,1048576,0,40,49283072,2252.996094,433.200363,432.640045,2700.721579
|
||||
baseline-1m-12,baseline,1048576,0,12,49283072,1972.113281,433.092522,432.438225,43542.687642
|
||||
scale-r1-16384,stream,16384,0,5,770048,285.015625,15.861481,16.081467,50.761773
|
||||
scale-r1-65536,stream,65536,0,5,3080192,335.550781,35.726349,35.942062,157.858161
|
||||
scale-r1-131072,stream,131072,0,5,6160384,408.824219,62.143188,62.368248,310.738504
|
||||
scale-r1-262144,stream,262144,0,5,12320768,545.914063,114.884155,115.117432,644.254398
|
||||
scale-r1-524288,stream,524288,0,5,24641536,822.574219,221.255302,221.496635,1324.132947
|
||||
scale-r1-1048576,stream,1048576,0,5,49283072,1375.195313,433.120430,433.388168,2745.030359
|
||||
scale-r2-1048576,stream,1048576,0,5,49283072,1362.007813,434.165672,434.432190,2731.779330
|
||||
scale-r2-524288,stream,524288,0,5,24641536,816.343750,221.280807,221.522964,1301.608581
|
||||
scale-r2-262144,stream,262144,0,5,12320768,546.683594,114.899414,115.131882,613.970581
|
||||
scale-r2-131072,stream,131072,0,5,6160384,408.570313,62.126694,62.352158,301.875663
|
||||
scale-r2-65536,stream,65536,0,5,3080192,333.687500,35.727631,35.943748,151.271004
|
||||
scale-r2-16384,stream,16384,0,5,770048,285.046875,15.860825,16.080383,49.948246
|
||||
scale-r3-262144,stream,262144,0,5,12320768,547.566406,115.672760,115.906494,617.657477
|
||||
scale-r3-16384,stream,16384,0,5,770048,286.113281,15.859398,16.079651,50.132029
|
||||
scale-r3-1048576,stream,1048576,0,5,49283072,1354.042969,433.098320,433.365562,2688.166619
|
||||
scale-r3-65536,stream,65536,0,5,3080192,336.796875,35.741119,35.960281,150.155641
|
||||
scale-r3-524288,stream,524288,0,5,24641536,822.070313,221.264832,221.506134,1263.216974
|
||||
scale-r3-131072,stream,131072,0,5,6160384,408.785156,62.141579,62.367043,295.693047
|
||||
width-r1-128,stream,65536,128,5,28246016,445.675781,59.605537,59.822823,198.336899
|
||||
width-r1-512,stream,65536,512,5,103743488,662.609375,131.409729,131.627014,321.027283
|
||||
width-r2-128,stream,65536,128,5,28246016,445.824219,59.614334,59.831619,200.038170
|
||||
width-r2-512,stream,65536,512,5,103743488,661.734375,131.443489,131.660637,323.354956
|
||||
width-r3-128,stream,65536,128,5,28246016,443.546875,59.607437,59.825127,196.609874
|
||||
width-r3-512,stream,65536,512,5,103743488,663.210938,131.439034,131.656319,322.631212
|
||||
|
15833
docs/optimize/identity-reader-fix-20261103.json
Normal file
15833
docs/optimize/identity-reader-fix-20261103.json
Normal file
File diff suppressed because one or more lines are too long
84
docs/optimize/identity-reader-fix-20261103.md
Normal file
84
docs/optimize/identity-reader-fix-20261103.md
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# Identity reader memory fix — 2026-11-03
|
||||
|
||||
Status: implementation and validation complete. Local changes are not published.
|
||||
|
||||
PR #3805 remains at `6aeeace8e96603fb062dcc709e834042d5a6df74`. This follow-up starts at `9952270d81fc1f79bb8de34d43991b169d104289`, whose runtime is identical to that PR head. The compiled candidate hashes and complete measurements are in the accompanying JSON artifact. Node v22.23.3, LadybugDB 0.22.0, Linux x64.
|
||||
|
||||
## Change
|
||||
|
||||
Identity reconciliation consumes each row immediately through a direct native query. It no longer builds a JavaScript array for a complete table. The existing connection lock covers execution, synchronous row reads, cooperative yields every 1,024 rows, and cursor close. A read, validation or cleanup failure still prevents freshness publication. If validation and cleanup both fail, the original error is preserved.
|
||||
|
||||
Both checks remain in place: after COPY/checkpoint and immediately before freshness publication after the final checkpoint. The queries still scan each included label without filters and check every ID, name, path and source range, including unchanged nodes. Existing CSV normalization and exclusions are preserved. The general executeQuery/executePrepared/streamQuery behavior is unchanged.
|
||||
|
||||
Direct execution follows the evidence in [the preceding memory study](identity-memory-scaling-20261103.md): the prepared cooperative probe exhausted the fixed native pool on read 9 even with cursor closes; the direct-query probe completed 40 reads. This does not establish the precise C++ ownership defect or fix prepared execution globally.
|
||||
|
||||
## Memory evidence
|
||||
|
||||
All **172 complete identity verifications across 26 fresh reader processes passed**. Measurements use the same seeded fixtures, a 512 MiB native pool and a 4 GiB JavaScript heap limit. No GC is forced between reads. Initial/final GC is used only to measure retained memory. RSS includes native allocations and uses the OS high-water mark as well as periodic samples.
|
||||
|
||||
Two comparisons answer different questions:
|
||||
|
||||
- **Original PR implementation, matched first 12 million-node reads:** 1972.1 → 1926.7 MiB peak RSS (-2.3%). Median verification time: 43.54 → 2.68 seconds. This is one fresh paired memory comparison; an earlier original-reader run peaked at 2,223.8 MiB, so the small RSS difference should not be generalized beyond these runs.
|
||||
- **Previous fast direct-query array probe, matched 40-read soak:** 2900.2 → 2253.0 MiB (-22.3%). The new reader completes all 40 million-node reads. Retained heap returns from 433.2 to 432.6 MiB after final GC. RSS rises and falls with normal GC; a flat RSS line is not claimed.
|
||||
|
||||
The following table compares the previous direct-query array probe with the implemented streaming reader. Each point is the median of three fresh processes, each performing five reads; all ranges are shown for the new reader.
|
||||
|
||||
| Nodes | Previous array probe peak MiB | Streaming peak MiB | Streaming range MiB | Change |
|
||||
| --------: | ----------------------------: | -----------------: | ------------------: | -----: |
|
||||
| 16,384 | 300.5 | 285.0 | 285.0–286.1 | -5.1% |
|
||||
| 65,536 | 410.5 | 335.6 | 333.7–336.8 | -18.3% |
|
||||
| 131,072 | 556.3 | 408.8 | 408.6–408.8 | -26.5% |
|
||||
| 262,144 | 806.2 | 546.7 | 545.9–547.6 | -32.2% |
|
||||
| 524,288 | 1283.9 | 822.1 | 816.3–822.6 | -36.0% |
|
||||
| 1,048,576 | 2306.2 | 1362.0 | 1354.0–1375.2 | -40.9% |
|
||||
|
||||
Fit only through 524,288 nodes:
|
||||
|
||||
`peak RSS ≈ 268.12 MiB + 1,109.57 bytes × node count`
|
||||
|
||||
R² = 0.99994720. The held-out 1,048,576-node result is 1362.0 MiB versus 1377.7 MiB predicted (-1.14%). Retained heap fits approximately 9.38 MiB + 424.01 bytes/node, including the original graph.
|
||||
|
||||
Long identities are checked separately at 65,536 nodes. Adding 128 or 512 ASCII bytes to each ID/name/path gives 26.94 or 98.94 MiB of identity input and median RSS peaks of 445.7 and 662.6 MiB. Memory depends on identity bytes as well as node count.
|
||||
|
||||

|
||||
|
||||
## What is established
|
||||
|
||||
The JavaScript live-data bound follows from the implementation: expected-node maps contain at most one entry per included graph node; each successful visit deletes one entry; the reader holds one decoded row and never accumulates rows. Auxiliary live JavaScript memory is O(N_expected + normalized expected-ID bytes + maximum row size). The original graph remains resident. There is no per-read global cache or retained result list.
|
||||
|
||||
Native query results still consume memory for the current label. The finite RSS measurements support approximately linear growth over 16,384–1,048,576 fixed-width nodes under the stated pool, heap and read count. They are not a mathematical proof about every native allocator, unlimited string widths, unlimited repetitions, or the full indexing pipeline. The 40-read soak separately checks repeated use; the held-out size and string-width cases guard against fitting only a convenient small range.
|
||||
|
||||
## End-to-end comparison
|
||||
|
||||
Three independent sessions each contain three paired edits in fresh processes and three in long-lived processes. Initial runtime order alternates between sessions, and edit order alternates within each session. Both runtimes use two parsing workers, a 4 GiB heap and FTS. The sparse 862-file corpus is pinned to `4916e0d5e1f4d6d70d41c1f39cd244bca2c7b390`. Each edit changes one file and uses incremental writes.
|
||||
|
||||
All 18 timed pairs have equal semantic graph digests and expected persisted call targets. Cold seeds are excluded from timing medians.
|
||||
|
||||
| Session | Process mode | Original median | Streaming median | Change |
|
||||
| ------: | ------------ | --------------: | ---------------: | ------: |
|
||||
| 1 | fresh | 19.164s | 17.356s | -9.43% |
|
||||
| 1 | persistent | 18.397s | 16.569s | -9.94% |
|
||||
| 2 | fresh | 19.144s | 17.377s | -9.23% |
|
||||
| 2 | persistent | 18.504s | 16.805s | -9.18% |
|
||||
| 3 | fresh | 20.554s | 17.635s | -14.20% |
|
||||
| 3 | persistent | 20.385s | 17.766s | -12.85% |
|
||||
|
||||
Across all nine edit runs per mode, fresh-process medians are **19.164s → 17.435s (−9.02%)**, and long-lived-process medians are **18.826s → 16.805s (−10.73%)**. Median paired reductions are 9.23% and 10.76%, respectively. All 18 individual pairs improved, with visible variation; the host is shared. These results establish this corpus/workload, not a universal full-repository or watch-mode speedup.
|
||||
|
||||
Whole-process RSS high-water marks across timed samples are 1,947.6 → 1,937.7 MiB for fresh processes and 3,104.4 → 2,898.3 MiB for long-lived processes. The latter includes allocations from seeding and previous edits; these values must not be interpreted as isolated verifier memory.
|
||||
|
||||
## Validation
|
||||
|
||||
- Build, source typecheck and test typecheck passed. Changed-file ESLint completed with 0 errors and 44 warnings (including adapter/native-mock `any` types and an existing FTS-fixture non-null assertion).
|
||||
- 39 focused tests passed: corruption rejection and recovery, CSV Unicode/null parity, row-at-a-time consumption, connection serialization across yields, cursor cleanup, incomplete/multiple results, and primary-error preservation.
|
||||
- Native benchmark passed its normalized-scaling budget (0.713, budget 2), missing/swapped-field controls, and independent tuple oracle through selective DELETE/COPY, checkpoint and reopen.
|
||||
- All 18 end-to-end timing pairs passed semantic graph and persisted-target checks.
|
||||
- Full regression run: 31,574 passed, 103 skipped, five failures in one FTS crash-marker mock (4,788.40 s). The mock still wired the old row-array API; after adapting it to the visitor, all 38 tests in that file passed (50.04 s), as did test typechecking. The full run was not repeated after this test-only correction. Production sources and compiled runtime hashes remained identical to those measured.
|
||||
- The local graph was refreshed. Final change analysis includes the new reader and completed without a partial or truncated result.
|
||||
- Sequential source review found no actionable defect. No independent reviewer was dispatched under the user's AGENTS delegation mapping; this is not an independent review receipt.
|
||||
|
||||
## Reproduction
|
||||
|
||||
See [the benchmark instructions](../../gitnexus/bench/incremental-write-integrity/README.md#memory-scaling). The committed `memory.mjs` runs the actual compiled baseline or streaming implementation without prototype patches. Use separate processes and identical fixtures/limits for comparisons. The accompanying JSON preserves every raw run, fixture fingerprints, exact identity bytes, runtime hashes and per-read samples; CSV and PNG/PDF provide the plotted values.
|
||||
|
||||
[Raw evidence and scripts](identity-reader-fix-20261103.json) · [Process samples CSV](identity-reader-fix-20261103.csv) · [Figure PDF](identity-reader-fix-20261103.pdf)
|
||||
BIN
docs/optimize/identity-reader-fix-20261103.pdf
Normal file
BIN
docs/optimize/identity-reader-fix-20261103.pdf
Normal file
Binary file not shown.
BIN
docs/optimize/identity-reader-fix-20261103.png
Normal file
BIN
docs/optimize/identity-reader-fix-20261103.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 221 KiB |
124
docs/optimize/lexical-3805-report.md
Normal file
124
docs/optimize/lexical-3805-report.md
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
# PR #3805 optimization assessment — 2026-11-03
|
||||
|
||||
No additional runtime optimization met the acceptance limits. The final runtime
|
||||
is unchanged at `6aeeace8e96603fb062dcc709e834042d5a6df74`. The local
|
||||
`optimize/lexical-3805` branch adds complete baseline collection, reversible
|
||||
runtime ordering, regression tests, and measurement receipts.
|
||||
|
||||
## Measurements and decisions
|
||||
|
||||
Each session ran three real edits per lifecycle against matched unchanged-PR
|
||||
controls. Cold seeds were excluded. A second session reversed both lifecycle
|
||||
and initial runtime order. Values below are medians of session medians; with
|
||||
two sessions, the outer median is their midpoint. Negative changes mean less
|
||||
elapsed time.
|
||||
|
||||
| Candidate | Sessions | Fresh: reference → candidate | Change | Warm: reference → candidate | Change | Decision |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | --- |
|
||||
| Reuse claim buckets and specialize singleton selection | 1 | 29.799s → 32.233s | +8.17% | 34.280s → 33.238s | −3.04% | Rejected: fresh regression |
|
||||
| Replace intermediate claim-selection arrays with scans | 2 | 26.347s → 25.941s | −1.54% | 23.526s → 23.491s | −0.15% | Inconclusive |
|
||||
|
||||
The limits were a repeatable warm-process improvement greater than 5%, at most
|
||||
3% fresh-process regression, and three independent confirmation sessions before
|
||||
any keep. Neither candidate reached confirmation eligibility. The second
|
||||
candidate's first session improved neither objective; adding another session
|
||||
cannot satisfy the configured requirement that every paired session improve
|
||||
beyond the threshold. The decision helper returned terminal inconclusive after
|
||||
the requested second sample. There is no confirmed additional speedup, and no
|
||||
final before/after speedup is claimed for the unchanged runtime.
|
||||
|
||||
The [experiment receipt](../../gitnexus/bench/lexical-lookup/optimization-20261103.json)
|
||||
contains all samples, graph hashes, call targets, memory/load diagnostics,
|
||||
candidate patch hashes, forecasts, and decision outputs. It records 18 paired
|
||||
edits across the two experiments: 36 measured analyses plus 12 cold seeds.
|
||||
|
||||
## Baseline identity and uncertainty
|
||||
|
||||
Both experiment runtimes used Node 22.23.3, LadybugDB 0.22.0, two workers and a
|
||||
4 GiB heap. The sparse ingestion corpus was pinned to
|
||||
`4916e0d5e1f4d6d70d41c1f39cd244bca2c7b390`. The host was shared; other workloads
|
||||
were observable during measurement. The first large apparent warm gain in the
|
||||
second experiment disappeared in its completed session median.
|
||||
|
||||
The earlier [PR-versus-base receipt](../../gitnexus/bench/lexical-lookup/baseline-20261103.json)
|
||||
used PR `6b487a5` and base `4916e0d`, before the PR merged a Kotlin grammar
|
||||
update. Its three independent sessions measured 28.494s → 27.552s fresh and
|
||||
30.276s → 29.814s warm. Candidate session variation was 5.02% fresh and 17.01%
|
||||
warm, so those small gains are inconclusive. Those older timings were never
|
||||
used to score the newer experiments. The remote PR still pointed to `6aeeace8`
|
||||
when checked during wrap-up.
|
||||
|
||||
Selection and reporting used the same sparse corpus. No holdout was configured,
|
||||
and these results establish neither a full-repository nor a watch-mode speedup.
|
||||
|
||||
## What profiling justified trying
|
||||
|
||||
The unchanged PR's diagnostic CPU profile attributed about 7.8% of elapsed time
|
||||
to lexical lookup, including claim selection. Garbage collection occupied 8.5%.
|
||||
The CPU profile overlapped indexing, so it was used to locate costs, not to score
|
||||
candidates. A separate allocation profile attributed approximately 1.6 GiB of
|
||||
temporary allocations to claim selection, about 20% of sampled allocations.
|
||||
These are cumulative allocation estimates, not live heap or peak RSS.
|
||||
|
||||
The first candidate's forecast was roughly 0.3–1.5s per 30s edit, plus unknown
|
||||
GC savings. Its warm gain was smaller than the required threshold and fresh
|
||||
time regressed. The second forecast was roughly 0.5–1.5s; its measured aggregate
|
||||
warm reduction was only 0.035s. Allocation reduction alone did not establish an
|
||||
elapsed-time benefit.
|
||||
|
||||
Remaining binding-filter and cycle-detection work has smaller attributed cost.
|
||||
Its elapsed benefit remains unconfirmed and is unlikely to clear this run's 5%
|
||||
threshold independently. A quieter benchmark host would help evaluate smaller
|
||||
changes; broader pipeline changes require a separately scoped experiment.
|
||||
|
||||
## Correctness and retained changes
|
||||
|
||||
All 18 experiment edit pairs matched source/semantic graph hashes and persisted
|
||||
call targets. Every edit changed the target between `save` and `load`; the runs
|
||||
rejected no-op, skipped-FTS, collapsed-graph, or non-incremental shortcuts.
|
||||
Derived communities/processes are excluded from the semantic digest, as in the
|
||||
existing benchmark protocol.
|
||||
|
||||
The first candidate passed 3,472 scope tests and 72,000 differential comparisons.
|
||||
The second passed 3,471 scope tests, 72,000 lookup comparisons, and 50,000
|
||||
additional selector comparisons covering ordering, merges, hoisting, activation,
|
||||
and non-finite positions. These counts describe separate runs and should not be
|
||||
added as distinct tests. Both experimental source patches remain isolated.
|
||||
|
||||
The retained benchmark changes add `GITNEXUS_LEXICAL_BENCH_SCREEN=0` to retain
|
||||
all pairs and `GITNEXUS_LEXICAL_BENCH_BASE_FIRST=0` to reverse initial ordering.
|
||||
Defaults are preserved. Deterministic fresh/persistent regression tests verify
|
||||
that even a slower candidate produces every requested sample and that temporary
|
||||
control files are removed.
|
||||
|
||||
Final source and test typechecks, targeted tests, ESLint, formatting, and staged
|
||||
diff checks passed. Complete graph change analysis reported 18 changed symbols
|
||||
across six files, low risk, and no partial or truncated result. The benchmark's
|
||||
external CI uses were also checked; an empty graph caller set was not treated
|
||||
as proof that it was unused.
|
||||
|
||||
The full suite ran for 93 minutes: 31,567 tests passed, four failed, and 103
|
||||
were skipped. All four failures were in unchanged Codex setup tests whose
|
||||
temporary home directories were bypassed by the inherited `CODEX_HOME` setting.
|
||||
Rerunning both affected files with child-process-only `env -u CODEX_HOME`
|
||||
passed all 62 tests. The full suite was not repeated with that environment
|
||||
correction; its original failure result remains in the validation receipt.
|
||||
|
||||
No final three-session timing confirmation was run: no runtime candidate
|
||||
qualified for retention, and the remaining experiment budget could not fit
|
||||
another confirmation cycle. Final elapsed-time values therefore remain
|
||||
unconfirmed; unchanged runtime source is not a measured zero-percent gain.
|
||||
|
||||
## Run accounting
|
||||
|
||||
The optimization loop ran for 46 minutes 9 seconds. Kept: 0; rejected: 1;
|
||||
inconclusive: 1; not selected: 0; censored: 0; errors/timeouts: 0. The remaining
|
||||
hypothesis was deferred because it lacked a supported gain above the threshold
|
||||
and the remaining approved time could not fit another full confirmation cycle.
|
||||
No dependencies were added. No judges or subagents were used; monetary cost
|
||||
telemetry was unavailable.
|
||||
|
||||
The resumable local ledger is
|
||||
`.context/compound-engineering/ce-optimize/lexical-3805/experiment-log.yaml`,
|
||||
with the approved `spec.yaml` beside it. The receipts linked above preserve the
|
||||
measurements in the branch. No PR updates or pushes were made by this run.
|
||||
120
docs/optimize/pipeline-3805-report.md
Normal file
120
docs/optimize/pipeline-3805-report.md
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
# PR #3805: identity-read optimization experiment
|
||||
|
||||
This report records the rejected batching experiment. The subsequent streaming
|
||||
implementation and its validation are tracked in the
|
||||
[identity-reader fix report](identity-reader-fix-20261103.md).
|
||||
|
||||
The additional batching candidate was rejected because it doubled peak resident
|
||||
memory in the controlled large-table probe. No runtime change was retained.
|
||||
At that decision, the runtime remained `6aeeace8e96603fb062dcc709e834042d5a6df74`; this report and
|
||||
the [measurement receipt](pipeline-3805-results.json) preserve the evidence.
|
||||
|
||||
## Matched baseline and candidate
|
||||
|
||||
Two independent sessions each measured three real edits in fresh processes and
|
||||
three in a persistent process. Session two reversed lifecycle and initial runtime
|
||||
order. Cold seeds were excluded. Every measured pair matched the semantic graph
|
||||
and persisted call targets; all edits used incremental writes.
|
||||
|
||||
| Session | Fresh reference → batch | Warm reference → batch |
|
||||
| ------------------------- | ----------------------- | ---------------------- |
|
||||
| 1 | 22.182s → 21.639s | 22.827s → 20.864s |
|
||||
| 2 | 23.181s → 21.174s | 19.852s → 18.743s |
|
||||
| Median of session medians | 22.682s → 21.406s | 21.339s → 19.803s |
|
||||
|
||||
The aggregate reductions were 1.275s (5.62%) fresh and 1.536s (7.20%) warm.
|
||||
The forecast was approximately 1.5–2.5s per 24-second warm edit; the measured
|
||||
warm reduction is near its lower bound, but cannot qualify as a retained gain.
|
||||
The required third session was not run after the memory failure. These are
|
||||
diagnostic candidate results, not confirmed improvements to the final branch.
|
||||
|
||||
Both runtimes used Node 22.23.3, LadybugDB 0.22.0, two parsing workers, and a
|
||||
4 GiB heap. The sparse ingestion corpus was pinned to
|
||||
`4916e0d5e1f4d6d70d41c1f39cd244bca2c7b390`. The comparison reference was the
|
||||
unchanged PR runtime; candidate patch SHA-256 was
|
||||
`8350e17daba98dfbf3042ab233ae7b9c2b924cfcfaa2ccdd979bcb9795812a25`.
|
||||
The host is shared, and session variation is visible in the table. No holdout
|
||||
was configured. No full-repository or watch-mode improvement is established.
|
||||
|
||||
## Why batching was rejected
|
||||
|
||||
The candidate kept both full identity checks and collected complete identity
|
||||
structs per table, avoiding one asynchronous native call per row. All 33 targeted
|
||||
tests passed, including corruption detection at both publication boundaries.
|
||||
Build, source/test typechecks, and changed-file ESLint passed. The native
|
||||
16K/65K-node benchmark also passed missing/swapped-field controls and checked
|
||||
selective delete/COPY, checkpoint, and reopen against an independent tuple oracle.
|
||||
|
||||
Timing and graph equality did not expose the memory cost. A separate probe used
|
||||
65,536 Function nodes, a fixed 256 MiB database buffer pool, explicit GC, four
|
||||
verifier calls, and separate reference/candidate processes on the same seeded
|
||||
database. Reference peak RSS was 346,552 KiB (338.4 MiB); unbounded collection
|
||||
reached 709,272 KiB (692.6 MiB). Retained JavaScript heap remained about 30 MB in
|
||||
both. The native allocation mechanism was not established.
|
||||
|
||||
Two scratch alternatives were also unsuitable. Ordered paging failed closed on
|
||||
a duplicate identity in the healthy fixture; no engine root cause is claimed.
|
||||
Grouping identities by physical offset preserved the fixture but still reached
|
||||
675,416 KiB. Neither was incorporated into runtime source.
|
||||
|
||||
The correctness review found the measured RSS regression. It was sequential;
|
||||
the independent adversarial lane did not run under the repository's solo mapping.
|
||||
Its verdict is **Not ready**. The full suite was not rerun for this rejected
|
||||
candidate. The earlier unchanged-runtime full-suite result and its setup-test
|
||||
environment limitation remain documented in [the prior report](lexical-3805-report.md).
|
||||
|
||||
## Next measured opportunity
|
||||
|
||||
The installed driver exposes `getNextSync()`. A process-local probe retained the
|
||||
original unfiltered queries and consumed rows synchronously, yielding with
|
||||
`setImmediate` every 1,024 rows. No installed dependency or runtime source was
|
||||
modified by this probe.
|
||||
|
||||
| Controlled 65K-node probe | Original reader | Cooperative synchronous reader |
|
||||
| ----------------------------- | --------------: | -----------------------------: |
|
||||
| Median of four verifier calls | 3,395.766ms | 166.648ms |
|
||||
| Peak RSS | 346,552 KiB | 348,804 KiB |
|
||||
|
||||
The standalone native benchmark measured 50.930ms at 16K nodes and 146.986ms at
|
||||
65K nodes. Both sizes passed the independent oracle after selective writes,
|
||||
checkpointing, and read-only reopening, and rejected missing/swapped fields.
|
||||
These are local verifier measurements, not end-to-end timing confirmation.
|
||||
|
||||
A next implementation should make this reader opt-in at the two identity-check
|
||||
call sites, preserving existing connection locking, error propagation, cursor
|
||||
closing, and default query behavior. It needs yielding/error/cleanup tests,
|
||||
larger-table memory and responsiveness checks, then three matched fresh/warm
|
||||
sessions and the full suite. No new dependency is needed.
|
||||
|
||||
Subsequent [memory-scaling tests](identity-memory-scaling-20261103.md) found that
|
||||
the cooperative prepared path exhausts the native pool on its ninth repeated
|
||||
million-node read without forced GC. A separate direct-query probe passed 40
|
||||
reads and its natural-GC size sweep, but showed higher peak RSS than the original
|
||||
reader. The prepared-path proposal above is therefore superseded by that evidence.
|
||||
|
||||
Fresh graph impact for the shared `readQueryRows` helper is **CRITICAL**:
|
||||
27 affected symbols, six direct callers and six processes, including query
|
||||
handling, embedding reads, statistics, and analysis. This expands the approved
|
||||
mutable scope beyond the reconciliation module and requires maintainer sign-off
|
||||
under GUARDRAILS.md. A global replacement is not proposed.
|
||||
|
||||
## Run state
|
||||
|
||||
The one approved implementation experiment completed and was rejected; kept 0,
|
||||
rejected 1, not selected 0, inconclusive 0, censored 0, experiment errors/timeouts 0.
|
||||
Scratch paging failure is recorded separately above. The optimization loop took
|
||||
about 30 minutes before wrap-up; the durable ledger records exact active time.
|
||||
No judges, subagents, new dependencies, pushes, or PR comments were used. Monetary
|
||||
cost telemetry was unavailable.
|
||||
|
||||
At this experiment's closure, no final speedup or measured zero-percent result
|
||||
was claimed for the unchanged runtime. The greater-than-5% warm target remained
|
||||
unmet by a retained change; fresh timing likewise had no new confirmed value.
|
||||
The adapter opportunity was measured but unimplemented. Its subsequent work is
|
||||
recorded separately in the follow-up report linked above.
|
||||
|
||||
Resumable state is under
|
||||
`.context/compound-engineering/ce-optimize/pipeline-3805/`, including
|
||||
`experiment-log.yaml`, `spec.yaml`, archived candidate patch and raw measurements.
|
||||
The tracked receipt beside this report preserves the paired timings, graph
|
||||
checks, memory samples, native results, and impact report outside that local state.
|
||||
2172
docs/optimize/pipeline-3805-results.json
Normal file
2172
docs/optimize/pipeline-3805-results.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -29,11 +29,15 @@ export interface NameClaimSources {
|
|||
readonly bindingsAt?: (scope: Scope, name: string) => readonly BindingRef[];
|
||||
readonly importsAt?: (scope: Scope) => readonly ImportEdge[];
|
||||
}
|
||||
const NO_BINDINGS: readonly BindingRef[] = Object.freeze([]);
|
||||
const NO_CLAIMS: readonly NameClaim[] = Object.freeze([]);
|
||||
const NO_IMPORTS: readonly ImportEdge[] = Object.freeze([]);
|
||||
const NO_SELECTION: ScopeClaimSelection = Object.freeze({ status: 'absent', claims: NO_CLAIMS });
|
||||
const ABSENT: NameClaimResult = Object.freeze({
|
||||
status: 'absent',
|
||||
bindings: [],
|
||||
imports: [],
|
||||
claims: [],
|
||||
bindings: NO_BINDINGS,
|
||||
imports: NO_IMPORTS,
|
||||
claims: NO_CLAIMS,
|
||||
});
|
||||
// Published scope arrays are readonly. Cache by the array, not the scope ID:
|
||||
// finalization and cache hydration can replace a scope's import/claim snapshot.
|
||||
|
|
@ -42,8 +46,6 @@ const importsByName = new WeakMap<
|
|||
readonly ImportEdge[],
|
||||
ReadonlyMap<string, readonly ImportEdge[]>
|
||||
>();
|
||||
const NO_CLAIMS: readonly NameClaim[] = Object.freeze([]);
|
||||
const NO_IMPORTS: readonly ImportEdge[] = Object.freeze([]);
|
||||
|
||||
export function nameClaimsFor(scope: Scope, name: string): readonly NameClaim[] {
|
||||
const claims = scope.nameClaims;
|
||||
|
|
@ -93,11 +95,13 @@ export function selectClaimsAtScope(
|
|||
name: string,
|
||||
options: NameLookupOptions = {},
|
||||
): ScopeClaimSelection {
|
||||
const candidates = nameClaimsFor(scope, name);
|
||||
if (candidates.length === 0) return NO_SELECTION;
|
||||
const purpose = options.purpose ?? 'value';
|
||||
const claims = nameClaimsFor(scope, name).filter(
|
||||
const claims = candidates.filter(
|
||||
(c) => c.purpose === undefined || c.purpose === 'both' || c.purpose === purpose,
|
||||
);
|
||||
if (claims.length === 0) return { status: 'absent', claims };
|
||||
if (claims.length === 0) return NO_SELECTION;
|
||||
const active = claims.filter(
|
||||
(c) =>
|
||||
c.availableFrom === undefined ||
|
||||
|
|
@ -192,6 +196,9 @@ export function lookupLexicalName(
|
|||
): NameClaimResult {
|
||||
let id: ScopeId | null = startScope;
|
||||
let position = options.position;
|
||||
// Position is cleared when crossing deferred/redirected ownership. Reuse
|
||||
// the caller's readonly options until that happens, then copy only once.
|
||||
let selectionOptions = options;
|
||||
let forcedClaims: readonly NameClaim[] | undefined;
|
||||
let carriedConstructor: TypeRef | undefined;
|
||||
const visited = new Set<ScopeId>();
|
||||
|
|
@ -209,9 +216,10 @@ export function lookupLexicalName(
|
|||
id = scope.lookupPolicy?.parentScope ?? scope.parent;
|
||||
continue;
|
||||
}
|
||||
if (selectionOptions.position !== position) selectionOptions = { ...options, position };
|
||||
const selected: ScopeClaimSelection =
|
||||
forcedClaims === undefined
|
||||
? selectClaimsAtScope(scope, name, { ...options, position })
|
||||
? selectClaimsAtScope(scope, name, selectionOptions)
|
||||
: { status: 'selected', claims: forcedClaims };
|
||||
forcedClaims = undefined;
|
||||
const redirected = selected.claims.find((c) => c.redirect !== undefined);
|
||||
|
|
@ -245,38 +253,47 @@ export function lookupLexicalName(
|
|||
return { ...ABSENT, status: 'blocked', scope, claims: selected.claims };
|
||||
const nameClaims = nameClaimsFor(scope, name);
|
||||
const hasClaimsForName = nameClaims.length > 0;
|
||||
const allBindings = sources.bindingsAt?.(scope, name) ?? scope.bindings.get(name) ?? [];
|
||||
const allBindings =
|
||||
sources.bindingsAt?.(scope, name) ?? scope.bindings.get(name) ?? NO_BINDINGS;
|
||||
const allImports = importsFor(sources.importsAt?.(scope) ?? scope.imports ?? NO_IMPORTS, name);
|
||||
let bindings = allBindings.filter(
|
||||
(b) =>
|
||||
b.availableFrom === undefined ||
|
||||
position === undefined ||
|
||||
compare(b.availableFrom, position) <= 0,
|
||||
);
|
||||
let bindings =
|
||||
allBindings.length === 0
|
||||
? NO_BINDINGS
|
||||
: allBindings.filter(
|
||||
(b) =>
|
||||
b.availableFrom === undefined ||
|
||||
position === undefined ||
|
||||
compare(b.availableFrom, position) <= 0,
|
||||
);
|
||||
let imports = allImports;
|
||||
if (selected.status === 'selected') {
|
||||
bindings = bindings.filter((b) =>
|
||||
selected.claims.some((c) =>
|
||||
c.kind === 'import'
|
||||
? b.via !== undefined && matchesClaim(b.via.atRange, c)
|
||||
: c.kind === 'binding' &&
|
||||
(b.origin === 'local' || b.declarationRange !== undefined) &&
|
||||
matchesBindingClaim(b.declarationRange, c),
|
||||
),
|
||||
);
|
||||
imports = allImports.filter((e) =>
|
||||
selected.claims.some((c) => c.kind === 'import' && matchesClaim(e.atRange, c)),
|
||||
);
|
||||
if (bindings.length > 0)
|
||||
bindings = bindings.filter((b) =>
|
||||
selected.claims.some((c) =>
|
||||
c.kind === 'import'
|
||||
? b.via !== undefined && matchesClaim(b.via.atRange, c)
|
||||
: c.kind === 'binding' &&
|
||||
(b.origin === 'local' || b.declarationRange !== undefined) &&
|
||||
matchesBindingClaim(b.declarationRange, c),
|
||||
),
|
||||
);
|
||||
imports =
|
||||
allImports.length === 0
|
||||
? NO_IMPORTS
|
||||
: allImports.filter((e) =>
|
||||
selected.claims.some((c) => c.kind === 'import' && matchesClaim(e.atRange, c)),
|
||||
);
|
||||
} else if (hasClaimsForName) {
|
||||
// A declaration in another namespace, or a not-yet-visible declaration,
|
||||
// must not sneak back through the unfiltered legacy binding channels.
|
||||
bindings = bindings.filter(
|
||||
(b) =>
|
||||
b.availableFrom !== undefined &&
|
||||
b.declarationRange !== undefined &&
|
||||
!nameClaims.some((c) => rangesOverlap(c.range, b.declarationRange!)),
|
||||
);
|
||||
imports = [];
|
||||
if (bindings.length > 0)
|
||||
bindings = bindings.filter(
|
||||
(b) =>
|
||||
b.availableFrom !== undefined &&
|
||||
b.declarationRange !== undefined &&
|
||||
!nameClaims.some((c) => rangesOverlap(c.range, b.declarationRange!)),
|
||||
);
|
||||
imports = NO_IMPORTS;
|
||||
}
|
||||
const ownType = scope.typeBindings.get(name);
|
||||
const typeFactOwnsName =
|
||||
|
|
@ -328,9 +345,12 @@ export function lookupLexicalName(
|
|||
(typeBinding === undefined || isInferredFact(typeBinding.source))
|
||||
)
|
||||
typeBinding = carriedConstructor;
|
||||
const usable = bindings.filter(
|
||||
(b) => (options.purpose ?? 'value') !== 'value' || b.via?.typeOnly !== true,
|
||||
);
|
||||
const usable =
|
||||
bindings.length === 0
|
||||
? NO_BINDINGS
|
||||
: bindings.filter(
|
||||
(b) => (options.purpose ?? 'value') !== 'value' || b.via?.typeOnly !== true,
|
||||
);
|
||||
return {
|
||||
status:
|
||||
usable.length > 0 ||
|
||||
|
|
@ -358,6 +378,8 @@ export function lookupLexicalName(
|
|||
}
|
||||
// A write to an undeclared name declares nothing, but it still carries the
|
||||
// value's type to reads inside the writing scope.
|
||||
const carried = carriedConstructor === undefined ? {} : { typeBinding: carriedConstructor };
|
||||
if (carriedConstructor === undefined)
|
||||
return id === null ? ABSENT : { ...ABSENT, status: 'blocked' };
|
||||
const carried = { typeBinding: carriedConstructor };
|
||||
return id === null ? { ...ABSENT, ...carried } : { ...ABSENT, ...carried, status: 'blocked' };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# Analyze phase breakdown — where the time actually goes
|
||||
|
||||
Updated 2026-11-01 with scope-emission measurements after #3796 merged, plus
|
||||
production-hardening measurements on LadybugDB **0.22.0** for unchanged dirty checkouts.
|
||||
Updated 2026-11-02 with the lexical-lookup allocation experiment below.
|
||||
Records what the `analyze`
|
||||
pipeline costs per phase and the results of optimization experiments.
|
||||
The September measurements remain below as historical
|
||||
|
|
@ -14,6 +13,192 @@ dispatch path is `bench/parse-dispatch-rounds/`.
|
|||
|
||||
---
|
||||
|
||||
## Lexical-lookup allocation experiment — 2026-11-02
|
||||
|
||||
**Decision: keep experimental; the performance requirement for both workflows
|
||||
has not passed.** Fresh-process edits improved in all three pairs (8.95% lower
|
||||
median), but the first repeated-process pair took 12.68% longer and triggered
|
||||
the harness's early rejection gate. Its host load rose sharply. That does not
|
||||
prove a patch-induced regression, and it also does not establish a warm-process
|
||||
speedup. Do not promote this as a production optimization on this evidence.
|
||||
|
||||
The patch reduces temporary allocation in `lookupNameClaim`,
|
||||
`selectClaimsAtScope` and `lookupLexicalName`: reuse frozen empty results, skip
|
||||
filters over empty inputs, avoid concatenating empty binding channels, and copy
|
||||
lookup options only when the effective position changes. Nonempty binding
|
||||
results retain snapshot semantics. Provider hooks, declaration ownership,
|
||||
shadowing, redirects and position checks still run. No semantic-result cache,
|
||||
new invalidation mechanism or native dependency is introduced.
|
||||
|
||||
### Separate-process edit measurements
|
||||
|
||||
| Measurement | Baseline | Candidate | Less elapsed time |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| Edit 1: `save` → `load` | 122.065s | 111.135s | 8.95% |
|
||||
| Edit 2: `load` → `save` | 114.006s | 101.730s | 10.77% |
|
||||
| Edit 3: `save` → `load` | 131.285s | 111.303s | 15.22% |
|
||||
| Median edit | 122.065s | 111.135s | 8.95% |
|
||||
| All three edits | 367.356s | 324.168s | 11.76% |
|
||||
| Median scope-resolution phase | 68.197s | 62.344s | 8.58% |
|
||||
| Highest process peak RSS across edits | 4.29 GiB | 4.25 GiB | — |
|
||||
|
||||
All three scope-resolution phases improve, but not all wall-time differences
|
||||
belong to that phase. For example, the first pair's process-detection interval
|
||||
falls from 5.889s to 1.475s, and the third pair's post-`done` work falls from
|
||||
16.551s to 7.974s. The patch does not directly change either stage. Treat the
|
||||
observed end-to-end reductions as a small shared-host screen, not a universal
|
||||
speedup or a precise attribution of saved time. One-minute host load ranges
|
||||
from 7.79 to 11.43 at the start of these measured edits.
|
||||
|
||||
### Repeated-process screen
|
||||
|
||||
| Measurement | Baseline | Candidate | Less elapsed time |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| First edit: `save` → `load` | 109.483s | 123.364s | −12.68% |
|
||||
| Scope-resolution phase | 67.013s | 65.739s | 1.90% |
|
||||
| Ladybug write intervals | 8.814s | 12.902s | — |
|
||||
| FTS phase | 10.600s | 14.766s | — |
|
||||
| Work after `done` progress | 8.947s | 14.858s | — |
|
||||
| Process peak RSS, including setup | 5.49 GiB | 5.62 GiB | — |
|
||||
|
||||
These intervals come from progress callbacks; they include all work between
|
||||
adjacent callbacks rather than instrumenting native operations individually.
|
||||
Baseline one-minute load fell **6.35 → 3.81**; candidate load rose
|
||||
**3.81 → 12.23**. Scope resolution improved slightly while several other
|
||||
stages slowed. Only this first pair was run, as specified by the screening
|
||||
gate; there is no three-pair warm-process median. Both graphs and persisted
|
||||
call targets matched. Persistent mode retains one process per variant and
|
||||
models repeated analysis, not watch-event scheduling or debounce latency.
|
||||
|
||||
Separate excluded cold setup runs took 218.219s/187.148s in fresh mode and
|
||||
158.148s/153.951s in persistent mode (baseline/candidate). Each lifecycle uses
|
||||
independent index directories. Setup order is fixed and host load changes,
|
||||
so these are not controlled cold-index speedup claims. The
|
||||
[measurement receipt](lexical-lookup/measurements.json) includes every sample,
|
||||
phase duration, peak memory, graph hash, load average and runtime checksum.
|
||||
|
||||
### Method and correctness boundaries
|
||||
|
||||
Baseline runtime and both disposable corpus checkouts are pinned to
|
||||
`347cc99b4a237f9a71c0084167bcbcbd9d06e610`; the candidate runtime adds only the
|
||||
lookup patch. Each corpus has 3,384 indexed files / 3,120 parseable files,
|
||||
including the same control file. Node **v22.23.3**, LadybugDB **0.22.0**,
|
||||
Linux x64, Intel i9-12900K (24 logical CPUs), 16 parse workers, 8 GiB V8 heap.
|
||||
Samples run serially; paired order is baseline/candidate, candidate/baseline,
|
||||
baseline/candidate after excluded initialization runs. Runtime builds stay
|
||||
fixed throughout each comparison, with no tests or other benchmarks launched
|
||||
by this experiment concurrently. The machine is shared with other workloads.
|
||||
|
||||
The [harness](lexical-lookup/measure.mjs) times `runFullAnalysis`, including
|
||||
incremental persistence and FTS. Process startup, graph hashing and subsequent
|
||||
persisted-target verification are outside the timer. CPU profiling and heap
|
||||
probes are off; the existing scope phase logger remains on for both variants.
|
||||
Each measured edit changes one file, reparses 22 files, hits the parser cache
|
||||
for 3,098 files, uses incremental writes and has no rebuild reasons. FTS must
|
||||
remain available. This is a real edit benchmark, not an up-to-date fast return.
|
||||
|
||||
Every pair compares canonical hashes of **114,413 source/semantic nodes and
|
||||
318,798 relationships**. Derived Folder/Community/Process nodes and
|
||||
CONTAINS/MEMBER_OF/STEP_IN_PROCESS edges are excluded. Hashes cover the graph
|
||||
returned after persistence; cleared source content is not independently
|
||||
verified by these hashes. A separate LadybugDB query after each sample verifies
|
||||
that the control CALLS edge follows `save`/`load`. Matching graphs preserve the
|
||||
baseline's limits; they do not establish complete resolution of the corpus.
|
||||
|
||||
Targeted tests cover mutation after a missing lookup, result snapshots, reused
|
||||
frozen options and deferred activation. The initial validation passed 137 tests
|
||||
across lexical ownership, TypeScript type ownership, block shadowing and
|
||||
TypeScript/Python/C++ native rename suites. A seeded differential screen also
|
||||
matched the original implementation in 72,000 generated lexical lookups.
|
||||
The broader scope-resolution suite then passed **3,467 tests / 175 files**
|
||||
(overlaps the initial targeted tests). The parameterized differential harness
|
||||
also passed when rerun against the same two builds.
|
||||
Build, test typecheck and ESLint passed (existing non-null assertion warnings
|
||||
remain). These were the checks completed alongside the benchmark; PR validation is recorded separately.
|
||||
|
||||
To reproduce, build the baseline and candidate runtimes first, prepare two
|
||||
clean disposable corpus checkouts at the same commit, then run from
|
||||
`gitnexus/` (the runtime arguments name repository roots):
|
||||
|
||||
```sh
|
||||
node bench/lexical-lookup/differential.mjs BASE_RUNTIME CANDIDATE_RUNTIME
|
||||
node bench/lexical-lookup/measure.mjs BASE_RUNTIME CANDIDATE_RUNTIME BASE_CORPUS CANDIDATE_CORPUS OUTPUT_FRESH fresh
|
||||
node bench/lexical-lookup/measure.mjs BASE_RUNTIME CANDIDATE_RUNTIME BASE_CORPUS CANDIDATE_CORPUS OUTPUT_PERSISTENT persistent
|
||||
```
|
||||
|
||||
By default, the timing harness stops after the first edit pair if the candidate
|
||||
is not faster. To establish a baseline, set `GITNEXUS_LEXICAL_BENCH_SCREEN=0`
|
||||
to collect all three pairs regardless of their timings. Repeat each lifecycle
|
||||
in independent sessions, setting `GITNEXUS_LEXICAL_BENCH_BASE_FIRST=0` in
|
||||
alternating sessions to reverse seed and edit ordering. Both settings accept
|
||||
only `0` or `1` and are recorded in `results.json`. Report every sample and
|
||||
the variation between sessions; a screening result alone is not a baseline.
|
||||
|
||||
The harness waits for native workers to exit normally and removes only its
|
||||
control source file. Keep builds fixed and do not run tests concurrently.
|
||||
The receipt's harness checksums identify the versions used for measurement.
|
||||
The checked-in scripts were subsequently formatted, and cleanup now attempts
|
||||
both worker shutdowns and removes the control files even if a worker fails.
|
||||
These changes are outside the timed analysis interval.
|
||||
|
||||
### CI execution
|
||||
|
||||
`.github/workflows/ci-tests.yml` runs the differential check and both process
|
||||
lifecycles in a dedicated job. It builds the candidate and its first parent
|
||||
(the base branch on a pull-request merge checkout), then uses two clean sparse
|
||||
corpus checkouts at that parent. The CI corpus contains the ingestion and shared
|
||||
scope-resolution source trees, plus Git's cone-mode parent files. It is smaller
|
||||
than the full-repository corpus measured above; its timings are a separate series.
|
||||
|
||||
CI sets `GITNEXUS_LEXICAL_BENCH_WORKERS=2` and
|
||||
`GITNEXUS_LEXICAL_BENCH_HEAP_MIB=4096`; the local defaults remain 16 workers and
|
||||
8192 MiB. Each lifecycle requires a new output directory and retains its raw
|
||||
results and logs in the `lexical-lookup-performance` artifact. Graph and persisted
|
||||
call mismatches fail the job. The existing first-pair speed screen reports and
|
||||
stops a non-improving run; it does not make one noisy timing pair a CI failure.
|
||||
|
||||
### Repeated baseline, 2026-11-03 (CI corpus)
|
||||
|
||||
The [baseline receipt](lexical-lookup/baseline-20261103.json) records three
|
||||
independent sessions per lifecycle, each with three paired edits. Every pair
|
||||
was retained, including slower candidates. Runtime order was reversed in the
|
||||
middle session. The corpus is the sparse CI source tree at
|
||||
`4916e0d5e1f4d6d70d41c1f39cd244bca2c7b390`; baseline and candidate runtimes are
|
||||
`4916e0d5e1f4d6d70d41c1f39cd244bca2c7b390` and
|
||||
`6b487a529e14d59e51a9df9a44524e98705a6b71`. Both use Node 22.23.3,
|
||||
LadybugDB 0.22.0, two workers and a 4 GiB heap on the same shared host.
|
||||
|
||||
These are medians of the three session medians; cold seeds are excluded:
|
||||
|
||||
| Lifecycle | Baseline | PR candidate | Change |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| Separate processes | 28.494s | 27.552s | 3.31% less time |
|
||||
| Repeated processes | 30.276s | 29.814s | 1.53% less time |
|
||||
|
||||
The candidate's session medians have coefficients of variation of **5.02%**
|
||||
and **17.01%**, respectively. These small aggregate gains are inconclusive on
|
||||
this host. The receipt also retains every paired percentage change, graph hash,
|
||||
persisted call target and host-load reading. All 18 edit pairs matched 23,641
|
||||
source/semantic nodes and 64,292 relationships, and the seeded differential
|
||||
check passed 72,000 comparisons per session. Selection and reporting share the
|
||||
same corpus; no holdout was used. This does not establish a full-repository or
|
||||
watch-mode speedup.
|
||||
|
||||
During measurement, the PR advanced to
|
||||
`6aeeace8e96603fb062dcc709e834042d5a6df74` by merging a Kotlin grammar update.
|
||||
The lookup implementations and lockfile are unchanged, but the newer native
|
||||
grammar and revision were not measured in that baseline. This receipt remains
|
||||
pinned to the earlier builds.
|
||||
|
||||
The subsequent [optimization report](../../docs/optimize/lexical-3805-report.md)
|
||||
and [experiment receipt](lexical-lookup/optimization-20261103.json) compare two
|
||||
additional candidates against unchanged `6aeeace8` controls. Neither met the
|
||||
required repeatable 5% warm-process improvement with at most 3% fresh-process
|
||||
regression. The runtime remains unchanged; the retained changes make complete,
|
||||
opposite-order baseline collection reproducible.
|
||||
|
||||
---
|
||||
|
||||
## Current measurements — 2026-10-31
|
||||
|
||||
### Environment and timing boundaries
|
||||
|
|
|
|||
|
|
@ -84,3 +84,46 @@ An in-place verification failure keeps a non-FTS dirty phase: an FTS-only
|
|||
repair cannot clear the marker and recertify the inconsistent graph.
|
||||
The same protection applies when FTS returns but analyzer finalization fails
|
||||
before the final identity scan: recovery still rebuilds the graph.
|
||||
|
||||
## Memory scaling
|
||||
|
||||
`memory.mjs` measures the compiled production reader in a fresh process, without
|
||||
forcing GC between reads. It records full identity verification, elapsed time,
|
||||
sampled heap/RSS, the process RSS high-water mark, and initial/final retained heap.
|
||||
The native pool is 512 MiB for measurements and 1 GiB for fixture creation.
|
||||
|
||||
Build the runtime with `npm run build`. Save this configuration outside the repo:
|
||||
|
||||
```json
|
||||
{
|
||||
"runtime": "/absolute/path/to/GitNexus",
|
||||
"storage": "/tmp/gnx-identity-memory",
|
||||
"output": "/tmp/gnx-identity-memory/seed.json",
|
||||
"count": 1048576,
|
||||
"width": 0,
|
||||
"mode": "seed",
|
||||
"reader": "stream",
|
||||
"iterations": 40
|
||||
}
|
||||
```
|
||||
|
||||
Run from `gitnexus/`:
|
||||
|
||||
```sh
|
||||
node --expose-gc --max-old-space-size=4096 bench/incremental-write-integrity/memory.mjs /tmp/config.json
|
||||
```
|
||||
|
||||
Then change `mode` to `measure` and choose a new output file. Run each measurement
|
||||
in a new process and use one process per database at a time. To compare the
|
||||
original array-based implementation, set `runtime` to its separately built
|
||||
checkout and `reader` to `baseline`; keep the same fixture and memory limits.
|
||||
|
||||
Vary `count` with separate fixtures to measure node scaling. `width` adds that
|
||||
many ASCII bytes to each of ID, name and path, so it measures identity size
|
||||
independently of node count. Repeat each point and hold the largest size out of
|
||||
the fitted curve. A finite RSS fit is evidence over that range, not a guarantee
|
||||
about every native allocator or the rest of the indexing pipeline.
|
||||
|
||||
The streaming verifier retains expected-node maps and one decoded row rather
|
||||
than a full table of JavaScript rows. The expected maps grow linearly with graph
|
||||
size; the native query result and the original graph still consume memory.
|
||||
|
|
|
|||
|
|
@ -66,9 +66,9 @@ const source = (file) => import(pathToFileURL(join(ROOT, file)).href);
|
|||
await adapter.tryFlushWAL();
|
||||
const samples = [];
|
||||
let calls = 0;
|
||||
const query = async (sql) => {
|
||||
const query = async (sql, onRow) => {
|
||||
calls++;
|
||||
return adapter.executeQuery(sql);
|
||||
await adapter.forEachQueryRow(sql, onRow);
|
||||
};
|
||||
for (let run = 0; run < 9; run++) {
|
||||
calls = 0;
|
||||
|
|
@ -80,17 +80,18 @@ const source = (file) => import(pathToFileURL(join(ROOT, file)).href);
|
|||
if (run >= 2) samples.push(elapsed);
|
||||
}
|
||||
const first = graph.iterNodes().next().value;
|
||||
const alteredQuery = async (sql) => {
|
||||
const rows = await adapter.executeQuery(sql);
|
||||
if (sql.includes('(n:`Function`)')) rows.find((r) => r.id === first.id).startLine++;
|
||||
return rows;
|
||||
const alteredQuery = async (sql, onRow) => {
|
||||
await adapter.forEachQueryRow(sql, (row) => {
|
||||
if (sql.includes('(n:`Function`)') && row.id === first.id) row.startLine++;
|
||||
onRow(row);
|
||||
});
|
||||
};
|
||||
await assert.rejects(
|
||||
reconcileGraphNodeIdentities(graph, alteredQuery, 'negative-control'),
|
||||
/startLine/,
|
||||
);
|
||||
await assert.rejects(
|
||||
reconcileGraphNodeIdentities(graph, async () => [], 'negative-control'),
|
||||
reconcileGraphNodeIdentities(graph, async () => {}, 'negative-control'),
|
||||
/missing ID/,
|
||||
);
|
||||
|
||||
|
|
@ -124,12 +125,12 @@ const source = (file) => import(pathToFileURL(join(ROOT, file)).href);
|
|||
rows.length !== size.nodes || mismatched > 0 || actual.size !== wanted.size;
|
||||
if (rejected) {
|
||||
await assert.rejects(
|
||||
reconcileGraphNodeIdentities(graph, adapter.executeQuery, phase),
|
||||
reconcileGraphNodeIdentities(graph, adapter.forEachQueryRow, phase),
|
||||
/Graph identity reconciliation/,
|
||||
);
|
||||
} else {
|
||||
assert.equal(
|
||||
(await reconcileGraphNodeIdentities(graph, adapter.executeQuery, phase)).nodes,
|
||||
(await reconcileGraphNodeIdentities(graph, adapter.forEachQueryRow, phase)).nodes,
|
||||
size.nodes,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
143
gitnexus/bench/incremental-write-integrity/memory.mjs
Normal file
143
gitnexus/bench/incremental-write-integrity/memory.mjs
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
// Build the runtime, then: node --expose-gc --max-old-space-size=4096 memory.mjs CONFIG.json
|
||||
// Config: { runtime, storage, output, count, width, mode: "seed"|"measure",
|
||||
// reader: "baseline"|"stream", iterations }. Each measurement is a fresh process.
|
||||
// No forced GC between reads. Initial/final GC measure retained graph memory only.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
const config = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
|
||||
const { runtime, storage, output, count, width = 0, mode, reader, iterations } = config;
|
||||
assert(Number.isSafeInteger(count) && count >= 0);
|
||||
assert(Number.isSafeInteger(width) && width >= 0);
|
||||
assert(Number.isSafeInteger(iterations) && iterations > 0);
|
||||
assert(['seed', 'measure'].includes(mode));
|
||||
assert(['baseline', 'stream'].includes(reader));
|
||||
assert(global.gc, 'Run with --expose-gc');
|
||||
const bufferPoolBytes = (mode === 'seed' ? 1024 : 512) * 1048576;
|
||||
process.env.GITNEXUS_LBUG_BUFFER_POOL_SIZE = String(bufferPoolBytes);
|
||||
process.env.GITNEXUS_SHARED_STORE = 'off';
|
||||
process.env.GITNEXUS_HOME = path.join(path.dirname(output), 'home');
|
||||
const source = (file) => import(pathToFileURL(path.join(runtime, 'gitnexus/dist', file)).href);
|
||||
const { createKnowledgeGraph } = await source('core/graph/graph.js');
|
||||
const { reconcileGraphNodeIdentities } = await source('core/incremental/write-reconciliation.js');
|
||||
const adapter = await source('core/lbug/lbug-adapter.js');
|
||||
const graph = createKnowledgeGraph();
|
||||
const padding = 'x'.repeat(width);
|
||||
const fingerprint = createHash('sha256');
|
||||
let identityBytes = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const index = String(i).padStart(7, '0');
|
||||
const id = `Function:node${index}${padding}`;
|
||||
const name = `fn${index}${padding}`;
|
||||
const filePath = `src/file${index}.ts${padding}`;
|
||||
const startLine = (i % 32) * 4;
|
||||
const endLine = startLine + 2;
|
||||
identityBytes += Buffer.byteLength(id) + Buffer.byteLength(name) + Buffer.byteLength(filePath);
|
||||
fingerprint.update(JSON.stringify([id, name, filePath, startLine, endLine]) + '\n');
|
||||
graph.addNode({ id, label: 'Function', properties: { name, filePath, startLine, endLine } });
|
||||
}
|
||||
const result = {
|
||||
config,
|
||||
node: process.version,
|
||||
driver: JSON.parse(
|
||||
fs.readFileSync(path.join(runtime, 'gitnexus/node_modules/@ladybugdb/core/package.json')),
|
||||
).version,
|
||||
bufferPoolBytes,
|
||||
identityBytes,
|
||||
identityFingerprint: fingerprint.digest('hex'),
|
||||
iterations: [],
|
||||
completed: false,
|
||||
};
|
||||
let peakRss = 0,
|
||||
peakHeap = 0,
|
||||
currentPeakRss = 0,
|
||||
currentPeakHeap = 0;
|
||||
function sample() {
|
||||
const memory = process.memoryUsage();
|
||||
peakRss = Math.max(peakRss, memory.rss);
|
||||
peakHeap = Math.max(peakHeap, memory.heapUsed);
|
||||
currentPeakRss = Math.max(currentPeakRss, memory.rss);
|
||||
currentPeakHeap = Math.max(currentPeakHeap, memory.heapUsed);
|
||||
return memory;
|
||||
}
|
||||
function save() {
|
||||
result.peakRssBytes = Math.max(peakRss, process.resourceUsage().maxRSS * 1024);
|
||||
result.sampledPeakHeapBytes = peakHeap;
|
||||
fs.writeFileSync(output, JSON.stringify(result, null, 2) + '\n');
|
||||
}
|
||||
let timer;
|
||||
try {
|
||||
await adapter.initLbug(path.join(storage, 'lbug'), { skipFts: true });
|
||||
if (mode === 'seed') {
|
||||
await adapter.loadGraphToLbug(graph, storage, storage, undefined, undefined, undefined, 'none');
|
||||
await adapter.tryFlushWAL();
|
||||
} else {
|
||||
global.gc();
|
||||
result.initial = sample();
|
||||
timer = setInterval(sample, 20);
|
||||
timer.unref();
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
currentPeakRss = 0;
|
||||
currentPeakHeap = 0;
|
||||
const before = sample();
|
||||
let queries = 0;
|
||||
const readRows =
|
||||
reader === 'baseline'
|
||||
? async (query) => {
|
||||
queries++;
|
||||
sample();
|
||||
const rows = await adapter.executeQuery(query);
|
||||
sample();
|
||||
return rows;
|
||||
}
|
||||
: async (query, visit) => {
|
||||
queries++;
|
||||
sample();
|
||||
await adapter.forEachQueryRow(query, visit);
|
||||
sample();
|
||||
};
|
||||
const start = performance.now();
|
||||
const receipt = await reconcileGraphNodeIdentities(graph, readRows, 'memory-scaling');
|
||||
const ms = performance.now() - start;
|
||||
assert.equal(receipt.nodes, count);
|
||||
assert.equal(queries, receipt.tables);
|
||||
const after = sample();
|
||||
result.iterations.push({
|
||||
i,
|
||||
ms,
|
||||
queries,
|
||||
verifiedNodes: receipt.nodes,
|
||||
before,
|
||||
after,
|
||||
peakRssBytes: currentPeakRss,
|
||||
sampledPeakHeapBytes: currentPeakHeap,
|
||||
maxRssKiB: process.resourceUsage().maxRSS,
|
||||
});
|
||||
save();
|
||||
}
|
||||
clearInterval(timer);
|
||||
global.gc();
|
||||
result.finalAfterGc = sample();
|
||||
}
|
||||
result.completed = true;
|
||||
} catch (error) {
|
||||
result.error = String(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
clearInterval(timer);
|
||||
await adapter.closeLbug();
|
||||
save();
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
count,
|
||||
width,
|
||||
reader,
|
||||
completed: result.completed,
|
||||
iterations: result.iterations.length,
|
||||
peakRssMiB: result.peakRssBytes / 1048576,
|
||||
}),
|
||||
);
|
||||
1277
gitnexus/bench/lexical-lookup/baseline-20261103.json
Normal file
1277
gitnexus/bench/lexical-lookup/baseline-20261103.json
Normal file
File diff suppressed because it is too large
Load diff
109
gitnexus/bench/lexical-lookup/differential.mjs
Normal file
109
gitnexus/bench/lexical-lookup/differential.mjs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
const [baseline, candidate] = process.argv.slice(2);
|
||||
assert(baseline && candidate, 'Expected BASE_RUNTIME CANDIDATE_RUNTIME');
|
||||
const moduleAt = (root) =>
|
||||
pathToFileURL(path.resolve(root, 'gitnexus/dist/_shared/scope-resolution/name-claims.js')).href;
|
||||
const { lookupLexicalName: before } = await import(moduleAt(baseline));
|
||||
const { lookupLexicalName: after } = await import(moduleAt(candidate));
|
||||
let seed = 48161;
|
||||
const random = (n) => {
|
||||
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
|
||||
return seed % n;
|
||||
};
|
||||
const pos = (n) => ({ startLine: n, startCol: 0 });
|
||||
const range = (n) => ({ ...pos(n), endLine: n, endCol: 20 });
|
||||
const names = ['a', 'b', 'missing'];
|
||||
let comparisons = 0;
|
||||
for (let trial = 0; trial < 3000; trial++) {
|
||||
const scopes = new Map();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const claims = [];
|
||||
const bindings = new Map();
|
||||
const types = new Map();
|
||||
const imports = [];
|
||||
for (const name of names.slice(0, 2)) {
|
||||
if (random(3) === 0)
|
||||
claims.push({
|
||||
name,
|
||||
kind: ['binding', 'import', 'blocked'][random(3)],
|
||||
range: range(2 + random(8)),
|
||||
purpose: ['value', 'type', 'both', undefined][random(4)],
|
||||
availableFrom: random(2) ? pos(2 + random(8)) : undefined,
|
||||
inactive: ['outer', 'blocked', 'module'][random(3)],
|
||||
redirect: random(8) === 0 ? 'module' : undefined,
|
||||
hoisted: random(2) === 0,
|
||||
merge: random(3) === 0,
|
||||
});
|
||||
if (random(2))
|
||||
bindings.set(name, [
|
||||
{
|
||||
def: { nodeId: `${i}:${name}`, type: 'Function', filePath: 'a.ts' },
|
||||
origin: random(2) ? 'local' : 'import',
|
||||
availableFrom: random(2) ? pos(5) : undefined,
|
||||
declarationRange: range(5),
|
||||
},
|
||||
]);
|
||||
if (random(3) === 0)
|
||||
imports.push({
|
||||
kind: 'namespace',
|
||||
localName: name,
|
||||
targetFile: random(2) ? 'b.ts' : null,
|
||||
targetExportedName: '',
|
||||
atRange: range(4),
|
||||
linkStatus: random(2) ? 'resolved' : 'unresolved',
|
||||
});
|
||||
if (random(3) === 0)
|
||||
types.set(name, {
|
||||
rawName: 'Value',
|
||||
source: [
|
||||
'annotation',
|
||||
'constructor-inferred',
|
||||
'assignment-inferred',
|
||||
'receiver-propagated',
|
||||
][random(4)],
|
||||
declaredAtScope: String(i),
|
||||
bindingRange: random(2) ? range(5) : undefined,
|
||||
});
|
||||
}
|
||||
scopes.set(String(i), {
|
||||
id: String(i),
|
||||
parent: i === 0 ? null : String(i - 1),
|
||||
kind: i === 0 ? 'Module' : ['Function', 'Block', 'Class', 'Object'][random(4)],
|
||||
filePath: 'a.ts',
|
||||
range: range(i),
|
||||
bindings,
|
||||
imports,
|
||||
ownedDefs: [],
|
||||
typeBindings: types,
|
||||
nameClaims: claims,
|
||||
lexicalNames: random(3) === 0 ? new Set(['a']) : undefined,
|
||||
ownsReceivers: random(3) === 0 ? new Set(['b']) : undefined,
|
||||
lookupPolicy: {
|
||||
nameClaimsComplete: random(2) === 0,
|
||||
deferParentActivation: random(2) === 0,
|
||||
skipFromChildren: random(4) === 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
if (trial % 20 === 0) scopes.get('0').parent = '4';
|
||||
const sources = { scopes: { getScope: (id) => scopes.get(id) } };
|
||||
for (const name of names)
|
||||
for (const purpose of ['value', 'type'])
|
||||
for (const position of [undefined, pos(1), pos(6), pos(12)]) {
|
||||
const options = Object.freeze({
|
||||
purpose,
|
||||
position,
|
||||
skipEnclosingClasses: trial % 2 === 0,
|
||||
preservePosition: trial % 3 === 0,
|
||||
});
|
||||
assert.deepEqual(
|
||||
after('4', name, sources, options),
|
||||
before('4', name, sources, options),
|
||||
`Trial ${trial}, ${name}, ${purpose}, line ${position?.startLine}`,
|
||||
);
|
||||
comparisons++;
|
||||
}
|
||||
}
|
||||
console.log({ comparisons, status: 'passed' });
|
||||
303
gitnexus/bench/lexical-lookup/measure.mjs
Normal file
303
gitnexus/bench/lexical-lookup/measure.mjs
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
// Build both runtimes, then run with Node:
|
||||
// measure.mjs BASE_RUNTIME CANDIDATE_RUNTIME BASE_CORPUS CANDIDATE_CORPUS OUTPUT [fresh|persistent]
|
||||
// Corpus checkouts must be clean/disposable and at the same commit. OUTPUT must not exist.
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync, fork } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
const [baselineRuntime, candidateRuntime, baseline, candidate, output] = process.argv
|
||||
.slice(2, 7)
|
||||
.map((value) => path.resolve(value));
|
||||
const lifecycle = process.argv[7] ?? 'fresh';
|
||||
const workers = Number(process.env.GITNEXUS_LEXICAL_BENCH_WORKERS ?? 16);
|
||||
const heapMiB = Number(process.env.GITNEXUS_LEXICAL_BENCH_HEAP_MIB ?? 8192);
|
||||
const screenSetting = process.env.GITNEXUS_LEXICAL_BENCH_SCREEN ?? '1';
|
||||
const baseFirstSetting = process.env.GITNEXUS_LEXICAL_BENCH_BASE_FIRST ?? '1';
|
||||
assert(Number.isSafeInteger(workers) && workers > 0, 'Workers must be a positive integer');
|
||||
assert(Number.isSafeInteger(heapMiB) && heapMiB > 0, 'Heap MiB must be a positive integer');
|
||||
assert(['0', '1'].includes(screenSetting), 'Screen must be 0 or 1');
|
||||
assert(['0', '1'].includes(baseFirstSetting), 'Base first must be 0 or 1');
|
||||
const screen = screenSetting === '1';
|
||||
const baseFirst = baseFirstSetting === '1';
|
||||
assert(['fresh', 'persistent'].includes(lifecycle));
|
||||
assert(
|
||||
baseline && candidate && output,
|
||||
'Expected BASE_RUNTIME CANDIDATE_RUNTIME BASE_CORPUS CANDIDATE_CORPUS OUTPUT [fresh|persistent]',
|
||||
);
|
||||
assert.notEqual(baseline, candidate);
|
||||
const isWorker = process.env.GITNEXUS_LEXICAL_BENCH_WORKER === '1';
|
||||
assert(baselineRuntime && candidateRuntime);
|
||||
let runFullAnalysis, initLbug, executeQuery, closeLbug;
|
||||
if (!isWorker) assert(!fs.existsSync(output), 'Output already exists');
|
||||
const control = 'gitnexus/src/semantic-bench-control.ts';
|
||||
const source =
|
||||
'export class SemanticBenchStore { save(): void {} load(): void {} }\nexport function semanticBenchUse(s: SemanticBenchStore) { s.save(); }\n';
|
||||
const head = (repo) =>
|
||||
execFileSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).trim();
|
||||
assert.equal(head(baseline), head(candidate));
|
||||
for (const repo of isWorker ? [] : [baseline, candidate]) {
|
||||
assert.equal(
|
||||
execFileSync('git', ['status', '--porcelain'], { cwd: repo, encoding: 'utf8' }),
|
||||
'',
|
||||
'Checkout must be clean',
|
||||
);
|
||||
assert(!fs.existsSync(path.join(repo, control)));
|
||||
}
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
process.env.GITNEXUS_HOME = path.join(output, 'home');
|
||||
process.env.GITNEXUS_STORAGE_ROOT = path.join(output, 'indexes');
|
||||
delete process.env.GITNEXUS_STORAGE_PATH;
|
||||
process.env.GITNEXUS_SHARED_STORE = 'off';
|
||||
process.env.PROF_SCOPE_RESOLUTION = '1';
|
||||
const records = [];
|
||||
if (isWorker) {
|
||||
const runtime = process.env.GITNEXUS_BENCH_RUNTIME;
|
||||
({ runFullAnalysis } = await import(path.join(runtime, 'gitnexus/dist/core/run-analyze.js')));
|
||||
({ initLbug, executeQuery, closeLbug } = await import(
|
||||
path.join(runtime, 'gitnexus/dist/core/lbug/lbug-adapter.js')
|
||||
));
|
||||
const { configureLoggerSink } = await import(path.join(runtime, 'gitnexus/dist/core/logger.js'));
|
||||
configureLoggerSink({
|
||||
destination: {
|
||||
write(line) {
|
||||
try {
|
||||
records.push(JSON.parse(line));
|
||||
} catch {
|
||||
/* non-JSON diagnostic */
|
||||
}
|
||||
process.stderr.write(line);
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
const results = [];
|
||||
const canonical = (value) =>
|
||||
Array.isArray(value)
|
||||
? value.map(canonical)
|
||||
: value && typeof value === 'object'
|
||||
? Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, canonical(value[key])]),
|
||||
)
|
||||
: value;
|
||||
const digest = (rows) => {
|
||||
const hash = createHash('sha256');
|
||||
for (const row of rows.sort()) hash.update(row).update('\n');
|
||||
return { count: rows.length, sha256: hash.digest('hex') };
|
||||
};
|
||||
|
||||
async function sample(repo, reuse, name) {
|
||||
process.env.GITNEXUS_EXPERIMENTAL_SEMANTIC_CACHE = '0';
|
||||
global.gc?.();
|
||||
records.length = 0;
|
||||
const logs = [];
|
||||
const progress = [];
|
||||
const loadBefore = os.loadavg();
|
||||
const started = performance.now();
|
||||
const result = await runFullAnalysis(
|
||||
repo,
|
||||
{
|
||||
skipAgentsMd: true,
|
||||
skipSkills: true,
|
||||
workerPoolSize: workers,
|
||||
verbose: true,
|
||||
},
|
||||
{
|
||||
onProgress(phase, percent, message) {
|
||||
if (progress.at(-1)?.phase !== phase)
|
||||
progress.push({ ms: performance.now() - started, phase, percent, message });
|
||||
},
|
||||
onLog(message) {
|
||||
logs.push(message);
|
||||
},
|
||||
},
|
||||
);
|
||||
const ms = performance.now() - started;
|
||||
assert(!result.alreadyUpToDate, 'A no-op is not an edit benchmark');
|
||||
assert(!result.graphWriteCollapsed);
|
||||
assert(!result.ftsSkipped, 'This benchmark includes FTS');
|
||||
const graph = result.pipelineResult.graph;
|
||||
const nodes = [],
|
||||
edges = [];
|
||||
for (const node of graph.iterNodes()) {
|
||||
// Derived clustering is stochastic; compare all source/semantic properties.
|
||||
if (['Community', 'Process', 'Folder'].includes(node.label)) continue;
|
||||
nodes.push(JSON.stringify(canonical(node)));
|
||||
}
|
||||
let controlCalls = 0;
|
||||
for (const edge of graph.iterRelationships()) {
|
||||
if (['MEMBER_OF', 'STEP_IN_PROCESS', 'CONTAINS'].includes(edge.type)) continue;
|
||||
edges.push(JSON.stringify(canonical(edge)));
|
||||
if (edge.type === 'CALLS' && edge.sourceId.includes('semanticBenchUse')) controlCalls++;
|
||||
}
|
||||
assert(controlCalls > 0, 'The changed call must resolve');
|
||||
const expectedTarget = fs.readFileSync(path.join(repo, control), 'utf8').includes('s.load()')
|
||||
? 'load'
|
||||
: 'save';
|
||||
let persistedTargets;
|
||||
try {
|
||||
await initLbug(path.join(result.storagePath, 'lbug'), { skipFts: true });
|
||||
persistedTargets = await executeQuery(
|
||||
"MATCH (a:Function)-[r:CodeRelation]->(b:Method) WHERE a.name = 'semanticBenchUse' AND r.type = 'CALLS' RETURN b.name AS target",
|
||||
);
|
||||
assert.deepEqual(
|
||||
persistedTargets.map((row) => row.target),
|
||||
[expectedTarget],
|
||||
'Persisted call must follow the edit',
|
||||
);
|
||||
} finally {
|
||||
await closeLbug();
|
||||
}
|
||||
const report = {
|
||||
name,
|
||||
variant: reuse ? 'candidate' : 'baseline',
|
||||
ms,
|
||||
loadBefore,
|
||||
loadAfter: os.loadavg(),
|
||||
maxRssKiB: process.resourceUsage().maxRSS,
|
||||
stats: result.stats,
|
||||
incremental: result.incrementalStats,
|
||||
rebuildReasons: result.rebuildReasons,
|
||||
reparsedFiles: result.pipelineResult.reparsedFileCount,
|
||||
cachedFiles: result.pipelineResult.parseCacheHitFileCount,
|
||||
graph: { nodes: digest(nodes), edges: digest(edges) },
|
||||
logs,
|
||||
progress,
|
||||
persistedTargets,
|
||||
records: [...records],
|
||||
};
|
||||
console.log(
|
||||
'LEXICAL_BENCH ' +
|
||||
JSON.stringify({ name, ms, incremental: report.incremental, graph: report.graph }),
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
// Separate runtimes isolate implementation and JIT state. Fresh mode exits
|
||||
// after every sample; persistent mode retains a process across real edits.
|
||||
function worker(repo, reuse) {
|
||||
let child, pending, exited;
|
||||
const start = () => {
|
||||
child = fork(fileURLToPath(import.meta.url), process.argv.slice(2), {
|
||||
execArgv: [`--max-old-space-size=${heapMiB}`, '--expose-gc'],
|
||||
env: {
|
||||
...process.env,
|
||||
GITNEXUS_LEXICAL_BENCH_WORKER: '1',
|
||||
GITNEXUS_BENCH_RUNTIME: reuse ? candidateRuntime : baselineRuntime,
|
||||
},
|
||||
stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
|
||||
});
|
||||
exited = new Promise((resolve) => child.once('exit', resolve));
|
||||
child.on('message', (message) => {
|
||||
const current = pending;
|
||||
pending = undefined;
|
||||
if (message.error) current?.reject(new Error(message.error));
|
||||
else current?.resolve(message.report);
|
||||
});
|
||||
child.on('error', (error) => pending?.reject(error));
|
||||
child.on('exit', (code) => pending?.reject(new Error(`Benchmark worker exited: ${code}`)));
|
||||
};
|
||||
const close = async () => {
|
||||
if (!child) return;
|
||||
if (child.connected) child.disconnect();
|
||||
const code = await exited;
|
||||
child = undefined;
|
||||
assert.equal(code, 0, 'Worker must exit normally after analysis');
|
||||
};
|
||||
return {
|
||||
async run(name) {
|
||||
assert(!pending, 'Samples must run serially');
|
||||
if (!child) start();
|
||||
const report = await new Promise((resolve, reject) => {
|
||||
pending = { resolve, reject };
|
||||
child.send({ repo, reuse, name });
|
||||
});
|
||||
results.push(report);
|
||||
fs.writeFileSync(
|
||||
path.join(output, 'results.json'),
|
||||
JSON.stringify(
|
||||
{
|
||||
head: head(repo),
|
||||
node: process.version,
|
||||
lifecycle,
|
||||
workers,
|
||||
heapMiB,
|
||||
screen,
|
||||
baseFirst,
|
||||
baselineRuntime,
|
||||
candidateRuntime,
|
||||
results,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
if (lifecycle === 'fresh') await close();
|
||||
return report;
|
||||
},
|
||||
close,
|
||||
};
|
||||
}
|
||||
|
||||
if (isWorker) {
|
||||
process.on('message', async ({ repo, reuse, name }) => {
|
||||
try {
|
||||
process.send({ report: await sample(repo, reuse, name) });
|
||||
} catch (error) {
|
||||
process.send({ error: error.stack ?? String(error) });
|
||||
}
|
||||
});
|
||||
} else {
|
||||
const beforeWorker = worker(baseline, false);
|
||||
const afterWorker = worker(candidate, true);
|
||||
try {
|
||||
for (const repo of [baseline, candidate]) {
|
||||
fs.mkdirSync(path.dirname(path.join(repo, control)), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, control), source);
|
||||
}
|
||||
let beforeSeed, afterSeed;
|
||||
if (baseFirst) {
|
||||
beforeSeed = await beforeWorker.run('baseline-seed');
|
||||
afterSeed = await afterWorker.run('candidate-seed');
|
||||
} else {
|
||||
afterSeed = await afterWorker.run('candidate-seed');
|
||||
beforeSeed = await beforeWorker.run('baseline-seed');
|
||||
}
|
||||
assert.deepEqual(afterSeed.graph, beforeSeed.graph, 'Seed semantic graphs differ');
|
||||
for (let pair = 1; pair <= 3; pair++) {
|
||||
const text = pair % 2 ? source.replace('s.save()', 's.load()') : source;
|
||||
for (const repo of [baseline, candidate]) fs.writeFileSync(path.join(repo, control), text);
|
||||
// Alternate ordering to reduce warm-up/load bias. Both indexes see a real edit.
|
||||
let before, after;
|
||||
if (Boolean(pair % 2) === baseFirst) {
|
||||
before = await beforeWorker.run(`baseline-${pair}`);
|
||||
after = await afterWorker.run(`candidate-${pair}`);
|
||||
} else {
|
||||
after = await afterWorker.run(`candidate-${pair}`);
|
||||
before = await beforeWorker.run(`baseline-${pair}`);
|
||||
}
|
||||
assert.deepEqual(
|
||||
after.graph,
|
||||
before.graph,
|
||||
'Semantic graph differs from a full semantic pass',
|
||||
);
|
||||
for (const entry of [before, after]) {
|
||||
assert.equal(entry.incremental?.writeMode, 'incremental');
|
||||
assert.equal(entry.incremental?.changedFiles, 1);
|
||||
assert.deepEqual(entry.rebuildReasons, []);
|
||||
}
|
||||
if (screen && pair === 1 && after.ms >= before.ms) {
|
||||
console.log('SCREEN REJECTED: no end-to-end improvement; stopping after one pair.');
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
const closed = await Promise.allSettled([beforeWorker.close(), afterWorker.close()]);
|
||||
for (const repo of [baseline, candidate]) fs.rmSync(path.join(repo, control), { force: true });
|
||||
for (const result of closed) if (result.status === 'rejected') throw result.reason;
|
||||
}
|
||||
}
|
||||
735
gitnexus/bench/lexical-lookup/measurements.json
Normal file
735
gitnexus/bench/lexical-lookup/measurements.json
Normal file
|
|
@ -0,0 +1,735 @@
|
|||
{
|
||||
"date": "2026-11-02",
|
||||
"baseCommit": "347cc99b4a237f9a71c0084167bcbcbd9d06e610",
|
||||
"node": "v22.23.3",
|
||||
"ladybug": "0.22.0",
|
||||
"hardware": {
|
||||
"cpu": "12th Gen Intel(R) Core(TM) i9-12900K",
|
||||
"logicalCpus": 24,
|
||||
"platform": "linux",
|
||||
"arch": "x64"
|
||||
},
|
||||
"settings": {
|
||||
"workers": 16,
|
||||
"heapMiB": 8192,
|
||||
"sharedHost": true,
|
||||
"scopePhaseLogging": true,
|
||||
"cpuProfile": false,
|
||||
"independentColdSeeds": true
|
||||
},
|
||||
"measurement": {
|
||||
"timed": "runFullAnalysis including graph persistence and FTS",
|
||||
"excluded": ["process startup", "graph hashing", "post-analysis persisted target query"],
|
||||
"editOrder": [
|
||||
"baseline-1",
|
||||
"candidate-1",
|
||||
"candidate-2",
|
||||
"baseline-2",
|
||||
"baseline-3",
|
||||
"candidate-3"
|
||||
],
|
||||
"graphExclusions": [
|
||||
"Folder",
|
||||
"Community",
|
||||
"Process",
|
||||
"CONTAINS",
|
||||
"MEMBER_OF",
|
||||
"STEP_IN_PROCESS"
|
||||
],
|
||||
"limitations": [
|
||||
"three fresh-process pairs and one repeated-process pair on a shared host",
|
||||
"graph hashes are post-persistence; source content may have been cleared",
|
||||
"persistent process exercises repeated analysis, not watch event scheduling",
|
||||
"baseline resolution limits also apply to candidate",
|
||||
"process maxRSS includes earlier samples in persistent mode"
|
||||
]
|
||||
},
|
||||
"pin": {
|
||||
"patchSha256": "a61b37f5ab42a95a62ae96510a5e285cb9b835a3f983ce4a96ecd0413504ea43",
|
||||
"runtimes": {
|
||||
"baseline": {
|
||||
"gitnexus/dist/_shared/scope-resolution/name-claims.js": "157fd1ff2870468d335f97fff41d0b002f4cad749db64634aa48469053d7cea3",
|
||||
"gitnexus/dist/core/ingestion/scope-resolution/scope/walkers.js": "15dba69d6ec871e43f4d0d1aadbad93e1645c13a9ad2e664e9f9bd679f73e55c"
|
||||
},
|
||||
"candidate": {
|
||||
"gitnexus/dist/_shared/scope-resolution/name-claims.js": "6f998066bcb3f17d69f2dbae7dba1a1d5af865bc8798d4b137aaed9387a81284",
|
||||
"gitnexus/dist/core/ingestion/scope-resolution/scope/walkers.js": "9400ed640a22343f6cc54e3515944af664022ac7f4a8c4306da0cc95df37f9e1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"harnessSha256": "dbd428561cd614b8f0fb2dda48ad5445e214bea02a9c39f201cc2488bd7a81c8",
|
||||
"harnessProvenance": "Checksums identify the scripts used for measurement, before repository formatting, preflight/cleanup fixes and configurable CI worker/heap limits. Local resource defaults are unchanged.",
|
||||
"differential": {
|
||||
"seed": 48161,
|
||||
"comparisons": 72000,
|
||||
"status": "passed",
|
||||
"harnessSha256": "8a34834c153531daa9f6000e178fc87458e48d3507c8c6e20e2738f739413e88"
|
||||
},
|
||||
"runs": {
|
||||
"fresh": {
|
||||
"rawResultSha256": "3da8095b17ecbb5099164220152f8c9582268868db6a7d23b2565d987f2a7d6d",
|
||||
"samples": [
|
||||
{
|
||||
"name": "baseline-seed",
|
||||
"variant": "baseline",
|
||||
"ms": 218218.857255,
|
||||
"loadBefore": [19.08, 16.18, 14.03],
|
||||
"loadAfter": [13.58, 16.53, 14.82],
|
||||
"maxRssKiB": 5929416,
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 3120,
|
||||
"cachedFiles": 0,
|
||||
"phaseMs": {
|
||||
"extracting": 307.5897699999996,
|
||||
"structure": 59.0564260000001,
|
||||
"parsing": 77980.037363,
|
||||
"scopeResolution": 80390.527192,
|
||||
"enriching": 36.09870200001751,
|
||||
"communities": 2342.1585379999888,
|
||||
"processes": 659.8627029999916,
|
||||
"complete": 0.06783200000063516,
|
||||
"lbug": 17391.16107600002,
|
||||
"fts": 29286.008015999978,
|
||||
"done": 6083.309720000019,
|
||||
"beforeFirstProgress": 3682.979917
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=21ms finalize=71ms propagate=78ms rangeBind=0.0ms resolve=175ms emit=1547ms total=1891ms (110 files)",
|
||||
"[scope-resolution prof] extract=2649ms finalize=1451ms propagate=1426ms rangeBind=0.0ms resolve=7468ms emit=41291ms total=54285ms (2893 files)",
|
||||
"[scope-resolution prof] extract=270ms finalize=65ms propagate=61ms rangeBind=0.0ms resolve=444ms emit=1835ms total=2675ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "595afccf9f3fa2725727f3439cb5fd7b1aadd18ae6aca4eaa478ef00b793fe21"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "save"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "candidate-seed",
|
||||
"variant": "candidate",
|
||||
"ms": 187147.92687899998,
|
||||
"loadBefore": [13.13, 16.39, 14.78],
|
||||
"loadAfter": [11.64, 15.07, 14.65],
|
||||
"maxRssKiB": 5912680,
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 3120,
|
||||
"cachedFiles": 0,
|
||||
"phaseMs": {
|
||||
"extracting": 248.5604829999993,
|
||||
"structure": 55.20889800000077,
|
||||
"parsing": 62435.270292,
|
||||
"scopeResolution": 61285.705398999984,
|
||||
"enriching": 31.93969999998808,
|
||||
"communities": 2074.6883160000143,
|
||||
"processes": 544.6516690000135,
|
||||
"complete": 0.05536599998595193,
|
||||
"lbug": 16684.85912899999,
|
||||
"fts": 29567.160881000018,
|
||||
"done": 5673.711943999981,
|
||||
"beforeFirstProgress": 8546.114802
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=18ms finalize=60ms propagate=52ms rangeBind=0.0ms resolve=118ms emit=1176ms total=1425ms (110 files)",
|
||||
"[scope-resolution prof] extract=2048ms finalize=1153ms propagate=1045ms rangeBind=0.0ms resolve=5119ms emit=31736ms total=41101ms (2893 files)",
|
||||
"[scope-resolution prof] extract=201ms finalize=62ms propagate=64ms rangeBind=0.0ms resolve=282ms emit=1469ms total=2078ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "595afccf9f3fa2725727f3439cb5fd7b1aadd18ae6aca4eaa478ef00b793fe21"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "save"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "baseline-1",
|
||||
"variant": "baseline",
|
||||
"ms": 122065.065072,
|
||||
"loadBefore": [11.43, 14.97, 14.62],
|
||||
"loadAfter": [8.47, 12.61, 13.79],
|
||||
"maxRssKiB": 4474272,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 221.9082469999994,
|
||||
"structure": 52.67533299999923,
|
||||
"parsing": 8765.232970000001,
|
||||
"scopeResolution": 65945.25450899999,
|
||||
"enriching": 121.54619799999637,
|
||||
"complete": 0.25131200000760145,
|
||||
"lbug": 10923.801725000012,
|
||||
"communities": 2019.391571,
|
||||
"processes": 5888.645534999989,
|
||||
"fts": 13240.190296,
|
||||
"done": 8270.738006,
|
||||
"beforeFirstProgress": 6615.429370000001
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=19ms finalize=60ms propagate=58ms rangeBind=0.0ms resolve=148ms emit=1294ms total=1578ms (110 files)",
|
||||
"[scope-resolution prof] extract=2031ms finalize=1178ms propagate=1108ms rangeBind=0.0ms resolve=6717ms emit=35251ms total=46284ms (2893 files)",
|
||||
"[scope-resolution prof] extract=232ms finalize=61ms propagate=59ms rangeBind=0.0ms resolve=379ms emit=1552ms total=2283ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "4ece58529c66b824a32ee5b1c3f642193279f98e39d95eea53bf2f5ec1700ae8"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "load"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "candidate-1",
|
||||
"variant": "candidate",
|
||||
"ms": 111134.852017,
|
||||
"loadBefore": [8.47, 12.61, 13.79],
|
||||
"loadAfter": [7.79, 11.1, 13.1],
|
||||
"maxRssKiB": 4406608,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 228.39688900000056,
|
||||
"structure": 58.946551,
|
||||
"parsing": 7914.542683,
|
||||
"scopeResolution": 62759.170073999994,
|
||||
"enriching": 126.18967100000009,
|
||||
"complete": 0.17001499999605585,
|
||||
"lbug": 9233.545855000018,
|
||||
"communities": 1969.2113839999947,
|
||||
"processes": 1475.3109669999976,
|
||||
"fts": 12857.877876999992,
|
||||
"done": 8347.120962,
|
||||
"beforeFirstProgress": 6164.369089
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=15ms finalize=57ms propagate=51ms rangeBind=0.0ms resolve=115ms emit=1221ms total=1459ms (110 files)",
|
||||
"[scope-resolution prof] extract=2192ms finalize=1268ms propagate=1122ms rangeBind=0.0ms resolve=5371ms emit=33985ms total=43937ms (2893 files)",
|
||||
"[scope-resolution prof] extract=176ms finalize=75ms propagate=55ms rangeBind=0.0ms resolve=253ms emit=1333ms total=1892ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "4ece58529c66b824a32ee5b1c3f642193279f98e39d95eea53bf2f5ec1700ae8"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "load"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "candidate-2",
|
||||
"variant": "candidate",
|
||||
"ms": 101730.29475,
|
||||
"loadBefore": [7.79, 11.1, 13.1],
|
||||
"loadAfter": [8.7, 10.56, 12.7],
|
||||
"maxRssKiB": 4447528,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 227.87885000000006,
|
||||
"structure": 50.07236400000011,
|
||||
"parsing": 6870.926647,
|
||||
"scopeResolution": 58457.57910199999,
|
||||
"enriching": 123.74990100000286,
|
||||
"complete": 0.12248400000680704,
|
||||
"lbug": 9597.999660000001,
|
||||
"communities": 1855.2971499999985,
|
||||
"processes": 1319.511560999992,
|
||||
"fts": 12526.206298000005,
|
||||
"done": 8028.759919000004,
|
||||
"beforeFirstProgress": 2672.190814
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=14ms finalize=61ms propagate=47ms rangeBind=0.0ms resolve=103ms emit=1077ms total=1302ms (110 files)",
|
||||
"[scope-resolution prof] extract=1860ms finalize=1052ms propagate=1064ms rangeBind=0.0ms resolve=4989ms emit=31810ms total=40774ms (2893 files)",
|
||||
"[scope-resolution prof] extract=189ms finalize=62ms propagate=52ms rangeBind=0.0ms resolve=258ms emit=1383ms total=1943ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "595afccf9f3fa2725727f3439cb5fd7b1aadd18ae6aca4eaa478ef00b793fe21"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "save"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "baseline-2",
|
||||
"variant": "baseline",
|
||||
"ms": 114005.953714,
|
||||
"loadBefore": [8.7, 10.56, 12.7],
|
||||
"loadAfter": [9.5, 10.18, 12.3],
|
||||
"maxRssKiB": 4501784,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 220.18439399999988,
|
||||
"structure": 56.05602999999974,
|
||||
"parsing": 6693.682359,
|
||||
"scopeResolution": 68196.749623,
|
||||
"enriching": 136.36504800000694,
|
||||
"complete": 0.24019299999054056,
|
||||
"lbug": 10199.119456000015,
|
||||
"communities": 1921.6023549999954,
|
||||
"processes": 1269.8549959999946,
|
||||
"fts": 12961.042190000007,
|
||||
"done": 8423.348173999999,
|
||||
"beforeFirstProgress": 3927.708896
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=15ms finalize=63ms propagate=59ms rangeBind=0.0ms resolve=129ms emit=1239ms total=1506ms (110 files)",
|
||||
"[scope-resolution prof] extract=2317ms finalize=1312ms propagate=1295ms rangeBind=0.0ms resolve=7098ms emit=36604ms total=48627ms (2893 files)",
|
||||
"[scope-resolution prof] extract=225ms finalize=58ms propagate=59ms rangeBind=0.0ms resolve=366ms emit=1623ms total=2331ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "595afccf9f3fa2725727f3439cb5fd7b1aadd18ae6aca4eaa478ef00b793fe21"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "save"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "baseline-3",
|
||||
"variant": "baseline",
|
||||
"ms": 131285.117027,
|
||||
"loadBefore": [9.94, 10.26, 12.32],
|
||||
"loadAfter": [11.35, 10.88, 12.28],
|
||||
"maxRssKiB": 4423784,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 246.84667200000013,
|
||||
"structure": 55.95557000000008,
|
||||
"parsing": 7195.49225,
|
||||
"scopeResolution": 72694.88498100001,
|
||||
"enriching": 136.15589299998828,
|
||||
"complete": 0.1802390000084415,
|
||||
"lbug": 13022.899677999987,
|
||||
"communities": 2414.3188599999994,
|
||||
"processes": 1639.1324740000127,
|
||||
"fts": 14523.853445,
|
||||
"done": 16550.80450099999,
|
||||
"beforeFirstProgress": 2804.592464
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=17ms finalize=67ms propagate=66ms rangeBind=0.0ms resolve=138ms emit=1370ms total=1658ms (110 files)",
|
||||
"[scope-resolution prof] extract=2113ms finalize=1144ms propagate=1129ms rangeBind=0.0ms resolve=7406ms emit=40471ms total=52262ms (2893 files)",
|
||||
"[scope-resolution prof] extract=255ms finalize=74ms propagate=68ms rangeBind=0.0ms resolve=393ms emit=1821ms total=2611ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "4ece58529c66b824a32ee5b1c3f642193279f98e39d95eea53bf2f5ec1700ae8"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "load"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "candidate-3",
|
||||
"variant": "candidate",
|
||||
"ms": 111303.319634,
|
||||
"loadBefore": [11, 10.82, 12.25],
|
||||
"loadAfter": [8.98, 10.27, 11.9],
|
||||
"maxRssKiB": 4458828,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 224.80108500000006,
|
||||
"structure": 54.754461999999876,
|
||||
"parsing": 9906.336681,
|
||||
"scopeResolution": 62344.37333,
|
||||
"enriching": 117.25856000000203,
|
||||
"complete": 0.1237049999908777,
|
||||
"lbug": 9196.625813000006,
|
||||
"communities": 2015.0293479999964,
|
||||
"processes": 3180.0593300000037,
|
||||
"fts": 12870.898965,
|
||||
"done": 7974.200893999994,
|
||||
"beforeFirstProgress": 3418.8574609999996
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=22ms finalize=74ms propagate=61ms rangeBind=0.0ms resolve=115ms emit=1269ms total=1540ms (110 files)",
|
||||
"[scope-resolution prof] extract=2053ms finalize=1125ms propagate=1129ms rangeBind=0.0ms resolve=5562ms emit=32778ms total=42647ms (2893 files)",
|
||||
"[scope-resolution prof] extract=215ms finalize=61ms propagate=54ms rangeBind=0.0ms resolve=267ms emit=1445ms total=2041ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "4ece58529c66b824a32ee5b1c3f642193279f98e39d95eea53bf2f5ec1700ae8"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "load"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"baseline": {
|
||||
"editCount": 3,
|
||||
"medianMs": 122065.065072,
|
||||
"sumMs": 367356.13581300003,
|
||||
"medianScopeMs": 68196.749623,
|
||||
"maxRssKiB": 4501784
|
||||
},
|
||||
"candidate": {
|
||||
"editCount": 3,
|
||||
"medianMs": 111134.852017,
|
||||
"sumMs": 324168.466401,
|
||||
"medianScopeMs": 62344.37333,
|
||||
"maxRssKiB": 4458828
|
||||
},
|
||||
"medianReductionPercent": 8.954415457488041,
|
||||
"aggregateReductionPercent": 11.756349003514243,
|
||||
"scopeMedianReductionPercent": 8.581605905490585
|
||||
}
|
||||
},
|
||||
"persistent": {
|
||||
"rawResultSha256": "b09e4f487c1933281a4f875620b5ac5a23794e56246d6c317dd6d3196762293e",
|
||||
"samples": [
|
||||
{
|
||||
"name": "baseline-seed",
|
||||
"variant": "baseline",
|
||||
"ms": 158147.924033,
|
||||
"loadBefore": [3.28, 6.72, 10.14],
|
||||
"loadAfter": [5.3, 7.19, 9.87],
|
||||
"maxRssKiB": 5757604,
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 3120,
|
||||
"cachedFiles": 0,
|
||||
"phaseMs": {
|
||||
"extracting": 205.49587699999938,
|
||||
"structure": 49.78513100000055,
|
||||
"parsing": 58223.494267999995,
|
||||
"scopeResolution": 62109.850408000006,
|
||||
"enriching": 31.760779000003822,
|
||||
"communities": 1872.6300760000013,
|
||||
"processes": 506.77787499999977,
|
||||
"complete": 0.04477599999518134,
|
||||
"lbug": 10664.06607499998,
|
||||
"fts": 18818.957672999997,
|
||||
"done": 2247.716622000007,
|
||||
"beforeFirstProgress": 3417.344473
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=16ms finalize=59ms propagate=66ms rangeBind=0.0ms resolve=132ms emit=1181ms total=1454ms (110 files)",
|
||||
"[scope-resolution prof] extract=1988ms finalize=1096ms propagate=1099ms rangeBind=0.0ms resolve=6393ms emit=33758ms total=44335ms (2893 files)",
|
||||
"[scope-resolution prof] extract=168ms finalize=64ms propagate=50ms rangeBind=0.0ms resolve=322ms emit=1375ms total=1978ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "595afccf9f3fa2725727f3439cb5fd7b1aadd18ae6aca4eaa478ef00b793fe21"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "save"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "candidate-seed",
|
||||
"variant": "candidate",
|
||||
"ms": 153951.283825,
|
||||
"loadBefore": [5.3, 7.19, 9.87],
|
||||
"loadAfter": [6.35, 7.83, 9.77],
|
||||
"maxRssKiB": 5734340,
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 3120,
|
||||
"cachedFiles": 0,
|
||||
"phaseMs": {
|
||||
"extracting": 226.53108299999963,
|
||||
"structure": 46.397985999999946,
|
||||
"parsing": 58254.662231,
|
||||
"scopeResolution": 61421.754931999996,
|
||||
"enriching": 35.70321300000069,
|
||||
"communities": 1928.893007000006,
|
||||
"processes": 529.0402800000011,
|
||||
"complete": 0.05887399999483023,
|
||||
"lbug": 10686.215123000002,
|
||||
"fts": 16689.450012999994,
|
||||
"done": 2236.283251999994,
|
||||
"beforeFirstProgress": 1896.2938310000002
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=16ms finalize=54ms propagate=49ms rangeBind=0.0ms resolve=108ms emit=1134ms total=1361ms (110 files)",
|
||||
"[scope-resolution prof] extract=2107ms finalize=1253ms propagate=1084ms rangeBind=0.0ms resolve=5107ms emit=32228ms total=41779ms (2893 files)",
|
||||
"[scope-resolution prof] extract=172ms finalize=65ms propagate=52ms rangeBind=0.0ms resolve=270ms emit=1373ms total=1932ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "595afccf9f3fa2725727f3439cb5fd7b1aadd18ae6aca4eaa478ef00b793fe21"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "save"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "baseline-1",
|
||||
"variant": "baseline",
|
||||
"ms": 109482.844232,
|
||||
"loadBefore": [6.35, 7.83, 9.77],
|
||||
"loadAfter": [3.81, 6.46, 9.05],
|
||||
"maxRssKiB": 5757604,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 211.5071590000298,
|
||||
"structure": 41.07579499995336,
|
||||
"parsing": 7984.799295000033,
|
||||
"scopeResolution": 67012.942599,
|
||||
"enriching": 124.94524899998214,
|
||||
"complete": 0.18859700002940372,
|
||||
"lbug": 8813.617234999954,
|
||||
"communities": 1921.8943519999739,
|
||||
"processes": 1619.4587620000239,
|
||||
"fts": 10599.878043000004,
|
||||
"done": 8947.475924000028,
|
||||
"beforeFirstProgress": 2205.0612219999894
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=12ms finalize=49ms propagate=55ms rangeBind=0.0ms resolve=133ms emit=1216ms total=1467ms (110 files)",
|
||||
"[scope-resolution prof] extract=2000ms finalize=1107ms propagate=1133ms rangeBind=0.0ms resolve=7186ms emit=37166ms total=48593ms (2893 files)",
|
||||
"[scope-resolution prof] extract=164ms finalize=57ms propagate=58ms rangeBind=0.0ms resolve=346ms emit=1455ms total=2081ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "4ece58529c66b824a32ee5b1c3f642193279f98e39d95eea53bf2f5ec1700ae8"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "load"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "candidate-1",
|
||||
"variant": "candidate",
|
||||
"ms": 123364.35062099999,
|
||||
"loadBefore": [3.81, 6.46, 9.05],
|
||||
"loadAfter": [12.23, 8.31, 9.37],
|
||||
"maxRssKiB": 5896624,
|
||||
"incremental": {
|
||||
"changedFiles": 1,
|
||||
"reparsedFiles": 22,
|
||||
"affectedDependents": 0,
|
||||
"deletedFiles": 0,
|
||||
"writeMode": "incremental"
|
||||
},
|
||||
"rebuildReasons": [],
|
||||
"reparsedFiles": 22,
|
||||
"cachedFiles": 3098,
|
||||
"phaseMs": {
|
||||
"extracting": 221.45905800000764,
|
||||
"structure": 46.565340999979526,
|
||||
"parsing": 7283.348788000003,
|
||||
"scopeResolution": 65739.48159400001,
|
||||
"enriching": 129.2759209999931,
|
||||
"complete": 0.21377500001108274,
|
||||
"lbug": 12901.81731199997,
|
||||
"communities": 2457.193612000032,
|
||||
"processes": 2556.050026999961,
|
||||
"fts": 14765.58123000001,
|
||||
"done": 14857.72775999998,
|
||||
"beforeFirstProgress": 2405.636203000031
|
||||
},
|
||||
"scopeLogs": [
|
||||
"[scope-resolution prof] extract=11ms finalize=47ms propagate=51ms rangeBind=0.0ms resolve=107ms emit=1124ms total=1340ms (110 files)",
|
||||
"[scope-resolution prof] extract=2046ms finalize=1134ms propagate=1135ms rangeBind=0.0ms resolve=5161ms emit=34994ms total=44470ms (2893 files)",
|
||||
"[scope-resolution prof] extract=407ms finalize=69ms propagate=71ms rangeBind=0.0ms resolve=369ms emit=2216ms total=3132ms (113 files)"
|
||||
],
|
||||
"graph": {
|
||||
"nodes": {
|
||||
"count": 114413,
|
||||
"sha256": "d2e2ee693005fee4770a654b1391f5348d7b03316d57bf0e7bceb4cf6787f9ad"
|
||||
},
|
||||
"edges": {
|
||||
"count": 318798,
|
||||
"sha256": "4ece58529c66b824a32ee5b1c3f642193279f98e39d95eea53bf2f5ec1700ae8"
|
||||
}
|
||||
},
|
||||
"persistedTargets": [
|
||||
{
|
||||
"target": "load"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"baseline": {
|
||||
"editCount": 1,
|
||||
"medianMs": 109482.844232,
|
||||
"sumMs": 109482.844232,
|
||||
"medianScopeMs": 67012.942599,
|
||||
"maxRssKiB": 5757604
|
||||
},
|
||||
"candidate": {
|
||||
"editCount": 1,
|
||||
"medianMs": 123364.35062099999,
|
||||
"sumMs": 123364.35062099999,
|
||||
"medianScopeMs": 65739.48159400001,
|
||||
"maxRssKiB": 5896624
|
||||
},
|
||||
"medianReductionPercent": -12.679161275335815,
|
||||
"aggregateReductionPercent": -12.679161275335815,
|
||||
"scopeMedianReductionPercent": 1.9003209762333162
|
||||
}
|
||||
}
|
||||
},
|
||||
"decision": "Experimental: fresh-process screen positive; persistent screen stopped after first non-improving pair; benefit for both workflows unproven",
|
||||
"validation": {
|
||||
"runtimePinsVerifiedAfterRuns": true,
|
||||
"focusedTests": {
|
||||
"files": 6,
|
||||
"tests": 137,
|
||||
"status": "passed"
|
||||
},
|
||||
"scopeTests": {
|
||||
"files": 175,
|
||||
"tests": 3467,
|
||||
"status": "passed",
|
||||
"overlapWithFocused": true
|
||||
},
|
||||
"differentialComparisons": 72000,
|
||||
"parameterizedDifferentialRerun": "passed",
|
||||
"build": "passed for both runtimes",
|
||||
"testTypecheck": "passed",
|
||||
"eslint": "passed; existing warnings",
|
||||
"fullSuite": "not run"
|
||||
}
|
||||
}
|
||||
6310
gitnexus/bench/lexical-lookup/optimization-20261103.json
Normal file
6310
gitnexus/bench/lexical-lookup/optimization-20261103.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -34,14 +34,15 @@ const storedIdentityField = (node: GraphNode, field: string): unknown => {
|
|||
* retained rows. A write-set-only probe misses corruption of an unchanged row.
|
||||
* Scan one label at a time, without path/range predicates: a corrupt field must
|
||||
* not be able to hide its own row. No multi-label scans (#3139), N+1 ID lookups,
|
||||
* source-content copies, or whole-graph row materialization.
|
||||
* source-content copies, or JS row-array materialization. The reader calls the
|
||||
* synchronous visitor for each row and releases its cursor before resolving.
|
||||
*
|
||||
* This certifies node identity, not relationships or native storage internals.
|
||||
* Call after COPY/checkpoint and after FTS/embedding/checkpoint, before metadata.
|
||||
*/
|
||||
export async function reconcileGraphNodeIdentities(
|
||||
graph: KnowledgeGraph,
|
||||
query: (cypher: string) => Promise<IdentityRow[]>,
|
||||
readRows: (cypher: string, onRow: (row: IdentityRow) => void) => Promise<void>,
|
||||
phase: string,
|
||||
): Promise<{ nodes: number; tables: number }> {
|
||||
const expected = new Map<NodeTableName, Map<string, GraphNode>>();
|
||||
|
|
@ -71,10 +72,25 @@ export async function reconcileGraphNodeIdentities(
|
|||
{ cause },
|
||||
);
|
||||
};
|
||||
let rows: IdentityRow[];
|
||||
try {
|
||||
rows = await query(
|
||||
await readRows(
|
||||
`MATCH (n:\`${table}\`) RETURN ${fields.map((f) => `n.${f} AS ${f}`).join(', ')}`,
|
||||
(row) => {
|
||||
const node = remaining.get(row.id);
|
||||
if (!node) throw new Error(`unexpected or duplicate ID ${JSON.stringify(row.id)}`);
|
||||
for (const field of identityFields) {
|
||||
const wanted = storedIdentityField(node, field);
|
||||
const actual =
|
||||
field === 'name' || field === 'filePath' ? (row[field] ?? '') : row[field];
|
||||
if (actual !== wanted) {
|
||||
throw new Error(
|
||||
`${row.id}.${field}: expected ${JSON.stringify(wanted)}, read ${JSON.stringify(actual)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
remaining.delete(row.id);
|
||||
nodes++;
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
fail(
|
||||
|
|
@ -82,21 +98,6 @@ export async function reconcileGraphNodeIdentities(
|
|||
error,
|
||||
);
|
||||
}
|
||||
for (const row of rows) {
|
||||
const node = remaining.get(row.id);
|
||||
if (!node) fail(`unexpected or duplicate ID ${JSON.stringify(row.id)}`);
|
||||
for (const field of identityFields) {
|
||||
const wanted = storedIdentityField(node, field);
|
||||
const actual = field === 'name' || field === 'filePath' ? (row[field] ?? '') : row[field];
|
||||
if (actual !== wanted) {
|
||||
fail(
|
||||
`${row.id}.${field}: expected ${JSON.stringify(wanted)}, read ${JSON.stringify(actual)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
remaining.delete(row.id);
|
||||
nodes++;
|
||||
}
|
||||
if (remaining.size) {
|
||||
fail(
|
||||
`${remaining.size} missing ID(s), including ${JSON.stringify(remaining.keys().next().value)}`,
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ export function lookupNameClaim(
|
|||
local,
|
||||
scopes.bindings.get(scope.id)?.get(spelling) ?? EMPTY_BINDINGS,
|
||||
) ?? EMPTY_BINDINGS;
|
||||
if (additional.length === 0 && augmented.length === 0) return local;
|
||||
return [
|
||||
...local,
|
||||
...compatibleClassBindings(additional, options.classCandidateFilter),
|
||||
|
|
@ -108,24 +109,23 @@ export function lookupNameClaim(
|
|||
),
|
||||
];
|
||||
}
|
||||
return [
|
||||
...local,
|
||||
...compatibleClassBindings(
|
||||
scopes.bindings.get(scope.id)?.get(spelling) ?? EMPTY_BINDINGS,
|
||||
options.classCandidateFilter,
|
||||
),
|
||||
...augmented,
|
||||
...(collectNamespaceFqnBindings(
|
||||
scope.id,
|
||||
spelling,
|
||||
scopes,
|
||||
options.classCandidateFilter,
|
||||
) ?? []),
|
||||
...compatibleClassBindings(
|
||||
scopes.workspaceFqnBindings?.get(spelling) ?? EMPTY_BINDINGS,
|
||||
options.classCandidateFilter,
|
||||
),
|
||||
];
|
||||
const indexed = compatibleClassBindings(
|
||||
scopes.bindings.get(scope.id)?.get(spelling) ?? EMPTY_BINDINGS,
|
||||
options.classCandidateFilter,
|
||||
);
|
||||
const namespace =
|
||||
collectNamespaceFqnBindings(scope.id, spelling, scopes, options.classCandidateFilter) ??
|
||||
EMPTY_BINDINGS;
|
||||
const workspace = compatibleClassBindings(
|
||||
scopes.workspaceFqnBindings?.get(spelling) ?? EMPTY_BINDINGS,
|
||||
options.classCandidateFilter,
|
||||
);
|
||||
if (
|
||||
local.length + indexed.length + augmented.length + namespace.length + workspace.length ===
|
||||
0
|
||||
)
|
||||
return EMPTY_BINDINGS;
|
||||
return [...local, ...indexed, ...augmented, ...namespace, ...workspace];
|
||||
},
|
||||
},
|
||||
options,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { createReadStream, createWriteStream, existsSync, constants as fsConstan
|
|||
import { createInterface } from 'readline';
|
||||
import { once } from 'events';
|
||||
import { finished } from 'stream/promises';
|
||||
import { setImmediate as yieldQueryRows } from 'node:timers/promises';
|
||||
import path from 'path';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import { closeQueryResults } from './query-result-utils.js';
|
||||
|
|
@ -1956,6 +1957,54 @@ export const executeQuery = async (cypher: string): Promise<any[]> => {
|
|||
return await executePrepared(cypher, {});
|
||||
};
|
||||
|
||||
/**
|
||||
* Visit a single trusted, parameter-free query without retaining a JS row array.
|
||||
* The visitor must be synchronous and must not issue another connection query.
|
||||
* Keep the connection locked through yields and cursor close so a checkpoint
|
||||
* cannot interleave with reading. Use direct execution: repeated prepared reads
|
||||
* exhausted the native pool in the normal-GC identity scaling probe.
|
||||
*/
|
||||
export const forEachQueryRow = async (cypher: string, onRow: (row: any) => void): Promise<void> => {
|
||||
warnIfQueryTextUnbounded(cypher, 'forEachQueryRow', (message) => logger.warn(message));
|
||||
const c = conn;
|
||||
if (!c) throw new Error('LadybugDB not initialized. Call initLbug first.');
|
||||
|
||||
await withConnLock(async () => {
|
||||
const queryResult = await c.query(cypher);
|
||||
const results = Array.isArray(queryResult) ? queryResult : [queryResult];
|
||||
let failed = false;
|
||||
let failure: unknown;
|
||||
try {
|
||||
if (results.length !== 1 || !results[0]) {
|
||||
throw new Error('forEachQueryRow requires exactly one query result');
|
||||
}
|
||||
const result = results[0];
|
||||
let rows = 0;
|
||||
while (result.hasNext()) {
|
||||
const row = result.getNextSync();
|
||||
if (row === null) throw new Error('Query cursor ended before its advertised next row');
|
||||
onRow(row);
|
||||
if (++rows % 1024 === 0) await yieldQueryRows();
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true;
|
||||
failure = error;
|
||||
} finally {
|
||||
for (const result of results) {
|
||||
try {
|
||||
result?.close();
|
||||
} catch (error) {
|
||||
if (!failed) {
|
||||
failed = true;
|
||||
failure = error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failed) throw failure;
|
||||
});
|
||||
};
|
||||
|
||||
export const streamQuery = async (
|
||||
cypher: string,
|
||||
onRow: (row: any) => void | Promise<void>,
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ import {
|
|||
loadGraphToLbug,
|
||||
getLbugStats,
|
||||
executeQuery,
|
||||
forEachQueryRow,
|
||||
executeWithReusedStatement,
|
||||
closeLbug,
|
||||
closeLbugBeforeExit,
|
||||
|
|
@ -4419,7 +4420,7 @@ async function runFullAnalysisInner(
|
|||
await markIncrementalGraphVerification();
|
||||
await reconcileGraphNodeIdentities(
|
||||
pipelineResult.graph,
|
||||
executeQuery,
|
||||
forEachQueryRow,
|
||||
'post-COPY/checkpoint',
|
||||
);
|
||||
}
|
||||
|
|
@ -5573,7 +5574,7 @@ async function runFullAnalysisInner(
|
|||
}
|
||||
await reconcileGraphNodeIdentities(
|
||||
pipelineResult.graph,
|
||||
executeQuery,
|
||||
forEachQueryRow,
|
||||
'pre-publish/checkpoint',
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@
|
|||
* use an empty (initialized) DB — they prove the lock is taken regardless of
|
||||
* whether any rows match.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import { withTestLbugDB } from '../helpers/test-indexed-db.js';
|
||||
import { NODE_TABLES } from '../../src/core/lbug/schema.js';
|
||||
|
||||
|
|
@ -40,6 +41,114 @@ import { closeQueryResults } from '../../src/core/lbug/query-result-utils.js';
|
|||
const closeSpy = vi.mocked(closeQueryResults);
|
||||
|
||||
withTestLbugDB('conn-serialization', () => {
|
||||
describe('incremental identity cursor lifetime', () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it('reads real rows without getAll and releases the cursor after a visitor failure', async () => {
|
||||
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
const getAll = vi.spyOn(lbug.QueryResult.prototype, 'getAll');
|
||||
const close = vi.spyOn(lbug.QueryResult.prototype, 'close');
|
||||
const seen: number[] = [];
|
||||
await adapter.forEachQueryRow('UNWIND [1, 2, 3] AS n RETURN n', (row) => {
|
||||
seen.push(row.n);
|
||||
});
|
||||
expect(seen).toEqual([1, 2, 3]);
|
||||
expect(getAll).not.toHaveBeenCalled();
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
const error = new Error('bad identity');
|
||||
await expect(
|
||||
adapter.forEachQueryRow('RETURN 1 AS n', () => {
|
||||
throw error;
|
||||
}),
|
||||
).rejects.toBe(error);
|
||||
expect(close).toHaveBeenCalledTimes(2);
|
||||
await expect(adapter.executeQuery('RETURN 2 AS n')).resolves.toEqual([{ n: 2 }]);
|
||||
});
|
||||
|
||||
it('yields while retaining the lock through the last row and cursor close', async () => {
|
||||
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
let read = 0;
|
||||
let visited = 0;
|
||||
let yieldedAt = 0;
|
||||
const close = vi.fn();
|
||||
const cursor = {
|
||||
hasNext: () => read < 2050,
|
||||
getNextSync: () => {
|
||||
expect(visited).toBe(read); // The previous row was consumed before reading another.
|
||||
return { n: ++read };
|
||||
},
|
||||
close,
|
||||
};
|
||||
const query = vi.spyOn(lbug.Connection.prototype, 'query').mockImplementation(async (sql) => {
|
||||
expect(sql).toBe('scan');
|
||||
return cursor as any;
|
||||
});
|
||||
let checkpoint: Promise<unknown> | undefined;
|
||||
// Schedule outside the reader's lock context, like the WAL driver timer.
|
||||
setImmediate(() => {
|
||||
yieldedAt = visited;
|
||||
checkpoint = adapter.executeQuery('CHECKPOINT');
|
||||
});
|
||||
const prepare = vi.spyOn(lbug.Connection.prototype, 'prepare');
|
||||
// executeQuery reaches prepare only once the reader has closed.
|
||||
prepare.mockImplementation(async () => {
|
||||
expect(visited).toBe(2050);
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
return { isSuccess: () => true } as any;
|
||||
});
|
||||
vi.spyOn(lbug.Connection.prototype, 'execute').mockResolvedValue({
|
||||
getAll: async () => [],
|
||||
close: () => {},
|
||||
} as any);
|
||||
await adapter.forEachQueryRow('scan', () => {
|
||||
visited++;
|
||||
});
|
||||
await checkpoint;
|
||||
expect(yieldedAt).toBeGreaterThan(0);
|
||||
expect(yieldedAt).toBeLessThan(2050);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
expect(prepare).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(['none', 'multiple'] as const)(
|
||||
'rejects %s result cursors and closes all returned cursors',
|
||||
async (kind) => {
|
||||
const { forEachQueryRow } = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
const close = vi.fn();
|
||||
const cursors = kind === 'none' ? [] : [{ close }, { close }];
|
||||
vi.spyOn(lbug.Connection.prototype, 'query').mockResolvedValue(cursors as any);
|
||||
const visit = vi.fn();
|
||||
await expect(forEachQueryRow('scan', visit)).rejects.toThrow('exactly one query result');
|
||||
expect(visit).not.toHaveBeenCalled();
|
||||
expect(close).toHaveBeenCalledTimes(cursors.length);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['read', 'null', 'close', 'read-and-close'] as const)(
|
||||
'fails closed on %s failure',
|
||||
async (stage) => {
|
||||
const { forEachQueryRow } = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
const readError = new Error('native read failure');
|
||||
const closeError = new Error('native close failure');
|
||||
const close = vi.fn(() => {
|
||||
if (stage.includes('close')) throw closeError;
|
||||
});
|
||||
vi.spyOn(lbug.Connection.prototype, 'query').mockResolvedValue({
|
||||
hasNext: () => stage !== 'close',
|
||||
getNextSync: () => {
|
||||
if (stage === 'null') return null;
|
||||
throw readError;
|
||||
},
|
||||
close,
|
||||
} as any);
|
||||
const outcome = forEachQueryRow('scan', vi.fn());
|
||||
if (stage === 'null') await expect(outcome).rejects.toThrow('advertised next row');
|
||||
else await expect(outcome).rejects.toBe(stage === 'close' ? closeError : readError);
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('singleton-conn helpers acquire withConnLock (P1 #2264)', () => {
|
||||
beforeEach(() => {
|
||||
// Setup (clear/seed/flush) already exercised the lock; reset so each
|
||||
|
|
|
|||
|
|
@ -52,7 +52,12 @@ it.each([
|
|||
nativePhases,
|
||||
size: { nodes: 1 },
|
||||
reconcileGraphNodeIdentities,
|
||||
adapter: { executeQuery: async (query: string) => (query.includes('Function') ? rows : []) },
|
||||
adapter: {
|
||||
executeQuery: async (query: string) => (query.includes('Function') ? rows : []),
|
||||
forEachQueryRow: async (query: string, onRow: (row: (typeof rows)[number]) => void) => {
|
||||
if (query.includes('Function')) rows.forEach(onRow);
|
||||
},
|
||||
},
|
||||
}) as (phase: string) => Promise<void>;
|
||||
await audit('controlled-scan');
|
||||
expect(nativePhases).toEqual([expect.objectContaining({ verdict, rows: rows.length })]);
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ describe('incremental graph identity before publication', () => {
|
|||
);
|
||||
await adapter.tryFlushWAL();
|
||||
await expect(
|
||||
reconcileGraphNodeIdentities(graph, adapter.executeQuery, 'native CSV parity'),
|
||||
reconcileGraphNodeIdentities(graph, adapter.forEachQueryRow, 'native CSV parity'),
|
||||
).resolves.toMatchObject({ nodes: 5 });
|
||||
} finally {
|
||||
await adapter.closeLbug();
|
||||
|
|
@ -264,7 +264,7 @@ describe('incremental graph identity before publication', () => {
|
|||
await expect(
|
||||
reconcileGraphNodeIdentities(
|
||||
recovered.pipelineResult.graph,
|
||||
adapter.executeQuery,
|
||||
adapter.forEachQueryRow,
|
||||
'recovered/reopened',
|
||||
),
|
||||
).resolves.toMatchObject({ nodes: expect.any(Number) });
|
||||
|
|
|
|||
177
gitnexus/test/unit/lexical-benchmark-preflight.test.ts
Normal file
177
gitnexus/test/unit/lexical-benchmark-preflight.test.ts
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
const harness = fileURLToPath(new URL('../../bench/lexical-lookup/measure.mjs', import.meta.url));
|
||||
const scratch: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const root of scratch.splice(0)) fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('lexical benchmark preflight', () => {
|
||||
it.each([
|
||||
['GITNEXUS_LEXICAL_BENCH_SCREEN', 'Screen must be 0 or 1'],
|
||||
['GITNEXUS_LEXICAL_BENCH_BASE_FIRST', 'Base first must be 0 or 1'],
|
||||
])('rejects invalid %s before touching corpora', (setting, message) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lexical-option-'));
|
||||
scratch.push(root);
|
||||
const output = path.join(root, 'output');
|
||||
const result = spawnSync(process.execPath, [harness, root, root, root, root, output], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, [setting]: 'false' },
|
||||
});
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain(message);
|
||||
expect(fs.existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it.each(['empty', 'indexes', 'results.json'])(
|
||||
'rejects existing %s output before touching corpora',
|
||||
(contents) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lexical-preflight-'));
|
||||
scratch.push(root);
|
||||
const output = path.join(root, 'output');
|
||||
fs.mkdirSync(output);
|
||||
if (contents === 'indexes') fs.mkdirSync(path.join(output, 'indexes'));
|
||||
if (contents === 'results.json')
|
||||
fs.writeFileSync(path.join(output, 'results.json'), 'prior result');
|
||||
const before = path.join(root, 'before');
|
||||
const after = path.join(root, 'after');
|
||||
const result = spawnSync(process.execPath, [harness, root, root, before, after, output], {
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
GITNEXUS_LEXICAL_BENCH_WORKER: '0',
|
||||
GITNEXUS_LEXICAL_BENCH_WORKERS: '2',
|
||||
GITNEXUS_LEXICAL_BENCH_HEAP_MIB: '4096',
|
||||
},
|
||||
});
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('Output already exists');
|
||||
expect(fs.existsSync(before)).toBe(false);
|
||||
expect(fs.existsSync(after)).toBe(false);
|
||||
expect(fs.readdirSync(output)).toEqual(contents === 'empty' ? [] : [contents]);
|
||||
if (contents === 'results.json')
|
||||
expect(fs.readFileSync(path.join(output, contents), 'utf8')).toBe('prior result');
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('lexical benchmark sample collection', () => {
|
||||
it.each(['fresh', 'persistent'])(
|
||||
'collects a complete reversed-order baseline with a slower candidate (%s)',
|
||||
(lifecycle) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lexical-samples-'));
|
||||
scratch.push(root);
|
||||
const corpora = ['before', 'after'].map((name) => {
|
||||
const repo = path.join(root, name);
|
||||
fs.mkdirSync(repo);
|
||||
execFileSync('git', ['init', '--quiet'], { cwd: repo });
|
||||
execFileSync(
|
||||
'git',
|
||||
[
|
||||
'-c',
|
||||
'user.name=Benchmark Test',
|
||||
'-c',
|
||||
'user.email=test@example.test',
|
||||
'-c',
|
||||
'commit.gpgsign=false',
|
||||
'commit',
|
||||
'--quiet',
|
||||
'--allow-empty',
|
||||
'-m',
|
||||
'fixture',
|
||||
],
|
||||
{
|
||||
cwd: repo,
|
||||
env: {
|
||||
...process.env,
|
||||
GIT_AUTHOR_DATE: '2020-01-01T00:00:00Z',
|
||||
GIT_COMMITTER_DATE: '2020-01-01T00:00:00Z',
|
||||
},
|
||||
},
|
||||
);
|
||||
return repo;
|
||||
});
|
||||
const runtimes = [1, 100].map((duration) => {
|
||||
const runtime = path.join(root, `runtime-${duration}`);
|
||||
const core = path.join(runtime, 'gitnexus/dist/core');
|
||||
fs.mkdirSync(path.join(core, 'lbug'), { recursive: true });
|
||||
fs.writeFileSync(path.join(runtime, 'package.json'), '{"type":"module"}');
|
||||
// A deterministic clock isolates collection/order behavior from machine load.
|
||||
fs.writeFileSync(
|
||||
path.join(core, 'run-analyze.js'),
|
||||
`
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
let clock = 0;
|
||||
Object.defineProperty(globalThis, 'performance', {value: {now: () => clock}});
|
||||
export async function runFullAnalysis(repo) {
|
||||
clock += ${duration};
|
||||
const target = fs.readFileSync(path.join(repo, 'gitnexus/src/semantic-bench-control.ts'), 'utf8').includes('s.load()') ? 'load' : 'save';
|
||||
return {
|
||||
storagePath: repo, stats: {}, rebuildReasons: [],
|
||||
incrementalStats: {writeMode: 'incremental', changedFiles: 1},
|
||||
pipelineResult: {graph: {
|
||||
*iterNodes() { yield {id: 'semanticBenchUse', label: 'Function'}; },
|
||||
*iterRelationships() { yield {sourceId: 'semanticBenchUse', targetId: target, type: 'CALLS'}; }
|
||||
}}
|
||||
};
|
||||
}
|
||||
`,
|
||||
);
|
||||
fs.writeFileSync(path.join(core, 'logger.js'), 'export function configureLoggerSink() {}');
|
||||
fs.writeFileSync(
|
||||
path.join(core, 'lbug/lbug-adapter.js'),
|
||||
`
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
let repo;
|
||||
export async function initLbug(db) { repo = path.dirname(db); }
|
||||
export async function executeQuery() {
|
||||
return [{target: fs.readFileSync(path.join(repo, 'gitnexus/src/semantic-bench-control.ts'), 'utf8').includes('s.load()') ? 'load' : 'save'}];
|
||||
}
|
||||
export async function closeLbug() {}
|
||||
`,
|
||||
);
|
||||
return runtime;
|
||||
});
|
||||
for (const screen of ['0', '1']) {
|
||||
const output = path.join(root, `output-${screen}`);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[harness, ...runtimes, ...corpora, output, lifecycle],
|
||||
{
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
GITNEXUS_LEXICAL_BENCH_WORKER: '0',
|
||||
GITNEXUS_LEXICAL_BENCH_WORKERS: '2',
|
||||
GITNEXUS_LEXICAL_BENCH_HEAP_MIB: '256',
|
||||
GITNEXUS_LEXICAL_BENCH_SCREEN: screen,
|
||||
GITNEXUS_LEXICAL_BENCH_BASE_FIRST: '0',
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const report = JSON.parse(fs.readFileSync(path.join(output, 'results.json'), 'utf8'));
|
||||
expect(report.screen).toBe(screen === '1');
|
||||
expect(report.baseFirst).toBe(false);
|
||||
expect(report.results.map((sample: { name: string }) => sample.name)).toEqual([
|
||||
'candidate-seed',
|
||||
'baseline-seed',
|
||||
'candidate-1',
|
||||
'baseline-1',
|
||||
...(screen === '0' ? ['baseline-2', 'candidate-2', 'candidate-3', 'baseline-3'] : []),
|
||||
]);
|
||||
for (const repo of corpora)
|
||||
expect(fs.existsSync(path.join(repo, 'gitnexus/src/semantic-bench-control.ts'))).toBe(
|
||||
false,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -120,13 +120,14 @@ const mockLbugAdapter = async () => {
|
|||
initLbug: vi.fn(async () => undefined),
|
||||
loadGraphToLbug: vi.fn(async () => undefined),
|
||||
getLbugStats: vi.fn(async () => ({ nodes: 1, edges: 0, communities: 0, processes: 0 })),
|
||||
executeQuery: vi.fn(async () => []),
|
||||
// The policy fixture has one stored File row. Publication now reconciles
|
||||
// that identity instead of trusting the synthetic node count alone.
|
||||
executeQuery: vi.fn(async (query: string) =>
|
||||
query.startsWith('MATCH (n:`File`) RETURN n.id AS id')
|
||||
? [{ id: 'file:src/a.ts', name: REL_FILE, filePath: REL_FILE }]
|
||||
: [],
|
||||
),
|
||||
forEachQueryRow: vi.fn<typeof actual.forEachQueryRow>(async (query, onRow) => {
|
||||
if (query.startsWith('MATCH (n:`File`) RETURN n.id AS id')) {
|
||||
onRow({ id: 'file:src/a.ts', name: REL_FILE, filePath: REL_FILE });
|
||||
}
|
||||
}),
|
||||
executeWithReusedStatement: vi.fn(async () => []),
|
||||
closeLbug: vi.fn(async () => undefined),
|
||||
wipeLbugDbFiles: vi.fn(async () => undefined),
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
findClassBindingInScope,
|
||||
findReceiverTypeBinding,
|
||||
isNamespaceNameShadowed,
|
||||
lookupNameClaim,
|
||||
} from '../../../src/core/ingestion/scope-resolution/scope/walkers.js';
|
||||
import { findCallableBindingsAndAdlBlocker } from '../../../src/core/ingestion/languages/cpp/callable-bindings.js';
|
||||
import { followChainPostFinalize } from '../../../src/core/ingestion/scope-resolution/passes/imported-return-types.js';
|
||||
|
|
@ -60,6 +61,40 @@ function indexes(
|
|||
}
|
||||
|
||||
describe('lexical name ownership', () => {
|
||||
it.each(['Function', 'Class'] as const)(
|
||||
'keeps %s lookup snapshots after source bindings change',
|
||||
(kind) => {
|
||||
const bindings = new Map<string, BindingRef[]>();
|
||||
const inner = scope('fn', 'module', { bindings, kind });
|
||||
const outer = scope('module', null);
|
||||
const state = { ...indexes(inner, outer), bindings: new Map([[inner.id, bindings]]) };
|
||||
const missing = lookupNameClaim(inner.id, 'helper', state);
|
||||
expect(missing.status).toBe('absent');
|
||||
const local: BindingRef[] = [binding];
|
||||
bindings.set('helper', local);
|
||||
const found = lookupNameClaim(inner.id, 'helper', state);
|
||||
expect(found.bindings).toEqual([binding]);
|
||||
local.push({ def: { ...target, nodeId: 'second:helper' }, origin: 'local' });
|
||||
expect(found.bindings).toEqual([binding]);
|
||||
expect(missing.bindings).toEqual([]);
|
||||
expect(lookupNameClaim(inner.id, 'helper', state).bindings).toHaveLength(2);
|
||||
},
|
||||
);
|
||||
|
||||
it('clears deferred activation without mutating reused caller options', () => {
|
||||
const late: BindingRef = { ...binding, availableFrom: { startLine: 8, startCol: 0 } };
|
||||
const outer = scope('module', null, { bindings: new Map([['helper', [late]]]) });
|
||||
const inner = scope('fn', outer.id, { lookupPolicy: { deferParentActivation: true } });
|
||||
const options = Object.freeze({ position: Object.freeze({ startLine: 1, startCol: 0 }) });
|
||||
const state = indexes(inner, outer);
|
||||
expect(lookupNameClaim(inner.id, 'helper', state, options).bindings).toEqual([late]);
|
||||
expect(lookupNameClaim(outer.id, 'helper', state, options).status).toBe('absent');
|
||||
expect(
|
||||
lookupNameClaim(inner.id, 'helper', state, { ...options, preservePosition: true }).status,
|
||||
).toBe('absent');
|
||||
expect(options.position).toEqual({ startLine: 1, startCol: 0 });
|
||||
});
|
||||
|
||||
it('keeps touching half-open import ranges separate', () => {
|
||||
const earlier = { startLine: 1, startCol: 0, endLine: 1, endCol: 10 };
|
||||
const later = { startLine: 1, startCol: 10, endLine: 1, endCol: 20 };
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ const rows = () => [
|
|||
},
|
||||
];
|
||||
const queryFor = (functions: ReturnType<typeof rows>) =>
|
||||
vi.fn(async (query: string) => (query.includes('(n:`Function`)') ? functions : []));
|
||||
vi.fn(async (query: string, onRow: (row: ReturnType<typeof rows>[number]) => void) => {
|
||||
if (query.includes('(n:`Function`)')) functions.forEach(onRow);
|
||||
});
|
||||
|
||||
describe('incremental identity reconciliation', () => {
|
||||
it('certifies every identity using one unfiltered query per label', async () => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue