From 2fa490ced97559e91c323289c83d286fc8f509aa Mon Sep 17 00:00:00 2001 From: y4ho0 <119646314+y4ho0@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:33:12 +0800 Subject: [PATCH] test(ruby): align gem benchmark baseline with ratio gates --- .../bench/ruby-gem-resolution/baseline.json | 36 ++++++++++++------- .../bench/ruby-gem-resolution/measure.mjs | 5 +-- .../ingestion/languages/ruby/import-target.ts | 14 ++++---- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/gitnexus/bench/ruby-gem-resolution/baseline.json b/gitnexus/bench/ruby-gem-resolution/baseline.json index 42f394ab5..8c618ad00 100644 --- a/gitnexus/bench/ruby-gem-resolution/baseline.json +++ b/gitnexus/bench/ruby-gem-resolution/baseline.json @@ -1,5 +1,6 @@ { - "_notes": "Synthetic Ruby manifest + ScopeResolver gate for #3096. Shape/fingerprints are exact; timings use min of 9 after 2 warmups. Scaling budgets are per import/project (4x workload normalized to 1), below the 4 expected from quadratic growth. Dense adds 8 -> 128 unrelated Gemfile declarations per project while keeping queries fixed. Absolute ceilings leave OS/filesystem headroom; they are not performance promises. Parser, DB, real gems and full-pipeline memory remain covered elsewhere, not by this microbenchmark.", + "_what": "Synthetic Ruby manifest + ScopeResolver gate for #3096, following bench/parse-dispatch-rounds/baselines.json: exact correctness floors and fingerprints, with ratios as the only timing gates. Calls the real config loader and resolver; parser, DB, installed gems and full-pipeline memory are outside this microbenchmark.", + "_triage": "Shape and fingerprint failures are deterministic: inspect the fixture and resolved targets, never rebaseline just to make CI green. For a timing failure, rerun on an idle machine and verify the report uses 15 samples after 2 warmups before investigating a regression. Millisecond observations are context, not gates.", "shapes": { "small": { "projects": 32, @@ -29,21 +30,32 @@ "fingerprint": "e37f4c81bc7f4bc8416732d0e8ebb00e16743b08a44646e1a6fa45c04283cfb3" } }, + "_shape_note": "Exact projects, declarations, scopes, files, imports and resolved counts are the workload FLOOR, not ceilings. They prevent a shrunken corpus or disconnected manifest loader from passing by doing less work. Small/large grow projects 4x; dense keeps files/imports fixed while growing extra Gemfile declarations from 8 to 128 per project.", + "_fingerprint_note": "Hashes pin every importer/require/target tuple, including external decoys, aliases, local path gem hits and misses, relative/local imports and sibling isolation. The runner also checks each expected target before hashing and proves the external decoy is reachable without config. An always-null resolver or missing config must fail, not get a new fingerprint.", "budgets": { "load_scaling_ratio": 2.2, "resolve_scaling_ratio": 2.2, - "dependency_count_ratio": 2, - "small_load_ms": 80, - "small_resolve_ms": 120, - "dense_load_ms": 160 + "dependency_count_ratio": 2 }, + "_scaling_note": "load_scaling_ratio and resolve_scaling_ratio are (t_4n/t_n)/4: about 1 is linear, about 4 is quadratic. Keep the existing 2.2 regression budgets; do not tighten them to local milliseconds. Filesystem loading and lookup are measured separately with fresh config/file Sets per pass.", + "_dependency_count_note": "dense.resolve_ms/small.resolve_ms should stay near 1: require lookup must scale with prefix/path depth, not all declared gems. The budget remains 2. Extra declaration parsing belongs to config loading, not this fixed-query lookup arm.", + "_negative_controls_note": "Disconnecting loadResolutionConfig fails the real-loader assertion. A wrapper scanning all scopes' external prefixes on every import leaves targets/fingerprints unchanged but fails resolve_scaling_ratio (3.427 > 2.2) and dependency_count_ratio (7.912 > 2) with 15 samples. These are deliberate regressions, not baseline observations.", "_measured": { "environment": "macOS arm64 / Node 25.8.0, 2026-09-10", - "small_load_ms": 5.385, - "small_resolve_ms": 16.837, - "large_load_ms": 21.151, - "large_resolve_ms": 70.497, - "dense_load_ms": 9.948, - "dense_resolve_ms": 16.888 - } + "load_scaling_ratio": 1.094, + "load_scaling_ratio_samples": [0.993, 1.06, 1.05, 1.019, 1.094], + "resolve_scaling_ratio": 1.066, + "resolve_scaling_ratio_samples": [1.058, 1.059, 1.056, 1.066, 1.061], + "dependency_count_ratio": 1.009, + "dependency_count_ratio_samples": [1.009, 1.003, 0.994, 0.999, 1.0], + "small_load_ms": 3.569, + "small_resolve_ms": 11.782, + "large_load_ms": 14.812, + "large_resolve_ms": 50.222, + "dense_load_ms": 7.27, + "dense_resolve_ms": 11.852, + "reps": 15, + "warmup": 2 + }, + "_measured_note": "Five consecutive local runs using the min-of-15 estimator from parse-dispatch-rounds, with 2 warmups here. Scalar ratios and millisecond values are per-metric maxima across those runs (rounded), not one combined run. Budgets retain about 2x headroom above observed ratios. Milliseconds are diagnostic context only, not gated or real-repository speed claims. Exact shapes and fingerprints are unchanged." } diff --git a/gitnexus/bench/ruby-gem-resolution/measure.mjs b/gitnexus/bench/ruby-gem-resolution/measure.mjs index 12871e264..8f7ef7875 100644 --- a/gitnexus/bench/ruby-gem-resolution/measure.mjs +++ b/gitnexus/bench/ruby-gem-resolution/measure.mjs @@ -23,7 +23,7 @@ import { rubyScopeResolver } from '../../src/core/ingestion/languages/ruby/scope const baselinePath = fileURLToPath(new URL('./baseline.json', import.meta.url)); const CHECK = process.argv.includes('--check'); const WARMUP = 2; -const REPS = 9; +const REPS = 15; const IMPORTS_PER_PROJECT = 256; const roots = []; @@ -159,9 +159,6 @@ try { load_scaling_ratio: timings.large.load_ms / timings.small.load_ms / scale, resolve_scaling_ratio: timings.large.resolve_ms / timings.small.resolve_ms / scale, dependency_count_ratio: timings.dense.resolve_ms / timings.small.resolve_ms, - small_load_ms: timings.small.load_ms, - small_resolve_ms: timings.small.resolve_ms, - dense_load_ms: timings.dense.load_ms, }; const report = { shapes, diff --git a/gitnexus/src/core/ingestion/languages/ruby/import-target.ts b/gitnexus/src/core/ingestion/languages/ruby/import-target.ts index 7c676d8eb..765590371 100644 --- a/gitnexus/src/core/ingestion/languages/ruby/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/ruby/import-target.ts @@ -3,8 +3,9 @@ * * Ruby import resolution rules: * - `require_relative './foo'` → resolve relative to the importing file's dir - * - `require 'foo'` → suffix-match via the existing Ruby import resolver - * - External gems → null (unresolvable within the repo) + * - `require 'foo'` → use scoped gem metadata before legacy suffix matching + * - Known local gems → resolve only within their declared load roots + * - Known external gems → null, even if an unrelated repo file suffix matches */ import { resolveRubyImportInternal } from '../../import-resolvers/ruby.js'; @@ -31,10 +32,11 @@ export interface RubyResolveContext { * against the importing file's directory, trying `.rb` and `/index.rb` * suffixes. * - * For bare requires (gem-style like `'json'`, `'serializable'`), delegates - * to the existing `resolveRubyImportInternal` which uses suffix matching. - * - * Returns `null` for external gems that have no matching file in the repo. + * For bare requires, scoped manifest metadata takes precedence: known local + * gems resolve only within their declared load roots (a miss returns `null`), + * and known external gem prefixes return `null` even if a repo suffix matches. + * Only requires without matching gem evidence delegate to the existing + * `resolveRubyImportInternal` suffix matcher. */ export function resolveRubyImportTarget( targetRaw: string,