test(ruby): align gem benchmark baseline with ratio gates

This commit is contained in:
y4ho0 2026-09-10 14:33:12 +08:00
parent 9c9d28c5cd
commit 2fa490ced9
3 changed files with 33 additions and 22 deletions

View file

@ -1,5 +1,6 @@
{
"_notes": "Synthetic Ruby manifest + ScopeResolver gate for #3096. Shape/fingerprints are exact; timings use min of 9 after 2 warmups. Scaling budgets are per import/project (4x workload normalized to 1), below the 4 expected from quadratic growth. Dense adds 8 -> 128 unrelated Gemfile declarations per project while keeping queries fixed. Absolute ceilings leave OS/filesystem headroom; they are not performance promises. Parser, DB, real gems and full-pipeline memory remain covered elsewhere, not by this microbenchmark.",
"_what": "Synthetic Ruby manifest + ScopeResolver gate for #3096, following bench/parse-dispatch-rounds/baselines.json: exact correctness floors and fingerprints, with ratios as the only timing gates. Calls the real config loader and resolver; parser, DB, installed gems and full-pipeline memory are outside this microbenchmark.",
"_triage": "Shape and fingerprint failures are deterministic: inspect the fixture and resolved targets, never rebaseline just to make CI green. For a timing failure, rerun on an idle machine and verify the report uses 15 samples after 2 warmups before investigating a regression. Millisecond observations are context, not gates.",
"shapes": {
"small": {
"projects": 32,
@ -29,21 +30,32 @@
"fingerprint": "e37f4c81bc7f4bc8416732d0e8ebb00e16743b08a44646e1a6fa45c04283cfb3"
}
},
"_shape_note": "Exact projects, declarations, scopes, files, imports and resolved counts are the workload FLOOR, not ceilings. They prevent a shrunken corpus or disconnected manifest loader from passing by doing less work. Small/large grow projects 4x; dense keeps files/imports fixed while growing extra Gemfile declarations from 8 to 128 per project.",
"_fingerprint_note": "Hashes pin every importer/require/target tuple, including external decoys, aliases, local path gem hits and misses, relative/local imports and sibling isolation. The runner also checks each expected target before hashing and proves the external decoy is reachable without config. An always-null resolver or missing config must fail, not get a new fingerprint.",
"budgets": {
"load_scaling_ratio": 2.2,
"resolve_scaling_ratio": 2.2,
"dependency_count_ratio": 2,
"small_load_ms": 80,
"small_resolve_ms": 120,
"dense_load_ms": 160
"dependency_count_ratio": 2
},
"_scaling_note": "load_scaling_ratio and resolve_scaling_ratio are (t_4n/t_n)/4: about 1 is linear, about 4 is quadratic. Keep the existing 2.2 regression budgets; do not tighten them to local milliseconds. Filesystem loading and lookup are measured separately with fresh config/file Sets per pass.",
"_dependency_count_note": "dense.resolve_ms/small.resolve_ms should stay near 1: require lookup must scale with prefix/path depth, not all declared gems. The budget remains 2. Extra declaration parsing belongs to config loading, not this fixed-query lookup arm.",
"_negative_controls_note": "Disconnecting loadResolutionConfig fails the real-loader assertion. A wrapper scanning all scopes' external prefixes on every import leaves targets/fingerprints unchanged but fails resolve_scaling_ratio (3.427 > 2.2) and dependency_count_ratio (7.912 > 2) with 15 samples. These are deliberate regressions, not baseline observations.",
"_measured": {
"environment": "macOS arm64 / Node 25.8.0, 2026-09-10",
"small_load_ms": 5.385,
"small_resolve_ms": 16.837,
"large_load_ms": 21.151,
"large_resolve_ms": 70.497,
"dense_load_ms": 9.948,
"dense_resolve_ms": 16.888
}
"load_scaling_ratio": 1.094,
"load_scaling_ratio_samples": [0.993, 1.06, 1.05, 1.019, 1.094],
"resolve_scaling_ratio": 1.066,
"resolve_scaling_ratio_samples": [1.058, 1.059, 1.056, 1.066, 1.061],
"dependency_count_ratio": 1.009,
"dependency_count_ratio_samples": [1.009, 1.003, 0.994, 0.999, 1.0],
"small_load_ms": 3.569,
"small_resolve_ms": 11.782,
"large_load_ms": 14.812,
"large_resolve_ms": 50.222,
"dense_load_ms": 7.27,
"dense_resolve_ms": 11.852,
"reps": 15,
"warmup": 2
},
"_measured_note": "Five consecutive local runs using the min-of-15 estimator from parse-dispatch-rounds, with 2 warmups here. Scalar ratios and millisecond values are per-metric maxima across those runs (rounded), not one combined run. Budgets retain about 2x headroom above observed ratios. Milliseconds are diagnostic context only, not gated or real-repository speed claims. Exact shapes and fingerprints are unchanged."
}

View file

@ -23,7 +23,7 @@ import { rubyScopeResolver } from '../../src/core/ingestion/languages/ruby/scope
const baselinePath = fileURLToPath(new URL('./baseline.json', import.meta.url));
const CHECK = process.argv.includes('--check');
const WARMUP = 2;
const REPS = 9;
const REPS = 15;
const IMPORTS_PER_PROJECT = 256;
const roots = [];
@ -159,9 +159,6 @@ try {
load_scaling_ratio: timings.large.load_ms / timings.small.load_ms / scale,
resolve_scaling_ratio: timings.large.resolve_ms / timings.small.resolve_ms / scale,
dependency_count_ratio: timings.dense.resolve_ms / timings.small.resolve_ms,
small_load_ms: timings.small.load_ms,
small_resolve_ms: timings.small.resolve_ms,
dense_load_ms: timings.dense.load_ms,
};
const report = {
shapes,

View file

@ -3,8 +3,9 @@
*
* Ruby import resolution rules:
* - `require_relative './foo'` → resolve relative to the importing file's dir
* - `require 'foo'` → suffix-match via the existing Ruby import resolver
* - External gems → null (unresolvable within the repo)
* - `require 'foo'` → use scoped gem metadata before legacy suffix matching
* - Known local gems → resolve only within their declared load roots
* - Known external gems → null, even if an unrelated repo file suffix matches
*/
import { resolveRubyImportInternal } from '../../import-resolvers/ruby.js';
@ -31,10 +32,11 @@ export interface RubyResolveContext {
* against the importing file's directory, trying `.rb` and `/index.rb`
* suffixes.
*
* For bare requires (gem-style like `'json'`, `'serializable'`), delegates
* to the existing `resolveRubyImportInternal` which uses suffix matching.
*
* Returns `null` for external gems that have no matching file in the repo.
* For bare requires, scoped manifest metadata takes precedence: known local
* gems resolve only within their declared load roots (a miss returns `null`),
* and known external gem prefixes return `null` even if a repo suffix matches.
* Only requires without matching gem evidence delegate to the existing
* `resolveRubyImportInternal` suffix matcher.
*/
export function resolveRubyImportTarget(
targetRaw: string,