GitNexus/gitnexus/bench/python-scope/measure.mjs

/**
 * Build-free measurement harness for `emitPythonScopeCaptures`
 * (ce-optimize: python-scope-capture).
 *
 * This is Python's counterpart to `bench/scope-capture/measure.mjs` (which
 * covers go/csharp/rust/php/ruby/cobol). Python lives here, NOT in that unified
 * harness, because this one ALSO covers import resolution
 * (`import-target-fingerprint.mjs`). Python's capture-scaling guard therefore
 * runs via `python-scope/measure.mjs --check`, not the unified harness — don't
 * remove either thinking the other covers Python.
 *
 * Mirrors the Go scope-capture harness (#1848). Imports the `.ts` hotpath
 * directly through tsx (`node --import tsx bench/python-scope/measure.mjs`):
 * a static `.ts` import works; a top-level `await import()` breaks tsx's lexer.
 *
 * Emits ONE JSON object on stdout with:
 *   - elapsed_ms_250 / elapsed_ms_800: median wall-clock (ms) of
 *     emitPythonScopeCaptures over a synthetic DAO-style source at that many
 *     top-level entities (warmed up first). 800/250 ~ 3.2x input; an O(n^2)
 *     path scales ~quadratically, an O(n) path ~linearly.
 *   - scaling_ratio: (t800/t250)/(800/250). ~3.2 = quadratic, ~1.0 = linear.
 *   - capture_groups_250 / capture_groups_800: match counts (a fast-but-empty
 *     regression can't pass — counts must stay > 0).
 *   - fingerprint: order-independent sha256 over emitPythonScopeCaptures output
 *     across the whole lang-resolution/python-* fixture corpus + a fixed
 *     20-entity synthetic DAO. This is the CORRECTNESS gate: any change to the
 *     captures changes the fingerprint. Entity-count-fixed so it is comparable
 *     across experiments regardless of the timing sizes.
 *   - capture_groups_fp / fixture_count: corpus sanity.
 */
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { emitPythonScopeCaptures } from '../../src/core/ingestion/languages/python/captures.ts';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const FIXTURE_ROOT = path.resolve(__dirname, '..', '..', 'test', 'fixtures', 'lang-resolution');

// ---- correctness fingerprint (order-independent, mirrors the Go golden) ----

function canonicalizeMatch(match) {
  const parts = [];
  for (const tag of Object.keys(match)) {
    // Scope-only lexical shadow metadata is correctness-tested separately and
    // does not alter capture matching or the benchmark's scaling contract.
    if (tag === '@scope.lexical-names') continue;
    const cap = match[tag];
    const r = cap.range;
    parts.push(`${tag}|${cap.text}|${r.startLine}:${r.startCol}-${r.endLine}:${r.endCol}`);
  }
  parts.sort();
  return parts.join(';');
}

function digestCaptures(matches) {
  const matchStrings = matches.map(canonicalizeMatch).sort();
  return crypto.createHash('sha256').update(matchStrings.join('\n')).digest('hex');
}

/** All `.py` files under `lang-resolution/python-*`, sorted by repo-relative key. */
function collectPythonFixtures() {
  const out = [];
  for (const entry of fs.readdirSync(FIXTURE_ROOT, { withFileTypes: true })) {
    if (!entry.isDirectory() || !entry.name.startsWith('python-')) continue;
    const stack = [path.join(FIXTURE_ROOT, entry.name)];
    while (stack.length) {
      const dir = stack.pop();
      for (const c of fs.readdirSync(dir, { withFileTypes: true })) {
        const p = path.join(dir, c.name);
        if (c.isDirectory()) stack.push(p);
        else if (c.name.endsWith('.py')) {
          out.push({ key: path.relative(FIXTURE_ROOT, p).split(path.sep).join('/'), absPath: p });
        }
      }
    }
  }
  out.sort((a, b) => a.key.localeCompare(b.key));
  return out;
}

/**
 * Synthetic DAO-style source: top-level imports + N classes (each with methods,
 * exercising @scope.function + @declaration.function + receiver binding) + N
 * module functions. Maximizes top-level children (rootChildren) AND function
 * matches, which is exactly the O(matches x rootChildren) shape #1848 hit.
 */
function generatePyDao(entityCount) {
  const lines = [];
  for (let i = 0; i < 12; i++) {
    lines.push(`from pkg.mod${i} import alpha${i}, beta${i}, gamma${i} as g${i}`);
    lines.push(`import top.level.module${i}`);
  }
  lines.push('');
  // Shared base + mixin so every Entity is heritage-bearing — exercises the
  // @reference.inherits synth (#1951) at scale (single + multiple inheritance),
  // not just the base capture loop.
  lines.push('class Base:', '    pass', '', 'class Mixin:', '    pass', '');
  for (let i = 0; i < entityCount; i++) {
    const n = String(i).padStart(4, '0');
    lines.push(
      `class Entity${n}(Base, Mixin):`,
      `    def __init__(self, id: int, name: str):`,
      `        self.id = id`,
      `        self.name = name`,
      `    def get_id(self) -> int:`,
      `        return self.id`,
      `    def set_name(self, name: str) -> None:`,
      `        self.name = name`,
      `    @classmethod`,
      `    def make(cls, id: int):`,
      `        return cls(id, "x")`,
      '',
      `def build_entity${n}(id: int, name: str) -> Entity${n}:`,
      `    return Entity${n}(id, name)`,
      '',
    );
  }
  return lines.join('\n');
}

// ---- timing ----

function timeOnce(src, filePath) {
  const start = process.hrtime.bigint();
  const matches = emitPythonScopeCaptures(src, filePath);
  const end = process.hrtime.bigint();
  return { ms: Number(end - start) / 1e6, count: matches.length };
}

function median(xs) {
  const s = [...xs].sort((a, b) => a - b);
  const m = Math.floor(s.length / 2);
  return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
}

function measureSize(entityCount, reps) {
  const src = generatePyDao(entityCount);
  // Warm up parser/query JIT (not counted).
  timeOnce(src, 'warmup.py');
  const samples = [];
  let count = 0;
  for (let i = 0; i < reps; i++) {
    const r = timeOnce(src, `bench-${entityCount}.py`);
    samples.push(r.ms);
    count = r.count;
  }
  return { ms: median(samples), count };
}

// ---- run ----

function computeFingerprint() {
  let groups = 0;
  const perFixtureDigests = [];
  for (const { key, absPath } of collectPythonFixtures()) {
    const src = fs.readFileSync(absPath, 'utf8');
    // Lexical policies carry scope IDs containing the input path. Keep the
    // fingerprint independent of the checkout's absolute location.
    const matches = emitPythonScopeCaptures(src, key);
    groups += matches.length;
    perFixtureDigests.push(`${key}\t${matches.length}\t${digestCaptures(matches)}`);
  }
  // Fixed 20-entity synthetic source so the fingerprint is comparable across
  // experiments independent of the timing sizes.
  const daoMatches = emitPythonScopeCaptures(generatePyDao(20), 'synthetic-dao-20.py');
  groups += daoMatches.length;
  perFixtureDigests.push(`synthetic:dao-20\t${daoMatches.length}\t${digestCaptures(daoMatches)}`);
  const fingerprint = crypto
    .createHash('sha256')
    .update(perFixtureDigests.sort().join('\n'))
    .digest('hex');
  return { fingerprint, groups, fixtureCount: perFixtureDigests.length };
}

// Higher rep count keeps the median stable on noisy shared CI runners.
const REPS = 7;
const SCALING_BUDGET = 1.5; // ~3.2 (quadratic) vs ~1.0 (linear); 1.5 has headroom.
const CHECK = process.argv.includes('--check');

const fp = computeFingerprint();
const small = measureSize(250, REPS);
const large = measureSize(800, REPS);
const scalingRatio = small.ms > 0 ? large.ms / small.ms / (800 / 250) : 0;

const result = {
  elapsed_ms_250: Number(small.ms.toFixed(2)),
  elapsed_ms_800: Number(large.ms.toFixed(2)),
  scaling_ratio: Number(scalingRatio.toFixed(3)),
  capture_groups_250: small.count,
  capture_groups_800: large.count,
  fingerprint: fp.fingerprint,
  capture_groups_fp: fp.groups,
  fixture_count: fp.fixtureCount,
};

if (!CHECK) {
  process.stdout.write(JSON.stringify(result) + '\n');
} else {
  // CI gate: capture output unchanged (fingerprint == committed baseline) AND
  // the path is still linear (scaling ratio under budget). Re-baseline a
  // legitimate capture change with `node --import tsx measure.mjs` (no --check)
  // and commit the new baseline-fingerprint.txt deliberately.
  // #3531: Tree-sitter 0.25 adds one zero-width module scope to each of the
  // 15 empty __init__.py fixtures (3982 -> 3997 groups). All non-empty fixture
  // captures and the synthetic DAO are unchanged; the golden test pins this.
  const baselinePath = path.resolve(__dirname, 'baseline-fingerprint.txt');
  const baseline = fs.readFileSync(baselinePath, 'utf8').trim();
  const failures = [];
  if (result.fingerprint !== baseline) {
    failures.push(
      `capture fingerprint drift: got ${result.fingerprint}, expected ${baseline} ` +
        `(emitPythonScopeCaptures output changed — re-baseline intentionally if expected)`,
    );
  }
  if (result.scaling_ratio >= SCALING_BUDGET) {
    failures.push(
      `capture scaling ratio ${result.scaling_ratio} >= ${SCALING_BUDGET} ` +
        `(possible O(n^2) regression; 250->800ms ${result.elapsed_ms_250}->${result.elapsed_ms_800})`,
    );
  }
  process.stdout.write(JSON.stringify(result) + '\n');
  if (failures.length > 0) {
    for (const f of failures) process.stderr.write(`[measure --check] FAIL: ${f}\n`);
    process.exit(1);
  }
  process.stderr.write('[measure --check] PASS (capture fingerprint + scaling)\n');
}