GitNexus/gitnexus/test/unit/cjk-segmentation.test.ts

import { afterEach, describe, expect, it, vi } from 'vitest';
import {
  applyCjkSegmentationIfEnabled,
  CJK_BIGRAM_WORST_CASE_GROWTH_FACTOR,
  cjkSegmentationModeMismatch,
  containsCjkIdeograph,
  containsSegmentableCjkRun,
  getSearchFTSCjkSegmentation,
  initialiseSearchFTSCjkSegmentation,
  segmentCjkSpans,
} from '../../src/core/search/cjk-segmentation.js';

afterEach(() => {
  vi.unstubAllEnvs();
});

describe('segmentCjkSpans', () => {
  it('segments a pure CJK phrase into overlapping bigrams', () => {
    // Issue #2331's own example: "purchase order automatic approval process"
    expect(segmentCjkSpans('采购订单自动审批流程')).toBe(
      '采购 购订 订单 单自 自动 动审 审批 批流 流程',
    );
  });

  it('inserts a boundary space between a non-CJK run and a CJK run', () => {
    expect(segmentCjkSpans('ERP审批流程')).toBe('ERP 审批 批流 流程');
  });

  it('leaves an exactly-2-character CJK run as the single unchanged bigram', () => {
    expect(segmentCjkSpans('审批')).toBe('审批');
  });

  it('leaves a single CJK character unchanged (no bigram possible)', () => {
    expect(segmentCjkSpans('审')).toBe('审');
  });

  it('does not produce a bigram spanning punctuation between two CJK runs', () => {
    const result = segmentCjkSpans('你好。世界');
    // The punctuation mark resets the run on both sides, so no two-character
    // token may fuse a pre-punctuation and post-punctuation character.
    expect(result).not.toContain('好。');
    expect(result).not.toContain('。世');
    expect(result).toBe('你好 。 世界');
  });

  it('passes pure ASCII/Latin text through unchanged (idempotent no-op)', () => {
    const text = 'the quick brown fox jumps over the lazy dog';
    expect(segmentCjkSpans(text)).toBe(text);
  });

  it('returns an empty string unchanged', () => {
    expect(segmentCjkSpans('')).toBe('');
  });

  it('does not double an existing whitespace boundary between scripts', () => {
    expect(segmentCjkSpans('ERP 审批')).toBe('ERP 审批');
  });

  it('does not double an existing whitespace boundary in the reverse direction', () => {
    expect(segmentCjkSpans('流程 ERP')).toBe('流程 ERP');
  });

  it('matches the ~7n/3-bytes-per-input-byte growth-factor formula for long CJK runs', () => {
    // Implementation Unit 3's CSV-flush margin math depends on this ratio —
    // a silent change to the expansion factor should fail this test loudly.
    const cjkChar = '采';
    const n = 10_000;
    const input = cjkChar.repeat(n);
    const inputBytes = Buffer.byteLength(input, 'utf8');
    const output = segmentCjkSpans(input);
    const outputBytes = Buffer.byteLength(output, 'utf8');
    const expectedBytes = inputBytes * CJK_BIGRAM_WORST_CASE_GROWTH_FACTOR;
    expect(outputBytes).toBeGreaterThan(expectedBytes * 0.95);
    expect(outputBytes).toBeLessThan(expectedBytes * 1.05);
  });

  it('scales linearly on realistic interleaved CJK/non-CJK content, not quadratically', () => {
    // Regression guard: an earlier implementation indexed into the growing
    // accumulated output string once per run boundary, which forces V8 to
    // flatten its internal rope representation on every access — O(n^2) on
    // content that alternates CJK and non-CJK runs (ordinary source code
    // with inline CJK comments, the feature's actual target). A single-run
    // input (like the growth-factor test above) never exercises this path,
    // since there is only one run boundary regardless of size.
    const unit = '采购订单自动审批流程 // approve the request after manual review\n';
    const build = (totalBytes: number) => unit.repeat(Math.ceil(totalBytes / unit.length));

    const small = build(64 * 1024);
    const large = build(512 * 1024); // 8x the input size

    const timeOf = (input: string) => {
      const start = performance.now();
      segmentCjkSpans(input);
      return performance.now() - start;
    };

    // Warm up the JIT before measuring either size.
    timeOf(small);
    timeOf(large);

    const smallMs = timeOf(small);
    const largeMs = timeOf(large);

    // Linear scaling means ~8x input takes roughly ~8x time, with headroom
    // for noise; quadratic scaling would mean ~64x time. 20x catches the
    // regression while tolerating CI timing variance.
    expect(largeMs).toBeLessThan(Math.max(smallMs, 1) * 20);
  });
});

describe('containsCjkIdeograph', () => {
  it('returns true for a CJK Unified Ideograph, including a single character', () => {
    expect(containsCjkIdeograph('审')).toBe(true);
    expect(containsCjkIdeograph('采购订单自动审批流程')).toBe(true);
  });

  it('returns false for Hiragana', () => {
    expect(containsCjkIdeograph('あ')).toBe(false);
  });

  it('returns false for Katakana', () => {
    expect(containsCjkIdeograph('ア')).toBe(false);
  });

  it('returns false for Hangul Syllables', () => {
    expect(containsCjkIdeograph('가')).toBe(false);
  });

  it('returns false for plain ASCII/Latin text', () => {
    expect(containsCjkIdeograph('hello world')).toBe(false);
  });

  it('returns false for an empty string', () => {
    expect(containsCjkIdeograph('')).toBe(false);
  });
});

describe('containsSegmentableCjkRun', () => {
  it('returns false for a single CJK character (no possible pairing)', () => {
    expect(containsSegmentableCjkRun('审')).toBe(false);
  });

  it('returns true for a 2+-character contiguous CJK run', () => {
    expect(containsSegmentableCjkRun('审批')).toBe(true);
    expect(containsSegmentableCjkRun('采购订单自动审批流程')).toBe(true);
  });

  it('returns false for non-CJK text', () => {
    expect(containsSegmentableCjkRun('hello world')).toBe(false);
  });

  it('is not stateful across repeated calls on the same input (regression guard)', () => {
    // A prior implementation called .test() on the shared, global-flagged
    // CJK_RUN_RE directly, which mutates lastIndex between calls and
    // alternates true/false/true on repeated calls with the same string.
    const text = '审批流程';
    expect(containsSegmentableCjkRun(text)).toBe(true);
    expect(containsSegmentableCjkRun(text)).toBe(true);
    expect(containsSegmentableCjkRun(text)).toBe(true);
  });
});

// NOTE ON ORDERING: `getSearchFTSCjkSegmentation`'s on-demand fallback only
// applies while the module-level cache is still unset. The describe blocks
// below are ordered so every test relying on that fallback (via `vi.stubEnv`)
// runs before `initialiseSearchFTSCjkSegmentation`'s "caches" test, which
// permanently sets the cache for the rest of this file — mirrors the same
// ordering constraint in fts-indexes.test.ts's sibling suite.

describe('getSearchFTSCjkSegmentation', () => {
  it('defaults to none when unset', () => {
    expect(getSearchFTSCjkSegmentation()).toBe('none');
  });

  it('normalizes a configured mode (case-insensitive, trimmed)', () => {
    vi.stubEnv('GITNEXUS_FTS_CJK_SEGMENTATION', ' Bigram ');
    expect(getSearchFTSCjkSegmentation()).toBe('bigram');
  });

  it('throws on an unsupported value, listing valid options', () => {
    vi.stubEnv('GITNEXUS_FTS_CJK_SEGMENTATION', 'jieba');
    expect(() => getSearchFTSCjkSegmentation()).toThrow('Invalid GITNEXUS_FTS_CJK_SEGMENTATION');
    expect(() => getSearchFTSCjkSegmentation()).toThrow('bigram, none');
  });
});

describe('applyCjkSegmentationIfEnabled', () => {
  it('is a no-op when the resolved mode is none (default)', () => {
    const text = '采购订单自动审批流程';
    expect(applyCjkSegmentationIfEnabled(text)).toBe(text);
  });

  it('delegates to segmentCjkSpans when the resolved mode is bigram', () => {
    vi.stubEnv('GITNEXUS_FTS_CJK_SEGMENTATION', 'bigram');
    expect(applyCjkSegmentationIfEnabled('审批流程')).toBe(segmentCjkSpans('审批流程'));
  });
});

describe('initialiseSearchFTSCjkSegmentation', () => {
  it('throws on an unsupported value without poisoning the cache', () => {
    vi.stubEnv('GITNEXUS_FTS_CJK_SEGMENTATION', 'jieba');
    expect(() => initialiseSearchFTSCjkSegmentation()).toThrow(
      'Invalid GITNEXUS_FTS_CJK_SEGMENTATION',
    );
  });

  it('resolves once so later reads ignore a changed env', () => {
    vi.stubEnv('GITNEXUS_FTS_CJK_SEGMENTATION', 'bigram');
    expect(initialiseSearchFTSCjkSegmentation()).toBe('bigram');

    vi.stubEnv('GITNEXUS_FTS_CJK_SEGMENTATION', 'none');
    expect(getSearchFTSCjkSegmentation()).toBe('bigram');
  });
});

describe('cjkSegmentationModeMismatch (#2331/#2339)', () => {
  it('legacy meta (no recorded stamp) + default live mode → no mismatch', () => {
    expect(cjkSegmentationModeMismatch(undefined, 'none')).toBe(false);
  });

  it('legacy meta + bigram live mode → mismatch (feature newly enabled)', () => {
    expect(cjkSegmentationModeMismatch(undefined, 'bigram')).toBe(true);
  });

  it('recorded bigram + live none → mismatch (on→off flip)', () => {
    expect(cjkSegmentationModeMismatch('bigram', 'none')).toBe(true);
  });

  it('recorded bigram + live bigram → no mismatch (unchanged)', () => {
    expect(cjkSegmentationModeMismatch('bigram', 'bigram')).toBe(false);
  });
});