GitNexus/gitnexus/test/unit/csv-escaping.test.ts

/**
 * P0 Unit Tests: CSV Escaping Functions
 *
 * Tests: escapeCSVField, escapeCSVNumber, sanitizeUTF8, isBinaryContent
 * Covers hardening fix #23 (keyword arrays with backslashes and commas)
 */
import { describe, it, expect } from 'vitest';
import { assertValidGraphNodeId } from '../../src/core/graph/node-identity.js';
import {
  escapeCSVField,
  escapeCSVNumber,
  sanitizeUTF8,
  isBinaryContent,
} from '../../src/core/lbug/csv-generator.js';

// ─── escapeCSVField ──────────────────────────────────────────────────

describe('escapeCSVField', () => {
  it('returns an unquoted NULL field for null', () => {
    expect(escapeCSVField(null)).toBe('');
  });

  it('returns an unquoted NULL field for undefined', () => {
    expect(escapeCSVField(undefined)).toBe('');
  });

  it('preserves the legacy NULL meaning of empty input', () => {
    expect(escapeCSVField('')).toBe('');
  });

  it('returns a NULL field when sanitization removes the entire value', () => {
    expect(escapeCSVField('\x00\x01')).toBe('');
  });

  it('keeps whitespace and zero as quoted non-null values', () => {
    expect(escapeCSVField(' ')).toBe('" "');
    expect(escapeCSVField(0)).toBe('"0"');
  });

  it('wraps simple string in quotes', () => {
    expect(escapeCSVField('hello')).toBe('"hello"');
  });

  it('doubles internal double quotes', () => {
    expect(escapeCSVField('say "hello"')).toBe('"say ""hello"""');
  });

  it('handles strings with commas', () => {
    expect(escapeCSVField('a,b,c')).toBe('"a,b,c"');
  });

  it('handles strings with newlines', () => {
    expect(escapeCSVField('line1\nline2')).toBe('"line1\nline2"');
  });

  it('converts numbers to quoted strings', () => {
    expect(escapeCSVField(42)).toBe('"42"');
  });

  it('handles strings with both quotes and commas', () => {
    expect(escapeCSVField('"hello",world')).toBe('"""hello"",world"');
  });

  // Hardening fix #23: keyword arrays with backslashes
  it('handles strings with backslashes', () => {
    const result = escapeCSVField('path\\to\\file');
    expect(result).toBe('"path\\to\\file"');
  });

  it('handles code content with special characters', () => {
    const code = 'function foo() {\n  return "bar";\n}';
    const result = escapeCSVField(code);
    expect(result).toContain('function foo()');
    expect(result).toContain('""bar""');
  });
});

// ─── escapeCSVNumber ─────────────────────────────────────────────────

describe('escapeCSVNumber', () => {
  it('returns default value for null', () => {
    expect(escapeCSVNumber(null)).toBe('-1');
  });

  it('returns default value for undefined', () => {
    expect(escapeCSVNumber(undefined)).toBe('-1');
  });

  it('returns custom default value', () => {
    expect(escapeCSVNumber(null, 0)).toBe('0');
  });

  it('returns string representation of number', () => {
    expect(escapeCSVNumber(42)).toBe('42');
  });

  it('handles zero', () => {
    expect(escapeCSVNumber(0)).toBe('0');
  });

  it('handles negative numbers', () => {
    expect(escapeCSVNumber(-5)).toBe('-5');
  });

  it('handles floating point', () => {
    expect(escapeCSVNumber(3.14)).toBe('3.14');
  });
});

// ─── sanitizeUTF8 ────────────────────────────────────────────────────

describe('sanitizeUTF8', () => {
  it('passes through clean strings unchanged', () => {
    expect(sanitizeUTF8('hello world')).toBe('hello world');
  });

  it('normalizes CRLF to LF', () => {
    expect(sanitizeUTF8('line1\r\nline2')).toBe('line1\nline2');
  });

  it('normalizes lone CR to LF', () => {
    expect(sanitizeUTF8('line1\rline2')).toBe('line1\nline2');
  });

  it('strips null bytes', () => {
    expect(sanitizeUTF8('hello\x00world')).toBe('helloworld');
  });

  it('strips control characters', () => {
    expect(sanitizeUTF8('hello\x01\x02\x03world')).toBe('helloworld');
  });

  it('preserves tabs', () => {
    expect(sanitizeUTF8('hello\tworld')).toBe('hello\tworld');
  });

  it('preserves newlines', () => {
    expect(sanitizeUTF8('hello\nworld')).toBe('hello\nworld');
  });

  it('strips lone surrogates', () => {
    expect(sanitizeUTF8('hello\uD800world')).toBe('helloworld');
  });

  it('preserves valid supplementary Unicode characters', () => {
    expect(sanitizeUTF8('hello😀𐐀world')).toBe('hello😀𐐀world');
  });

  it('strips BOM-like characters (FFFE/FFFF)', () => {
    expect(sanitizeUTF8('hello\uFFFEworld')).toBe('helloworld');
  });
});

describe('graph identity CSV fidelity', () => {
  it.each([
    ...Array.from({ length: 32 }, (_, code) => String.fromCharCode(code)).filter(
      (char) => char !== '\t' && char !== '\n',
    ),
    '\x7F',
    '\uD800',
    '\uDFFF',
    '\uFFFE',
    '\uFFFF',
  ])('rejects a character that CSV would alter: %j', (char) => {
    expect(() => assertValidGraphNodeId(`Function:source.ts:a${char}b`, 'Graph COPY')).toThrow(
      /invalid graph node ID/,
    );
  });

  it.each(['Function:café.ts:😀', 'Function:𐐀.ts:method', 'Function:a\tb.ts:line\nname'])(
    'preserves accepted identity %j through CSV escaping',
    (id) => {
      expect(() => assertValidGraphNodeId(id, 'Graph COPY')).not.toThrow();
      expect(escapeCSVField(id)).toBe(`"${id}"`);
    },
  );
});

// ─── isBinaryContent ─────────────────────────────────────────────────

describe('isBinaryContent', () => {
  it('returns false for empty string', () => {
    expect(isBinaryContent('')).toBe(false);
  });

  it('returns false for normal text', () => {
    expect(isBinaryContent('hello world\nline two')).toBe(false);
  });

  it('returns false for code content', () => {
    const code = 'function foo() {\n  return 42;\n}\n';
    expect(isBinaryContent(code)).toBe(false);
  });

  it('returns true when >10% non-printable characters', () => {
    // Create a string that's ~20% null bytes
    const binary = 'a'.repeat(80) + '\x00'.repeat(20);
    expect(isBinaryContent(binary)).toBe(true);
  });

  it('returns false when just under 10% threshold', () => {
    // 9% non-printable should not be binary
    const borderline = 'a'.repeat(91) + '\x01'.repeat(9);
    expect(isBinaryContent(borderline)).toBe(false);
  });

  it('only samples first 1000 characters', () => {
    // Binary content past 1000 chars should be ignored
    const text = 'a'.repeat(1000) + '\x00'.repeat(500);
    expect(isBinaryContent(text)).toBe(false);
  });

  // #2889 — every file enters through a lossy `utf-8` decode, so an embedded
  // binary payload reaches this function as U+FFFD, never as the raw bytes.
  it('returns true when >10% U+FFFD replacement characters', () => {
    const decoded = 'a'.repeat(80) + '<27>'.repeat(20);
    expect(isBinaryContent(decoded)).toBe(true);
  });

  it('counts U+FFFD toward the same threshold as control bytes', () => {
    // 5 control + 6 replacement = 11% of 100 — neither group crosses 10% alone.
    const mixed = 'a'.repeat(89) + '\x01'.repeat(5) + '<27>'.repeat(6);
    expect(isBinaryContent(mixed)).toBe(true);
  });

  it('returns false for text carrying a few replacement characters', () => {
    // A mis-decoded latin-1 comment in an otherwise clean file stays indexable.
    const mostlyText = 'a'.repeat(95) + '<27>'.repeat(5);
    expect(isBinaryContent(mostlyText)).toBe(false);
  });
});