GitNexus/gitnexus/test/unit/ingestion-utils.test.ts

import { describe, it, expect } from 'vitest';
import { getLanguageFromFilename, isBuiltInOrNoise, extractFunctionName } from '../../src/core/ingestion/utils.js';
import { getTreeSitterBufferSize, TREE_SITTER_BUFFER_SIZE, TREE_SITTER_MAX_BUFFER } from '../../src/core/ingestion/constants.js';
import { SupportedLanguages } from '../../src/config/supported-languages.js';
import Parser from 'tree-sitter';
import C from 'tree-sitter-c';
import CPP from 'tree-sitter-cpp';
import Python from 'tree-sitter-python';
import TypeScript from 'tree-sitter-typescript';

describe('getLanguageFromFilename', () => {
  describe('TypeScript', () => {
    it('detects .ts files', () => {
      expect(getLanguageFromFilename('index.ts')).toBe(SupportedLanguages.TypeScript);
    });

    it('detects .tsx files', () => {
      expect(getLanguageFromFilename('Component.tsx')).toBe(SupportedLanguages.TypeScript);
    });

    it('detects .ts files in paths', () => {
      expect(getLanguageFromFilename('src/core/utils.ts')).toBe(SupportedLanguages.TypeScript);
    });
  });

  describe('JavaScript', () => {
    it('detects .js files', () => {
      expect(getLanguageFromFilename('index.js')).toBe(SupportedLanguages.JavaScript);
    });

    it('detects .jsx files', () => {
      expect(getLanguageFromFilename('App.jsx')).toBe(SupportedLanguages.JavaScript);
    });
  });

  describe('Python', () => {
    it('detects .py files', () => {
      expect(getLanguageFromFilename('main.py')).toBe(SupportedLanguages.Python);
    });
  });

  describe('Java', () => {
    it('detects .java files', () => {
      expect(getLanguageFromFilename('Main.java')).toBe(SupportedLanguages.Java);
    });
  });

  describe('C', () => {
    it('detects .c files', () => {
      expect(getLanguageFromFilename('main.c')).toBe(SupportedLanguages.C);
    });
  });

  describe('C++', () => {
    it.each(['.cpp', '.cc', '.cxx', '.h', '.hpp', '.hxx', '.hh'])(
      'detects %s files',
      (ext) => {
        expect(getLanguageFromFilename(`file${ext}`)).toBe(SupportedLanguages.CPlusPlus);
      }
    );
  });

  describe('C#', () => {
    it('detects .cs files', () => {
      expect(getLanguageFromFilename('Program.cs')).toBe(SupportedLanguages.CSharp);
    });
  });

  describe('Go', () => {
    it('detects .go files', () => {
      expect(getLanguageFromFilename('main.go')).toBe(SupportedLanguages.Go);
    });
  });

  describe('Rust', () => {
    it('detects .rs files', () => {
      expect(getLanguageFromFilename('main.rs')).toBe(SupportedLanguages.Rust);
    });
  });

  describe('PHP', () => {
    it.each(['.php', '.phtml', '.php3', '.php4', '.php5', '.php8'])(
      'detects %s files',
      (ext) => {
        expect(getLanguageFromFilename(`file${ext}`)).toBe(SupportedLanguages.PHP);
      }
    );
  });

  describe('Swift', () => {
    it('detects .swift files', () => {
      expect(getLanguageFromFilename('App.swift')).toBe(SupportedLanguages.Swift);
    });
  });

  describe('Ruby', () => {
    it.each(['.rb', '.rake', '.gemspec'])(
      'detects %s files',
      (ext) => {
        expect(getLanguageFromFilename(`file${ext}`)).toBe(SupportedLanguages.Ruby);
      }
    );

    it('detects extensionless Rakefile', () => {
      expect(getLanguageFromFilename('Rakefile')).toBe(SupportedLanguages.Ruby);
    });

    it('detects extensionless Gemfile', () => {
      expect(getLanguageFromFilename('Gemfile')).toBe(SupportedLanguages.Ruby);
    });
  });

  describe('Kotlin', () => {
    it.each(['.kt', '.kts'])(
      'detects %s files',
      (ext) => {
        expect(getLanguageFromFilename(`file${ext}`)).toBe(SupportedLanguages.Kotlin);
      }
    );
  });

  describe('unsupported', () => {
    it.each(['.scala', '.r', '.lua', '.zig', '.txt', '.md', '.json', '.yaml'])(
      'returns null for %s files',
      (ext) => {
        expect(getLanguageFromFilename(`file${ext}`)).toBeNull();
      }
    );

    it('returns null for files without extension', () => {
      expect(getLanguageFromFilename('Makefile')).toBeNull();
    });

    it('returns null for empty string', () => {
      expect(getLanguageFromFilename('')).toBeNull();
    });
  });
});

describe('isBuiltInOrNoise', () => {
  describe('JavaScript/TypeScript', () => {
    it('filters console methods', () => {
      expect(isBuiltInOrNoise('console')).toBe(true);
      expect(isBuiltInOrNoise('log')).toBe(true);
      expect(isBuiltInOrNoise('warn')).toBe(true);
    });

    it('filters React hooks', () => {
      expect(isBuiltInOrNoise('useState')).toBe(true);
      expect(isBuiltInOrNoise('useEffect')).toBe(true);
      expect(isBuiltInOrNoise('useCallback')).toBe(true);
    });

    it('filters array methods', () => {
      expect(isBuiltInOrNoise('map')).toBe(true);
      expect(isBuiltInOrNoise('filter')).toBe(true);
      expect(isBuiltInOrNoise('reduce')).toBe(true);
    });
  });

  describe('Python', () => {
    it('filters built-in functions', () => {
      expect(isBuiltInOrNoise('print')).toBe(true);
      expect(isBuiltInOrNoise('len')).toBe(true);
      expect(isBuiltInOrNoise('range')).toBe(true);
    });
  });

  describe('PHP', () => {
    it('filters PHP built-in functions', () => {
      expect(isBuiltInOrNoise('echo')).toBe(true);
      expect(isBuiltInOrNoise('isset')).toBe(true);
      expect(isBuiltInOrNoise('date')).toBe(true);
      expect(isBuiltInOrNoise('json_encode')).toBe(true);
      expect(isBuiltInOrNoise('array_map')).toBe(true);
    });

    it('filters PHP string functions', () => {
      expect(isBuiltInOrNoise('strlen')).toBe(true);
      expect(isBuiltInOrNoise('substr')).toBe(true);
      expect(isBuiltInOrNoise('str_replace')).toBe(true);
    });
  });

  describe('C/C++', () => {
    it('filters standard library functions', () => {
      expect(isBuiltInOrNoise('printf')).toBe(true);
      expect(isBuiltInOrNoise('malloc')).toBe(true);
      expect(isBuiltInOrNoise('free')).toBe(true);
    });

    it('filters Linux kernel macros', () => {
      expect(isBuiltInOrNoise('container_of')).toBe(true);
      expect(isBuiltInOrNoise('ARRAY_SIZE')).toBe(true);
      expect(isBuiltInOrNoise('pr_info')).toBe(true);
    });
  });

  describe('Kotlin', () => {
    it('filters stdlib functions', () => {
      expect(isBuiltInOrNoise('println')).toBe(true);
      expect(isBuiltInOrNoise('listOf')).toBe(true);
      expect(isBuiltInOrNoise('TODO')).toBe(true);
    });

    it('filters coroutine functions', () => {
      expect(isBuiltInOrNoise('launch')).toBe(true);
      expect(isBuiltInOrNoise('async')).toBe(true);
    });
  });

  describe('Swift', () => {
    it('filters built-in functions', () => {
      expect(isBuiltInOrNoise('print')).toBe(true);
      expect(isBuiltInOrNoise('fatalError')).toBe(true);
    });

    it('filters UIKit methods', () => {
      expect(isBuiltInOrNoise('addSubview')).toBe(true);
      expect(isBuiltInOrNoise('reloadData')).toBe(true);
    });
  });

  describe('Rust', () => {
    it('filters Result/Option methods', () => {
      expect(isBuiltInOrNoise('unwrap')).toBe(true);
      expect(isBuiltInOrNoise('expect')).toBe(true);
      expect(isBuiltInOrNoise('unwrap_or')).toBe(true);
      expect(isBuiltInOrNoise('unwrap_or_else')).toBe(true);
      expect(isBuiltInOrNoise('unwrap_or_default')).toBe(true);
      expect(isBuiltInOrNoise('ok')).toBe(true);
      expect(isBuiltInOrNoise('err')).toBe(true);
      expect(isBuiltInOrNoise('is_ok')).toBe(true);
      expect(isBuiltInOrNoise('is_err')).toBe(true);
      expect(isBuiltInOrNoise('map_err')).toBe(true);
      expect(isBuiltInOrNoise('and_then')).toBe(true);
      expect(isBuiltInOrNoise('or_else')).toBe(true);
    });

    it('filters trait conversion methods', () => {
      expect(isBuiltInOrNoise('clone')).toBe(true);
      expect(isBuiltInOrNoise('to_string')).toBe(true);
      expect(isBuiltInOrNoise('to_owned')).toBe(true);
      expect(isBuiltInOrNoise('into')).toBe(true);
      expect(isBuiltInOrNoise('from')).toBe(true);
      expect(isBuiltInOrNoise('as_ref')).toBe(true);
      expect(isBuiltInOrNoise('as_mut')).toBe(true);
    });

    it('filters iterator methods', () => {
      expect(isBuiltInOrNoise('iter')).toBe(true);
      expect(isBuiltInOrNoise('into_iter')).toBe(true);
      expect(isBuiltInOrNoise('collect')).toBe(true);
      expect(isBuiltInOrNoise('fold')).toBe(true);
      expect(isBuiltInOrNoise('for_each')).toBe(true);
    });

    it('filters collection methods', () => {
      expect(isBuiltInOrNoise('len')).toBe(true);
      expect(isBuiltInOrNoise('is_empty')).toBe(true);
      expect(isBuiltInOrNoise('push')).toBe(true);
      expect(isBuiltInOrNoise('pop')).toBe(true);
      expect(isBuiltInOrNoise('insert')).toBe(true);
      expect(isBuiltInOrNoise('remove')).toBe(true);
      expect(isBuiltInOrNoise('contains')).toBe(true);
    });

    it('filters macro-like and panic functions', () => {
      expect(isBuiltInOrNoise('format')).toBe(true);
      expect(isBuiltInOrNoise('panic')).toBe(true);
      expect(isBuiltInOrNoise('unreachable')).toBe(true);
      expect(isBuiltInOrNoise('todo')).toBe(true);
      expect(isBuiltInOrNoise('unimplemented')).toBe(true);
      expect(isBuiltInOrNoise('vec')).toBe(true);
      expect(isBuiltInOrNoise('println')).toBe(true);
      expect(isBuiltInOrNoise('eprintln')).toBe(true);
      expect(isBuiltInOrNoise('dbg')).toBe(true);
    });

    it('filters sync primitives', () => {
      expect(isBuiltInOrNoise('lock')).toBe(true);
      expect(isBuiltInOrNoise('try_lock')).toBe(true);
      expect(isBuiltInOrNoise('spawn')).toBe(true);
      expect(isBuiltInOrNoise('join')).toBe(true);
      expect(isBuiltInOrNoise('sleep')).toBe(true);
    });

    it('filters enum constructors', () => {
      expect(isBuiltInOrNoise('Some')).toBe(true);
      expect(isBuiltInOrNoise('None')).toBe(true);
      expect(isBuiltInOrNoise('Ok')).toBe(true);
      expect(isBuiltInOrNoise('Err')).toBe(true);
    });

    it('does not filter user-defined Rust functions', () => {
      expect(isBuiltInOrNoise('process_request')).toBe(false);
      expect(isBuiltInOrNoise('handle_connection')).toBe(false);
      expect(isBuiltInOrNoise('build_response')).toBe(false);
    });
  });

  describe('C#/.NET', () => {
    it('filters Console I/O', () => {
      expect(isBuiltInOrNoise('Console')).toBe(true);
      expect(isBuiltInOrNoise('WriteLine')).toBe(true);
      expect(isBuiltInOrNoise('ReadLine')).toBe(true);
    });

    it('filters LINQ methods', () => {
      expect(isBuiltInOrNoise('Where')).toBe(true);
      expect(isBuiltInOrNoise('Select')).toBe(true);
      expect(isBuiltInOrNoise('GroupBy')).toBe(true);
      expect(isBuiltInOrNoise('OrderBy')).toBe(true);
      expect(isBuiltInOrNoise('FirstOrDefault')).toBe(true);
      expect(isBuiltInOrNoise('ToList')).toBe(true);
    });

    it('filters Task async methods', () => {
      expect(isBuiltInOrNoise('Task')).toBe(true);
      expect(isBuiltInOrNoise('Run')).toBe(true);
      expect(isBuiltInOrNoise('WhenAll')).toBe(true);
      expect(isBuiltInOrNoise('ConfigureAwait')).toBe(true);
    });

    it('filters Object base methods', () => {
      expect(isBuiltInOrNoise('ToString')).toBe(true);
      expect(isBuiltInOrNoise('GetType')).toBe(true);
      expect(isBuiltInOrNoise('Equals')).toBe(true);
      expect(isBuiltInOrNoise('GetHashCode')).toBe(true);
    });
  });

  describe('user-defined functions', () => {
    it('does not filter custom function names', () => {
      expect(isBuiltInOrNoise('myCustomFunction')).toBe(false);
      expect(isBuiltInOrNoise('processData')).toBe(false);
      expect(isBuiltInOrNoise('handleUserRequest')).toBe(false);
    });
  });
});

describe('extractFunctionName', () => {
  const parser = new Parser();

  describe('C', () => {
    it('extracts function name from C function definition', () => {
      parser.setLanguage(C);
      const code = `int main() { return 0; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('main');
      expect(result.label).toBe('Function');
    });

    it('extracts function name with parameters', () => {
      parser.setLanguage(C);
      const code = `void helper(int a, char* b) {}`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('helper');
      expect(result.label).toBe('Function');
    });
  });

  describe('C++', () => {
    it('extracts method name from C++ class method definition', () => {
      parser.setLanguage(CPP);
      const code = `int MyClass::OnEncryptData() { return 0; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('OnEncryptData');
      expect(result.label).toBe('Method');
    });

    it('extracts method name with namespace', () => {
      parser.setLanguage(CPP);
      const code = `void HuksListener::OnDataOprEvent(int type, DataInfo& info) {}`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('OnDataOprEvent');
      expect(result.label).toBe('Method');
    });

    it('extracts C function (not method)', () => {
      parser.setLanguage(CPP);
      const code = `void standalone_function() {}`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('standalone_function');
      expect(result.label).toBe('Function');
    });

    it('extracts method with parenthesized declarator', () => {
      parser.setLanguage(CPP);
      const code = `void (MyClass::handler)() {}`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('handler');
      expect(result.label).toBe('Method');
    });
  });

  describe('C pointer returns', () => {
    it('extracts name from function returning pointer', () => {
      parser.setLanguage(C);
      const code = `int* get_data() { return 0; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('get_data');
      expect(result.label).toBe('Function');
    });

    it('extracts name from function returning double pointer', () => {
      parser.setLanguage(C);
      const code = `char** get_strings() { return 0; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('get_strings');
      expect(result.label).toBe('Function');
    });

    it('extracts name from struct pointer return', () => {
      parser.setLanguage(C);
      const code = `struct Node* create_node(int val) { return 0; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('create_node');
      expect(result.label).toBe('Function');
    });
  });

  describe('C++ pointer/reference returns', () => {
    it('extracts name from method returning pointer', () => {
      parser.setLanguage(CPP);
      const code = `int* MyClass::getData() { return nullptr; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('getData');
      expect(result.label).toBe('Method');
    });

    it('extracts name from function returning reference', () => {
      parser.setLanguage(CPP);
      const code = `std::string& get_name() { static std::string s; return s; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('get_name');
      expect(result.label).toBe('Function');
    });

    it('extracts name from method returning reference', () => {
      parser.setLanguage(CPP);
      const code = `int& Container::at(int i) { return data[i]; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('at');
      expect(result.label).toBe('Method');
    });

    it('extracts name from method returning const reference', () => {
      parser.setLanguage(CPP);
      const code = `const std::string& Config::getName() const { return name_; }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('getName');
      expect(result.label).toBe('Method');
    });
  });

  describe('C++ destructors', () => {
    it('extracts destructor name from out-of-line definition', () => {
      parser.setLanguage(CPP);
      const code = `MyClass::~MyClass() { cleanup(); }`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      // destructor_name includes the ~ prefix
      expect(result.funcName).toBe('~MyClass');
      expect(result.label).toBe('Method');
    });
  });

  describe('TypeScript', () => {
    it('extracts arrow function name from variable declarator', () => {
      parser.setLanguage(TypeScript.typescript);
      const code = `const myHandler = () => { return 1; }`;
      const tree = parser.parse(code);
      const program = tree.rootNode;
      const varDecl = program.child(0);
      const declarator = varDecl!.namedChild(0);
      const arrowFunc = declarator!.namedChild(1);

      const result = extractFunctionName(arrowFunc);

      expect(result.funcName).toBe('myHandler');
      expect(result.label).toBe('Function');
    });

    it('extracts function expression name from variable declarator', () => {
      parser.setLanguage(TypeScript.typescript);
      const code = `const processItem = function() { }`;
      const tree = parser.parse(code);
      const program = tree.rootNode;
      const varDecl = program.child(0);
      const declarator = varDecl!.namedChild(0);
      const funcExpr = declarator!.namedChild(1);

      const result = extractFunctionName(funcExpr);

      expect(result.funcName).toBe('processItem');
      expect(result.label).toBe('Function');
    });
  });

  describe('Python', () => {
    it('extracts function name from Python function definition', () => {
      parser.setLanguage(Python);
      const code = `def hello_world():\n    pass`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('hello_world');
      expect(result.label).toBe('Function');
    });

    it('extracts function name with parameters', () => {
      parser.setLanguage(Python);
      const code = `def calculate_sum(a, b):\n    return a + b`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('calculate_sum');
      expect(result.label).toBe('Function');
    });

    it('extracts async function name', () => {
      parser.setLanguage(Python);
      const code = `async def fetch_data():\n    pass`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('fetch_data');
      expect(result.label).toBe('Function');
    });

    it('extracts function name with type hints', () => {
      parser.setLanguage(Python);
      const code = `def process_data(items: list[int]) -> bool:\n    return True`;
      const tree = parser.parse(code);
      const funcNode = tree.rootNode.child(0);

      const result = extractFunctionName(funcNode);

      expect(result.funcName).toBe('process_data');
      expect(result.label).toBe('Function');
    });

    it('extracts nested function name', () => {
      parser.setLanguage(Python);
      const code = `def outer():\n    def inner():\n        pass`;
      const tree = parser.parse(code);
      const outerFunc = tree.rootNode.child(0);
      const block = outerFunc!.child(4);
      const innerFunc = block!.namedChild(0);

      const result = extractFunctionName(innerFunc);

      expect(result.funcName).toBe('inner');
      expect(result.label).toBe('Function');
    });
  });
});

describe('getTreeSitterBufferSize', () => {
  it('returns minimum 512KB for small files', () => {
    expect(getTreeSitterBufferSize(100)).toBe(TREE_SITTER_BUFFER_SIZE);
    expect(getTreeSitterBufferSize(0)).toBe(TREE_SITTER_BUFFER_SIZE);
    expect(getTreeSitterBufferSize(1000)).toBe(TREE_SITTER_BUFFER_SIZE);
  });

  it('returns 2x content length when larger than minimum', () => {
    const size = 400 * 1024; // 400 KB — 2x = 800 KB > 512 KB min
    expect(getTreeSitterBufferSize(size)).toBe(size * 2);
  });

  it('caps at 32MB for very large files', () => {
    const huge = 20 * 1024 * 1024; // 20 MB — 2x = 40 MB > 32 MB cap
    expect(getTreeSitterBufferSize(huge)).toBe(32 * 1024 * 1024);
  });

  it('returns exactly 512KB at the boundary', () => {
    // 256KB * 2 = 512KB = minimum, so should return minimum
    expect(getTreeSitterBufferSize(256 * 1024)).toBe(TREE_SITTER_BUFFER_SIZE);
  });

  it('scales linearly between min and max', () => {
    const small = getTreeSitterBufferSize(300 * 1024);
    const medium = getTreeSitterBufferSize(1 * 1024 * 1024);
    const large = getTreeSitterBufferSize(5 * 1024 * 1024);
    expect(small).toBeLessThan(medium);
    expect(medium).toBeLessThan(large);
  });

  it('TREE_SITTER_MAX_BUFFER is 32MB', () => {
    expect(TREE_SITTER_MAX_BUFFER).toBe(32 * 1024 * 1024);
  });

  it('returns max buffer at exact boundary (16MB input)', () => {
    // 16MB * 2 = 32MB = max
    expect(getTreeSitterBufferSize(16 * 1024 * 1024)).toBe(TREE_SITTER_MAX_BUFFER);
  });

  it('file just over max returns max buffer', () => {
    // 17MB * 2 = 34MB > 32MB cap
    expect(getTreeSitterBufferSize(17 * 1024 * 1024)).toBe(TREE_SITTER_MAX_BUFFER);
  });

  it('handles files between old 512KB limit and new 32MB limit', () => {
    // This is the range that was previously silently skipped
    const sizes = [600 * 1024, 1024 * 1024, 5 * 1024 * 1024, 10 * 1024 * 1024];
    for (const size of sizes) {
      const bufSize = getTreeSitterBufferSize(size);
      expect(bufSize).toBeGreaterThanOrEqual(TREE_SITTER_BUFFER_SIZE);
      expect(bufSize).toBeLessThanOrEqual(TREE_SITTER_MAX_BUFFER);
    }
  });
});