scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-09 03:17:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(lbug): write csv in chunks to avoid V8 max string length limit

Browse source
This commit is contained in:
Vincent 2026-03-31 10:31:32 +07:00
parent c72890d59d
commit c91e82dad0
1 changed files with 12 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
gitnexus/src/core/lbug/lbug-adapter.ts
View file

@ -297,7 +297,18 @@ export const loadGraphToLbug = async (
pairIdx++;
const [fromLabel, toLabel] = pairKey.split('|');
const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8');
// Write in chunks to avoid V8 max string length limit
const { createWriteStream } = await import('fs');
await new Promise<void>((resolveWrite, rejectWrite) => {
const ws = createWriteStream(pairCsvPath, 'utf-8');
ws.on('error', rejectWrite);
ws.write(relHeader + '\n');
const CHUNK = 50000;
for (let ci = 0; ci < lines.length; ci += CHUNK) {
ws.write(lines.slice(ci, ci + CHUNK).join('\n') + '\n');
}
ws.end(() => resolveWrite());
});
const normalizedPath = normalizeCopyPath(pairCsvPath);
const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;