diff --git a/src/package.json b/src/package.json index 78806ce47b..443eb7591f 100644 --- a/src/package.json +++ b/src/package.json @@ -410,6 +410,7 @@ "pdf-parse": "^1.1.1", "pkce-challenge": "^5.0.0", "pretty-bytes": "^7.0.0", + "proper-lockfile": "^4.1.2", "ps-tree": "^1.2.0", "puppeteer-chromium-resolver": "^24.0.0", "puppeteer-core": "^23.4.0", @@ -419,6 +420,7 @@ "serialize-error": "^12.0.0", "simple-git": "^3.27.0", "sound-play": "^1.1.0", + "stream-json": "^1.8.0", "string-similarity": "^4.0.4", "strip-ansi": "^7.1.0", "strip-bom": "^5.0.0", @@ -446,7 +448,9 @@ "@types/node": "20.x", "@types/node-cache": "^4.1.3", "@types/node-ipc": "^9.2.3", + "@types/proper-lockfile": "^4.1.4", "@types/ps-tree": "^1.1.6", + "@types/stream-json": "^1.7.8", "@types/string-similarity": "^4.0.2", "@types/tmp": "^0.2.6", "@types/turndown": "^5.0.5", diff --git a/src/utils/safeWriteJson.ts b/src/utils/safeWriteJson.ts index b724bc5376..9d3e66309f 100644 --- a/src/utils/safeWriteJson.ts +++ b/src/utils/safeWriteJson.ts @@ -1,6 +1,9 @@ import * as fs from "fs/promises" +import * as fsSync from "fs" import * as path from "path" import * as lockfile from "proper-lockfile" +import Disassembler from "stream-json/Disassembler" +import Stringer from "stream-json/Stringer" /** * Safely writes JSON data to a file. @@ -13,12 +16,8 @@ import * as lockfile from "proper-lockfile" * @param {any} data - The data to serialize to JSON and write. * @returns {Promise} */ -async function safeWriteJson( - filePath: string, - data: any, - replacer?: (key: string, value: any) => any, - space: string | number = 2, -): Promise { + +async function safeWriteJson(filePath: string, data: any): Promise { const absoluteFilePath = path.resolve(filePath) const lockPath = `${absoluteFilePath}.lock` let releaseLock = async () => {} // Initialized to a no-op @@ -59,8 +58,8 @@ async function safeWriteJson( path.dirname(absoluteFilePath), `.${path.basename(absoluteFilePath)}.new_${Date.now()}_${Math.random().toString(36).substring(2)}.tmp`, ) - const jsonData = JSON.stringify(data, replacer, space) - await fs.writeFile(actualTempNewFilePath, jsonData, "utf8") + + await _streamDataToFile(actualTempNewFilePath, data) // Step 2: Check if the target file exists. If so, rename it to a backup path. try { @@ -159,4 +158,58 @@ async function safeWriteJson( } } +/** + * Helper function to stream JSON data to a file. + * @param targetPath The path to write the stream to. + * @param data The data to stream. + * @returns Promise + */ +async function _streamDataToFile(targetPath: string, data: any): Promise { + // Stream data to avoid high memory usage for large JSON objects. + const fileWriteStream = fsSync.createWriteStream(targetPath, { encoding: "utf8" }) + const disassembler = Disassembler.disassembler() + // Output will be compact JSON as standard Stringer is used. + const stringer = Stringer.stringer() + + return new Promise((resolve, reject) => { + let errorOccurred = false + const handleError = (_streamName: string) => (err: Error) => { + if (!errorOccurred) { + errorOccurred = true + if (!fileWriteStream.destroyed) { + fileWriteStream.destroy(err) + } + reject(err) + } + } + + disassembler.on("error", handleError("Disassembler")) + stringer.on("error", handleError("Stringer")) + fileWriteStream.on("error", (err: Error) => { + if (!errorOccurred) { + errorOccurred = true + reject(err) + } + }) + + fileWriteStream.on("finish", () => { + if (!errorOccurred) { + resolve() + } + }) + + disassembler.pipe(stringer).pipe(fileWriteStream) + + // stream-json's Disassembler might error if `data` is undefined. + // JSON.stringify(undefined) would produce the string "undefined" if it's the root value. + // Writing 'null' is a safer JSON representation for a root undefined value. + if (data === undefined) { + disassembler.write(null) + } else { + disassembler.write(data) + } + disassembler.end() + }) +} + export { safeWriteJson }