diff --git a/src/integrations/misc/__tests__/extract-text-unicode.spec.ts b/src/integrations/misc/__tests__/extract-text-unicode.spec.ts new file mode 100644 index 0000000000..de67340bc5 --- /dev/null +++ b/src/integrations/misc/__tests__/extract-text-unicode.spec.ts @@ -0,0 +1,245 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest" +import * as fs from "fs/promises" +import * as path from "path" +import { extractTextFromFile } from "../extract-text" +import { readLines } from "../read-lines" +import { tmpdir } from "os" + +describe("Unicode and UTF-8 handling", () => { + let tempDir: string + let testFilePath: string + + beforeEach(async () => { + // Create a temporary directory for test files + tempDir = await fs.mkdtemp(path.join(tmpdir(), "unicode-test-")) + }) + + afterEach(async () => { + // Clean up temporary directory + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + describe("extractTextFromFile with Unicode content", () => { + it("should handle Chinese characters correctly", async () => { + const content = `# 测试文件 +- **时间戳 (Timestamp)**: 2025-07-26 15:41 +- **任务/目标 (Task/Goal)**: 实现问卷的前端核心交互逻辑。` + + testFilePath = path.join(tempDir, "chinese.md") + await fs.writeFile(testFilePath, content, "utf8") + + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("时间戳") + expect(result).toContain("任务/目标") + expect(result).toContain("实现问卷的前端核心交互逻辑") + }) + + it("should handle mixed Unicode content (emoji, various languages)", async () => { + const content = `# Unicode Test +😀 Emoji test 🎉 +日本語: こんにちは世界 +한국어: 안녕하세요 세계 +العربية: مرحبا بالعالم +עברית: שלום עולם +Русский: Привет мир` + + testFilePath = path.join(tempDir, "mixed-unicode.md") + await fs.writeFile(testFilePath, content, "utf8") + + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("😀") + expect(result).toContain("🎉") + expect(result).toContain("こんにちは世界") + expect(result).toContain("안녕하세요") + expect(result).toContain("مرحبا بالعالم") + expect(result).toContain("שלום עולם") + expect(result).toContain("Привет мир") + }) + + it("should handle special Unicode characters", async () => { + const content = `Special chars: +Non-breaking space: test test +Zero-width space: test\u200Btest +Quotes: "test" 'test' „test" «test» +Math: ∑ ∏ ∫ √ ∞ ≈ ≠ ≤ ≥` + + testFilePath = path.join(tempDir, "special-chars.md") + await fs.writeFile(testFilePath, content, "utf8") + + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("Non-breaking space") + expect(result).toContain("Zero-width space") + expect(result).toContain('"test"') + expect(result).toContain("∑") + expect(result).toContain("∞") + }) + + it("should handle files with BOM (Byte Order Mark)", async () => { + const content = "Test content with BOM" + const bom = "\uFEFF" + + testFilePath = path.join(tempDir, "bom-file.txt") + await fs.writeFile(testFilePath, bom + content, "utf8") + + const result = await extractTextFromFile(testFilePath) + // Should handle BOM gracefully + expect(result).toContain("Test content with BOM") + }) + + it("should handle invalid UTF-8 sequences gracefully", async () => { + // Create a file with potentially problematic byte sequences + testFilePath = path.join(tempDir, "invalid-utf8.md") // Use .md extension + + // Write raw bytes that might cause UTF-8 decoding issues + const buffer = Buffer.from([ + 0x48, + 0x65, + 0x6c, + 0x6c, + 0x6f, // "Hello" + 0x20, // space + 0xff, + 0xfe, // Invalid UTF-8 sequence + 0x20, // space + 0x57, + 0x6f, + 0x72, + 0x6c, + 0x64, // "World" + ]) + await fs.writeFile(testFilePath, buffer) + + // Should not throw and should return readable content + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("Hello") + expect(result).toContain("World") + }) + }) + + describe("readLines with Unicode content", () => { + it("should handle Chinese characters in line ranges", async () => { + const content = `第一行 +第二行:时间戳 +第三行:任务/目标 +第四行:关键决策` + + testFilePath = path.join(tempDir, "chinese-lines.txt") + await fs.writeFile(testFilePath, content, "utf8") + + const result = await readLines(testFilePath, 2, 1) + expect(result).toContain("第二行") + expect(result).toContain("第三行") + expect(result).not.toContain("第一行") + expect(result).not.toContain("第四行") + }) + + it("should handle emoji in specific line ranges", async () => { + const content = `Line 1: Regular text +Line 2: 😀 Happy emoji +Line 3: 🚀 Rocket emoji +Line 4: Regular text again` + + testFilePath = path.join(tempDir, "emoji-lines.txt") + await fs.writeFile(testFilePath, content, "utf8") + + const result = await readLines(testFilePath, 2, 1) + expect(result).toContain("😀") + expect(result).toContain("🚀") + }) + + it("should handle mixed scripts in line reading", async () => { + const lines = [ + "English line", + "中文行", + "日本語の行", + "한국어 줄", + "Русская строка", + "שורה בעברית", + "سطر عربي", + ] + + testFilePath = path.join(tempDir, "mixed-scripts.txt") + await fs.writeFile(testFilePath, lines.join("\n"), "utf8") + + // Read middle lines + const result = await readLines(testFilePath, 4, 2) + expect(result).toContain("日本語の行") + expect(result).toContain("한국어 줄") + expect(result).toContain("Русская строка") + expect(result).not.toContain("English line") + expect(result).not.toContain("שורה בעברית") + }) + + it("should handle invalid UTF-8 in line reading gracefully", async () => { + testFilePath = path.join(tempDir, "invalid-utf8-lines.txt") + + // Create content with invalid sequences + const validLine1 = Buffer.from("First line\n", "utf8") + const invalidLine = Buffer.concat([ + Buffer.from("Second ", "utf8"), + Buffer.from([0xff, 0xfe]), // Invalid UTF-8 + Buffer.from(" line\n", "utf8"), + ]) + const validLine3 = Buffer.from("Third line\n", "utf8") + + await fs.writeFile(testFilePath, Buffer.concat([validLine1, invalidLine, validLine3])) + + // Should handle gracefully + const result = await readLines(testFilePath, 2, 0) + expect(result).toContain("First line") + expect(result).toContain("line") // Should contain parts of the second line + expect(result).toContain("Third line") + }) + }) + + describe("Edge cases and stress tests", () => { + it("should handle very long lines with Unicode", async () => { + // Create a very long line with repeated Unicode characters + const longLine = "测试".repeat(10000) + "\n" + "第二行" + + testFilePath = path.join(tempDir, "long-unicode.txt") + await fs.writeFile(testFilePath, longLine, "utf8") + + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("测试") + expect(result).toContain("第二行") + }) + + it("should handle empty files", async () => { + testFilePath = path.join(tempDir, "empty.txt") + await fs.writeFile(testFilePath, "", "utf8") + + const result = await extractTextFromFile(testFilePath) + expect(result).toBe("") + }) + + it("should handle files with only Unicode characters", async () => { + const content = "中文中文中文中文中文" + + testFilePath = path.join(tempDir, "only-unicode.txt") + await fs.writeFile(testFilePath, content, "utf8") + + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("中文中文中文中文中文") + }) + + it("should handle the exact reproduction case from the issue", async () => { + const content = `- **时间戳 (Timestamp)**: 2025-07-26 15:41 +- **任务/目标 (Task/Goal)**: 实现问卷的前端核心交互逻辑。 +- **关键决策/操作 (Key Decision/Action)**: + 1. 在 \`questionnaire-server/public/index.html\` 中添加了 \`
\` 作为动态内容的挂载点。 + 2. 在 \`questionnaire-server/public/style.css\` 中添加了完整的基础样式,确保界面干净、可用,并对问卷的各个部分(欢迎页、问题页、选项)进行了样式设置。` + + testFilePath = path.join(tempDir, "issue-reproduction.md") + await fs.writeFile(testFilePath, content, "utf8") + + // This should not throw or hang + const result = await extractTextFromFile(testFilePath) + expect(result).toContain("时间戳 (Timestamp)") + expect(result).toContain("任务/目标 (Task/Goal)") + expect(result).toContain("关键决策/操作") + expect(result).toContain("questionnaire-server/public/index.html") + expect(result).toContain("questionnaire-container") + }, 30000) // Increase timeout to 30 seconds + }) +}) diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index 8231c609be..a8424872b9 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -8,6 +8,44 @@ import { extractTextFromXLSX } from "./extract-text-from-xlsx" import { countFileLines } from "./line-counter" import { readLines } from "./read-lines" +/** + * Safely reads a text file with UTF-8 encoding, handling potential encoding errors gracefully. + * If the file contains invalid UTF-8 sequences, it attempts to read with error replacement. + * + * @param filePath - Path to the file to read + * @returns Promise resolving to the file content as a string + */ +async function safeReadTextFile(filePath: string): Promise { + try { + // First attempt: standard UTF-8 reading + return await fs.readFile(filePath, "utf8") + } catch (error: any) { + // If it's an encoding error, try reading as buffer and converting with replacement + if ( + error.code === "ERR_INVALID_ARG_TYPE" || + error.code === "ERR_ENCODING_INVALID_ENCODED_DATA" || + error.message?.includes("encoding") || + error.message?.includes("Invalid") || + error.errno === -92 + ) { + // EILSEQ error code for invalid byte sequence + + try { + // Read as buffer + const buffer = await fs.readFile(filePath) + // Convert to string, replacing invalid sequences with replacement character + // This ensures we can still read the file even with encoding issues + return buffer.toString("utf8") + } catch (bufferError) { + // If buffer reading also fails, throw the original error + throw error + } + } + // For non-encoding errors, throw as-is + throw error + } +} + async function extractTextFromPDF(filePath: string): Promise { const dataBuffer = await fs.readFile(filePath) const data = await pdf(dataBuffer) @@ -20,7 +58,7 @@ async function extractTextFromDOCX(filePath: string): Promise { } async function extractTextFromIPYNB(filePath: string): Promise { - const data = await fs.readFile(filePath, "utf8") + const data = await safeReadTextFile(filePath) const notebook = JSON.parse(data) let extractedText = "" @@ -86,9 +124,70 @@ export async function extractTextFromFile(filePath: string, maxReadFileLine?: nu } // Handle other files - const isBinary = await isBinaryFile(filePath).catch(() => false) + // For text-like extensions, always try to read as text regardless of binary detection + const textExtensions = [ + ".txt", + ".md", + ".markdown", + ".text", + ".log", + ".csv", + ".json", + ".xml", + ".html", + ".htm", + ".css", + ".js", + ".ts", + ".jsx", + ".tsx", + ".py", + ".java", + ".c", + ".cpp", + ".h", + ".hpp", + ".cs", + ".rb", + ".go", + ".rs", + ".php", + ".swift", + ".kt", + ".scala", + ".r", + ".m", + ".mm", + ".sh", + ".bash", + ".zsh", + ".fish", + ".ps1", + ".bat", + ".cmd", + ".yaml", + ".yml", + ".toml", + ".ini", + ".cfg", + ".conf", + ".properties", + ] + const isTextExtension = textExtensions.includes(fileExtension.toLowerCase()) - if (!isBinary) { + let isBinary = false + if (!isTextExtension) { + // Only check binary for non-text extensions + try { + isBinary = await isBinaryFile(filePath) + } catch (error) { + // If isBinaryFile throws an error, assume it's a text file and try to read it + console.warn(`Warning: isBinaryFile check failed for ${filePath}, treating as text file:`, error) + isBinary = false + } + } + + if (!isBinary || isTextExtension) { // Check if we need to apply line limit if (maxReadFileLine !== undefined && maxReadFileLine !== -1) { const totalLines = await countFileLines(filePath) @@ -103,7 +202,7 @@ export async function extractTextFromFile(filePath: string, maxReadFileLine?: nu } } // Read the entire file if no limit or file is within limit - return addLineNumbers(await fs.readFile(filePath, "utf8")) + return addLineNumbers(await safeReadTextFile(filePath)) } else { throw new Error(`Cannot read text for file type: ${fileExtension}`) } diff --git a/src/integrations/misc/read-lines.ts b/src/integrations/misc/read-lines.ts index 5a5eda9f83..fd05a5a530 100644 --- a/src/integrations/misc/read-lines.ts +++ b/src/integrations/misc/read-lines.ts @@ -52,19 +52,65 @@ export function readLines(filepath: string, endLine?: number, startLine?: number ) } - // Set up stream - const input = createReadStream(filepath) + // Set up stream with UTF-8 encoding + const input = createReadStream(filepath, { encoding: "utf8" }) let buffer = "" let lineCount = 0 let result = "" // Handle errors - input.on("error", reject) + input.on("error", (error: any) => { + // If it's an encoding error, try to handle it gracefully + if ( + error.code === "ERR_INVALID_ARG_TYPE" || + error.code === "ERR_ENCODING_INVALID_ENCODED_DATA" || + error.message?.includes("encoding") || + error.message?.includes("Invalid") || + error.errno === -92 + ) { + // EILSEQ error code for invalid byte sequence + + // Create a new stream without encoding to read as buffer + const bufferInput = createReadStream(filepath) + let bufferData = Buffer.alloc(0) + + bufferInput.on("error", reject) + + bufferInput.on("data", (chunk) => { + bufferData = Buffer.concat([bufferData, chunk as Buffer]) + }) + + bufferInput.on("end", () => { + // Convert buffer to string with UTF-8, replacing invalid sequences + const content = bufferData.toString("utf8") + const lines = content.split("\n") + + // Apply line range filtering + const effectiveEndLine = + endLine === undefined ? lines.length - 1 : Math.min(endLine, lines.length - 1) + + if (effectiveStartLine >= lines.length) { + reject(outOfRangeError(filepath, effectiveStartLine)) + } else { + const selectedLines = lines.slice(effectiveStartLine, effectiveEndLine + 1) + // Join with newlines, but don't add trailing newline if original didn't have one + const hasTrailingNewline = content.endsWith("\n") + let resultText = selectedLines.join("\n") + if (hasTrailingNewline || effectiveEndLine < lines.length - 1) { + resultText += "\n" + } + resolve(resultText) + } + }) + } else { + reject(error) + } + }) // Process data chunks directly input.on("data", (chunk) => { - // Add chunk to buffer - buffer += chunk.toString() + // Chunk is already a string due to encoding option + buffer += chunk as string let pos = 0 let nextNewline = buffer.indexOf("\n", pos) diff --git a/test-unicode.md b/test-unicode.md new file mode 100644 index 0000000000..1c8fb3c964 --- /dev/null +++ b/test-unicode.md @@ -0,0 +1,27 @@ +# Test File for Unicode Characters + +This file contains mixed-language content to test Unicode handling. + +- **时间戳 (Timestamp)**: 2025-07-26 15:41 +- **任务/目标 (Task/Goal)**: 实现问卷的前端核心交互逻辑。 +- **关键决策/操作 (Key Decision/Action)**: + 1. 在 `questionnaire-server/public/index.html` 中添加了 `
` 作为动态内容的挂载点。 + 2. 在 `questionnaire-server/public/style.css` 中添加了完整的基础样式,确保界面干净、可用,并对问卷的各个部分(欢迎页、问题页、选项)进行了样式设置。 + +## Additional Unicode Test Cases + +- Emoji: 😀 🎉 🚀 ✨ +- Japanese: こんにちは世界 +- Korean: 안녕하세요 세계 +- Arabic: مرحبا بالعالم +- Hebrew: שלום עולם +- Russian: Привет мир +- Greek: Γεια σου κόσμο +- Thai: สวัสดีชาวโลก + +## Special Characters + +- Non-breaking space: test test +- Zero-width space: test​test +- Various quotes: "test" 'test' „test" «test» +- Math symbols: ∑ ∏ ∫ √ ∞ ≈ ≠ ≤ ≥