From cb75197502f2068a4e9ece839497a357d3e154d1 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph Date: Wed, 14 Jan 2026 22:20:01 -0700 Subject: [PATCH] fix: align read_file tool with plan spec - Fix MAX_LINE_BYTES to 500 (was 2000), rename from MAX_LINE_LENGTH for clarity - Implement byte-based UTF-8 line truncation at safe boundaries (safeUtf8Truncate) - Rename FALLBACK_LIMIT to DEFAULT_LINE_LIMIT for clarity - Add maxLines property to indentation schema in tool definition - Add test for UTF-8 boundary truncation with emoji --- .../prompts/tools/native-tools/read_file.ts | 15 ++-- .../misc/__tests__/read-file-content.spec.ts | 21 ++++- src/integrations/misc/read-file-content.ts | 76 +++++++++++++++---- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/core/prompts/tools/native-tools/read_file.ts b/src/core/prompts/tools/native-tools/read_file.ts index d7acd867d1..c0faf01866 100644 --- a/src/core/prompts/tools/native-tools/read_file.ts +++ b/src/core/prompts/tools/native-tools/read_file.ts @@ -34,12 +34,7 @@ export interface CreateReadFileToolOptions { * @returns Native tool definition for read_file */ export function createReadFileTool(options: CreateReadFileToolOptions = {}): OpenAI.Chat.ChatCompletionTool { - const { - partialReadsEnabled = true, - maxReadFileLine, - maxConcurrentFileReads = 5, - supportsImages = false, - } = options + const { partialReadsEnabled = true, maxReadFileLine, maxConcurrentFileReads = 5, supportsImages = false } = options const isMultipleReadsEnabled = maxConcurrentFileReads > 1 // Build limit info for descriptions @@ -76,8 +71,7 @@ export function createReadFileTool(options: CreateReadFileToolOptions = {}): Ope ? `Example multiple files (within ${maxConcurrentFileReads}-file limit): { files: [{ path: 'file1.ts' }, { path: 'file2.ts' }] }` : "") - const description = - baseDescription + modeDescription + getReadFileSupportsNote(supportsImages) + " " + examples + const description = baseDescription + modeDescription + getReadFileSupportsNote(supportsImages) + " " + examples // Build the file properties object conditionally const fileProperties: Record = { @@ -135,6 +129,11 @@ export function createReadFileTool(options: CreateReadFileToolOptions = {}): Ope description: "Whether to include comment headers above the anchor block. Defaults to true.", default: true, }, + maxLines: { + type: ["integer", "null"], + description: "Hard cap on returned lines for indentation mode. Defaults to the limit parameter.", + minimum: 1, + }, }, additionalProperties: false, } diff --git a/src/integrations/misc/__tests__/read-file-content.spec.ts b/src/integrations/misc/__tests__/read-file-content.spec.ts index 74b4256be7..73ea210315 100644 --- a/src/integrations/misc/__tests__/read-file-content.spec.ts +++ b/src/integrations/misc/__tests__/read-file-content.spec.ts @@ -93,7 +93,7 @@ describe("read-file-content", () => { }) it("should truncate long lines", async () => { - const longLine = "x".repeat(600) // Longer than MAX_LINE_LENGTH (500) + const longLine = "x".repeat(600) // Longer than MAX_LINE_BYTES (500) await withTempFile("slice-long-line-test.txt", longLine, async (filepath) => { const result = await readSlice(filepath, 1, 1) // Line should be truncated to 500 characters + line number prefix @@ -101,6 +101,25 @@ describe("read-file-content", () => { }) }) + it("should truncate at UTF-8 byte boundary for multi-byte characters", async () => { + // Each emoji is 4 bytes in UTF-8, so 150 emojis = 600 bytes + // With MAX_LINE_BYTES = 500 bytes, we should get ~125 emojis max + const longEmojiLine = "😀".repeat(150) // 600 bytes total + await withTempFile("slice-utf8-truncate-test.txt", longEmojiLine, async (filepath) => { + const result = await readSlice(filepath, 1, 1) + // Extract just the content (after "1 | ") + const content = result.content.replace(/^\d+ \| /, "") + // Verify byte length is <= 500 bytes + const encoder = new TextEncoder() + const byteLength = encoder.encode(content).length + expect(byteLength).toBeLessThanOrEqual(500) + // Also verify we didn't split a multi-byte character (no replacement chars) + expect(content).not.toContain("�") + // Track truncation in metadata + expect(result.metadata.lineLengthTruncations).toContain(1) + }) + }) + it("should handle files with CRLF line endings", async () => { const content = "Line 1\r\nLine 2\r\nLine 3" await withTempFile("slice-crlf-test.txt", content, async (filepath) => { diff --git a/src/integrations/misc/read-file-content.ts b/src/integrations/misc/read-file-content.ts index 44551a9f59..4eace8e5b7 100644 --- a/src/integrations/misc/read-file-content.ts +++ b/src/integrations/misc/read-file-content.ts @@ -6,9 +6,43 @@ import { createReadStream } from "fs" // Configuration constants -const MAX_LINE_LENGTH = 500 // Truncate lines longer than this +const MAX_LINE_BYTES = 500 // Max bytes per line (UTF-8) - truncated at safe boundary per plan spec const TAB_WIDTH = 4 // Treat tabs as 4 spaces for indentation -const FALLBACK_LIMIT = 2000 // Fallback when no limit is specified and maxReadFileLine is not set +const DEFAULT_LINE_LIMIT = 2000 // Default max lines when no limit is specified and maxReadFileLine is not set + +/** + * Truncate a string at a UTF-8 boundary so that its byte length does not exceed maxBytes. + * Returns a prefix of `text` whose UTF-8 byte length is <= maxBytes and which does not + * split a multi-byte codepoint. + */ +function safeUtf8Truncate(text: string, maxBytes: number): string { + // Fast path: if the string's byte length is within limit, return as-is + const encoder = new TextEncoder() + const encoded = encoder.encode(text) + if (encoded.length <= maxBytes) { + return text + } + + // Binary search for the maximum character index that fits within maxBytes + let low = 0 + let high = text.length + let result = "" + + while (low <= high) { + const mid = Math.floor((low + high) / 2) + const substring = text.substring(0, mid) + const byteLength = encoder.encode(substring).length + + if (byteLength <= maxBytes) { + result = substring + low = mid + 1 + } else { + high = mid - 1 + } + } + + return result +} // Comment prefixes for header detection const COMMENT_PREFIXES = ["#", "//", "--", "/*", "*", "'''", '"""', "