From b14a2e9f90f864d368c13fc19072eb34502506c3 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Sun, 7 Dec 2025 18:01:51 +0000 Subject: [PATCH] feat: add token budget validation to simpleReadFileTool - Import validateFileTokenBudget and truncateFileContent helpers - Import getModelMaxOutputTokens and ANTHROPIC_DEFAULT_MAX_TOKENS - Add token budget validation in normal file read path - Truncate file content if it exceeds available context budget - Display appropriate notice when truncation occurs - Mirrors the behavior of ReadFileTool for consistency Fixes context overflow issues in Grok and other models using simpleReadFileTool --- src/core/tools/simpleReadFileTool.ts | 55 +++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/src/core/tools/simpleReadFileTool.ts b/src/core/tools/simpleReadFileTool.ts index 1b41e9e9d6..06686786e5 100644 --- a/src/core/tools/simpleReadFileTool.ts +++ b/src/core/tools/simpleReadFileTool.ts @@ -4,6 +4,7 @@ import { isBinaryFile } from "isbinaryfile" import { Task } from "../task/Task" import { ClineSayTool } from "../../shared/ExtensionMessage" import { formatResponse } from "../prompts/responses" +import { getModelMaxOutputTokens } from "../../shared/api" import { t } from "../../i18n" import { ToolUse, AskApproval, HandleError, PushToolResult, RemoveClosingTag } from "../../shared/tools" import { RecordSource } from "../context-tracking/FileContextTrackerTypes" @@ -13,7 +14,7 @@ import { countFileLines } from "../../integrations/misc/line-counter" import { readLines } from "../../integrations/misc/read-lines" import { extractTextFromFile, addLineNumbers, getSupportedBinaryFormats } from "../../integrations/misc/extract-text" import { parseSourceCodeDefinitionsForFile } from "../../services/tree-sitter" -import { ToolProtocol, isNativeProtocol } from "@roo-code/types" +import { ToolProtocol, isNativeProtocol, ANTHROPIC_DEFAULT_MAX_TOKENS } from "@roo-code/types" import { DEFAULT_MAX_IMAGE_FILE_SIZE_MB, DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB, @@ -21,6 +22,7 @@ import { validateImageForProcessing, processImageFile, } from "./helpers/imageHelpers" +import { validateFileTokenBudget, truncateFileContent } from "./helpers/fileTokenBudget" /** * Simplified read file tool for models that only support single file reads @@ -249,12 +251,53 @@ export async function simpleReadFileTool( } // Handle normal file read - const content = await extractTextFromFile(fullPath) - const lineRangeAttr = ` lines="1-${totalLines}"` - let xmlInfo = totalLines > 0 ? `\n${content}\n` : `` + // Get token budget information + const { id: modelId, info: modelInfo } = cline.api.getModel() + const { contextTokens } = cline.getTokenUsage() + const contextWindow = modelInfo.contextWindow - if (totalLines === 0) { - xmlInfo += `File is empty\n` + const maxOutputTokens = + getModelMaxOutputTokens({ + modelId, + model: modelInfo, + settings: cline.apiConfiguration, + }) ?? ANTHROPIC_DEFAULT_MAX_TOKENS + + // Validate if file content fits within token budget + const budgetResult = await validateFileTokenBudget( + fullPath, + contextWindow - maxOutputTokens, + contextTokens || 0, + ) + + let content = await extractTextFromFile(fullPath) + let xmlInfo = "" + + if (budgetResult.shouldTruncate && budgetResult.maxChars !== undefined) { + // File exceeds token budget - truncate it + const truncateResult = truncateFileContent( + content, + budgetResult.maxChars, + content.length, + budgetResult.isPreview, + ) + content = truncateResult.content + + let displayedLines = content.length === 0 ? 0 : content.split(/\r?\n/).length + if (displayedLines > 0 && content.endsWith("\n")) { + displayedLines-- + } + const lineRangeAttr = displayedLines > 0 ? ` lines="1-${displayedLines}"` : "" + xmlInfo = content.length > 0 ? `\n${content}\n` : `` + xmlInfo += `${truncateResult.notice}\n` + } else { + // File fits within budget - read normally + const lineRangeAttr = ` lines="1-${totalLines}"` + xmlInfo = totalLines > 0 ? `\n${content}\n` : `` + + if (totalLines === 0) { + xmlInfo += `File is empty\n` + } } // Track file read