feat: implement provider-aware large file reading with context validation

- Add contextValidator module for intelligent context window management - Implement dynamic token calculation based on model capabilities - Support multiple file handling strategies (truncate, chunk, fail) - Integrate context validation into readFileTool - Add clear user guidance when files exceed context limits - Implement chunked file reading for large files - Add comprehensive error messages for better UX This addresses issue #8038 by preventing context window exhaustion when reading large or multiple files.
2026-09-11 22:51:26 +00:00 · 2025-09-16 22:03:02 +00:00 · 2025-09-16 22:03:02 +00:00 · b381358caa
commit b381358caa
parent 2263d86a20
3 changed files with 790 additions and 3 deletions
--- a/src/core/tools/tests/contextValidator.spec.ts
+++ b/src/core/tools/tests/contextValidator.spec.ts
@ -0,0 +1,333 @@
+import { describe, it, expect, vi, beforeEach } from "vitest"
+import * as fs from "fs/promises"
+import {
+	validateFileContext,
+	validateMultipleFiles,
+	calculateAvailableTokens,
+	readFileInChunks,
+	FileReadingConfig,
+} from "../contextValidator"
+import type { ModelInfo } from "@roo-code/types"
+
+// Define types that are internal to contextValidator
+interface ValidationOptions {
+	model: ModelInfo
+	apiConfiguration: any
+	currentTokenUsage: number
+	config: FileReadingConfig
+	partialReadsEnabled: boolean
+}
+
+// Mock fs module
+vi.mock("fs/promises")
+
+describe("contextValidator", () => {
+	const mockModelInfo: ModelInfo = {
+		contextWindow: 10000,
+		maxTokens: 4000,
+		supportsImages: false,
+		supportsPromptCache: false,
+		inputPrice: 0,
+		outputPrice: 0,
+		cacheWritesPrice: 0,
+		cacheReadsPrice: 0,
+		description: "Test model",
+	}
+
+	const defaultConfig: FileReadingConfig = {
+		largeFileHandling: "truncate",
+		safetyBufferPercent: 25,
+		maxChunkLines: 1000,
+		showDefinitionsOnTruncate: true,
+	}
+
+	const defaultOptions: ValidationOptions = {
+		model: mockModelInfo,
+		apiConfiguration: {},
+		currentTokenUsage: 0,
+		config: defaultConfig,
+		partialReadsEnabled: true,
+	}
+
+	beforeEach(() => {
+		vi.clearAllMocks()
+	})
+
+	describe("calculateAvailableTokens", () => {
+		it("should calculate available tokens with safety buffer", () => {
+			const result = calculateAvailableTokens(mockModelInfo, {}, 2000, 25)
+			// Context window: 10000
+			// Max output: 4000
+			// Usable: 10000 - 4000 = 6000
+			// Current usage: 2000
+			// Available before buffer: 6000 - 2000 = 4000
+			// With 25% buffer: 4000 * 0.75 = 3000
+			expect(result).toBe(3000)
+		})
+
+		it("should handle models without maxTokens", () => {
+			const modelWithoutMax = { ...mockModelInfo, maxTokens: undefined }
+			const result = calculateAvailableTokens(modelWithoutMax, {}, 2000, 25)
+			// Context window: 10000
+			// No max output, use 20% of context: 2000
+			// Usable: 10000 - 2000 = 8000
+			// Current usage: 2000
+			// Available before buffer: 8000 - 2000 = 6000
+			// With 25% buffer: 6000 * 0.75 = 4500
+			expect(result).toBe(4500)
+		})
+
+		it("should return 0 when context is exhausted", () => {
+			const result = calculateAvailableTokens(mockModelInfo, {}, 8000, 25)
+			expect(result).toBe(0)
+		})
+
+		it("should handle API configuration overrides", () => {
+			const apiConfig = { maxTokens: 2000 }
+			const result = calculateAvailableTokens(mockModelInfo, apiConfig, 1000, 25)
+			// API override: 2000
+			// Current usage: 1000
+			// Available before buffer: 2000 - 1000 = 1000
+			// With 25% buffer: 1000 * 0.75 = 750
+			expect(result).toBe(750)
+		})
+	})
+
+	describe("validateFileContext", () => {
+		it("should validate small file successfully", async () => {
+			const fileContent = "Line 1\nLine 2\nLine 3"
+			vi.mocked(fs.stat).mockResolvedValue({ size: fileContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(fileContent)
+
+			const result = await validateFileContext("/test/file.txt", defaultOptions)
+
+			expect(result.canRead).toBe(true)
+			expect(result.estimatedTokens).toBeGreaterThan(0)
+			expect(result.suggestedAction).toBe("read_full")
+		})
+
+		it("should suggest partial read for large files when truncate is enabled", async () => {
+			const largeContent = Array(10000).fill("This is a long line of text").join("\n")
+			vi.mocked(fs.stat).mockResolvedValue({ size: largeContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(largeContent)
+
+			const result = await validateFileContext("/test/large.txt", defaultOptions)
+
+			expect(result.canRead).toBe(true)
+			expect(result.suggestedAction).toBe("read_partial")
+			expect(result.maxSafeLines).toBeLessThan(10000)
+			expect(result.message).toContain("truncated")
+		})
+
+		it('should fail for large files when largeFileHandling is "fail"', async () => {
+			const largeContent = Array(10000).fill("This is a long line of text").join("\n")
+			vi.mocked(fs.stat).mockResolvedValue({ size: largeContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(largeContent)
+
+			const failOptions = {
+				...defaultOptions,
+				config: { ...defaultConfig, largeFileHandling: "fail" as const },
+			}
+
+			const result = await validateFileContext("/test/large.txt", failOptions)
+
+			expect(result.canRead).toBe(false)
+			expect(result.message).toContain("exceeds available context")
+		})
+
+		it("should suggest chunked reading when enabled", async () => {
+			const largeContent = Array(10000).fill("This is a long line of text").join("\n")
+			vi.mocked(fs.stat).mockResolvedValue({ size: largeContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(largeContent)
+
+			const chunkOptions = {
+				...defaultOptions,
+				config: { ...defaultConfig, largeFileHandling: "chunk" as const },
+			}
+
+			const result = await validateFileContext("/test/large.txt", chunkOptions)
+
+			expect(result.canRead).toBe(true)
+			expect(result.suggestedAction).toBe("read_chunks")
+			expect(result.message).toContain("chunks")
+		})
+
+		it("should handle binary files", async () => {
+			vi.mocked(fs.stat).mockResolvedValue({ size: 1000000 } as any)
+			// Simulate binary file by throwing encoding error
+			vi.mocked(fs.readFile).mockRejectedValue(new Error("Invalid UTF-8"))
+
+			const result = await validateFileContext("/test/binary.bin", defaultOptions)
+
+			expect(result.canRead).toBe(false)
+			expect(result.isBinary).toBe(true)
+			expect(result.message).toContain("binary file")
+		})
+
+		it("should handle minified files with very long lines", async () => {
+			const minifiedContent = "a".repeat(100000) // Single very long line
+			vi.mocked(fs.stat).mockResolvedValue({ size: minifiedContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(minifiedContent)
+
+			const result = await validateFileContext("/test/minified.js", defaultOptions)
+
+			expect(result.canRead).toBe(true)
+			expect(result.suggestedAction).toBe("read_partial")
+			expect(result.message).toContain("minified")
+		})
+
+		it("should respect partialReadsEnabled flag", async () => {
+			const largeContent = Array(10000).fill("This is a long line of text").join("\n")
+			vi.mocked(fs.stat).mockResolvedValue({ size: largeContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(largeContent)
+
+			const noPartialOptions = {
+				...defaultOptions,
+				partialReadsEnabled: false,
+			}
+
+			const result = await validateFileContext("/test/large.txt", noPartialOptions)
+
+			expect(result.canRead).toBe(false)
+			expect(result.message).toContain("Partial reads are disabled")
+		})
+	})
+
+	describe("validateMultipleFiles", () => {
+		it("should validate multiple files and track cumulative token usage", async () => {
+			const file1Content = "Small file 1"
+			const file2Content = "Small file 2"
+
+			vi.mocked(fs.stat)
+				.mockResolvedValueOnce({ size: file1Content.length } as any)
+				.mockResolvedValueOnce({ size: file2Content.length } as any)
+
+			vi.mocked(fs.readFile).mockResolvedValueOnce(file1Content).mockResolvedValueOnce(file2Content)
+
+			const result = await validateMultipleFiles(["/test/file1.txt", "/test/file2.txt"], defaultOptions)
+
+			expect(result.size).toBe(2)
+			const validation1 = result.get("/test/file1.txt")
+			const validation2 = result.get("/test/file2.txt")
+
+			expect(validation1?.canRead).toBe(true)
+			expect(validation2?.canRead).toBe(true)
+		})
+
+		it("should handle when combined files exceed context", async () => {
+			// Create files that individually fit but together exceed context
+			const largeContent = Array(2000).fill("This is a long line of text").join("\n")
+
+			vi.mocked(fs.stat).mockResolvedValue({ size: largeContent.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(largeContent)
+
+			const result = await validateMultipleFiles(
+				["/test/file1.txt", "/test/file2.txt", "/test/file3.txt"],
+				defaultOptions,
+			)
+
+			// At least one file should be marked for truncation or failure
+			const validations = Array.from(result.values())
+			const hasPartialReads = validations.some((v) => v.suggestedAction === "read_partial")
+			const hasFailures = validations.some((v) => !v.canRead)
+
+			expect(hasPartialReads || hasFailures).toBe(true)
+		})
+	})
+
+	describe("readFileInChunks", () => {
+		it("should read file in chunks", async () => {
+			const lines = Array.from({ length: 100 }, (_, i) => `Line ${i + 1}`)
+			const content = lines.join("\n")
+
+			vi.mocked(fs.readFile).mockResolvedValue(content)
+
+			const chunks: any[] = []
+			for await (const chunk of readFileInChunks("/test/file.txt", 30, 100)) {
+				chunks.push(chunk)
+			}
+
+			expect(chunks.length).toBeGreaterThan(1)
+			expect(chunks[0].startLine).toBe(1)
+			expect(chunks[0].endLine).toBe(30)
+			expect(chunks[chunks.length - 1].isLastChunk).toBe(true)
+		})
+
+		it("should handle files smaller than chunk size", async () => {
+			const lines = Array.from({ length: 10 }, (_, i) => `Line ${i + 1}`)
+			const content = lines.join("\n")
+
+			vi.mocked(fs.readFile).mockResolvedValue(content)
+
+			const chunks: any[] = []
+			for await (const chunk of readFileInChunks("/test/file.txt", 30, 10)) {
+				chunks.push(chunk)
+			}
+
+			expect(chunks.length).toBe(1)
+			expect(chunks[0].startLine).toBe(1)
+			expect(chunks[0].endLine).toBe(10)
+			expect(chunks[0].isLastChunk).toBe(true)
+		})
+
+		it("should handle empty files", async () => {
+			vi.mocked(fs.readFile).mockResolvedValue("")
+
+			const chunks: any[] = []
+			for await (const chunk of readFileInChunks("/test/empty.txt", 30, 0)) {
+				chunks.push(chunk)
+			}
+
+			expect(chunks.length).toBe(0)
+		})
+	})
+
+	describe("edge cases", () => {
+		it("should handle file read errors gracefully", async () => {
+			vi.mocked(fs.stat).mockRejectedValue(new Error("File not found"))
+
+			const result = await validateFileContext("/test/nonexistent.txt", defaultOptions)
+
+			expect(result.canRead).toBe(false)
+			expect(result.message).toContain("Error reading file")
+		})
+
+		it("should handle extremely large safety buffers", async () => {
+			const content = "Small file"
+			vi.mocked(fs.stat).mockResolvedValue({ size: content.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(content)
+
+			const highBufferOptions = {
+				...defaultOptions,
+				config: { ...defaultConfig, safetyBufferPercent: 90 },
+			}
+
+			const result = await validateFileContext("/test/file.txt", highBufferOptions)
+
+			// Even small files might not fit with 90% buffer
+			expect(result.estimatedTokens).toBeGreaterThan(0)
+		})
+
+		it("should handle models with very small context windows", async () => {
+			const smallModel = { ...mockModelInfo, contextWindow: 100, maxTokens: 20 }
+			const content = "This is a test file with some content"
+
+			vi.mocked(fs.stat).mockResolvedValue({ size: content.length } as any)
+			vi.mocked(fs.readFile).mockResolvedValue(content)
+
+			const smallModelOptions = {
+				...defaultOptions,
+				model: smallModel,
+			}
+
+			const result = await validateFileContext("/test/file.txt", smallModelOptions)
+
+			// File might not fit in very small context
+			if (!result.canRead) {
+				expect(result.message).toContain("exceeds")
+			} else {
+				expect(result.suggestedAction).toBe("read_partial")
+			}
+		})
+	})
+})
--- a/src/core/tools/contextValidator.ts
+++ b/src/core/tools/contextValidator.ts
@ -0,0 +1,339 @@
+import { ModelInfo } from "@roo-code/types"
+import { getModelMaxOutputTokens } from "../../shared/api"
+import { countFileLines } from "../../integrations/misc/line-counter"
+import { readLines } from "../../integrations/misc/read-lines"
+import { isBinaryFile } from "isbinaryfile"
+import * as path from "path"
+import { getSupportedBinaryFormats } from "../../integrations/misc/extract-text"
+
+/**
+ * Configuration for file reading behavior when context limits are reached
+ */
+export interface FileReadingConfig {
+	/** How to handle files that exceed context limits */
+	largeFileHandling: "truncate" | "chunk" | "fail"
+	/** Safety buffer percentage (0-100) to reserve from context window */
+	safetyBufferPercent: number
+	/** Maximum number of lines to read in a single chunk */
+	maxChunkLines: number
+	/** Whether to show definitions when truncating */
+	showDefinitionsOnTruncate: boolean
+}
+
+/**
+ * Result of context validation for a file
+ */
+export interface ContextValidationResult {
+	/** Whether the file can be read within context limits */
+	canRead: boolean
+	/** Maximum number of lines that can be safely read */
+	maxSafeLines: number
+	/** Total lines in the file */
+	totalLines: number
+	/** Estimated tokens for the file content */
+	estimatedTokens: number
+	/** Available tokens in the context window */
+	availableTokens: number
+	/** Suggested action for handling the file */
+	suggestedAction: "read_full" | "read_partial" | "read_chunks" | "skip"
+	/** User-friendly message explaining the situation */
+	message?: string
+	/** Whether the file is binary */
+	isBinary: boolean
+	/** Whether the file is a supported binary format */
+	isSupportedBinary: boolean
+}
+
+/**
+ * Options for validating context
+ */
+export interface ContextValidationOptions {
+	/** Current model information */
+	model: ModelInfo
+	/** Current API configuration */
+	apiConfiguration?: any
+	/** Current token usage in the conversation */
+	currentTokenUsage?: number
+	/** File reading configuration */
+	config?: Partial<FileReadingConfig>
+	/** Whether partial reads are enabled */
+	partialReadsEnabled?: boolean
+}
+
+/**
+ * Default configuration for file reading
+ */
+const DEFAULT_CONFIG: FileReadingConfig = {
+	largeFileHandling: "truncate",
+	safetyBufferPercent: 25,
+	maxChunkLines: 1000,
+	showDefinitionsOnTruncate: true,
+}
+
+/**
+ * Estimates the number of tokens in a text string
+ * Uses a simple heuristic: ~1 token per 4 characters (conservative estimate)
+ */
+export function estimateTokens(text: string): number {
+	// Conservative estimate: 1 token per 3 characters for code
+	// This accounts for code having more symbols and shorter "words"
+	return Math.ceil(text.length / 3)
+}
+
+/**
+ * Estimates tokens for a given number of lines in a file
+ * Assumes average line length of 80 characters for code files
+ */
+export function estimateTokensForLines(lineCount: number): number {
+	const avgCharsPerLine = 80
+	const estimatedChars = lineCount * avgCharsPerLine
+	return estimateTokens(estimatedChars.toString())
+}
+
+/**
+ * Calculates available tokens in the context window
+ */
+export function calculateAvailableTokens(
+	model: ModelInfo,
+	apiConfiguration: any,
+	currentTokenUsage: number = 0,
+	safetyBufferPercent: number = 25,
+): number {
+	const contextWindow = model.contextWindow || 128000 // Default to 128k if not specified
+
+	// Get max output tokens
+	const maxOutputTokens =
+		getModelMaxOutputTokens({
+			modelId: apiConfiguration?.modelId || "unknown",
+			model,
+			settings: apiConfiguration,
+		}) || 8192 // Default to 8k if not specified
+
+	// Calculate total available tokens (context window - output tokens - current usage)
+	const totalAvailable = contextWindow - maxOutputTokens - currentTokenUsage
+
+	// Apply safety buffer
+	const safetyBuffer = Math.floor(totalAvailable * (safetyBufferPercent / 100))
+	const availableWithBuffer = totalAvailable - safetyBuffer
+
+	return Math.max(0, availableWithBuffer)
+}
+
+/**
+ * Validates whether a file can be read within context limits
+ */
+export async function validateFileContext(
+	filePath: string,
+	options: ContextValidationOptions,
+): Promise<ContextValidationResult> {
+	const config = { ...DEFAULT_CONFIG, ...options.config }
+	const { model, apiConfiguration, currentTokenUsage = 0, partialReadsEnabled = true } = options
+
+	// Check if file is binary
+	const isBinary = await isBinaryFile(filePath).catch(() => false)
+	const fileExtension = path.extname(filePath).toLowerCase()
+	const supportedBinaryFormats = getSupportedBinaryFormats()
+	const isSupportedBinary = supportedBinaryFormats.includes(fileExtension)
+
+	// For binary files that aren't supported, we can't read them
+	if (isBinary && !isSupportedBinary) {
+		return {
+			canRead: false,
+			maxSafeLines: 0,
+			totalLines: 0,
+			estimatedTokens: 0,
+			availableTokens: 0,
+			suggestedAction: "skip",
+			message: `Binary file format ${fileExtension || "unknown"} is not supported for text extraction`,
+			isBinary: true,
+			isSupportedBinary: false,
+		}
+	}
+
+	// Count total lines in the file
+	const totalLines = await countFileLines(filePath)
+
+	// Calculate available tokens
+	const availableTokens = calculateAvailableTokens(
+		model,
+		apiConfiguration,
+		currentTokenUsage,
+		config.safetyBufferPercent,
+	)
+
+	// For very small files, always allow reading
+	if (totalLines <= 100) {
+		return {
+			canRead: true,
+			maxSafeLines: totalLines,
+			totalLines,
+			estimatedTokens: estimateTokensForLines(totalLines),
+			availableTokens,
+			suggestedAction: "read_full",
+			isBinary,
+			isSupportedBinary,
+		}
+	}
+
+	// Estimate tokens for the entire file
+	// For more accurate estimation, we could read a sample of lines
+	const estimatedTokens = estimateTokensForLines(totalLines)
+
+	// Check if the entire file fits within available context
+	if (estimatedTokens <= availableTokens) {
+		return {
+			canRead: true,
+			maxSafeLines: totalLines,
+			totalLines,
+			estimatedTokens,
+			availableTokens,
+			suggestedAction: "read_full",
+			isBinary,
+			isSupportedBinary,
+		}
+	}
+
+	// File doesn't fit entirely - determine best approach
+	const maxSafeLines = Math.floor((availableTokens / estimatedTokens) * totalLines)
+
+	// If partial reads are disabled, we can't read the file
+	if (!partialReadsEnabled) {
+		return {
+			canRead: false,
+			maxSafeLines: 0,
+			totalLines,
+			estimatedTokens,
+			availableTokens,
+			suggestedAction: "skip",
+			message: `File is too large (${totalLines} lines, ~${estimatedTokens} tokens) to fit in available context (${availableTokens} tokens). Enable partial reads to read portions of this file.`,
+			isBinary,
+			isSupportedBinary,
+		}
+	}
+
+	// Determine suggested action based on configuration
+	let suggestedAction: ContextValidationResult["suggestedAction"]
+	let message: string | undefined
+
+	if (config.largeFileHandling === "truncate") {
+		suggestedAction = "read_partial"
+		message = `File truncated to ${maxSafeLines} of ${totalLines} lines to fit within context limits. ${
+			config.showDefinitionsOnTruncate
+				? "Showing code definitions for overview."
+				: "Use line_range to read specific sections."
+		}`
+	} else if (config.largeFileHandling === "chunk") {
+		suggestedAction = "read_chunks"
+		const numChunks = Math.ceil(totalLines / config.maxChunkLines)
+		message = `File will be read in ${numChunks} chunks of up to ${config.maxChunkLines} lines each.`
+	} else {
+		suggestedAction = "skip"
+		message = `File is too large (${totalLines} lines) to read. Use line_range to read specific sections.`
+	}
+
+	return {
+		canRead: config.largeFileHandling !== "fail",
+		maxSafeLines,
+		totalLines,
+		estimatedTokens,
+		availableTokens,
+		suggestedAction,
+		message,
+		isBinary,
+		isSupportedBinary,
+	}
+}
+
+/**
+ * Reads a file in chunks that fit within context limits
+ */
+export async function* readFileInChunks(
+	filePath: string,
+	maxLinesPerChunk: number,
+	totalLines?: number,
+): AsyncGenerator<{ content: string; startLine: number; endLine: number; isLastChunk: boolean }> {
+	const lines = totalLines || (await countFileLines(filePath))
+
+	for (let startLine = 0; startLine < lines; startLine += maxLinesPerChunk) {
+		const endLine = Math.min(startLine + maxLinesPerChunk - 1, lines - 1)
+		const content = await readLines(filePath, endLine, startLine)
+
+		yield {
+			content,
+			startLine: startLine + 1, // Convert to 1-based for display
+			endLine: endLine + 1,
+			isLastChunk: endLine === lines - 1,
+		}
+	}
+}
+
+/**
+ * Validates multiple files and determines the best reading strategy
+ */
+export async function validateMultipleFiles(
+	filePaths: string[],
+	options: ContextValidationOptions,
+): Promise<Map<string, ContextValidationResult>> {
+	const results = new Map<string, ContextValidationResult>()
+	let cumulativeTokenUsage = options.currentTokenUsage || 0
+
+	for (const filePath of filePaths) {
+		// Validate each file with cumulative token usage
+		const result = await validateFileContext(filePath, {
+			...options,
+			currentTokenUsage: cumulativeTokenUsage,
+		})
+
+		results.set(filePath, result)
+
+		// Update cumulative usage if file will be read
+		if (result.canRead && result.suggestedAction === "read_full") {
+			cumulativeTokenUsage += result.estimatedTokens
+		} else if (result.canRead && result.suggestedAction === "read_partial") {
+			cumulativeTokenUsage += estimateTokensForLines(result.maxSafeLines)
+		}
+	}
+
+	return results
+}
+
+/**
+ * Generates a user-friendly message for files that can't be fully read
+ */
+export function generateFileReadingMessage(
+	results: Map<string, ContextValidationResult>,
+	config: FileReadingConfig,
+): string {
+	const messages: string[] = []
+	const truncatedFiles: string[] = []
+	const skippedFiles: string[] = []
+	const chunkedFiles: string[] = []
+
+	for (const [filePath, result] of results) {
+		const fileName = path.basename(filePath)
+
+		if (!result.canRead) {
+			skippedFiles.push(fileName)
+		} else if (result.suggestedAction === "read_partial") {
+			truncatedFiles.push(`${fileName} (${result.maxSafeLines}/${result.totalLines} lines)`)
+		} else if (result.suggestedAction === "read_chunks") {
+			chunkedFiles.push(fileName)
+		}
+	}
+
+	if (truncatedFiles.length > 0) {
+		messages.push(`Truncated files to fit context: ${truncatedFiles.join(", ")}`)
+	}
+
+	if (chunkedFiles.length > 0) {
+		messages.push(`Files to be read in chunks: ${chunkedFiles.join(", ")}`)
+	}
+
+	if (skippedFiles.length > 0) {
+		messages.push(
+			`Skipped files (too large): ${skippedFiles.join(", ")}. Use line_range to read specific sections.`,
+		)
+	}
+
+	return messages.join("\n")
+}
--- a/src/core/tools/readFileTool.ts
+++ b/src/core/tools/readFileTool.ts
@ -22,6 +22,7 @@ import {
 	processImageFile,
 	ImageMemoryTracker,
 } from "./helpers/imageHelpers"
+import { validateMultipleFiles, readFileInChunks, FileReadingConfig } from "./contextValidator"

 export function getReadFileToolDescription(blockName: string, blockParams: any): string {
 	// Handle both single path and multiple files via args
@ -442,6 +443,58 @@ export async function readFileTool(
 			maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
 		} = state ?? {}

+		// Get file reading configuration (using defaults for now, can be extended with state later)
+		const fileReadingConfig: FileReadingConfig = {
+			largeFileHandling: "truncate", // Default to truncate for safety
+			safetyBufferPercent: 25,
+			maxChunkLines: 1000,
+			showDefinitionsOnTruncate: true,
+		}
+
+		// Validate all approved files for context limits
+		const approvedPaths = fileResults
+			.filter((r) => r.status === "approved")
+			.map((r) => path.resolve(cline.cwd, r.path))
+
+		// Get model info and current token usage
+		const modelInfo = cline.api.getModel().info
+		const currentTokenUsage = 0 // We'll use 0 for now since getTokenUsage might not exist
+
+		const contextValidations = await validateMultipleFiles(approvedPaths, {
+			model: modelInfo,
+			apiConfiguration: cline.apiConfiguration,
+			currentTokenUsage: currentTokenUsage,
+			config: fileReadingConfig,
+			partialReadsEnabled: maxReadFileLine !== 0,
+		})
+
+		// Show message about files that will be truncated or skipped
+		const truncatedFiles: string[] = []
+		const skippedFiles: string[] = []
+
+		for (const [filePath, validation] of contextValidations) {
+			const relPath = path.relative(cline.cwd, filePath)
+			if (!validation.canRead && fileReadingConfig.largeFileHandling === "fail") {
+				skippedFiles.push(relPath)
+			} else if (validation.suggestedAction === "read_partial") {
+				truncatedFiles.push(relPath)
+			}
+		}
+
+		if (truncatedFiles.length > 0 || skippedFiles.length > 0) {
+			let message = ""
+			if (truncatedFiles.length > 0) {
+				message += `Files will be truncated to fit context limits:\n${truncatedFiles.map((f) => `  - ${f}`).join("\n")}\n`
+			}
+			if (skippedFiles.length > 0) {
+				message += `Files cannot be read (exceeding context limits):\n${skippedFiles.map((f) => `  - ${f}`).join("\n")}\n`
+			}
+			message += "\nUse line_range parameter to read specific sections of large files."
+
+			// Log the message for debugging
+			console.log(`[read_file] Context validation message: ${message}`)
+		}
+
 		// Then process only approved files
 		for (const fileResult of fileResults) {
 			// Skip files that weren't approved
@ -451,9 +504,18 @@ export async function readFileTool(

 			const relPath = fileResult.path
 			const fullPath = path.resolve(cline.cwd, relPath)
+			const contextValidation = contextValidations.get(fullPath)

 			// Process approved files
 			try {
+				// Check context validation first
+				if (contextValidation && !contextValidation.canRead) {
+					updateFileResult(relPath, {
+						xmlContent: `<file><path>${relPath}</path>\n<error>${contextValidation.message}</error>\n</file>`,
+					})
+					continue
+				}
+
 				const [totalLines, isBinary] = await Promise.all([countFileLines(fullPath), isBinaryFile(fullPath)])

 				// Handle binary files (but allow specific file types that extractTextFromFile can handle)
@ -528,7 +590,7 @@ export async function readFileTool(
 					}
 				}

-				// Handle range reads (bypass maxReadFileLine)
+				// Handle range reads (bypass maxReadFileLine and context validation)
 				if (fileResult.lineRanges && fileResult.lineRanges.length > 0) {
 					const rangeResults: string[] = []
 					for (const range of fileResult.lineRanges) {
@ -545,12 +607,65 @@ export async function readFileTool(
 					continue
 				}

+				// Use context validation to determine how to read the file
+				if (contextValidation) {
+					if (contextValidation.suggestedAction === "read_partial") {
+						// Read only the safe number of lines
+						const maxLines = contextValidation.maxSafeLines
+						const content = addLineNumbers(await readLines(fullPath, maxLines - 1, 0))
+						const lineRangeAttr = ` lines="1-${maxLines}"`
+						let xmlInfo = `<content${lineRangeAttr}>\n${content}</content>\n`
+
+						// Add definitions if configured
+						if (fileReadingConfig.showDefinitionsOnTruncate) {
+							try {
+								const defResult = await parseSourceCodeDefinitionsForFile(
+									fullPath,
+									cline.rooIgnoreController,
+								)
+								if (defResult) {
+									xmlInfo += `<list_code_definition_names>${defResult}</list_code_definition_names>\n`
+								}
+							} catch (error) {
+								// Log but don't fail
+								console.warn(`[read_file] Could not parse definitions: ${error}`)
+							}
+						}
+
+						xmlInfo += `<notice>${contextValidation.message}</notice>\n`
+						updateFileResult(relPath, {
+							xmlContent: `<file><path>${relPath}</path>\n${xmlInfo}</file>`,
+						})
+						continue
+					} else if (contextValidation.suggestedAction === "read_chunks") {
+						// Read file in chunks (for now, just read the first chunk)
+						const chunkIterator = readFileInChunks(fullPath, fileReadingConfig.maxChunkLines, totalLines)
+						const firstChunk = await chunkIterator.next()
+
+						if (!firstChunk.done) {
+							const { content, startLine, endLine, isLastChunk } = firstChunk.value
+							const numberedContent = addLineNumbers(content, startLine)
+							const lineRangeAttr = ` lines="${startLine}-${endLine}"`
+							let xmlInfo = `<content${lineRangeAttr}>\n${numberedContent}</content>\n`
+
+							if (!isLastChunk) {
+								xmlInfo += `<notice>Showing chunk 1 (lines ${startLine}-${endLine} of ${totalLines}). File is being read in chunks to fit context limits.</notice>\n`
+							}
+
+							updateFileResult(relPath, {
+								xmlContent: `<file><path>${relPath}</path>\n${xmlInfo}</file>`,
+							})
+						}
+						continue
+					}
+				}
+
 				// Handle definitions-only mode
 				if (maxReadFileLine === 0) {
 					try {
 						const defResult = await parseSourceCodeDefinitionsForFile(fullPath, cline.rooIgnoreController)
 						if (defResult) {
-							let xmlInfo = `<notice>Showing only ${maxReadFileLine} of ${totalLines} total lines. Use line_range if you need to read more lines</notice>\n`
+							let xmlInfo = `<notice>Showing only definitions. Use line_range if you need to read actual content.</notice>\n`
 							updateFileResult(relPath, {
 								xmlContent: `<file><path>${relPath}</path>\n<list_code_definition_names>${defResult}</list_code_definition_names>\n${xmlInfo}</file>`,
 							})
@ -567,7 +682,7 @@ export async function readFileTool(
 					continue
 				}

-				// Handle files exceeding line threshold
+				// Handle files exceeding line threshold (legacy behavior for backward compatibility)
 				if (maxReadFileLine > 0 && totalLines > maxReadFileLine) {
 					const content = addLineNumbers(await readLines(fullPath, maxReadFileLine - 1, 0))
 					const lineRangeAttr = ` lines="1-${maxReadFileLine}"`