diff --git a/src/core/tools/__tests__/contextValidator.test.ts b/src/core/tools/__tests__/contextValidator.test.ts index e8497f6bcf..bb708f54e9 100644 --- a/src/core/tools/__tests__/contextValidator.test.ts +++ b/src/core/tools/__tests__/contextValidator.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest" import { validateFileSizeForContext } from "../contextValidator" import { Task } from "../../task/Task" import { promises as fs } from "fs" +import * as fsPromises from "fs/promises" import { readLines } from "../../../integrations/misc/read-lines" import * as sharedApi from "../../../shared/api" @@ -11,12 +12,17 @@ vi.mock("fs", () => ({ }, })) +vi.mock("fs/promises", () => ({ + stat: vi.fn(), +})) + vi.mock("../../../integrations/misc/read-lines", () => ({ readLines: vi.fn(), })) vi.mock("../../../shared/api", () => ({ getModelMaxOutputTokens: vi.fn(), + getFormatForProvider: vi.fn().mockReturnValue("anthropic"), })) describe("contextValidator", () => { @@ -25,6 +31,11 @@ describe("contextValidator", () => { beforeEach(() => { vi.clearAllMocks() + // Default file size mock (1MB - large enough to trigger validation) + vi.mocked(fs.stat).mockResolvedValue({ + size: 1024 * 1024, // 1MB + } as any) + // Mock Task instance mockTask = { api: { @@ -55,26 +66,27 @@ describe("contextValidator", () => { }) describe("validateFileSizeForContext", () => { - it("should apply 25% buffer to remaining context and read incrementally", async () => { + it("should apply 25% buffer to remaining context and use character-based reading", async () => { const mockStats = { size: 50000 } vi.mocked(fs.stat).mockResolvedValue(mockStats as any) - // Mock readLines to return content in batches - // Each batch is 100 lines, returning content that results in 1200 tokens per batch + // Mock readLines to return content in larger batches (500 lines) vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { const start = startLine ?? 0 - const end = endLine ?? 99 - const lines = end - start + 1 - return `test content line\n`.repeat(lines) + const end = endLine ?? 499 + const lines = [] + for (let i = start; i <= end; i++) { + // Each line is ~60 chars to simulate real code + lines.push(`const variable${i} = "test content line with enough characters";`) + } + return lines.join("\n") }) - // Mock token count - 12 tokens per line (1200 per 100-line batch) - let callCount = 0 + // Mock token count based on character count (using ~3 chars per token) mockTask.api.countTokens = vi.fn().mockImplementation(async (content) => { - callCount++ const text = content[0].text - const lines = text.split("\n").length - 1 - return lines * 12 // 12 tokens per line + // Approximate 3 characters per token + return Math.ceil(text.length / 3) }) const result = await validateFileSizeForContext( @@ -88,13 +100,14 @@ describe("contextValidator", () => { // Context window = 100k, current usage = 10k // Remaining = 90k // With 25% buffer on remaining: usable = 90k * 0.75 = 67.5k - // Reserved for response ~2k - // Available should be around 65.5k tokens - // File needs 12k tokens total (1000 lines * 12 tokens) + // Reserved for response = 4096 + // Available = 67.5k - 4096 ≈ 63.4k tokens + // Target limit = 63.4k * 0.9 ≈ 57k tokens + // File content: 1000 lines * 60 chars = 60k chars ≈ 20k tokens expect(result.shouldLimit).toBe(false) - // Verify readLines was called multiple times (incremental reading) - expect(readLines).toHaveBeenCalled() + // Should make fewer API calls with character-based approach + expect(mockTask.api.countTokens).toHaveBeenCalledTimes(1) // Verify the new calculation approach const remaining = 100000 - 10000 // 90k remaining @@ -106,19 +119,24 @@ describe("contextValidator", () => { const mockStats = { size: 50000 } vi.mocked(fs.stat).mockResolvedValue(mockStats as any) - // Mock readLines + // Mock readLines with larger batches vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { const start = startLine ?? 0 - const end = endLine ?? 99 - const lines = end - start + 1 - return `test content line\n`.repeat(lines) + const end = endLine ?? 499 + const lines = [] + for (let i = start; i <= end && i < 2000; i++) { + // Dense content - 150 chars per line + lines.push( + `const longVariable${i} = "This is a much longer line of content to simulate dense code with many characters per line";`, + ) + } + return lines.join("\n") }) - // Mock token count - 50 tokens per line + // Mock token count based on character count mockTask.api.countTokens = vi.fn().mockImplementation(async (content) => { const text = content[0].text - const lines = text.split("\n").length - 1 - return lines * 50 + return Math.ceil(text.length / 3) }) // Test with 50% context already used @@ -134,12 +152,16 @@ describe("contextValidator", () => { ) // With 50k remaining and 25% buffer: 50k * 0.75 = 37.5k usable - // Minus ~2k for response = ~35.5k available - // File needs 100k tokens (2000 lines * 50 tokens) + // Minus 4096 for response = ~33.4k available + // Target limit = 33.4k * 0.9 ≈ 30k tokens + // File content: 2000 lines * 150 chars = 300k chars ≈ 100k tokens // Should limit the file expect(result.shouldLimit).toBe(true) expect(result.safeMaxLines).toBeLessThan(2000) expect(result.reason).toContain("exceeds available context space") + + // Should use character-based approach with fewer API calls + expect(mockTask.api.countTokens).toHaveBeenCalled() }) it("should limit file when it exceeds available space with buffer", async () => { @@ -147,19 +169,26 @@ describe("contextValidator", () => { const mockStats = { size: 500000 } // Large file vi.mocked(fs.stat).mockResolvedValue(mockStats as any) - // Mock readLines to return content in batches + // Mock readLines to return dense content vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { const start = startLine ?? 0 - const end = endLine ?? 99 - const lines = end - start + 1 - return `large content line\n`.repeat(lines) + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < 10000; i++) { + // Very dense content - 300 chars per line + lines.push( + `const veryLongVariable${i} = "This is an extremely long line of content that simulates very dense code with many characters, such as minified JavaScript or long string literals that would consume many tokens";`, + ) + } + return lines.join("\n") }) - // Mock large token count - 100 tokens per line + // Mock token count based on character count + let apiCallCount = 0 mockTask.api.countTokens = vi.fn().mockImplementation(async (content) => { + apiCallCount++ const text = content[0].text - const lines = text.split("\n").length - 1 - return lines * 100 // 100 tokens per line + return Math.ceil(text.length / 3) }) const result = await validateFileSizeForContext( @@ -173,6 +202,9 @@ describe("contextValidator", () => { expect(result.safeMaxLines).toBeGreaterThan(0) expect(result.safeMaxLines).toBeLessThan(10000) // Should stop before reading all lines expect(result.reason).toContain("exceeds available context space") + + // Should make 1-2 API calls with character-based approach + expect(apiCallCount).toBeLessThanOrEqual(2) }) it("should handle very large files through incremental reading", async () => { @@ -180,19 +212,25 @@ describe("contextValidator", () => { const mockStats = { size: 60_000_000 } // 60MB file vi.mocked(fs.stat).mockResolvedValue(mockStats as any) - // Mock readLines to return content in batches + // Mock readLines to return dense content in larger batches vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { const start = startLine ?? 0 - const end = endLine ?? 99 - const lines = end - start + 1 - return `large file content line\n`.repeat(lines) + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < 100000; i++) { + // Very dense content - 300 chars per line + lines.push( + `const veryLongVariable${i} = "This is an extremely long line of content that simulates very dense code with many characters, such as minified JavaScript or long string literals that would consume many tokens";`, + ) + } + return lines.join("\n") }) - // Mock very high token count per line (simulating dense content) + // Mock token count based on character count mockTask.api.countTokens = vi.fn().mockImplementation(async (content) => { const text = content[0].text - const lines = text.split("\n").length - 1 - return lines * 200 // 200 tokens per line for very large file + // Return high token count to trigger limit + return Math.ceil(text.length / 2) // More tokens per char for dense content }) const result = await validateFileSizeForContext( @@ -205,8 +243,9 @@ describe("contextValidator", () => { expect(result.shouldLimit).toBe(true) // Should have attempted to read the file incrementally expect(readLines).toHaveBeenCalled() - // Should stop early due to token limits - expect(result.safeMaxLines).toBeLessThan(1000) + // With character-based approach, it reads more lines before hitting limit + expect(result.safeMaxLines).toBeGreaterThan(0) + expect(result.safeMaxLines).toBeLessThan(10000) // But still limited expect(result.reason).toContain("exceeds available context space") }) @@ -215,7 +254,9 @@ describe("contextValidator", () => { vi.mocked(fs.stat).mockResolvedValue(mockStats as any) // Mock readLines to fail - vi.mocked(readLines).mockRejectedValue(new Error("Read error")) + vi.mocked(readLines).mockImplementation(async () => { + throw new Error("Read error") + }) const result = await validateFileSizeForContext( "/test/problematic.ts", @@ -236,24 +277,26 @@ describe("contextValidator", () => { // Set very high context usage // With new calculation: 100k - 95k = 5k remaining // 5k * 0.75 = 3.75k usable - // Minus ~2k for response = ~1.75k available + // Minus 4096 for response = negative available space mockTask.getTokenUsage = vi.fn().mockReturnValue({ contextTokens: 95000, // 95% of context used }) - // Mock small token count + // Mock token count to exceed available space immediately mockTask.api.countTokens = vi.fn().mockImplementation(async (content) => { - const text = content[0].text - const lines = text.split("\n").length - 1 - return lines * 10 // 10 tokens per line + // Return tokens that exceed available space + return 5000 // More than available }) // Mock readLines vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { const start = startLine ?? 0 - const end = endLine ?? 99 - const lines = end - start + 1 - return `test line\n`.repeat(lines) + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < 500; i++) { + lines.push(`const var${i} = "test line";`) + } + return lines.join("\n") }) const result = await validateFileSizeForContext( @@ -264,10 +307,10 @@ describe("contextValidator", () => { ) expect(result.shouldLimit).toBe(true) - // With the new calculation using full model max tokens (4096), - // we have less space available, so we get the minimum 50 lines - expect(result.safeMaxLines).toBe(50) - expect(result.reason).toContain("Very limited context space") + // With the new implementation, when content exceeds limit even after cutback, + // it returns a very small number (10) as specified in the safety check + expect(result.safeMaxLines).toBe(10) + expect(result.reason).toContain("File too large for available context") }) it("should handle negative available space gracefully", async () => { @@ -277,11 +320,25 @@ describe("contextValidator", () => { // Set extremely high context usage // With 100k - 99k = 1k remaining // 1k * 0.75 = 750 tokens usable - // Minus 2k for response = negative available space + // Minus 4096 for response = negative available space mockTask.getTokenUsage = vi.fn().mockReturnValue({ contextTokens: 99000, // 99% of context used }) + // Mock token count to always exceed limit + mockTask.api.countTokens = vi.fn().mockResolvedValue(10000) + + // Mock readLines + vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { + const start = startLine ?? 0 + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < 500; i++) { + lines.push(`const var${i} = "test line";`) + } + return lines.join("\n") + }) + const result = await validateFileSizeForContext( "/test/smallfile.ts", 500, // totalLines @@ -290,10 +347,9 @@ describe("contextValidator", () => { ) expect(result.shouldLimit).toBe(true) - expect(result.safeMaxLines).toBe(50) // Should be limited to minimum useful lines - expect(result.reason).toContain("Very limited context space") - // With negative available space, readLines won't be called - expect(readLines).not.toHaveBeenCalled() + // When available space is negative, it returns minimal safe value + expect(result.safeMaxLines).toBe(10) // Minimal safe value from safety check + expect(result.reason).toContain("File too large for available context") }) it("should limit file when it is too large and would be truncated", async () => { @@ -306,11 +362,19 @@ describe("contextValidator", () => { contextTokens: 90000, // 90% of context used }) - // Mock token counting to simulate a large file - mockTask.api.countTokens = vi.fn().mockResolvedValue(1000) // Each batch is 1000 tokens + // Mock token counting to exceed limit on first call + mockTask.api.countTokens = vi.fn().mockResolvedValue(20000) // Exceeds available space - // Mock readLines to return some content - vi.mocked(readLines).mockResolvedValue("line content") + // Mock readLines to return content + vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { + const start = startLine ?? 0 + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < totalLines; i++) { + lines.push(`line content ${i} with enough characters`) + } + return lines.join("\n") + }) const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) @@ -332,15 +396,24 @@ describe("contextValidator", () => { }) // Mock token counting to quickly exceed limit - mockTask.api.countTokens = vi.fn().mockResolvedValue(500) // Each batch uses a lot of tokens + mockTask.api.countTokens = vi.fn().mockResolvedValue(5000) // Exceeds available space immediately - vi.mocked(readLines).mockResolvedValue("line content") + vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { + const start = startLine ?? 0 + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < totalLines; i++) { + lines.push(`line content ${i}`) + } + return lines.join("\n") + }) const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) expect(result.shouldLimit).toBe(true) - expect(result.reason).toContain("Very limited context space") - expect(result.reason).toContain("Consider using search_files or line_range") + // With the new implementation, when space is very limited and content exceeds, + // it returns the minimal safe value + expect(result.reason).toContain("File too large for available context") }) it("should not limit when file fits within context", async () => { @@ -351,7 +424,21 @@ describe("contextValidator", () => { // Mock low token usage mockTask.api.countTokens = vi.fn().mockResolvedValue(10) // Small token count per batch - vi.mocked(readLines).mockResolvedValue("line content") + vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { + const start = startLine ?? 0 + const end = endLine ?? 0 + + // For sampling phase (first 50 lines), return normal length content + if (start === 0 && end === 49) { + const lines = [] + for (let i = 0; i <= end; i++) { + lines.push(`line content with enough characters to avoid heuristic skip`) + } + return lines.join("\n") + } + + return "line content" + }) const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) @@ -377,4 +464,158 @@ describe("contextValidator", () => { expect(result.reason).toContain("Large file detected") }) }) + + describe("heuristic optimization", () => { + it("should skip validation for files with less than 100 lines", async () => { + const filePath = "/test/small-file.ts" + const totalLines = 50 // Less than 100 lines + const currentMaxReadFileLine = -1 + + // Mock file size to be small (3KB) + vi.mocked(fs.stat).mockResolvedValue({ + size: 3 * 1024, // 3KB + } as any) + + const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) + + // Should not limit small files + expect(result.shouldLimit).toBe(false) + expect(result.safeMaxLines).toBe(currentMaxReadFileLine) + // Should not call countTokens for small files + expect(mockTask.api.countTokens).not.toHaveBeenCalled() + // Should not even attempt to read the file + expect(readLines).not.toHaveBeenCalled() + }) + + it("should skip validation for small files", async () => { + const filePath = "/test/small-file.ts" + const totalLines = 500 + const currentMaxReadFileLine = -1 + + // Mock file size to be small (3KB) + vi.mocked(fsPromises.stat).mockResolvedValueOnce({ + size: 3 * 1024, // 3KB + } as any) + + const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) + + // Small files should skip validation + expect(result.shouldLimit).toBe(false) + expect(result.safeMaxLines).toBe(currentMaxReadFileLine) + // Should not call readLines for validation + expect(readLines).not.toHaveBeenCalled() + // Should not call countTokens + expect(mockTask.api.countTokens).not.toHaveBeenCalled() + // Verify fs.stat was called + expect(fsPromises.stat).toHaveBeenCalledWith(filePath) + }) + + it("should skip validation for moderate files when context is mostly empty", async () => { + const filePath = "/test/moderate-file.ts" + const totalLines = 2000 + const currentMaxReadFileLine = -1 + + // Mock file size to be moderate (80KB - below 100KB threshold) + vi.mocked(fsPromises.stat).mockResolvedValueOnce({ + size: 80 * 1024, // 80KB + } as any) + + // Mock context to be mostly empty (30% used - below 50% threshold) + mockTask.getTokenUsage = vi.fn().mockReturnValue({ + contextTokens: 30000, // 30% of 100000 + }) + + const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) + + // Should skip validation when context is mostly empty and file is moderate + expect(result.shouldLimit).toBe(false) + expect(result.safeMaxLines).toBe(currentMaxReadFileLine) + expect(readLines).not.toHaveBeenCalled() + expect(mockTask.api.countTokens).not.toHaveBeenCalled() + // Verify fs.stat was called + expect(fsPromises.stat).toHaveBeenCalledWith(filePath) + }) + + it("should perform validation for larger files", async () => { + const filePath = "/test/large-file.ts" + const totalLines = 1000 + const currentMaxReadFileLine = -1 + + // Mock file size to be large (1MB) + vi.mocked(fs.stat).mockResolvedValue({ + size: 1024 * 1024, // 1MB + } as any) + + // Mock readLines to return normal content + vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { + const start = startLine ?? 0 + const end = endLine ?? 0 + + // For sampling phase, return normal code lines + if (start === 0 && end === 49) { + const lines = [] + for (let i = 0; i <= 49; i++) { + lines.push(`const variable${i} = "This is a normal length line of code";`) + } + return lines.join("\n") + } + + // For actual reading + const lines = [] + for (let i = start; i <= end; i++) { + lines.push(`const variable${i} = "This is a normal length line of code";`) + } + return lines.join("\n") + }) + + // Mock token counting + mockTask.api.countTokens = vi.fn().mockResolvedValue(100) + + const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) + + // Should perform normal validation + expect(readLines).toHaveBeenCalled() + expect(mockTask.api.countTokens).toHaveBeenCalled() + }) + + it("should handle cutback strategy when content exceeds limit", async () => { + const filePath = "/test/cutback-test.ts" + const totalLines = 1000 + const currentMaxReadFileLine = -1 + + // Mock readLines to return content + vi.mocked(readLines).mockImplementation(async (path, endLine, startLine) => { + const start = startLine ?? 0 + const end = Math.min(endLine ?? 499, start + 499) + const lines = [] + for (let i = start; i <= end && i < totalLines; i++) { + lines.push(`const variable${i} = "This is a line of content";`) + } + return lines.join("\n") + }) + + // Mock token counting to exceed limit on first call, then succeed after cutback + let apiCallCount = 0 + mockTask.api.countTokens = vi.fn().mockImplementation(async (content) => { + apiCallCount++ + const text = content[0].text + const charCount = text.length + + // First call: return tokens that exceed the limit + if (apiCallCount === 1) { + return 70000 // Exceeds available tokens + } + // After cutback: return acceptable amount + return Math.ceil(charCount / 3) + }) + + const result = await validateFileSizeForContext(filePath, totalLines, currentMaxReadFileLine, mockTask) + + // Should apply cutback strategy + expect(mockTask.api.countTokens).toHaveBeenCalledTimes(2) // Initial + after cutback + expect(result.shouldLimit).toBe(true) + expect(result.safeMaxLines).toBeLessThan(totalLines) + expect(result.safeMaxLines).toBeGreaterThan(0) + }) + }) }) diff --git a/src/core/tools/__tests__/readFileTool.spec.ts b/src/core/tools/__tests__/readFileTool.spec.ts index c0f50e59d5..37d9f3cafb 100644 --- a/src/core/tools/__tests__/readFileTool.spec.ts +++ b/src/core/tools/__tests__/readFileTool.spec.ts @@ -12,6 +12,10 @@ import { readFileTool } from "../readFileTool" import { formatResponse } from "../../prompts/responses" import * as contextValidatorModule from "../contextValidator" +vi.mock("../../../i18n", () => ({ + t: vi.fn((key: string) => key), +})) + vi.mock("path", async () => { const originalPath = await vi.importActual("path") return { @@ -242,8 +246,7 @@ describe("read_file tool with maxReadFileLine setting", () => { expect(result).toContain(``) // Verify XML structure - expect(result).toContain("Showing only 0 of 5 total lines") - expect(result).toContain("") + expect(result).toContain("tools.readFile.showingOnlyLines") expect(result).toContain("") expect(result).toContain(sourceCodeDef.trim()) expect(result).toContain("") @@ -269,7 +272,7 @@ describe("read_file tool with maxReadFileLine setting", () => { expect(result).toContain(`${testFilePath}`) expect(result).toContain(``) expect(result).toContain(``) - expect(result).toContain("Showing only 3 of 5 total lines") + expect(result).toContain("tools.readFile.showingOnlyLines") }) }) @@ -565,11 +568,7 @@ describe("read_file tool XML output structure", () => { // Verify the result contains the inline instructions expect(result).toContain("") expect(result).toContain("File exceeds available context space") - expect(result).toContain("To read specific sections of this file, use the following format:") - expect(result).toContain("start-end") - expect(result).toContain("For example, to read lines 2001-3000:") - expect(result).toContain("2001-3000") - expect(result).toContain("large-file.ts") + expect(result).toContain("tools.readFile.contextLimitInstructions") }) it("should not show any special notice when file fits in context", async () => { diff --git a/src/core/tools/contextValidator.ts b/src/core/tools/contextValidator.ts index f8191dfafe..298aaf6e7d 100644 --- a/src/core/tools/contextValidator.ts +++ b/src/core/tools/contextValidator.ts @@ -1,6 +1,7 @@ import { Task } from "../task/Task" import { readLines } from "../../integrations/misc/read-lines" -import { getModelMaxOutputTokens } from "../../shared/api" +import { getModelMaxOutputTokens, getFormatForProvider } from "../../shared/api" +import * as fs from "fs/promises" /** * More aggressive buffer percentage specifically for file reading validation. @@ -15,9 +16,61 @@ export interface ContextValidationResult { reason?: string } +/** + * Determines if we should skip the expensive token-based validation. + * Returns true if we're confident the file can be read without limits. + * Prioritizes accuracy - only skips when very confident. + */ +async function shouldSkipValidation(filePath: string, totalLines: number, cline: Task): Promise { + // Heuristic 1: Very small files by line count (< 100 lines) + if (totalLines < 100) { + console.log( + `[shouldSkipValidation] Skipping validation for ${filePath} - small line count (${totalLines} lines)`, + ) + return true + } + + try { + // Get file size + const stats = await fs.stat(filePath) + const fileSizeBytes = stats.size + const fileSizeMB = fileSizeBytes / (1024 * 1024) + + // Heuristic 2: Very small files by size (< 5KB) - definitely safe to skip validation + if (fileSizeBytes < 5 * 1024) { + console.log( + `[shouldSkipValidation] Skipping validation for ${filePath} - small file size (${(fileSizeBytes / 1024).toFixed(1)}KB)`, + ) + return true + } + + // For larger files, check if context is mostly empty + const modelInfo = cline.api.getModel().info + const { contextTokens: currentContextTokens } = cline.getTokenUsage() + const contextWindow = modelInfo.contextWindow + + // Calculate context usage percentage + const contextUsagePercent = (currentContextTokens || 0) / contextWindow + + // Heuristic 3: If context is mostly empty (< 50% used) and file is not too big (< 100KB), + // we can skip validation as there's plenty of room + if (contextUsagePercent < 0.5 && fileSizeBytes < 100 * 1024) { + console.log( + `[validateFileSizeForContext] Skipping validation for ${filePath} - context mostly empty (${Math.round(contextUsagePercent * 100)}% used) and file is moderate size (${fileSizeMB.toFixed(2)}MB)`, + ) + return true + } + } catch (error) { + // If we can't check file size or context state, don't skip validation + console.warn(`[validateFileSizeForContext] Could not check file size or context state: ${error}`) + } + + return false +} + /** * Validates if a file can be safely read based on its size and current runtime context state. - * Reads lines incrementally and counts tokens as it goes, stopping when reaching the token limit. + * Uses a 2-phase approach: character-based estimation followed by actual token validation. * Returns a safe maxReadFileLine value to prevent context overflow. */ export async function validateFileSizeForContext( @@ -27,6 +80,11 @@ export async function validateFileSizeForContext( cline: Task, ): Promise { try { + // Check if we can skip validation + if (await shouldSkipValidation(filePath, totalLines, cline)) { + return { shouldLimit: false, safeMaxLines: currentMaxReadFileLine } + } + // Get actual runtime state from the task const modelInfo = cline.api.getModel().info const { contextTokens: currentContextTokens } = cline.getTokenUsage() @@ -37,22 +95,8 @@ export async function validateFileSizeForContext( const apiProvider = cline.apiConfiguration.apiProvider const settings = await cline.providerRef.deref()?.getState() - // Map apiProvider to the format expected by getModelMaxOutputTokens - let format: "anthropic" | "openai" | "gemini" | "openrouter" | undefined - if ( - apiProvider === "anthropic" || - apiProvider === "bedrock" || - apiProvider === "vertex" || - apiProvider === "claude-code" - ) { - format = "anthropic" - } else if (apiProvider === "openrouter") { - format = "openrouter" - } else if (apiProvider === "openai" || apiProvider === "openai-native") { - format = "openai" - } else if (apiProvider === "gemini" || apiProvider === "gemini-cli") { - format = "gemini" - } + // Use the centralized utility function to get the format + const format = getFormatForProvider(apiProvider) const maxResponseTokens = getModelMaxOutputTokens({ modelId, model: modelInfo, settings, format }) @@ -73,91 +117,151 @@ export async function validateFileSizeForContext( // Calculate available tokens for file content const availableTokensForFile = usableRemainingContext - reservedForResponse - // Now read lines incrementally and count tokens until we reach the limit - const BATCH_SIZE = 100 // Read 100 lines at a time - let currentLine = 0 - let totalTokensSoFar = 0 - let safeMaxLines = 0 - // Use 90% of available space to leave some margin const targetTokenLimit = Math.floor(availableTokensForFile * 0.9) - while (currentLine < totalLines && totalTokensSoFar < targetTokenLimit) { - // Calculate the end line for this batch - const batchEndLine = Math.min(currentLine + BATCH_SIZE - 1, totalLines - 1) + // Constants for the 2-phase approach + const CHARS_PER_TOKEN_ESTIMATE = 3 + const CUTBACK_PERCENTAGE = 0.2 // 20% reduction when over limit + const READ_BATCH_SIZE = 100 // Read 100 lines at a time for efficiency + + // Phase 1: Read content up to estimated safe character limit + const estimatedSafeChars = targetTokenLimit * CHARS_PER_TOKEN_ESTIMATE + + let accumulatedContent = "" + let currentLine = 0 + let lineToCharMap: Map = new Map() // Maps line number to character position + + // Track the start position of each line for potential cutback + lineToCharMap.set(0, 0) + + // Read until we hit our estimated character limit or EOF + while (currentLine < totalLines && accumulatedContent.length < estimatedSafeChars) { + const batchEndLine = Math.min(currentLine + READ_BATCH_SIZE - 1, totalLines - 1) try { - // Read the next batch of lines const batchContent = await readLines(filePath, batchEndLine, currentLine) - // Count tokens for this batch - const batchTokens = await cline.api.countTokens([{ type: "text", text: batchContent }]) - - // Check if adding this batch would exceed our limit - if (totalTokensSoFar + batchTokens > targetTokenLimit) { - // This batch would exceed the limit - // Try to find a more precise cutoff within this batch - if (batchEndLine - currentLine > 10) { - // Read smaller chunks to find a more precise cutoff - const FINE_BATCH_SIZE = 10 - let fineLine = currentLine - - while (fineLine <= batchEndLine && totalTokensSoFar < targetTokenLimit) { - const fineEndLine = Math.min(fineLine + FINE_BATCH_SIZE - 1, batchEndLine) - const fineContent = await readLines(filePath, fineEndLine, fineLine) - const fineTokens = await cline.api.countTokens([{ type: "text", text: fineContent }]) - - if (totalTokensSoFar + fineTokens > targetTokenLimit) { - // Even this fine batch exceeds the limit - break - } - - totalTokensSoFar += fineTokens - safeMaxLines = fineEndLine + 1 // Convert to 1-based line count - fineLine = fineEndLine + 1 - } + // Track line positions within the accumulated content + let localPos = 0 + for (let lineNum = currentLine; lineNum <= batchEndLine; lineNum++) { + const nextNewline = batchContent.indexOf("\n", localPos) + if (nextNewline !== -1) { + lineToCharMap.set(lineNum + 1, accumulatedContent.length + nextNewline + 1) + localPos = nextNewline + 1 } - // Stop processing more batches - break } - // Add this batch's tokens to our total - totalTokensSoFar += batchTokens - safeMaxLines = batchEndLine + 1 // Convert to 1-based line count + accumulatedContent += batchContent currentLine = batchEndLine + 1 } catch (error) { - // If we encounter an error reading a batch, stop here + console.warn(`[validateFileSizeForContext] Error reading batch: ${error}`) break } } + // Phase 2: Validate with actual API and cutback if needed + let finalContent = accumulatedContent + let finalLineCount = currentLine + let apiCallCount = 0 + const maxApiCalls = 5 // Safety limit to prevent infinite loops + + while (apiCallCount < maxApiCalls) { + apiCallCount++ + + // Make the actual API call to count tokens + const actualTokens = await cline.api.countTokens([{ type: "text", text: finalContent }]) + + console.log( + `[validateFileSizeForContext] API call ${apiCallCount}: ${actualTokens} tokens for ${finalContent.length} chars (${finalLineCount} lines)`, + ) + + if (actualTokens <= targetTokenLimit) { + // We're under the limit, we're done! + break + } + + // We're over the limit - cut back by 20% + const targetLength = Math.floor(finalContent.length * (1 - CUTBACK_PERCENTAGE)) + + // Find the line that gets us closest to the target length + let cutoffLine = 0 + for (const [lineNum, charPos] of lineToCharMap.entries()) { + if (charPos > targetLength) { + break + } + cutoffLine = lineNum + } + + // Ensure we don't cut back too far + if (cutoffLine < 10) { + console.warn( + `[validateFileSizeForContext] Cutback resulted in too few lines (${cutoffLine}), using minimum`, + ) + cutoffLine = Math.min(50, totalLines) + } + + // Get the character position for the cutoff line + const cutoffCharPos = lineToCharMap.get(cutoffLine) || 0 + finalContent = accumulatedContent.substring(0, cutoffCharPos) + finalLineCount = cutoffLine + + // Safety check + if (finalContent.length === 0) { + return { + shouldLimit: true, + safeMaxLines: 10, + reason: `File too large for available context. Even minimal content exceeds token limit.`, + } + } + } + + // Log final statistics + console.log( + `[validateFileSizeForContext] Final: ${finalLineCount} lines, ${finalContent.length} chars, ${apiCallCount} API calls`, + ) + // Ensure we provide at least a minimum useful amount const minUsefulLines = 50 - const finalSafeMaxLines = Math.max(minUsefulLines, safeMaxLines) + const finalSafeMaxLines = Math.max(minUsefulLines, finalLineCount) // If we read the entire file without exceeding the limit, no limitation needed - if (safeMaxLines >= totalLines) { + if (finalLineCount >= totalLines) { return { shouldLimit: false, safeMaxLines: currentMaxReadFileLine } } // If we couldn't read even the minimum useful lines - if (safeMaxLines < minUsefulLines) { + if (finalLineCount < minUsefulLines) { return { shouldLimit: true, safeMaxLines: finalSafeMaxLines, - reason: `Very limited context space. Could only safely read ${safeMaxLines} lines before exceeding token limit. Context: ${currentlyUsed}/${contextWindow} tokens used (${Math.round((currentlyUsed / contextWindow) * 100)}%). Limited to ${finalSafeMaxLines} lines. Consider using search_files or line_range for specific sections.`, + reason: `Very limited context space. Could only safely read ${finalLineCount} lines before exceeding token limit. Context: ${currentlyUsed}/${contextWindow} tokens used (${Math.round((currentlyUsed / contextWindow) * 100)}%). Limited to ${finalSafeMaxLines} lines. Consider using search_files or line_range for specific sections.`, } } return { shouldLimit: true, safeMaxLines: finalSafeMaxLines, - reason: `File exceeds available context space. Safely read ${finalSafeMaxLines} lines (${totalTokensSoFar} tokens) out of ${totalLines} total lines. Context usage: ${currentlyUsed}/${contextWindow} tokens (${Math.round((currentlyUsed / contextWindow) * 100)}%). Use line_range to read specific sections.`, + reason: `File exceeds available context space. Safely read ${finalSafeMaxLines} lines out of ${totalLines} total lines. Context usage: ${currentlyUsed}/${contextWindow} tokens (${Math.round((currentlyUsed / contextWindow) * 100)}%). Use line_range to read specific sections.`, } } catch (error) { // If we can't get runtime state, fall back to conservative estimation console.warn(`[validateFileSizeForContext] Error accessing runtime state: ${error}`) + // In error cases, we can't check context state, so use simple file size heuristics + try { + const stats = await fs.stat(filePath) + const fileSizeBytes = stats.size + + // Very small files are safe + if (fileSizeBytes < 5 * 1024) { + return { shouldLimit: false, safeMaxLines: currentMaxReadFileLine } + } + } catch (statError) { + // If we can't even stat the file, proceed with conservative defaults + console.warn(`[validateFileSizeForContext] Could not stat file: ${statError}`) + } + if (totalLines > 10000) { return { shouldLimit: true, diff --git a/src/shared/__tests__/providerFormat.spec.ts b/src/shared/__tests__/providerFormat.spec.ts new file mode 100644 index 0000000000..a23cf39e72 --- /dev/null +++ b/src/shared/__tests__/providerFormat.spec.ts @@ -0,0 +1,85 @@ +import { describe, it, expect } from "vitest" +import { getFormatForProvider, isVertexAnthropicModel } from "../api" +import { ProviderName } from "@roo-code/types" + +describe("providerFormat", () => { + describe("getFormatForProvider", () => { + it("should return 'anthropic' for Anthropic-based providers", () => { + const anthropicProviders: ProviderName[] = ["anthropic", "bedrock", "vertex", "claude-code", "requesty"] + + anthropicProviders.forEach((provider) => { + expect(getFormatForProvider(provider)).toBe("anthropic") + }) + }) + + it("should return 'openai' for OpenAI-based providers", () => { + const openaiProviders: ProviderName[] = [ + "openai", + "openai-native", + "deepseek", + "moonshot", + "xai", + "groq", + "chutes", + "mistral", + "ollama", + "lmstudio", + "litellm", + "huggingface", + "glama", + "unbound", + "vscode-lm", + "human-relay", + "fake-ai", + ] + + openaiProviders.forEach((provider) => { + expect(getFormatForProvider(provider)).toBe("openai") + }) + }) + + it("should return 'gemini' for Gemini-based providers", () => { + const geminiProviders: ProviderName[] = ["gemini", "gemini-cli"] + + geminiProviders.forEach((provider) => { + expect(getFormatForProvider(provider)).toBe("gemini") + }) + }) + + it("should return 'openrouter' for OpenRouter provider", () => { + expect(getFormatForProvider("openrouter")).toBe("openrouter") + }) + + it("should return undefined for undefined provider", () => { + expect(getFormatForProvider(undefined)).toBeUndefined() + }) + + it("should return undefined for unknown providers", () => { + // Test with a provider that doesn't exist in the switch statement + // by casting to bypass TypeScript type checking + expect(getFormatForProvider("unknown-provider" as ProviderName)).toBeUndefined() + }) + }) + + describe("isVertexAnthropicModel", () => { + it("should return true for Claude models", () => { + expect(isVertexAnthropicModel("claude-3-opus")).toBe(true) + expect(isVertexAnthropicModel("claude-3-sonnet")).toBe(true) + expect(isVertexAnthropicModel("claude-3-haiku")).toBe(true) + expect(isVertexAnthropicModel("CLAUDE-3-OPUS")).toBe(true) // Case insensitive + expect(isVertexAnthropicModel("anthropic.claude-v2")).toBe(true) + }) + + it("should return false for non-Claude models", () => { + expect(isVertexAnthropicModel("gemini-pro")).toBe(false) + expect(isVertexAnthropicModel("gemini-1.5-pro")).toBe(false) + expect(isVertexAnthropicModel("palm-2")).toBe(false) + expect(isVertexAnthropicModel("gpt-4")).toBe(false) + }) + + it("should return false for undefined or empty model ID", () => { + expect(isVertexAnthropicModel(undefined)).toBe(false) + expect(isVertexAnthropicModel("")).toBe(false) + }) + }) +}) diff --git a/src/shared/api.ts b/src/shared/api.ts index 8cbfc72133..05ccde74c6 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -1,10 +1,82 @@ import { type ModelInfo, type ProviderSettings, + type ProviderName, ANTHROPIC_DEFAULT_MAX_TOKENS, CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS, } from "@roo-code/types" +// Provider Format Mapping + +/** + * Maps API provider names to their corresponding format for model parameter handling. + * This centralizes the provider-to-format mapping logic used across the codebase. + * + * @param apiProvider - The API provider name + * @returns The format string used by getModelParams and getModelMaxOutputTokens, or undefined if not mapped + */ +export function getFormatForProvider( + apiProvider: ProviderName | undefined, +): "anthropic" | "openai" | "gemini" | "openrouter" | undefined { + if (!apiProvider) { + return undefined + } + + switch (apiProvider) { + // Anthropic-based providers + case "anthropic": + case "bedrock": + case "vertex": // Note: vertex can use either anthropic or gemini format depending on the model + case "claude-code": + case "requesty": // Uses anthropic format based on code analysis + return "anthropic" + + // OpenAI-based providers + case "openai": + case "openai-native": + case "deepseek": + case "moonshot": + case "xai": + case "groq": + case "chutes": + case "mistral": + case "ollama": + case "lmstudio": + case "litellm": + case "huggingface": + case "glama": + case "unbound": + case "vscode-lm": + case "human-relay": + case "fake-ai": + return "openai" + + // Gemini-based providers + case "gemini": + case "gemini-cli": + return "gemini" + + // OpenRouter + case "openrouter": + return "openrouter" + + // Providers that don't have a specific format mapping + default: + return undefined + } +} + +/** + * Special case: Vertex provider can use either anthropic or gemini format depending on the model. + * This function checks if a vertex model should use anthropic format. + * + * @param modelId - The model ID to check + * @returns true if the model should use anthropic format + */ +export function isVertexAnthropicModel(modelId?: string): boolean { + return modelId?.toLowerCase().includes("claude") ?? false +} + // ApiHandlerOptions export type ApiHandlerOptions = Omit @@ -70,14 +142,17 @@ export const getModelMaxOutputTokens = ({ return settings.claudeCodeMaxOutputTokens || CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS } + // If format is not provided, derive it from the provider settings + const effectiveFormat = format ?? getFormatForProvider(settings?.apiProvider) + if (shouldUseReasoningBudget({ model, settings })) { return settings?.modelMaxTokens || DEFAULT_HYBRID_REASONING_MODEL_MAX_TOKENS } const isAnthropicContext = modelId.includes("claude") || - format === "anthropic" || - (format === "openrouter" && modelId.startsWith("anthropic/")) + effectiveFormat === "anthropic" || + (effectiveFormat === "openrouter" && modelId.startsWith("anthropic/")) // For "Hybrid" reasoning models, discard the model's actual maxTokens for Anthropic contexts if (model.supportsReasoningBudget && isAnthropicContext) { @@ -95,7 +170,7 @@ export const getModelMaxOutputTokens = ({ } // For non-Anthropic formats without explicit maxTokens, return undefined - if (format) { + if (effectiveFormat) { return undefined }