From 0a70e046e77b18f6e6f2dcdb24dd1f433981709a Mon Sep 17 00:00:00 2001 From: Hannes Rudolph Date: Sat, 24 Jan 2026 13:44:20 -0700 Subject: [PATCH] feat: add read_command_output tool for retrieving truncated command output Implements a new tool that allows the LLM to retrieve full command output when execute_command produces output exceeding the preview threshold. Key components: - ReadCommandOutputTool: Reads persisted output with search/pagination - OutputInterceptor: Intercepts and persists large command outputs to disk - Terminal settings UI: Configuration for output interception behavior - Type definitions for output interception settings The tool supports: - Reading full output beyond the truncated preview - Search/filtering with regex patterns (like grep) - Pagination through large outputs using offset/limit Includes comprehensive tests for ReadCommandOutputTool and OutputInterceptor. --- packages/types/src/global-settings.ts | 16 +- packages/types/src/vscode-extension-host.ts | 2 + pnpm-lock.yaml | 11 + .../tools/native-tools/read_command_output.ts | 26 +- src/core/tools/ExecuteCommandTool.ts | 9 +- src/core/tools/ReadCommandOutputTool.ts | 158 ++-------- .../__tests__/ReadCommandOutputTool.test.ts | 50 ++- .../terminal/OutputInterceptor.ts | 244 +++------------ .../__tests__/OutputInterceptor.test.ts | 285 +++++++----------- src/integrations/terminal/index.ts | 57 ++++ src/package.json | 1 + .../src/components/settings/SettingsView.tsx | 3 + .../components/settings/TerminalSettings.tsx | 25 ++ .../src/context/ExtensionStateContext.tsx | 8 + webview-ui/src/i18n/locales/en/settings.json | 6 +- 15 files changed, 346 insertions(+), 555 deletions(-) create mode 100644 src/integrations/terminal/index.ts diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index d57ec616ff..65adaefb9d 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -29,9 +29,9 @@ export const DEFAULT_WRITE_DELAY_MS = 1000 * the LLM decides to retrieve more via `read_command_output`. Larger previews * mean more immediate context but consume more of the context window. * - * - `small`: 5KB preview - Best for long-running commands with verbose output - * - `medium`: 10KB preview - Balanced default for most use cases - * - `large`: 20KB preview - Best when commands produce critical info early + * - `small`: 2KB preview - Best for long-running commands with verbose output + * - `medium`: 4KB preview - Balanced default for most use cases + * - `large`: 8KB preview - Best when commands produce critical info early * * @see OutputInterceptor - Uses this setting to determine when to spill to disk * @see PersistedCommandOutput - Contains the resulting preview and artifact reference @@ -46,14 +46,14 @@ export type TerminalOutputPreviewSize = "small" | "medium" | "large" * to disk and made available via the `read_command_output` tool. */ export const TERMINAL_PREVIEW_BYTES: Record = { - small: 5 * 1024, // 5KB - medium: 10 * 1024, // 10KB - large: 20 * 1024, // 20KB + small: 2048, // 2KB + medium: 4096, // 4KB + large: 8192, // 8KB } /** * Default terminal output preview size. - * The "medium" (10KB) setting provides a good balance between immediate + * The "medium" (4KB) setting provides a good balance between immediate * visibility and context window conservation for most use cases. */ export const DEFAULT_TERMINAL_OUTPUT_PREVIEW_SIZE: TerminalOutputPreviewSize = "medium" @@ -176,6 +176,8 @@ export const globalSettingsSchema = z.object({ maxImageFileSize: z.number().optional(), maxTotalImageSize: z.number().optional(), + terminalOutputLineLimit: z.number().optional(), + terminalOutputCharacterLimit: z.number().optional(), terminalOutputPreviewSize: z.enum(["small", "medium", "large"]).optional(), terminalShellIntegrationTimeout: z.number().optional(), terminalShellIntegrationDisabled: z.boolean().optional(), diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index 7ae89e8777..ce0d337d91 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -302,6 +302,8 @@ export type ExtensionState = Pick< | "soundEnabled" | "soundVolume" | "maxConcurrentFileReads" + | "terminalOutputLineLimit" + | "terminalOutputCharacterLimit" | "terminalOutputPreviewSize" | "terminalShellIntegrationTimeout" | "terminalShellIntegrationDisabled" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b8ca01240b..b0a904f457 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1017,6 +1017,9 @@ importers: '@types/glob': specifier: ^8.1.0 version: 8.1.0 + '@types/json-stream-stringify': + specifier: ^2.0.4 + version: 2.0.4 '@types/lodash.debounce': specifier: ^4.0.9 version: 4.0.9 @@ -4302,6 +4305,10 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/json-stream-stringify@2.0.4': + resolution: {integrity: sha512-xSFsVnoQ8Y/7BiVF3/fEIwRx9RoGzssDKVwhy1g23wkA4GAmA3v8lsl6CxsmUD6vf4EiRd+J0ULLkMbAWRSsgQ==} + deprecated: This is a stub types definition. json-stream-stringify provides its own type definitions, so you do not need this installed. + '@types/katex@0.16.7': resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} @@ -14316,6 +14323,10 @@ snapshots: '@types/json-schema@7.0.15': {} + '@types/json-stream-stringify@2.0.4': + dependencies: + json-stream-stringify: 3.1.6 + '@types/katex@0.16.7': {} '@types/lodash.debounce@4.0.9': diff --git a/src/core/prompts/tools/native-tools/read_command_output.ts b/src/core/prompts/tools/native-tools/read_command_output.ts index 44c069be1e..0bab31be9e 100644 --- a/src/core/prompts/tools/native-tools/read_command_output.ts +++ b/src/core/prompts/tools/native-tools/read_command_output.ts @@ -20,15 +20,15 @@ The tool supports two modes: Parameters: - artifact_id: (required) The artifact filename from the truncated output message (e.g., "cmd-1706119234567.txt") -- search: (optional) Pattern to filter lines. Supports regex or literal strings. Case-insensitive. **Omit this parameter entirely if you don't need to filter - do not pass null or empty string.** +- search: (optional) Pattern to filter lines. Supports regex or literal strings. Case-insensitive. - offset: (optional) Byte offset to start reading from. Default: 0. Use for pagination. -- limit: (optional) Maximum bytes to return. Default: 40KB. +- limit: (optional) Maximum bytes to return. Default: 32KB. Example: Reading truncated command output { "artifact_id": "cmd-1706119234567.txt" } -Example: Reading with pagination (after first 40KB) -{ "artifact_id": "cmd-1706119234567.txt", "offset": 40960 } +Example: Reading with pagination (after first 32KB) +{ "artifact_id": "cmd-1706119234567.txt", "offset": 32768 } Example: Searching for errors in build output { "artifact_id": "cmd-1706119234567.txt", "search": "error|failed|Error" } @@ -38,22 +38,18 @@ Example: Finding specific test failures const ARTIFACT_ID_DESCRIPTION = `The artifact filename from the truncated command output (e.g., "cmd-1706119234567.txt")` -const SEARCH_DESCRIPTION = `Optional regex or literal pattern to filter lines (case-insensitive, like grep). Omit this parameter if not searching - do not pass null or empty string.` +const SEARCH_DESCRIPTION = `Optional regex or literal pattern to filter lines (case-insensitive, like grep)` const OFFSET_DESCRIPTION = `Byte offset to start reading from (default: 0, for pagination)` -const LIMIT_DESCRIPTION = `Maximum bytes to return (default: 40KB)` +const LIMIT_DESCRIPTION = `Maximum bytes to return (default: 32KB)` export default { type: "function", function: { name: "read_command_output", description: READ_COMMAND_OUTPUT_DESCRIPTION, - // Note: strict mode is intentionally disabled for this tool. - // With strict: true, OpenAI requires ALL properties to be in the 'required' array, - // which forces the LLM to always provide explicit values (even null) for optional params. - // This creates verbose tool calls and poor UX. By disabling strict mode, the LLM can - // omit optional parameters entirely, making the tool easier to use. + strict: true, parameters: { type: "object", properties: { @@ -62,19 +58,19 @@ export default { description: ARTIFACT_ID_DESCRIPTION, }, search: { - type: "string", + type: ["string", "null"], description: SEARCH_DESCRIPTION, }, offset: { - type: "number", + type: ["number", "null"], description: OFFSET_DESCRIPTION, }, limit: { - type: "number", + type: ["number", "null"], description: LIMIT_DESCRIPTION, }, }, - required: ["artifact_id"], + required: ["artifact_id", "search", "offset", "limit"], additionalProperties: false, }, }, diff --git a/src/core/tools/ExecuteCommandTool.ts b/src/core/tools/ExecuteCommandTool.ts index fca3cf7a31..28957fc868 100644 --- a/src/core/tools/ExecuteCommandTool.ts +++ b/src/core/tools/ExecuteCommandTool.ts @@ -4,7 +4,12 @@ import * as vscode from "vscode" import delay from "delay" -import { CommandExecutionStatus, DEFAULT_TERMINAL_OUTPUT_PREVIEW_SIZE, PersistedCommandOutput } from "@roo-code/types" +import { + CommandExecutionStatus, + DEFAULT_TERMINAL_OUTPUT_CHARACTER_LIMIT, + DEFAULT_TERMINAL_OUTPUT_PREVIEW_SIZE, + PersistedCommandOutput, +} from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { Task } from "../task/Task" @@ -196,6 +201,7 @@ export async function executeCommandInTerminal( const providerState = await provider?.getState() const terminalOutputPreviewSize = providerState?.terminalOutputPreviewSize ?? DEFAULT_TERMINAL_OUTPUT_PREVIEW_SIZE + const terminalCompressProgressBar = providerState?.terminalCompressProgressBar ?? true interceptor = new OutputInterceptor({ executionId, @@ -203,6 +209,7 @@ export async function executeCommandInTerminal( command, storageDir, previewSize: terminalOutputPreviewSize, + compressProgressBar: terminalCompressProgressBar, }) } diff --git a/src/core/tools/ReadCommandOutputTool.ts b/src/core/tools/ReadCommandOutputTool.ts index 9d3bbd35dd..d81352c30a 100644 --- a/src/core/tools/ReadCommandOutputTool.ts +++ b/src/core/tools/ReadCommandOutputTool.ts @@ -6,8 +6,8 @@ import { getTaskDirectoryPath } from "../../utils/storage" import { BaseTool, ToolCallbacks } from "./BaseTool" -/** Default byte limit for read operations (40KB) */ -const DEFAULT_LIMIT = 40 * 1024 // 40KB default limit +/** Default byte limit for read operations (32KB) */ +const DEFAULT_LIMIT = 32 * 1024 // 32KB default limit /** * Parameters accepted by the read_command_output tool. @@ -159,38 +159,15 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { } let result: string - let readStart = 0 - let readEnd = 0 - let matchCount: number | undefined if (search) { // Search mode: filter lines matching the pattern - const searchResult = await this.searchInArtifact(artifactPath, search, totalSize, limit) - result = searchResult.content - matchCount = searchResult.matchCount - // For search, we're scanning the whole file - readStart = 0 - readEnd = totalSize + result = await this.searchInArtifact(artifactPath, search, totalSize, limit) } else { // Normal read mode with offset/limit result = await this.readArtifact(artifactPath, offset, limit, totalSize) - // Calculate actual read range - readStart = offset - readEnd = Math.min(offset + limit, totalSize) } - // Report to UI that we read command output - await task.say( - "tool", - JSON.stringify({ - tool: "readCommandOutput", - readStart, - readEnd, - totalBytes: totalSize, - ...(search && { searchPattern: search, matchCount }), - }), - ) - task.consecutiveMistakeCount = 0 pushToolResult(result) } catch (error) { @@ -246,10 +223,14 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { const { bytesRead } = await fileHandle.read(buffer, 0, buffer.length, offset) const content = buffer.slice(0, bytesRead).toString("utf8") - // Calculate line numbers based on offset using chunked reading to avoid large allocations + // Calculate line numbers based on offset let startLineNumber = 1 if (offset > 0) { - startLineNumber = await this.countNewlinesBeforeOffset(fileHandle, offset) + // Count newlines before offset to determine starting line number + const prefixBuffer = Buffer.alloc(offset) + await fileHandle.read(prefixBuffer, 0, offset, 0) + const prefix = prefixBuffer.toString("utf8") + startLineNumber = (prefix.match(/\n/g) || []).length + 1 } const endOffset = offset + bytesRead @@ -272,14 +253,10 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { } /** - * Search artifact content for lines matching a pattern using chunked streaming. + * Search artifact content for lines matching a pattern. * - * Performs grep-like searching through the artifact file using bounded memory. - * Instead of loading the entire file into memory, this reads in fixed-size chunks - * and processes lines as they are encountered. This keeps memory usage predictable - * even for very large command outputs (e.g., 100MB+ build logs). - * - * The pattern is treated as a case-insensitive regex. If the pattern is invalid + * Performs grep-like searching through the artifact file. The pattern + * is treated as a case-insensitive regex. If the pattern is invalid * regex syntax, it's escaped and treated as a literal string. * * Results are limited by the byte limit to prevent excessive output. @@ -296,8 +273,10 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { pattern: string, totalSize: number, limit: number, - ): Promise<{ content: string; matchCount: number }> { - const CHUNK_SIZE = 64 * 1024 // 64KB chunks for bounded memory + ): Promise { + // Read the entire file for search (we need all content to search) + const content = await fs.readFile(artifactPath, "utf8") + const lines = content.split("\n") // Create case-insensitive regex for search let regex: RegExp @@ -308,89 +287,45 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { regex = new RegExp(this.escapeRegExp(pattern), "i") } - const fileHandle = await fs.open(artifactPath, "r") + // Find matching lines with their line numbers const matches: Array<{ lineNumber: number; content: string }> = [] let totalMatchBytes = 0 - let lineNumber = 0 - let partialLine = "" // Holds incomplete line from previous chunk - let bytesRead = 0 - let hitLimit = false - try { - while (bytesRead < totalSize && !hitLimit) { - const chunkSize = Math.min(CHUNK_SIZE, totalSize - bytesRead) - const buffer = Buffer.alloc(chunkSize) - const result = await fileHandle.read(buffer, 0, chunkSize, bytesRead) + for (let i = 0; i < lines.length; i++) { + if (regex.test(lines[i])) { + const lineContent = lines[i] + const lineBytes = Buffer.byteLength(lineContent, "utf8") - if (result.bytesRead === 0) { + // Stop if we've exceeded the byte limit + if (totalMatchBytes + lineBytes > limit) { break } - const chunk = buffer.slice(0, result.bytesRead).toString("utf8") - bytesRead += result.bytesRead - - // Combine with partial line from previous chunk - const combined = partialLine + chunk - const lines = combined.split("\n") - - // Last element may be incomplete (no trailing newline), save for next iteration - partialLine = lines.pop() ?? "" - - // Process complete lines - for (const line of lines) { - lineNumber++ - - if (regex.test(line)) { - const lineBytes = Buffer.byteLength(line, "utf8") - - // Stop if we've exceeded the byte limit - if (totalMatchBytes + lineBytes > limit) { - hitLimit = true - break - } - - matches.push({ lineNumber, content: line }) - totalMatchBytes += lineBytes - } - } + matches.push({ lineNumber: i + 1, content: lineContent }) + totalMatchBytes += lineBytes } - - // Process any remaining partial line at end of file - if (!hitLimit && partialLine.length > 0) { - lineNumber++ - if (regex.test(partialLine)) { - const lineBytes = Buffer.byteLength(partialLine, "utf8") - if (totalMatchBytes + lineBytes <= limit) { - matches.push({ lineNumber, content: partialLine }) - } - } - } - } finally { - await fileHandle.close() } const artifactId = path.basename(artifactPath) if (matches.length === 0) { - const content = [ + return [ `[Command Output: ${artifactId}] (search: "${pattern}")`, `Total size: ${this.formatBytes(totalSize)}`, "", "No matches found for the search pattern.", ].join("\n") - return { content, matchCount: 0 } } // Format matches with line numbers const matchedLines = matches.map((m) => `${String(m.lineNumber).padStart(5)} | ${m.content}`).join("\n") - const content = [ + return [ `[Command Output: ${artifactId}] (search: "${pattern}")`, `Total matches: ${matches.length} | Showing first ${matches.length}`, "", matchedLines, ].join("\n") - return { content, matchCount: matches.length } } /** @@ -439,45 +374,6 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { private escapeRegExp(string: string): string { return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") } - - /** - * Count newlines before a given byte offset using fixed-size chunks. - * - * This avoids allocating a buffer of size `offset` which could be huge - * for large files. Instead, we read in 64KB chunks and count newlines. - * - * @param fileHandle - Open file handle for reading - * @param offset - The byte offset to count newlines up to - * @returns The line number at the given offset (1-indexed) - * @private - */ - private async countNewlinesBeforeOffset(fileHandle: fs.FileHandle, offset: number): Promise { - const CHUNK_SIZE = 64 * 1024 // 64KB chunks - let newlineCount = 0 - let bytesRead = 0 - - while (bytesRead < offset) { - const chunkSize = Math.min(CHUNK_SIZE, offset - bytesRead) - const buffer = Buffer.alloc(chunkSize) - const result = await fileHandle.read(buffer, 0, chunkSize, bytesRead) - - if (result.bytesRead === 0) { - break - } - - // Count newlines in this chunk - for (let i = 0; i < result.bytesRead; i++) { - if (buffer[i] === 0x0a) { - // '\n' - newlineCount++ - } - } - - bytesRead += result.bytesRead - } - - return newlineCount + 1 // Line numbers are 1-indexed - } } /** Singleton instance of the ReadCommandOutputTool */ diff --git a/src/core/tools/__tests__/ReadCommandOutputTool.test.ts b/src/core/tools/__tests__/ReadCommandOutputTool.test.ts index 11f85e67c0..a2e3147cc6 100644 --- a/src/core/tools/__tests__/ReadCommandOutputTool.test.ts +++ b/src/core/tools/__tests__/ReadCommandOutputTool.test.ts @@ -159,16 +159,16 @@ describe("ReadCommandOutputTool", () => { }) describe("Pagination (offset/limit)", () => { - it("should use default limit of 40KB", async () => { + it("should use default limit of 32KB", async () => { const artifactId = "cmd-1706119234567.txt" const largeContent = "x".repeat(50 * 1024) // 50KB const fileSize = Buffer.byteLength(largeContent, "utf8") vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) - // Mock read to return only up to default limit (40KB) + // Mock read to return only up to default limit (32KB) mockFileHandle.read.mockImplementation((buf: Buffer) => { - const defaultLimit = 40 * 1024 + const defaultLimit = 32 * 1024 const bytesToRead = Math.min(buf.length, defaultLimit) buf.write(largeContent.slice(0, bytesToRead)) return Promise.resolve({ bytesRead: bytesToRead }) @@ -276,31 +276,13 @@ describe("ReadCommandOutputTool", () => { }) describe("Search filtering", () => { - // Helper to setup file handle mock for search (which now uses streaming) - const setupSearchMock = (content: string) => { - const buffer = Buffer.from(content) - const fileSize = buffer.length - vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) - - // Mock streaming read - return entire content in one chunk (simulates small file) - mockFileHandle.read.mockImplementation( - (buf: Buffer, bufOffset: number, length: number, position: number | null) => { - const pos = position ?? 0 - if (pos >= fileSize) { - return Promise.resolve({ bytesRead: 0 }) - } - const bytesToRead = Math.min(length, fileSize - pos) - buffer.copy(buf, 0, pos, pos + bytesToRead) - return Promise.resolve({ bytesRead: bytesToRead }) - }, - ) - } - it("should filter lines matching pattern", async () => { const artifactId = "cmd-1706119234567.txt" const content = "Line 1: error occurred\nLine 2: success\nLine 3: error found\nLine 4: complete\n" + const fileSize = Buffer.byteLength(content, "utf8") - setupSearchMock(content) + vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) + vi.mocked(fs.readFile).mockResolvedValue(content) await tool.execute({ artifact_id: artifactId, search: "error" }, mockTask, mockCallbacks) @@ -314,8 +296,10 @@ describe("ReadCommandOutputTool", () => { it("should use case-insensitive matching", async () => { const artifactId = "cmd-1706119234567.txt" const content = "ERROR: Something bad\nwarning: minor issue\nERROR: Another problem\n" + const fileSize = Buffer.byteLength(content, "utf8") - setupSearchMock(content) + vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) + vi.mocked(fs.readFile).mockResolvedValue(content) await tool.execute({ artifact_id: artifactId, search: "error" }, mockTask, mockCallbacks) @@ -327,8 +311,10 @@ describe("ReadCommandOutputTool", () => { it("should show match count and line numbers", async () => { const artifactId = "cmd-1706119234567.txt" const content = "Line 1\nError on line 2\nLine 3\nError on line 4\n" + const fileSize = Buffer.byteLength(content, "utf8") - setupSearchMock(content) + vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) + vi.mocked(fs.readFile).mockResolvedValue(content) await tool.execute({ artifact_id: artifactId, search: "Error" }, mockTask, mockCallbacks) @@ -341,8 +327,10 @@ describe("ReadCommandOutputTool", () => { it("should handle empty search results gracefully", async () => { const artifactId = "cmd-1706119234567.txt" const content = "Line 1\nLine 2\nLine 3\n" + const fileSize = Buffer.byteLength(content, "utf8") - setupSearchMock(content) + vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) + vi.mocked(fs.readFile).mockResolvedValue(content) await tool.execute({ artifact_id: artifactId, search: "NOTFOUND" }, mockTask, mockCallbacks) @@ -353,8 +341,10 @@ describe("ReadCommandOutputTool", () => { it("should handle regex patterns in search", async () => { const artifactId = "cmd-1706119234567.txt" const content = "test123\ntest456\nabc789\ntest000\n" + const fileSize = Buffer.byteLength(content, "utf8") - setupSearchMock(content) + vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) + vi.mocked(fs.readFile).mockResolvedValue(content) await tool.execute({ artifact_id: artifactId, search: "test\\d+" }, mockTask, mockCallbacks) @@ -368,8 +358,10 @@ describe("ReadCommandOutputTool", () => { it("should handle invalid regex patterns by treating as literal", async () => { const artifactId = "cmd-1706119234567.txt" const content = "Line with [brackets]\nLine without\n" + const fileSize = Buffer.byteLength(content, "utf8") - setupSearchMock(content) + vi.mocked(fs.stat).mockResolvedValue({ size: fileSize } as any) + vi.mocked(fs.readFile).mockResolvedValue(content) // Invalid regex but valid as literal string await tool.execute({ artifact_id: artifactId, search: "[" }, mockTask, mockCallbacks) diff --git a/src/integrations/terminal/OutputInterceptor.ts b/src/integrations/terminal/OutputInterceptor.ts index d1725c6426..c9e984ff69 100644 --- a/src/integrations/terminal/OutputInterceptor.ts +++ b/src/integrations/terminal/OutputInterceptor.ts @@ -3,6 +3,8 @@ import * as path from "path" import { TerminalOutputPreviewSize, TERMINAL_PREVIEW_BYTES, PersistedCommandOutput } from "@roo-code/types" +import { processCarriageReturns, processBackspaces } from "../misc/extract-text" + /** * Configuration options for creating an OutputInterceptor instance. */ @@ -17,6 +19,8 @@ export interface OutputInterceptorOptions { storageDir: string /** Size category for the preview buffer (small/medium/large) */ previewSize: TerminalOutputPreviewSize + /** Whether to compress progress bar output using carriage return processing */ + compressProgressBar: boolean } /** @@ -26,14 +30,13 @@ export interface OutputInterceptorOptions { * files, with only a preview shown to the LLM. The LLM can then use the `read_command_output` * tool to retrieve full contents or search through the output. * - * The interceptor uses a **head/tail buffer** strategy (inspired by Codex): - * - 50% of the preview budget is allocated to the "head" (beginning of output) - * - 50% of the preview budget is allocated to the "tail" (end of output) - * - Middle content is dropped when output exceeds the preview threshold + * The interceptor operates in two modes: + * 1. **Buffer mode**: Output is accumulated in memory until it exceeds the preview threshold + * 2. **Spill mode**: Once threshold is exceeded, output is streamed directly to disk * - * This approach ensures the LLM sees both: - * - The beginning (command startup, environment info, early errors) - * - The end (final results, exit codes, error summaries) + * This approach prevents large command outputs (like build logs, test results, or verbose + * operations) from overwhelming the context window while still allowing the LLM to access + * the full output when needed. * * @example * ```typescript @@ -43,6 +46,7 @@ export interface OutputInterceptorOptions { * command: 'npm test', * storageDir: '/path/to/task/command-output', * previewSize: 'medium', + * compressProgressBar: true * }); * * // Write output chunks as they arrive @@ -51,38 +55,18 @@ export interface OutputInterceptorOptions { * * // Finalize and get the result * const result = interceptor.finalize(); - * // result.preview contains head + [omitted] + tail for display + * // result.preview contains truncated output for display * // result.artifactPath contains path to full output if truncated * ``` */ export class OutputInterceptor { - /** Buffer for the head (beginning) of output */ - private headBuffer: string = "" - /** Buffer for the tail (end) of output - rolling buffer that drops front when full */ - private tailBuffer: string = "" - /** Number of bytes currently in the head buffer */ - private headBytes: number = 0 - /** Number of bytes currently in the tail buffer */ - private tailBytes: number = 0 - /** Number of bytes omitted from the middle */ - private omittedBytes: number = 0 - - /** - * Pending chunks accumulated before spilling to disk. - * These contain ALL content (lossless) until we decide to spill. - * Once spilled, this array is cleared and subsequent writes go directly to disk. - */ - private pendingChunks: string[] = [] - + private buffer: string = "" private writeStream: fs.WriteStream | null = null private artifactPath: string private totalBytes: number = 0 private spilledToDisk: boolean = false private readonly previewBytes: number - /** Budget for the head buffer (50% of total preview) */ - private readonly headBudget: number - /** Budget for the tail buffer (50% of total preview) */ - private readonly tailBudget: number + private readonly compressProgressBar: boolean /** * Creates a new OutputInterceptor instance. @@ -91,19 +75,16 @@ export class OutputInterceptor { */ constructor(private readonly options: OutputInterceptorOptions) { this.previewBytes = TERMINAL_PREVIEW_BYTES[options.previewSize] - this.headBudget = Math.floor(this.previewBytes / 2) - this.tailBudget = this.previewBytes - this.headBudget + this.compressProgressBar = options.compressProgressBar this.artifactPath = path.join(options.storageDir, `cmd-${options.executionId}.txt`) } /** * Write a chunk of output to the interceptor. * - * Output is first added to the head buffer until it's full (50% of preview budget). - * Subsequent output goes to a rolling tail buffer that keeps the most recent content. - * - * If the total output exceeds the preview threshold, the interceptor spills to disk - * for full output storage while maintaining head/tail buffers for the preview. + * If the accumulated output exceeds the preview threshold, the interceptor + * automatically spills to disk and switches to streaming mode. Subsequent + * chunks are written directly to the disk file. * * @param chunk - The output string to write * @@ -117,15 +98,10 @@ export class OutputInterceptor { const chunkBytes = Buffer.byteLength(chunk, "utf8") this.totalBytes += chunkBytes - // Always update the head/tail preview buffers - this.addToPreviewBuffers(chunk) - - // Handle disk spilling for full output preservation if (!this.spilledToDisk) { - // Accumulate ALL chunks for lossless disk storage - this.pendingChunks.push(chunk) + this.buffer += chunk - if (this.totalBytes > this.previewBytes) { + if (Buffer.byteLength(this.buffer, "utf8") > this.previewBytes) { this.spillToDisk() } } else { @@ -134,127 +110,6 @@ export class OutputInterceptor { } } - /** - * Add a chunk to the head/tail preview buffers using 50/50 split strategy. - * - * Fill head first until budget exhausted, then maintain a rolling tail buffer. - * - * @private - */ - private addToPreviewBuffers(chunk: string): void { - let remaining = chunk - let remainingBytes = Buffer.byteLength(chunk, "utf8") - - // First, fill the head buffer if there's room - if (this.headBytes < this.headBudget) { - const headRoom = this.headBudget - this.headBytes - if (remainingBytes <= headRoom) { - // Entire chunk fits in head - this.headBuffer += remaining - this.headBytes += remainingBytes - return - } - // Split: part goes to head, rest goes to tail - const headPortion = this.sliceByBytes(remaining, headRoom) - this.headBuffer += headPortion - this.headBytes += headRoom - remaining = remaining.slice(headPortion.length) - remainingBytes = Buffer.byteLength(remaining, "utf8") - } - - // Add remainder to tail buffer - this.addToTailBuffer(remaining, remainingBytes) - } - - /** - * Add content to the rolling tail buffer, dropping old content as needed. - * - * @private - */ - private addToTailBuffer(chunk: string, chunkBytes: number): void { - if (this.tailBudget === 0) { - this.omittedBytes += chunkBytes - return - } - - // If this single chunk is larger than the tail budget, keep only the last tailBudget bytes - if (chunkBytes >= this.tailBudget) { - const dropped = this.tailBytes + (chunkBytes - this.tailBudget) - this.omittedBytes += dropped - this.tailBuffer = this.sliceByBytesFromEnd(chunk, this.tailBudget) - this.tailBytes = this.tailBudget - return - } - - // Append to tail - this.tailBuffer += chunk - this.tailBytes += chunkBytes - - // Trim from front if over budget - this.trimTailToFit() - } - - /** - * Trim the tail buffer from the front to fit within the tail budget. - * - * @private - */ - private trimTailToFit(): void { - while (this.tailBytes > this.tailBudget && this.tailBuffer.length > 0) { - const excess = this.tailBytes - this.tailBudget - // Remove characters from the front until we're under budget - // We need to be careful with multi-byte characters - let removed = 0 - let removeChars = 0 - while (removed < excess && removeChars < this.tailBuffer.length) { - const charBytes = Buffer.byteLength(this.tailBuffer[removeChars], "utf8") - removed += charBytes - removeChars++ - } - this.omittedBytes += removed - this.tailBytes -= removed - this.tailBuffer = this.tailBuffer.slice(removeChars) - } - } - - /** - * Slice a string to get approximately the first N bytes (UTF-8). - * - * @private - */ - private sliceByBytes(str: string, maxBytes: number): string { - let bytes = 0 - let i = 0 - while (i < str.length && bytes < maxBytes) { - const charBytes = Buffer.byteLength(str[i], "utf8") - if (bytes + charBytes > maxBytes) { - break - } - bytes += charBytes - i++ - } - return str.slice(0, i) - } - - /** - * Slice a string to get approximately the last N bytes (UTF-8). - * - * @private - */ - private sliceByBytesFromEnd(str: string, maxBytes: number): string { - let bytes = 0 - let i = str.length - 1 - while (i >= 0 && bytes < maxBytes) { - const charBytes = Buffer.byteLength(str[i], "utf8") - if (bytes + charBytes > maxBytes) { - break - } - bytes += charBytes - i-- - } - return str.slice(i + 1) - } - /** * Spill buffered content to disk and switch to streaming mode. * @@ -272,36 +127,30 @@ export class OutputInterceptor { } this.writeStream = fs.createWriteStream(this.artifactPath) - - // Write ALL pending chunks to disk for lossless storage. - // This ensures no content is lost, even if the preview buffers have dropped middle content. - for (const chunk of this.pendingChunks) { - this.writeStream.write(chunk) - } - - // Clear pending chunks to free memory - subsequent writes go directly to disk - this.pendingChunks = [] - + this.writeStream.write(this.buffer) this.spilledToDisk = true + + // Keep only preview portion in memory + this.buffer = this.buffer.slice(0, this.previewBytes) } /** * Finalize the interceptor and return the persisted output result. * - * Closes any open file streams and waits for them to fully flush before returning. - * This ensures the artifact file is completely written and ready for reading. - * - * Returns a summary object containing: - * - A preview of the output (head + [omitted indicator] + tail) + * Closes any open file streams and returns a summary object containing: + * - A preview of the output (truncated to preview size) * - The total byte count of all output * - The path to the full output file (if truncated) * - A flag indicating whether the output was truncated * + * If `compressProgressBar` was enabled, the preview will have carriage returns + * and backspaces processed to show only final line states. + * * @returns The persisted command output summary * * @example * ```typescript - * const result = await interceptor.finalize(); + * const result = interceptor.finalize(); * console.log(`Preview: ${result.preview}`); * console.log(`Total bytes: ${result.totalBytes}`); * if (result.truncated) { @@ -309,24 +158,19 @@ export class OutputInterceptor { * } * ``` */ - async finalize(): Promise { - // Close write stream if open and wait for it to fully flush. - // This ensures the artifact is completely written before we advertise the artifact_id. + finalize(): PersistedCommandOutput { + // Close write stream if open if (this.writeStream) { - await new Promise((resolve, reject) => { - this.writeStream!.end(() => resolve()) - this.writeStream!.on("error", reject) - }) + this.writeStream.end() } - // Prepare preview: head + [omission indicator] + tail - let preview: string - if (this.omittedBytes > 0) { - const omissionIndicator = `\n[...${this.omittedBytes} bytes omitted...]\n` - preview = this.headBuffer + omissionIndicator + this.tailBuffer - } else { - // No truncation, just combine head and tail (or head alone if tail is empty) - preview = this.headBuffer + this.tailBuffer + // Prepare preview + let preview = this.buffer.slice(0, this.previewBytes) + + // Apply compression to preview only (for readability) + if (this.compressProgressBar) { + preview = processCarriageReturns(preview) + preview = processBackspaces(preview) } return { @@ -340,15 +184,13 @@ export class OutputInterceptor { /** * Get the current buffer content for UI display. * - * Returns the combined head + tail content for real-time UI updates. - * Note: Does not include the omission indicator to avoid flickering during streaming. + * Returns the in-memory buffer which contains either all output (if not spilled) + * or just the preview portion (if spilled to disk). * * @returns The current buffer content as a string */ getBufferForUI(): string { - // For UI, return combined head + tail without omission indicator - // This provides a smoother streaming experience - return this.headBuffer + this.tailBuffer + return this.buffer } /** diff --git a/src/integrations/terminal/__tests__/OutputInterceptor.test.ts b/src/integrations/terminal/__tests__/OutputInterceptor.test.ts index ed308cff13..9268854208 100644 --- a/src/integrations/terminal/__tests__/OutputInterceptor.test.ts +++ b/src/integrations/terminal/__tests__/OutputInterceptor.test.ts @@ -32,16 +32,12 @@ describe("OutputInterceptor", () => { beforeEach(() => { vi.clearAllMocks() - storageDir = path.normalize("/tmp/test-storage") + storageDir = "/tmp/test-storage" - // Setup mock write stream with callback support for end() + // Setup mock write stream mockWriteStream = { write: vi.fn(), - end: vi.fn((callback?: () => void) => { - // Immediately call the callback to simulate stream flush completing - if (callback) callback() - }), - on: vi.fn(), + end: vi.fn(), } vi.mocked(fs.existsSync).mockReturnValue(true) @@ -53,13 +49,14 @@ describe("OutputInterceptor", () => { }) describe("Buffering behavior", () => { - it("should keep small output in memory without spilling to disk", async () => { + it("should keep small output in memory without spilling to disk", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "echo test", storageDir, - previewSize: "small", // 5KB + previewSize: "small", // 2KB + compressProgressBar: false, }) const smallOutput = "Hello World\n" @@ -68,7 +65,7 @@ describe("OutputInterceptor", () => { expect(interceptor.hasSpilledToDisk()).toBe(false) expect(fs.createWriteStream).not.toHaveBeenCalled() - const result = await interceptor.finalize() + const result = interceptor.finalize() expect(result.preview).toBe(smallOutput) expect(result.truncated).toBe(false) expect(result.artifactPath).toBe(null) @@ -81,45 +78,44 @@ describe("OutputInterceptor", () => { taskId: "task-1", command: "echo test", storageDir, - previewSize: "small", // 5KB = 5120 bytes + previewSize: "small", // 2KB = 2048 bytes + compressProgressBar: false, }) - // Write enough data to exceed 5KB threshold - const chunk = "x".repeat(2 * 1024) // 2KB chunk + // Write enough data to exceed 2KB threshold + const chunk = "x".repeat(1024) // 1KB chunk + interceptor.write(chunk) // 1KB - should stay in memory + expect(interceptor.hasSpilledToDisk()).toBe(false) + interceptor.write(chunk) // 2KB - should stay in memory expect(interceptor.hasSpilledToDisk()).toBe(false) - interceptor.write(chunk) // 4KB - should stay in memory - expect(interceptor.hasSpilledToDisk()).toBe(false) - - interceptor.write(chunk) // 6KB - should trigger spill + interceptor.write(chunk) // 3KB - should trigger spill expect(interceptor.hasSpilledToDisk()).toBe(true) expect(fs.createWriteStream).toHaveBeenCalledWith(path.join(storageDir, "cmd-12345.txt")) expect(mockWriteStream.write).toHaveBeenCalled() }) - it("should truncate preview after spilling to disk using head/tail split", async () => { + it("should truncate preview after spilling to disk", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "echo test", storageDir, - previewSize: "small", // 5KB + previewSize: "small", // 2KB + compressProgressBar: false, }) // Write data that exceeds threshold - const chunk = "x".repeat(6000) + const chunk = "x".repeat(3000) interceptor.write(chunk) expect(interceptor.hasSpilledToDisk()).toBe(true) - const result = await interceptor.finalize() + const result = interceptor.finalize() expect(result.truncated).toBe(true) expect(result.artifactPath).toBe(path.join(storageDir, "cmd-12345.txt")) - // Preview is head (1024) + omission indicator + tail (1024) - // The omission indicator adds some extra bytes - expect(result.preview).toContain("[...") - expect(result.preview).toContain("bytes omitted...]") + expect(Buffer.byteLength(result.preview, "utf8")).toBeLessThanOrEqual(2048) }) it("should write subsequent chunks directly to disk after spilling", () => { @@ -129,10 +125,11 @@ describe("OutputInterceptor", () => { command: "echo test", storageDir, previewSize: "small", + compressProgressBar: false, }) - // Trigger spill (must exceed 5KB = 5120 bytes) - const largeChunk = "x".repeat(6000) + // Trigger spill + const largeChunk = "x".repeat(3000) interceptor.write(largeChunk) expect(interceptor.hasSpilledToDisk()).toBe(true) @@ -148,56 +145,59 @@ describe("OutputInterceptor", () => { }) describe("Threshold settings", () => { - it("should handle small (5KB) threshold correctly", () => { + it("should handle small (2KB) threshold correctly", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) - // Write exactly 5KB - interceptor.write("x".repeat(5 * 1024)) + // Write exactly 2KB + interceptor.write("x".repeat(2048)) expect(interceptor.hasSpilledToDisk()).toBe(false) - // Write more to exceed 5KB + // Write more to exceed 2KB interceptor.write("x") expect(interceptor.hasSpilledToDisk()).toBe(true) }) - it("should handle medium (10KB) threshold correctly", () => { + it("should handle medium (4KB) threshold correctly", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, previewSize: "medium", + compressProgressBar: false, }) - // Write exactly 10KB - interceptor.write("x".repeat(10 * 1024)) + // Write exactly 4KB + interceptor.write("x".repeat(4096)) expect(interceptor.hasSpilledToDisk()).toBe(false) - // Write more to exceed 10KB + // Write more to exceed 4KB interceptor.write("x") expect(interceptor.hasSpilledToDisk()).toBe(true) }) - it("should handle large (20KB) threshold correctly", () => { + it("should handle large (8KB) threshold correctly", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, previewSize: "large", + compressProgressBar: false, }) - // Write exactly 20KB - interceptor.write("x".repeat(20 * 1024)) + // Write exactly 8KB + interceptor.write("x".repeat(8192)) expect(interceptor.hasSpilledToDisk()).toBe(false) - // Write more to exceed 20KB + // Write more to exceed 8KB interceptor.write("x") expect(interceptor.hasSpilledToDisk()).toBe(true) }) @@ -213,10 +213,11 @@ describe("OutputInterceptor", () => { command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) - // Trigger spill (must exceed 5KB = 5120 bytes) - interceptor.write("x".repeat(6000)) + // Trigger spill + interceptor.write("x".repeat(3000)) expect(fs.mkdirSync).toHaveBeenCalledWith(storageDir, { recursive: true }) }) @@ -229,31 +230,30 @@ describe("OutputInterceptor", () => { command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) - // Trigger spill (must exceed 5KB = 5120 bytes) - interceptor.write("x".repeat(6000)) + // Trigger spill + interceptor.write("x".repeat(3000)) expect(fs.createWriteStream).toHaveBeenCalledWith(path.join(storageDir, `cmd-${executionId}.txt`)) }) - it("should write head and tail buffers to artifact when spilling", () => { + it("should write full output to artifact, not truncated", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, - previewSize: "small", // 5KB = 5120 bytes, so head=2560, tail=2560 + previewSize: "small", + compressProgressBar: false, }) - const fullOutput = "x".repeat(10000) + const fullOutput = "x".repeat(5000) interceptor.write(fullOutput) - // The write stream should receive the head buffer content first - // (spillToDisk writes head + tail that existed at spill time) - expect(mockWriteStream.write).toHaveBeenCalled() - // Verify that we're writing to disk - expect(interceptor.hasSpilledToDisk()).toBe(true) + // The write stream should receive the full buffer content + expect(mockWriteStream.write).toHaveBeenCalledWith(fullOutput) }) it("should get artifact path from getArtifactPath() method", () => { @@ -264,6 +264,7 @@ describe("OutputInterceptor", () => { command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) const expectedPath = path.join(storageDir, `cmd-${executionId}.txt`) @@ -272,19 +273,20 @@ describe("OutputInterceptor", () => { }) describe("finalize() method", () => { - it("should return preview output for small commands", async () => { + it("should return preview output for small commands", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "echo hello", storageDir, previewSize: "small", + compressProgressBar: false, }) const output = "Hello World\n" interceptor.write(output) - const result = await interceptor.finalize() + const result = interceptor.finalize() expect(result.preview).toBe(output) expect(result.totalBytes).toBe(Buffer.byteLength(output, "utf8")) @@ -292,61 +294,61 @@ describe("OutputInterceptor", () => { expect(result.truncated).toBe(false) }) - it("should return PersistedCommandOutput for large commands with head/tail preview", async () => { + it("should return PersistedCommandOutput for large commands", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, - previewSize: "small", // 5KB = 5120, head=2560, tail=2560 + previewSize: "small", + compressProgressBar: false, }) - const largeOutput = "x".repeat(10000) + const largeOutput = "x".repeat(5000) interceptor.write(largeOutput) - const result = await interceptor.finalize() + const result = interceptor.finalize() expect(result.truncated).toBe(true) expect(result.artifactPath).toBe(path.join(storageDir, "cmd-12345.txt")) expect(result.totalBytes).toBe(Buffer.byteLength(largeOutput, "utf8")) - // Preview should contain head + omission indicator + tail - expect(result.preview).toContain("[...") - expect(result.preview).toContain("bytes omitted...]") + expect(Buffer.byteLength(result.preview, "utf8")).toBeLessThanOrEqual(2048) }) - it("should close write stream when finalizing", async () => { + it("should close write stream when finalizing", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) - // Trigger spill (must exceed 5KB = 5120 bytes) - interceptor.write("x".repeat(6000)) - await interceptor.finalize() + // Trigger spill + interceptor.write("x".repeat(3000)) + interceptor.finalize() expect(mockWriteStream.end).toHaveBeenCalled() }) - it("should include correct metadata (artifactId, size, truncated flag)", async () => { + it("should include correct metadata (artifactId, size, truncated flag)", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) - // Must exceed 5KB = 5120 bytes to trigger truncation - const output = "x".repeat(6000) + const output = "x".repeat(5000) interceptor.write(output) - const result = await interceptor.finalize() + const result = interceptor.finalize() expect(result).toHaveProperty("preview") - expect(result).toHaveProperty("totalBytes", 6000) + expect(result).toHaveProperty("totalBytes", 5000) expect(result).toHaveProperty("artifactPath") expect(result).toHaveProperty("truncated", true) expect(result.artifactPath).toMatch(/cmd-12345\.txt$/) @@ -401,6 +403,46 @@ describe("OutputInterceptor", () => { }) }) + describe("Progress bar compression", () => { + it("should apply compression when compressProgressBar is true", () => { + const interceptor = new OutputInterceptor({ + executionId: "12345", + taskId: "task-1", + command: "test", + storageDir, + previewSize: "small", + compressProgressBar: true, + }) + + // Output with carriage returns (simulating progress bar) + const output = "Progress: 10%\rProgress: 50%\rProgress: 100%\n" + interceptor.write(output) + + const result = interceptor.finalize() + + // Preview should be compressed (carriage returns processed) + // The processCarriageReturns function should keep only the last line before \r + expect(result.preview).not.toBe(output) + }) + + it("should not apply compression when compressProgressBar is false", () => { + const interceptor = new OutputInterceptor({ + executionId: "12345", + taskId: "task-1", + command: "test", + storageDir, + previewSize: "small", + compressProgressBar: false, + }) + + const output = "Line 1\nLine 2\n" + interceptor.write(output) + + const result = interceptor.finalize() + expect(result.preview).toBe(output) + }) + }) + describe("getBufferForUI() method", () => { it("should return current buffer for UI updates", () => { const interceptor = new OutputInterceptor({ @@ -409,6 +451,7 @@ describe("OutputInterceptor", () => { command: "test", storageDir, previewSize: "small", + compressProgressBar: false, }) const output = "Hello World" @@ -417,116 +460,22 @@ describe("OutputInterceptor", () => { expect(interceptor.getBufferForUI()).toBe(output) }) - it("should return head + tail buffer after spilling to disk", () => { + it("should return truncated buffer after spilling to disk", () => { const interceptor = new OutputInterceptor({ executionId: "12345", taskId: "task-1", command: "test", storageDir, - previewSize: "small", // 5KB = 5120, head=2560, tail=2560 + previewSize: "small", + compressProgressBar: false, }) // Trigger spill - const largeOutput = "x".repeat(10000) + const largeOutput = "x".repeat(5000) interceptor.write(largeOutput) const buffer = interceptor.getBufferForUI() - // Buffer for UI is head + tail (no omission indicator for smooth streaming) - expect(Buffer.byteLength(buffer, "utf8")).toBeLessThanOrEqual(5120) - }) - }) - - describe("Head/Tail split behavior", () => { - it("should preserve first 50% and last 50% of output", async () => { - const interceptor = new OutputInterceptor({ - executionId: "12345", - taskId: "task-1", - command: "test", - storageDir, - previewSize: "small", // 5KB = 5120, head=2560, tail=2560 - }) - - // Create identifiable head and tail content - const headContent = "HEAD".repeat(750) // 3000 bytes - const middleContent = "M".repeat(6000) // 6000 bytes (will be omitted) - const tailContent = "TAIL".repeat(750) // 3000 bytes - - interceptor.write(headContent) - interceptor.write(middleContent) - interceptor.write(tailContent) - - const result = await interceptor.finalize() - - // Should start with HEAD content (first 2560 bytes of head budget) - expect(result.preview.startsWith("HEAD")).toBe(true) - // Should end with TAIL content (last 2560 bytes) - expect(result.preview.endsWith("TAIL")).toBe(true) - // Should have omission indicator - expect(result.preview).toContain("[...") - expect(result.preview).toContain("bytes omitted...]") - }) - - it("should not add omission indicator when output fits in budget", async () => { - const interceptor = new OutputInterceptor({ - executionId: "12345", - taskId: "task-1", - command: "test", - storageDir, - previewSize: "small", // 5KB - }) - - const smallOutput = "Hello World\n" - interceptor.write(smallOutput) - - const result = await interceptor.finalize() - - // No omission indicator for small output - expect(result.preview).toBe(smallOutput) - expect(result.preview).not.toContain("[...") - }) - - it("should handle output that exactly fills head budget", async () => { - const interceptor = new OutputInterceptor({ - executionId: "12345", - taskId: "task-1", - command: "test", - storageDir, - previewSize: "small", // 5KB = 5120, head=2560 - }) - - // Write exactly 2560 bytes (head budget) - const exactHeadContent = "x".repeat(2560) - interceptor.write(exactHeadContent) - - const result = await interceptor.finalize() - - // Should fit entirely in head, no truncation - expect(result.preview).toBe(exactHeadContent) - expect(result.truncated).toBe(false) - }) - - it("should split single large chunk across head and tail", async () => { - const interceptor = new OutputInterceptor({ - executionId: "12345", - taskId: "task-1", - command: "test", - storageDir, - previewSize: "small", // 5KB = 5120, head=2560, tail=2560 - }) - - // Write a single chunk larger than preview budget - // First 2560 chars go to head, last 2560 chars go to tail - const content = "A".repeat(2560) + "B".repeat(4000) + "C".repeat(2560) - interceptor.write(content) - - const result = await interceptor.finalize() - - // Head should have A's - expect(result.preview.startsWith("A")).toBe(true) - // Tail should have C's - expect(result.preview.endsWith("C")).toBe(true) - // Should have omission indicator - expect(result.preview).toContain("[...") + expect(Buffer.byteLength(buffer, "utf8")).toBeLessThanOrEqual(2048) }) }) }) diff --git a/src/integrations/terminal/index.ts b/src/integrations/terminal/index.ts new file mode 100644 index 0000000000..afd05bb1e5 --- /dev/null +++ b/src/integrations/terminal/index.ts @@ -0,0 +1,57 @@ +/** + * Terminal Output Handling Module + * + * This module provides utilities for capturing, persisting, and retrieving + * command output from terminal executions. + * + * ## Overview + * + * When the LLM executes commands via `execute_command`, the output can be + * very large (build logs, test output, etc.). To prevent context window + * overflow while still allowing access to full output, this module + * implements a "persisted output" pattern: + * + * 1. **OutputInterceptor**: Buffers command output during execution. If + * output exceeds a configurable threshold, it "spills" to disk and + * keeps only a preview in memory. + * + * 2. **Artifact Storage**: Full outputs are stored as text files in the + * task's `command-output/` directory with names like `cmd-{timestamp}.txt`. + * + * 3. **ReadCommandOutputTool**: Allows the LLM to retrieve the full output + * later via the `read_command_output` tool, with support for search + * and pagination. + * + * ## Data Flow + * + * ``` + * execute_command + * │ + * ▼ + * OutputInterceptor.write() ──► Buffer accumulates + * │ + * ▼ (threshold exceeded) + * OutputInterceptor.spillToDisk() ──► Artifact file created + * │ + * ▼ + * OutputInterceptor.finalize() ──► Returns PersistedCommandOutput + * │ + * ▼ + * LLM receives preview + artifact_id + * │ + * ▼ (if needs full output) + * read_command_output(artifact_id) ──► Full content/search results + * ``` + * + * ## Configuration + * + * Preview size is controlled by `terminalOutputPreviewSize` setting: + * - `small`: 2KB preview + * - `medium`: 4KB preview (default) + * - `large`: 8KB preview + * + * @module terminal + */ + +export { OutputInterceptor } from "./OutputInterceptor" +export type { OutputInterceptorOptions } from "./OutputInterceptor" diff --git a/src/package.json b/src/package.json index bf4a009a94..674ab06e18 100644 --- a/src/package.json +++ b/src/package.json @@ -541,6 +541,7 @@ "@types/diff": "^5.2.1", "@types/diff-match-patch": "^1.0.36", "@types/glob": "^8.1.0", + "@types/json-stream-stringify": "^2.0.4", "@types/lodash.debounce": "^4.0.9", "@types/mocha": "^10.0.10", "@types/node": "20.x", diff --git a/webview-ui/src/components/settings/SettingsView.tsx b/webview-ui/src/components/settings/SettingsView.tsx index b84a9dd3a3..054047284c 100644 --- a/webview-ui/src/components/settings/SettingsView.tsx +++ b/webview-ui/src/components/settings/SettingsView.tsx @@ -179,6 +179,8 @@ const SettingsView = forwardRef(({ onDone, t ttsSpeed, soundVolume, telemetrySetting, + terminalOutputLineLimit, + terminalOutputCharacterLimit, terminalOutputPreviewSize, terminalShellIntegrationTimeout, terminalShellIntegrationDisabled, // Added from upstream @@ -397,6 +399,7 @@ const SettingsView = forwardRef(({ onDone, t terminalZshOhMy, terminalZshP10k, terminalZdotdir, + terminalCompressProgressBar, terminalOutputPreviewSize: terminalOutputPreviewSize ?? "medium", mcpEnabled, maxOpenTabsContext: Math.min(Math.max(0, maxOpenTabsContext ?? 20), 500), diff --git a/webview-ui/src/components/settings/TerminalSettings.tsx b/webview-ui/src/components/settings/TerminalSettings.tsx index 07f062cc01..881058caf2 100644 --- a/webview-ui/src/components/settings/TerminalSettings.tsx +++ b/webview-ui/src/components/settings/TerminalSettings.tsx @@ -124,6 +124,31 @@ export const TerminalSettings = ({ {t("settings:terminal.outputPreviewSize.description")} + + + setCachedStateField("terminalCompressProgressBar", e.target.checked) + } + data-testid="terminal-compress-progress-bar-checkbox"> + {t("settings:terminal.compressProgressBar.label")} + +
+ + + {" "} + + +
+ diff --git a/webview-ui/src/context/ExtensionStateContext.tsx b/webview-ui/src/context/ExtensionStateContext.tsx index d37f09bbc5..01bc032cd3 100644 --- a/webview-ui/src/context/ExtensionStateContext.tsx +++ b/webview-ui/src/context/ExtensionStateContext.tsx @@ -96,6 +96,10 @@ export interface ExtensionStateContextType extends ExtensionState { setWriteDelayMs: (value: number) => void screenshotQuality?: number setScreenshotQuality: (value: number) => void + terminalOutputLineLimit?: number + setTerminalOutputLineLimit: (value: number) => void + terminalOutputCharacterLimit?: number + setTerminalOutputCharacterLimit: (value: number) => void terminalOutputPreviewSize?: "small" | "medium" | "large" setTerminalOutputPreviewSize: (value: "small" | "medium" | "large") => void mcpEnabled: boolean @@ -537,6 +541,10 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode setState((prevState) => ({ ...prevState, browserViewportSize: value })), setWriteDelayMs: (value) => setState((prevState) => ({ ...prevState, writeDelayMs: value })), setScreenshotQuality: (value) => setState((prevState) => ({ ...prevState, screenshotQuality: value })), + setTerminalOutputLineLimit: (value) => + setState((prevState) => ({ ...prevState, terminalOutputLineLimit: value })), + setTerminalOutputCharacterLimit: (value) => + setState((prevState) => ({ ...prevState, terminalOutputCharacterLimit: value })), setTerminalOutputPreviewSize: (value) => setState((prevState) => ({ ...prevState, terminalOutputPreviewSize: value })), setTerminalShellIntegrationTimeout: (value) => diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 63f4056d66..f53aa48a1f 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -737,9 +737,9 @@ "label": "Command output preview size", "description": "Controls how much command output Roo sees directly. Full output is always saved and accessible when needed.", "options": { - "small": "Small (5KB)", - "medium": "Medium (10KB)", - "large": "Large (20KB)" + "small": "Small (2KB)", + "medium": "Medium (4KB)", + "large": "Large (8KB)" } }, "shellIntegrationTimeout": {