From 525bf9f3add4527659b3e22f3177bdf7234b5a8e Mon Sep 17 00:00:00 2001 From: Hannes Rudolph Date: Sat, 24 Jan 2026 14:43:58 -0700 Subject: [PATCH] fix: address PR review comments - Read terminalOutputPreviewSize from providerState instead of hardcoded default - Fix native tool schema to only require artifact_id (optional params no longer required) - Fix Buffer allocation for line numbers using chunked 64KB reads to avoid memory blowup --- Roo-EXTRACTION-terminal-shell-integration.md | 409 ++++++ claude-code.md | 51 + codex-extract-terminal-spawning-tool.md | 1206 +++++++++++++++++ .../tools/native-tools/read_command_output.ts | 2 +- src/core/tools/ReadCommandOutputTool.ts | 47 +- 5 files changed, 1708 insertions(+), 7 deletions(-) create mode 100644 Roo-EXTRACTION-terminal-shell-integration.md create mode 100644 claude-code.md create mode 100644 codex-extract-terminal-spawning-tool.md diff --git a/Roo-EXTRACTION-terminal-shell-integration.md b/Roo-EXTRACTION-terminal-shell-integration.md new file mode 100644 index 0000000000..4b44c2e230 --- /dev/null +++ b/Roo-EXTRACTION-terminal-shell-integration.md @@ -0,0 +1,409 @@ +# Terminal/Shell Integration - Agent Context Document + +--- + +Feature: Terminal/Shell Integration +Last Updated: 2025-01-24 +Status: Stable +Audience: Agents/Developers + +--- + +## Overview + +Roo Code's terminal integration enables the `execute_command` tool to run shell commands and capture their output. The system supports two execution providers: + +1. **VSCode Terminal Provider** (`vscode`) - Uses VSCode's native shell integration APIs for command execution with real-time output streaming and exit code detection +2. **Execa Provider** (`execa`) - A fallback that runs commands via Node.js's `execa` library without VSCode terminal UI integration + +## File Structure + +### Core Terminal Integration Files + +``` +src/integrations/terminal/ +├── BaseTerminal.ts # Abstract base class for terminal implementations +├── BaseTerminalProcess.ts # Abstract base class for process implementations +├── Terminal.ts # VSCode terminal provider implementation +├── TerminalProcess.ts # VSCode terminal process implementation +├── ExecaTerminal.ts # Execa provider implementation +├── ExecaTerminalProcess.ts # Execa process implementation +├── TerminalRegistry.ts # Singleton registry managing terminal instances +├── ShellIntegrationManager.ts # Manages zsh shell integration workarounds +├── mergePromise.ts # Utility for merging process with promise +└── types.ts # Type definitions for terminal interfaces +``` + +### Related Files + +| File | Purpose | +| -------------------------------------------------------------------------------- | ---------------------------------- | +| [`src/core/tools/ExecuteCommandTool.ts`](src/core/tools/ExecuteCommandTool.ts) | The `execute_command` tool handler | +| [`src/integrations/misc/extract-text.ts`](src/integrations/misc/extract-text.ts) | Output compression utilities | +| [`packages/types/src/terminal.ts`](packages/types/src/terminal.ts) | CommandExecutionStatus schema | +| [`packages/types/src/global-settings.ts`](packages/types/src/global-settings.ts) | Terminal configuration defaults | + +--- + +## Architecture + +### Class Hierarchy + +``` +BaseTerminal (abstract) +├── Terminal (vscode provider) +└── ExecaTerminal (execa provider) + +BaseTerminalProcess (abstract) +├── TerminalProcess (vscode provider) +└── ExecaTerminalProcess (execa provider) +``` + +### Key Interfaces + +**[`RooTerminal`](src/integrations/terminal/types.ts:5)** - Main terminal interface: + +```typescript +interface RooTerminal { + provider: "vscode" | "execa" + id: number + busy: boolean + running: boolean + taskId?: string + process?: RooTerminalProcess + getCurrentWorkingDirectory(): string + isClosed: () => boolean + runCommand: (command: string, callbacks: RooTerminalCallbacks) => RooTerminalProcessResultPromise + setActiveStream(stream: AsyncIterable | undefined, pid?: number): void + shellExecutionComplete(exitDetails: ExitCodeDetails): void + getProcessesWithOutput(): RooTerminalProcess[] + getUnretrievedOutput(): string + getLastCommand(): string + cleanCompletedProcessQueue(): void +} +``` + +**[`RooTerminalCallbacks`](src/integrations/terminal/types.ts:23)** - Callbacks for command execution: + +```typescript +interface RooTerminalCallbacks { + onLine: (line: string, process: RooTerminalProcess) => void + onCompleted: (output: string | undefined, process: RooTerminalProcess) => void + onShellExecutionStarted: (pid: number | undefined, process: RooTerminalProcess) => void + onShellExecutionComplete: (details: ExitCodeDetails, process: RooTerminalProcess) => void + onNoShellIntegration?: (message: string, process: RooTerminalProcess) => void +} +``` + +**[`ExitCodeDetails`](src/integrations/terminal/types.ts:55)** - Exit information: + +```typescript +interface ExitCodeDetails { + exitCode: number | undefined + signal?: number | undefined + signalName?: string + coreDumpPossible?: boolean +} +``` + +--- + +## Command Execution Flow + +### 1. Tool Invocation + +When the LLM uses `execute_command`, [`ExecuteCommandTool.execute()`](src/core/tools/ExecuteCommandTool.ts:32) is called: + +1. Validates the `command` parameter exists +2. Checks `.rooignore` rules via `task.rooIgnoreController?.validateCommand(command)` +3. Requests user approval via `askApproval("command", unescapedCommand)` +4. Determines provider based on `terminalShellIntegrationDisabled` setting +5. Calls [`executeCommandInTerminal()`](src/core/tools/ExecuteCommandTool.ts:154) + +### 2. Terminal Selection + +[`TerminalRegistry.getOrCreateTerminal()`](src/integrations/terminal/TerminalRegistry.ts:152) selects a terminal: + +1. First priority: Terminal already assigned to this task with matching CWD +2. Second priority: Any available terminal with matching CWD +3. Fallback: Creates new terminal via [`TerminalRegistry.createTerminal()`](src/integrations/terminal/TerminalRegistry.ts:130) + +### 3. Command Execution + +**VSCode Provider Flow** ([`Terminal.runCommand()`](src/integrations/terminal/Terminal.ts:43)): + +1. Sets terminal as busy +2. Creates [`TerminalProcess`](src/integrations/terminal/TerminalProcess.ts:9) instance +3. Waits for shell integration with timeout (default 5s, configurable) +4. If shell integration available: executes via `terminal.shellIntegration.executeCommand()` +5. If shell integration unavailable: emits `no_shell_integration` event + +**Execa Provider Flow** ([`ExecaTerminal.runCommand()`](src/integrations/terminal/ExecaTerminal.ts:18)): + +1. Sets terminal as busy +2. Creates [`ExecaTerminalProcess`](src/integrations/terminal/ExecaTerminalProcess.ts:8) instance +3. Executes command via `execa` with `shell: true` +4. Streams output via async iterable + +### 4. Output Processing + +Output is processed through callbacks: + +- [`onLine`](src/core/tools/ExecuteCommandTool.ts:197) - Called as output streams in +- [`onCompleted`](src/core/tools/ExecuteCommandTool.ts:226) - Called when command completes +- [`onShellExecutionStarted`](src/core/tools/ExecuteCommandTool.ts:236) - Called when shell execution begins (with PID) +- [`onShellExecutionComplete`](src/core/tools/ExecuteCommandTool.ts:240) - Called when shell execution ends (with exit code) + +Output is compressed via [`Terminal.compressTerminalOutput()`](src/integrations/terminal/BaseTerminal.ts:275): + +1. Process carriage returns (progress bars) +2. Process backspaces +3. Apply run-length encoding for repeated lines +4. Truncate to line/character limits + +--- + +## VSCode Shell Integration Details + +### OSC 633 Protocol + +VSCode uses OSC 633 escape sequences for shell integration. Key markers: + +| Sequence | Meaning | +| ------------------------------------ | ----------------------------------------- | +| `\x1b]633;A` | Mark prompt start | +| `\x1b]633;B` | Mark prompt end | +| `\x1b]633;C` | Mark pre-execution (command output start) | +| `\x1b]633;D[;]` | Mark execution finished | +| `\x1b]633;E;[;]` | Explicitly set command line | + +The [`TerminalProcess`](src/integrations/terminal/TerminalProcess.ts) class parses these markers: + +- [`matchAfterVsceStartMarkers()`](src/integrations/terminal/TerminalProcess.ts:396) - Finds content after C marker +- [`matchBeforeVsceEndMarkers()`](src/integrations/terminal/TerminalProcess.ts:405) - Finds content before D marker + +### Shell Integration Event Handlers + +Registered in [`TerminalRegistry.initialize()`](src/integrations/terminal/TerminalRegistry.ts:26): + +- [`onDidStartTerminalShellExecution`](src/integrations/terminal/TerminalRegistry.ts:49) - Captures stream and marks terminal busy +- [`onDidEndTerminalShellExecution`](src/integrations/terminal/TerminalRegistry.ts:76) - Processes exit code and signals completion + +--- + +## Configuration Options + +All settings are stored in extension state and managed via [`ClineProvider`](src/core/webview/ClineProvider.ts:752). + +### Terminal Settings + +| Setting | Type | Default | Description | +| ---------------------------------- | --------- | -------- | ----------------------------------------------------------------------- | +| `terminalShellIntegrationDisabled` | `boolean` | `true` | When true, uses execa provider instead of VSCode terminal | +| `terminalShellIntegrationTimeout` | `number` | `30000` | Milliseconds to wait for shell integration init (VSCode provider only) | +| `terminalOutputLineLimit` | `number` | `500` | Maximum lines to keep in compressed output | +| `terminalOutputCharacterLimit` | `number` | `100000` | Maximum characters to keep in compressed output | +| `terminalCommandDelay` | `number` | `0` | Milliseconds to delay after command (workaround for VSCode bug #237208) | + +### Shell-Specific Settings + +| Setting | Type | Default | Description | +| ----------------------------- | --------- | ------- | ----------------------------------------------------------------------------- | +| `terminalZshClearEolMark` | `boolean` | `true` | Clear ZSH EOL mark (`PROMPT_EOL_MARK=""`) | +| `terminalZshOhMy` | `boolean` | `true` | Enable Oh My Zsh integration (`ITERM_SHELL_INTEGRATION_INSTALLED=Yes`) | +| `terminalZshP10k` | `boolean` | `false` | Enable Powerlevel10k integration (`POWERLEVEL9K_TERM_SHELL_INTEGRATION=true`) | +| `terminalZdotdir` | `boolean` | `true` | Use ZDOTDIR workaround for zsh shell integration | +| `terminalPowershellCounter` | `boolean` | `false` | Add counter workaround for PowerShell | +| `terminalCompressProgressBar` | `boolean` | `true` | Process carriage returns to compress progress bar output | + +### VSCode Configuration + +The tool also reads from VSCode configuration: + +- `roo-cline.commandExecutionTimeout` - Seconds to auto-abort commands (0 = disabled) +- `roo-cline.commandTimeoutAllowlist` - Command prefixes exempt from timeout + +--- + +## Environment Variables + +The [`Terminal.getEnv()`](src/integrations/terminal/Terminal.ts:153) method sets environment variables for shell integration: + +| Variable | Value | Purpose | +| ------------------------------------- | --------------------------- | ----------------------------------------- | +| `PAGER` | `cat` (non-Windows) | Prevent pager interruption | +| `VTE_VERSION` | `0` | Disable VTE prompt command interference | +| `ITERM_SHELL_INTEGRATION_INSTALLED` | `Yes` (if enabled) | Oh My Zsh compatibility | +| `POWERLEVEL9K_TERM_SHELL_INTEGRATION` | `true` (if enabled) | Powerlevel10k compatibility | +| `PROMPT_COMMAND` | `sleep X` (if delay > 0) | Workaround for VSCode output race | +| `PROMPT_EOL_MARK` | `""` (if enabled) | Prevent ZSH EOL mark issues | +| `ZDOTDIR` | Temp directory (if enabled) | Load shell integration before user config | + +--- + +## Fallback Mechanism + +When VSCode shell integration fails: + +1. [`ShellIntegrationError`](src/core/tools/ExecuteCommandTool.ts:22) is thrown +2. User sees `shell_integration_warning` message +3. Command is re-executed with `terminalShellIntegrationDisabled: true` +4. Execa provider runs command without terminal UI + +Fallback triggers: + +- Shell integration timeout exceeded +- OSC 633;C marker not received +- Stream did not start within timeout + +--- + +## Process State Management + +### Terminal States + +| Property | Type | Description | +| -------------- | --------- | ------------------------------------------------------ | +| `busy` | `boolean` | Terminal is executing or waiting for shell integration | +| `running` | `boolean` | Command is actively executing | +| `streamClosed` | `boolean` | Output stream has ended | + +### Process States + +| Property | Type | Description | +| -------------------- | --------- | ---------------------------------------------------------- | +| `isHot` | `boolean` | Process recently produced output (affects request timing) | +| `isListening` | `boolean` | Process is still accepting output events | +| `fullOutput` | `string` | Complete accumulated output | +| `lastRetrievedIndex` | `number` | Index of last retrieved output (for incremental retrieval) | + +### Hot Timer + +The [`startHotTimer()`](src/integrations/terminal/BaseTerminalProcess.ts:157) method marks a process as "hot" after receiving output: + +- Normal output: 2 second hot period +- Compilation output (detected via markers): 15 second hot period + +Compilation markers: `compiling`, `building`, `bundling`, `transpiling`, `generating`, `starting` + +--- + +## Command Execution Status Updates + +The webview receives status updates via [`CommandExecutionStatus`](packages/types/src/terminal.ts:7): + +| Status | When | Data | +| ---------- | ---------------------- | --------------------- | +| `started` | Shell execution begins | `pid`, `command` | +| `output` | Output received | `output` (compressed) | +| `exited` | Command completes | `exitCode` | +| `fallback` | Switching to execa | - | +| `timeout` | Command timed out | - | + +--- + +## Key Implementation Details + +### PowerShell Workarounds + +In [`TerminalProcess.run()`](src/integrations/terminal/TerminalProcess.ts:109): + +- Counter workaround: Appends `; "(Roo/PS Workaround: N)" > $null` to ensure unique commands +- Delay workaround: Appends `; start-sleep -milliseconds X` for output timing + +### ZDOTDIR Workaround + +[`ShellIntegrationManager.zshInitTmpDir()`](src/integrations/terminal/ShellIntegrationManager.ts:13): + +1. Creates temporary directory +2. Creates `.zshrc` that sources VSCode's shell integration script +3. Sources user's original zsh config files +4. Cleans up after shell integration succeeds or times out + +### Signal Handling + +[`BaseTerminalProcess.interpretExitCode()`](src/integrations/terminal/BaseTerminalProcess.ts:16) translates exit codes: + +- Exit codes > 128 indicate signal termination +- Signal number = exit code - 128 +- Maps to signal names (SIGINT, SIGTERM, etc.) +- Identifies signals that may produce core dumps + +--- + +## Testing + +Test files are located in `src/integrations/terminal/__tests__/`: + +| File | Coverage | +| ------------------------------------------ | ----------------------------------- | +| `TerminalProcess.spec.ts` | VSCode terminal process logic | +| `TerminalRegistry.spec.ts` | Terminal registration and selection | +| `ExecaTerminal.spec.ts` | Execa terminal provider | +| `ExecaTerminalProcess.spec.ts` | Execa process execution | +| `TerminalProcessExec.*.spec.ts` | Shell-specific execution tests | +| `TerminalProcessInterpretExitCode.spec.ts` | Exit code interpretation | + +Execute_command tool tests: `src/core/tools/__tests__/executeCommand*.spec.ts` + +--- + +## Common Issues and Debugging + +### Shell Integration Not Available + +**Symptoms**: `no_shell_integration` event emitted, fallback to execa + +**Causes**: + +- Shell doesn't support OSC 633 sequences +- User's shell config overrides VSCode's integration +- Timeout too short for slow shell startup + +**Resolution**: + +- Increase `terminalShellIntegrationTimeout` +- Enable `terminalZdotdir` for zsh +- Check for conflicting shell plugins + +### Output Missing or Truncated + +**Symptoms**: Incomplete command output + +**Causes**: + +- VSCode bug #237208 (race between completion and output) +- Output exceeds line/character limits + +**Resolution**: + +- Enable `terminalCommandDelay` setting +- Increase `terminalOutputLineLimit` or `terminalOutputCharacterLimit` + +### Progress Bars Garbled + +**Symptoms**: Multiple lines of progress instead of single updating line + +**Causes**: + +- `terminalCompressProgressBar` disabled +- Multi-byte characters in progress output + +**Resolution**: + +- Enable `terminalCompressProgressBar` +- Check [`processCarriageReturns()`](src/integrations/misc/extract-text.ts:355) handling + +--- + +## Related Features + +- **Terminal Actions** ([`packages/types/src/vscode.ts:17`](packages/types/src/vscode.ts:17)): Context menu actions for terminal output + + - `terminalAddToContext` + - `terminalFixCommand` + - `terminalExplainCommand` + +- **Background Terminals**: Terminals can continue running after task completion, tracked via [`TerminalRegistry.getBackgroundTerminals()`](src/integrations/terminal/TerminalRegistry.ts:255) + +- **Output Retrieval**: Unretrieved output can be retrieved incrementally via [`getUnretrievedOutput()`](src/integrations/terminal/BaseTerminal.ts:133) for background process monitoring diff --git a/claude-code.md b/claude-code.md new file mode 100644 index 0000000000..614210ebf1 --- /dev/null +++ b/claude-code.md @@ -0,0 +1,51 @@ + { + "name": "Bash", + "description": "Executes a given bash command in a persistent shell session with optional timeout, ensuring proper handling and security measures.\n\nIMPORTANT: This tool is for terminal operations like git, npm, docker, etc. DO NOT use it for file operations (reading, writing, editing, searching, finding files) - use the specialized tools for this instead.\n\nBefore executing the command, please follow these steps:\n\n1. Directory Verification:\n - If the command will create new directories or files, first use `ls` to verify the parent directory exists and is the correct location\n - For example, before running \"mkdir foo/bar\", first use `ls foo` to check that \"foo\" exists and is the intended parent directory\n\n2. Command Execution:\n - Always quote file paths that contain spaces with double quotes (e.g., cd \"path with spaces/file.txt\")\n - Examples of proper quoting:\n - cd \"/Users/name/My Documents\" (correct)\n - cd /Users/name/My Documents (incorrect - will fail)\n - python \"/path/with spaces/script.py\" (correct)\n - python /path/with spaces/script.py (incorrect - will fail)\n - After ensuring proper quoting, execute the command.\n - Capture the output of the command.\n\nUsage notes:\n - The command argument is required.\n - You can specify an optional timeout in milliseconds (up to 600000ms / 10 minutes). If not specified, commands will timeout after 120000ms (2 minutes).\n - It is very helpful if you write a clear, concise description of what this command does. For simple commands, keep it brief (5-10 words). For complex commands (piped commands, obscure flags, or anything hard to understand at a glance), add enough context to clarify what it does.\n - If the output exceeds 30000 characters, output will be truncated before being returned to you.\n \n - You can use the `run_in_background` parameter to run the command in the background. Only use this if you don't need the result immediately and are OK being notified when the command completes later. You do not need to check the output right away - you'll be notified when it finishes. You do not need to use '&' at the end of the command when using this parameter.\n \n - Avoid using Bash with the `find`, `grep`, `cat`, `head`, `tail`, `sed`, `awk`, or `echo` commands, unless explicitly instructed or when these commands are truly necessary for the task. Instead, always prefer using the dedicated tools for these commands:\n - File search: Use Glob (NOT find or ls)\n - Content search: Use Grep (NOT grep or rg)\n - Read files: Use Read (NOT cat/head/tail)\n - Edit files: Use Edit (NOT sed/awk)\n - Write files: Use Write (NOT echo >/cat <\n pytest /foo/bar/tests\n \n \n cd /foo/bar && pytest tests\n \n\n# Committing changes with git\n\nOnly create commits when requested by the user. If unclear, ask first. When the user asks you to create a new git commit, follow these steps carefully:\n\nGit Safety Protocol:\n- NEVER update the git config\n- NEVER run destructive/irreversible git commands (like push --force, hard reset, etc) unless the user explicitly requests them\n- NEVER skip hooks (--no-verify, --no-gpg-sign, etc) unless the user explicitly requests it\n- NEVER run force push to main/master, warn the user if they request it\n- CRITICAL: ALWAYS create NEW commits. NEVER use git commit --amend, unless the user explicitly requests it\n- NEVER commit changes unless the user explicitly asks you to. It is VERY IMPORTANT to only commit when explicitly asked, otherwise the user will feel that you are being too proactive.\n\n1. You can call multiple tools in a single response. When multiple independent pieces of information are requested and all commands are likely to succeed, run multiple tool calls in parallel for optimal performance. run the following bash commands in parallel, each using the Bash tool:\n - Run a git status command to see all untracked files. IMPORTANT: Never use the -uall flag as it can cause memory issues on large repos.\n - Run a git diff command to see both staged and unstaged changes that will be committed.\n - Run a git log command to see recent commit messages, so that you can follow this repository's commit message style.\n2. Analyze all staged changes (both previously staged and newly added) and draft a commit message:\n - Summarize the nature of the changes (eg. new feature, enhancement to an existing feature, bug fix, refactoring, test, docs, etc.). Ensure the message accurately reflects the changes and their purpose (i.e. \"add\" means a wholly new feature, \"update\" means an enhancement to an existing feature, \"fix\" means a bug fix, etc.).\n - Do not commit files that likely contain secrets (.env, credentials.json, etc). Warn the user if they specifically request to commit those files\n - Draft a concise (1-2 sentences) commit message that focuses on the \"why\" rather than the \"what\"\n - Ensure it accurately reflects the changes and their purpose\n3. You can call multiple tools in a single response. When multiple independent pieces of information are requested and all commands are likely to succeed, run multiple tool calls in parallel for optimal performance. run the following commands:\n - Add relevant untracked files to the staging area.\n - Create the commit with a message ending with:\n Co-Authored-By: Claude Opus 4.5 \n - Run git status after the commit completes to verify success.\n Note: git status depends on the commit completing, so run it sequentially after the commit.\n4. If the commit fails due to pre-commit hook: fix the issue and create a NEW commit\n\nImportant notes:\n- NEVER run additional commands to read or explore code, besides git bash commands\n- NEVER use the TodoWrite or Task tools\n- DO NOT push to the remote repository unless the user explicitly asks you to do so\n- IMPORTANT: Never use git commands with the -i flag (like git rebase -i or git add -i) since they require interactive input which is not supported.\n- If there are no changes to commit (i.e., no untracked files and no modifications), do not create an empty commit\n- In order to ensure good formatting, ALWAYS pass the commit message via a HEREDOC, a la this example:\n\ngit commit -m \"$(cat <<'EOF'\n Commit message here.\n\n Co-Authored-By: Claude Opus 4.5 \n EOF\n )\"\n\n\n# Creating pull requests\nUse the gh command via the Bash tool for ALL GitHub-related tasks including working with issues, pull requests, checks, and releases. If given a Github URL use the gh command to get the information needed.\n\nIMPORTANT: When the user asks you to create a pull request, follow these steps carefully:\n\n1. You can call multiple tools in a single response. When multiple independent pieces of information are requested and all commands are likely to succeed, run multiple tool calls in parallel for optimal performance. run the following bash commands in parallel using the Bash tool, in order to understand the current state of the branch since it diverged from the main branch:\n - Run a git status command to see all untracked files (never use -uall flag)\n - Run a git diff command to see both staged and unstaged changes that will be committed\n - Check if the current branch tracks a remote branch and is up to date with the remote, so you know if you need to push to the remote\n - Run a git log command and `git diff [base-branch]...HEAD` to understand the full commit history for the current branch (from the time it diverged from the base branch)\n2. Analyze all changes that will be included in the pull request, making sure to look at all relevant commits (NOT just the latest commit, but ALL commits that will be included in the pull request!!!), and draft a pull request summary\n3. You can call multiple tools in a single response. When multiple independent pieces of information are requested and all commands are likely to succeed, run multiple tool calls in parallel for optimal performance. run the following commands in parallel:\n - Create new branch if needed\n - Push to remote with -u flag if needed\n - Create PR using gh pr create with the format below. Use a HEREDOC to pass the body to ensure correct formatting.\n\ngh pr create --title \"the pr title\" --body \"$(cat <<'EOF'\n## Summary\n<1-3 bullet points>\n\n## Test plan\n[Bulleted markdown checklist of TODOs for testing the pull request...]\n\n🤖 Generated with [Claude Code](https://claude.com/claude-code)\nEOF\n)\"\n\n\nImportant:\n- DO NOT use the TodoWrite or Task tools\n- Return the PR URL when you're done, so the user can see it\n\n# Other common operations\n- View comments on a Github PR: gh api repos/foo/bar/pulls/123/comments", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "command": { + "description": "The command to execute", + "type": "string" + }, + "timeout": { + "description": "Optional timeout in milliseconds (max 600000)", + "type": "number" + }, + "description": { + "description": "Clear, concise description of what this command does in active voice. Never use words like \"complex\" or \"risk\" in the description - just describe what it does.\n\nFor simple commands (git, npm, standard CLI tools), keep it brief (5-10 words):\n- ls → \"List files in current directory\"\n- git status → \"Show working tree status\"\n- npm install → \"Install package dependencies\"\n\nFor commands that are harder to parse at a glance (piped commands, obscure flags, etc.), add enough context to clarify what it does:\n- find . -name \"*.tmp\" -exec rm {} \\; → \"Find and delete all .tmp files recursively\"\n- git reset --hard origin/main → \"Discard all local changes and match remote main\"\n- curl -s url | jq '.data[]' → \"Fetch JSON from URL and extract data array elements\"", + "type": "string" + }, + "run_in_background": { + "description": "Set to true to run this command in the background. Use TaskOutput to read the output later.", + "type": "boolean" + }, + "dangerouslyDisableSandbox": { + "description": "Set this to true to dangerously override sandbox mode and run commands without sandboxing.", + "type": "boolean" + }, + "_simulatedSedEdit": { + "description": "Internal: pre-computed sed edit result from preview", + "type": "object", + "properties": { + "filePath": { + "type": "string" + }, + "newContent": { + "type": "string" + } + }, + "required": [ + "filePath", + "newContent" + ], + "additionalProperties": false + } + }, + "required": [ + "command" + ], + "additionalProperties": false + } + }, diff --git a/codex-extract-terminal-spawning-tool.md b/codex-extract-terminal-spawning-tool.md new file mode 100644 index 0000000000..aef70d584c --- /dev/null +++ b/codex-extract-terminal-spawning-tool.md @@ -0,0 +1,1206 @@ +# Executive Summary + +This document specifies the **Terminal Spawning Tool** feature—a system that enables an AI agent to execute shell commands on a host machine with comprehensive support for: + +- **Multiple spawn modes**: PTY-based interactive sessions or pipe-based non-interactive processes +- **Shell abstraction**: Cross-platform shell detection and command translation (Bash, Zsh, PowerShell, sh, cmd) +- **Sandbox enforcement**: Platform-native sandboxing (macOS Seatbelt, Linux seccomp/Landlock, Windows restricted tokens) +- **Approval workflows**: Configurable human-in-the-loop approval for dangerous operations +- **Process lifecycle management**: Output buffering, timeout handling, cancellation, and cleanup +- **Interactive sessions**: Persistent PTY processes that maintain state across multiple tool calls + +The feature is designed for AI coding assistants that need to execute commands while balancing autonomy with safety through layered sandboxing and approval mechanisms. + +--- + +# Glossary + +| Term | Definition | +| ----------------------- | ---------------------------------------------------------------------------------------------------- | +| **ToolHandler** | Registry entry that matches incoming tool calls by name and dispatches execution | +| **ToolRuntime** | Execution backend that runs a specific request type under sandbox orchestration | +| **ToolOrchestrator** | Central coordinator managing approval → sandbox selection → execution → retry | +| **ExecParams** | Portable command specification: command vector, working directory, environment, timeout | +| **ExecEnv** | Transformed execution environment ready for spawning (includes sandbox wrapper commands) | +| **SandboxPolicy** | Session-level filesystem/network access policy (ReadOnly, WorkspaceWrite, DangerFullAccess) | +| **SandboxPermissions** | Per-call override (UseDefault, RequireEscalated) | +| **SandboxType** | Platform-specific sandbox implementation (None, MacosSeatbelt, LinuxSeccomp, WindowsRestrictedToken) | +| **ProcessHandle** | Abstraction over a spawned process providing stdin writer, output receiver, and termination | +| **SpawnedProcess** | Return value from PTY/pipe spawn containing ProcessHandle, output channel, and exit receiver | +| **UnifiedExecProcess** | Managed process wrapper with output buffering, sandbox awareness, and lifecycle hooks | +| **ApprovalRequirement** | Classification of a command: Skip, NeedsApproval, or Forbidden | +| **Shell** | Detected user shell with type (Bash/Zsh/PowerShell/sh/cmd), path, and optional environment snapshot | + +--- + +# Feature Overview & Boundaries + +## What the Feature Does + +The Terminal Spawning Tool enables an AI agent to: + +1. **Execute shell commands** by translating high-level requests into platform-appropriate shell invocations +2. **Manage interactive sessions** where a PTY process persists across multiple tool calls, maintaining shell state +3. **Enforce security policies** through configurable sandboxing and human approval workflows +4. **Stream output** with intelligent truncation and buffering for token-efficient responses +5. **Handle timeouts and cancellation** gracefully, cleaning up process trees + +## Boundaries + +**In Scope:** + +- Shell command execution (one-shot and interactive) +- Cross-platform shell detection and argument translation +- Sandbox policy enforcement with platform-native mechanisms +- Approval caching and retry-without-sandbox flows +- Output buffering with head/tail preservation +- Process group management for clean termination + +**Out of Scope:** + +- GUI application launching +- Network service management +- Container orchestration +- Remote execution + +--- + +# System Architecture (High Level) + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Agent / LLM Interface │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ Tool Invocation Layer │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ ShellHandler │ │ShellCommandHandler│ │ UnifiedExec │ │ +│ │ (shell tool) │ │ (shell_command) │ │ (exec_command) │ │ +│ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ │ +│ │ │ │ │ +│ └────────────────────┴────────────────────┘ │ +│ │ │ +│ ┌─────────────────────────────▼─────────────────────────────────────┐ │ +│ │ ToolOrchestrator │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │ │ +│ │ │ Approval │ │ Sandbox │ │ Retry on Sandbox Denial │ │ │ +│ │ │ Workflow │ │ Selection │ │ (with re-approval) │ │ │ +│ │ └──────┬──────┘ └──────┬──────┘ └────────────┬────────────┘ │ │ +│ └─────────┴────────────────┴──────────────────────┴─────────────────┘ │ +│ │ │ +│ ┌─────────────────────────────▼─────────────────────────────────────┐ │ +│ │ SandboxManager │ │ +│ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │ +│ │ │ Seatbelt │ │ Landlock/ │ │ Windows │ │ │ +│ │ │ (macOS) │ │ seccomp (Linux)│ │ Restricted │ │ │ +│ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +│ │ │ +├────────────────────────────────▼────────────────────────────────────────────┤ +│ Process Spawning Layer │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ PTY Spawn │ │ Pipe Spawn │ │ spawn_child_async│ │ +│ │ (interactive) │ │ (non-interactive)│ │ (direct) │ │ +│ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +# Core Data Model & Schemas + +## ShellToolCallParams + +Parameters for the `shell` tool (command as array): + +```typescript +interface ShellToolCallParams { + command: string[] // e.g., ["ls", "-la"] + workdir?: string // Working directory (relative to session cwd) + timeout_ms?: number // Maximum execution time (default: 10000) + sandbox_permissions?: "use_default" | "require_escalated" + justification?: string // Reason for escalated permissions +} +``` + +## ShellCommandToolCallParams + +Parameters for the `shell_command` tool (command as freeform string): + +```typescript +interface ShellCommandToolCallParams { + command: string // e.g., "ls -la | grep foo" + workdir?: string + login?: boolean // Use login shell semantics (default: true) + timeout_ms?: number + sandbox_permissions?: "use_default" | "require_escalated" + justification?: string +} +``` + +## ExecParams (Internal) + +Portable execution parameters after initial processing: + +```typescript +interface ExecParams { + command: string[] // Full command vector including shell + cwd: PathBuf // Resolved absolute working directory + expiration: ExecExpiration // Timeout | DefaultTimeout | Cancellation + env: Map // Environment variables + sandbox_permissions: SandboxPermissions + justification?: string + arg0?: string // Optional argv[0] override +} +``` + +## ExecEnv (Sandbox-Transformed) + +Ready-to-spawn environment after sandbox transformation: + +```typescript +interface ExecEnv { + command: string[] // May include sandbox wrapper (e.g., sandbox-exec) + cwd: PathBuf + env: Map // Includes CODEX_SANDBOX_* variables + expiration: ExecExpiration + sandbox: SandboxType // None | MacosSeatbelt | LinuxSeccomp | WindowsRestrictedToken + sandbox_permissions: SandboxPermissions + justification?: string + arg0?: string +} +``` + +## Shell + +Detected user shell configuration: + +```typescript +interface Shell { + shell_type: "Zsh" | "Bash" | "PowerShell" | "Sh" | "Cmd" + shell_path: PathBuf // e.g., "/bin/zsh" + shell_snapshot?: ShellSnapshot // Optional environment snapshot for login shell emulation +} +``` + +## ProcessHandle + +Abstraction over a running process: + +```typescript +interface ProcessHandle { + writer_sender(): Sender // stdin channel + output_receiver(): BroadcastReceiver // stdout+stderr + has_exited(): boolean + exit_code(): number | null + terminate(): void +} +``` + +## SpawnedProcess + +Return value from spawn functions: + +```typescript +interface SpawnedProcess { + session: ProcessHandle + output_rx: BroadcastReceiver // Initial output subscription + exit_rx: OneshotReceiver // Exit code notification +} +``` + +## ExecToolCallOutput + +Result of command execution: + +```typescript +interface ExecToolCallOutput { + exit_code: number + stdout: StreamOutput + stderr: StreamOutput + aggregated_output: StreamOutput // Combined stdout + stderr + duration: Duration + timed_out: boolean +} + +interface StreamOutput { + text: T + truncated_after_lines?: number +} +``` + +--- + +# Public Interfaces + +## Tool Registration + +Tools are registered with a handler that implements: + +```typescript +interface ToolHandler { + kind(): ToolKind // Function | Custom | MCP + matches_kind(payload: ToolPayload): boolean // Can handle this payload type? + is_mutating(invocation: ToolInvocation): Promise // Affects filesystem? + handle(invocation: ToolInvocation): Promise +} +``` + +## ToolInvocation + +Context passed to handlers: + +```typescript +interface ToolInvocation { + session: Session // Global session state + turn: TurnContext // Current conversation turn + tracker: TurnDiffTracker // File change tracking + call_id: string // Unique identifier for this call + tool_name: string + payload: ToolPayload // Function | Custom | LocalShell | MCP +} +``` + +## ToolPayload Variants + +```typescript +type ToolPayload = + | { type: "Function"; arguments: string } // JSON arguments + | { type: "Custom"; input: string } // Raw input + | { type: "LocalShell"; params: ShellToolCallParams } + | { type: "Mcp"; server: string; tool: string; raw_arguments: string } +``` + +## ToolOutput + +Return value from handlers: + +```typescript +type ToolOutput = + | { type: "Function"; content: string; content_items?: ContentItem[]; success?: boolean } + | { type: "Mcp"; result: Result } +``` + +--- + +# Runtime Flow (End-to-End) + +```mermaid +sequenceDiagram + participant Agent + participant Handler as ShellHandler + participant Orchestrator as ToolOrchestrator + participant Runtime as ShellRuntime + participant Sandbox as SandboxManager + participant Spawner as spawn_child_async + + Agent->>Handler: handle(invocation) + Handler->>Handler: parse arguments to ExecParams + Handler->>Orchestrator: run(runtime, request, ctx) + + Orchestrator->>Orchestrator: check ExecApprovalRequirement + alt NeedsApproval + Orchestrator->>Agent: request_command_approval() + Agent-->>Orchestrator: ReviewDecision + end + + Orchestrator->>Sandbox: select_initial(policy, preference) + Sandbox-->>Orchestrator: SandboxType + + Orchestrator->>Runtime: run(request, attempt, ctx) + Runtime->>Runtime: build CommandSpec + Runtime->>Sandbox: transform(spec, policy, sandbox_type) + Sandbox-->>Runtime: ExecEnv + Runtime->>Spawner: spawn_child_async(program, args, cwd, env) + Spawner-->>Runtime: Child process + Runtime->>Runtime: consume_truncated_output(child, timeout) + Runtime-->>Orchestrator: ExecToolCallOutput + + alt Sandbox Denied & escalate_on_failure + Orchestrator->>Agent: request approval for no-sandbox retry + Agent-->>Orchestrator: Approved + Orchestrator->>Runtime: run(request, attempt{sandbox: None}) + Runtime-->>Orchestrator: ExecToolCallOutput + end + + Orchestrator-->>Handler: ExecToolCallOutput + Handler->>Handler: format output as ToolOutput + Handler-->>Agent: ToolOutput +``` + +--- + +# Initialization, Discovery, and Registration (If Applicable) + +## Shell Detection + +At session startup, the system detects the user's default shell: + +```mermaid +sequenceDiagram + participant Session + participant ShellDetector + participant System + + Session->>ShellDetector: default_user_shell() + ShellDetector->>System: getpwuid(getuid()).pw_shell [Unix] + System-->>ShellDetector: "/bin/zsh" + ShellDetector->>ShellDetector: detect_shell_type("/bin/zsh") + ShellDetector-->>Session: Shell { type: Zsh, path: "/bin/zsh" } +``` + +**Detection Algorithm:** + +1. On Unix: Read `pw_shell` from `getpwuid(getuid())` +2. Map shell path to type by matching basename (zsh → Zsh, bash → Bash, etc.) +3. Validate shell exists via `which` or fallback paths +4. On Windows: Default to PowerShell, fallback to cmd.exe + +## Tool Handler Registration + +Handlers are registered in a static registry: + +```typescript +// Pseudocode for handler registration +const TOOL_REGISTRY = { + shell: new ShellHandler(), + "container.exec": new ShellHandler(), // Alias + shell_command: new ShellCommandHandler(), + exec_command: new UnifiedExecHandler(), + write_stdin: new WriteStdinHandler(), +} +``` + +--- + +# Invocation, Routing, and Orchestration + +## Invocation Entry Points + +### 1. `shell` Tool (Vector Command) + +The agent provides a command as an array: + +```json +{ + "name": "shell", + "arguments": "{\"command\": [\"ls\", \"-la\"], \"workdir\": \"src\"}" +} +``` + +**Flow:** + +1. `ShellHandler.handle()` parses `ShellToolCallParams` +2. Converts to `ExecParams` (command vector used as-is) +3. Delegates to `run_exec_like()` + +### 2. `shell_command` Tool (Freeform String) + +The agent provides a shell command string: + +```json +{ + "name": "shell_command", + "arguments": "{\"command\": \"grep -r 'TODO' src/\"}" +} +``` + +**Flow:** + +1. `ShellCommandHandler.handle()` parses `ShellCommandToolCallParams` +2. Calls `derive_exec_args()` on the session's detected shell +3. For Bash/Zsh: `["/bin/zsh", "-lc", "grep -r 'TODO' src/"]` +4. For PowerShell: `["pwsh", "-Command", "grep -r 'TODO' src/"]` + +### 3. `exec_command` Tool (Interactive/Unified Exec) + +For interactive sessions that persist: + +```json +{ + "name": "exec_command", + "arguments": "{\"command\": [\"bash\", \"-i\"], \"process_id\": \"1234\", \"yield_time_ms\": 2500}" +} +``` + +**Flow:** + +1. `UnifiedExecHandler` allocates or retrieves process by ID +2. Opens PTY session if new +3. Collects output until yield time or process exit +4. Returns output with optional `process_id` for continuation + +### 4. `write_stdin` Tool (Send Input to Existing Process) + +```json +{ + "name": "write_stdin", + "arguments": "{\"process_id\": \"1234\", \"input\": \"export FOO=bar\\n\"}" +} +``` + +## Orchestration Flow + +The `ToolOrchestrator` coordinates the execution: + +``` +1. APPROVAL PHASE + ├─ Check ExecApprovalRequirement from exec_policy + ├─ If Skip: proceed immediately + ├─ If Forbidden: reject with error + └─ If NeedsApproval: + ├─ Check approval cache + ├─ If cached ApprovedForSession: proceed + └─ Else: prompt user, cache decision + +2. SANDBOX SELECTION PHASE + ├─ Check sandbox_mode_for_first_attempt(request) + ├─ If BypassSandboxFirstAttempt: use SandboxType::None + └─ Else: select_initial(policy, preference) + ├─ DangerFullAccess → None + ├─ ExternalSandbox → None + └─ ReadOnly/WorkspaceWrite → platform sandbox + +3. EXECUTION PHASE + ├─ Transform CommandSpec → ExecEnv via SandboxManager + ├─ Spawn process with spawn_child_async or PTY + └─ Collect output with timeout + +4. RETRY PHASE (on sandbox denial) + ├─ Detect denial via is_likely_sandbox_denied() + ├─ If escalate_on_failure && approval_policy allows: + │ ├─ Prompt for no-sandbox approval + │ └─ Re-execute with SandboxType::None + └─ Else: return error +``` + +--- + +# Permissions, Guardrails, and Validation + +## Approval Policies + +| Policy | Behavior | +| --------------- | ------------------------------------------------------ | +| `Never` | Never prompt; agent has full autonomy | +| `UnlessTrusted` | Always prompt unless command matches trusted patterns | +| `OnFailure` | Prompt only if command fails in sandbox | +| `OnRequest` | Prompt for all commands unless DangerFullAccess policy | + +## Sandbox Policies + +| Policy | Read | Write | Network | +| ------------------ | ---------------------------- | -------------------- | ------------ | +| `ReadOnly` | Anywhere | Nowhere | Blocked | +| `WorkspaceWrite` | Anywhere | cwd + writable_roots | Configurable | +| `DangerFullAccess` | Anywhere | Anywhere | Full | +| `ExternalSandbox` | Delegated to external system | | | + +## Safe Command Detection + +Commands are classified as "safe" (non-mutating) via `is_known_safe_command()`: + +```typescript +// Safe command patterns (no approval needed even in strict modes) +const SAFE_PATTERNS = [ + /^ls\b/, + /^cat\b/, + /^head\b/, + /^tail\b/, + /^grep\b/, + /^find\b/, + /^pwd$/, + /^echo\b/, + /^env$/, + // ... etc +] +``` + +## Sandbox Denial Detection + +After execution, output is scanned for sandbox denial indicators: + +```typescript +const SANDBOX_DENIED_KEYWORDS = [ + "operation not permitted", + "permission denied", + "read-only file system", + "seccomp", + "sandbox", + "landlock", + "failed to write file", +] +``` + +--- + +# Error Model, Retries, Timeouts, and Cancellation + +## Error Types + +```typescript +type ExecError = + | { type: "Timeout"; output: ExecToolCallOutput } // Command exceeded timeout + | { type: "Denied"; output: ExecToolCallOutput } // Sandbox blocked operation + | { type: "Signal"; signal: number } // Killed by signal + | { type: "IoError"; message: string } // Spawn/read failure + | { type: "Rejected"; reason: string } // User denied approval +``` + +## Timeout Handling + +```typescript +const DEFAULT_EXEC_COMMAND_TIMEOUT_MS = 10_000; +const EXEC_TIMEOUT_EXIT_CODE = 124; // Conventional timeout exit code + +async function consume_truncated_output(child, expiration) { + select! { + status = child.wait() => (status, timed_out: false), + _ = expiration.wait() => { + kill_child_process_group(child); + child.start_kill(); + (EXIT_CODE_SIGNAL_BASE + TIMEOUT_CODE, timed_out: true) + }, + _ = ctrl_c() => { + kill_child_process_group(child); + child.start_kill(); + (EXIT_CODE_SIGNAL_BASE + SIGKILL_CODE, timed_out: false) + } + } +} +``` + +## Cancellation + +Commands support cancellation via `CancellationToken`: + +```typescript +interface ExecExpiration { + type: "Timeout" | "DefaultTimeout" | "Cancellation" + duration?: Duration // For Timeout + token?: CancellationToken // For Cancellation +} +``` + +## Retry Logic + +On sandbox denial (detected via exit code + keywords): + +1. Check `escalate_on_failure()` on runtime → true for shell +2. Check approval policy allows retry → not Never/OnRequest +3. Prompt user with denial reason +4. If approved, re-execute with `SandboxType::None` + +--- + +# Async, Streaming, and Concurrency + +## Output Streaming + +Output is streamed via events during execution: + +```typescript +interface ExecCommandOutputDeltaEvent { + call_id: string + stream: "Stdout" | "Stderr" + chunk: bytes +} +``` + +Streaming is capped to prevent event flooding: + +```typescript +const MAX_EXEC_OUTPUT_DELTAS_PER_CALL = 10_000 +``` + +## Output Buffering + +For interactive sessions, a `HeadTailBuffer` preserves both beginning and end of output: + +```typescript +const UNIFIED_EXEC_OUTPUT_MAX_BYTES = 1024 * 1024 // 1 MiB + +class HeadTailBuffer { + head: bytes[] // First chunks + tail: bytes[] // Last chunks + total_bytes: number + + push_chunk(chunk: bytes) { + if (total_bytes >= MAX_BYTES) { + // Evict from middle, keep head + tail + } + } + + snapshot_chunks(): bytes[] { + return [...head, ...tail] + } +} +``` + +## Concurrent Process Management + +The `UnifiedExecProcessManager` tracks up to 64 concurrent interactive processes: + +```typescript +const MAX_UNIFIED_EXEC_PROCESSES = 64 +const WARNING_UNIFIED_EXEC_PROCESSES = 60 + +class ProcessStore { + processes: Map + reserved_process_ids: Set +} + +// Pruning policy when at capacity: +// 1. Prefer exited processes outside "recently used" set (last 8) +// 2. Fallback to LRU process outside protected set +``` + +## Process Group Management + +Child processes are placed in their own process group for clean termination: + +```typescript +// In pre_exec (Unix): +function detach_from_tty() { + setsid() // Start new session +} + +function set_parent_death_signal(parent_pid) { + // Linux only + prctl(PR_SET_PDEATHSIG, SIGTERM) + if (getppid() != parent_pid) raise(SIGTERM) // Race check +} + +// Termination: +function kill_process_group(pgid) { + killpg(pgid, SIGKILL) +} +``` + +--- + +# Logging, Metrics, and Telemetry + +## Event Emission + +Tool execution emits lifecycle events: + +```typescript +// Begin event +ToolEmitter.shell(command, cwd, source, freeform).begin(ctx) + +// End event (on completion) +emitter.finish(ctx, result) +``` + +## Telemetry Preview + +Output is truncated for telemetry: + +```typescript +const TELEMETRY_PREVIEW_MAX_BYTES = 2048 +const TELEMETRY_PREVIEW_MAX_LINES = 50 +const TELEMETRY_PREVIEW_TRUNCATION_NOTICE = "[output truncated]" +``` + +## Approval Metrics + +```typescript +otel.counter("codex.approval.requested", 1, { + tool: "shell", + approved: decision.to_opaque_string(), +}) +``` + +## Sandbox Environment Variables + +Set on spawned processes for observability: + +```typescript +// When network access is restricted: +CODEX_SANDBOX_NETWORK_DISABLED = 1 + +// When running under platform sandbox: +CODEX_SANDBOX = seatbelt // macOS +``` + +--- + +# Configuration + +## Session-Level Configuration + +```typescript +interface SessionConfig { + sandbox_policy: SandboxPolicy + approval_policy: AskForApproval + shell_environment_policy: ShellEnvironmentPolicy // env vars to inherit + codex_linux_sandbox_exe?: PathBuf // Path to Landlock sandbox binary +} +``` + +## Per-Turn Context + +```typescript +interface TurnContext { + cwd: PathBuf + sandbox_policy: SandboxPolicy + approval_policy: AskForApproval + shell_environment_policy: ShellEnvironmentPolicy + codex_linux_sandbox_exe?: PathBuf +} +``` + +## Environment Variables for Spawned Processes + +Interactive sessions (`exec_command`) inject: + +```typescript +const UNIFIED_EXEC_ENV = { + NO_COLOR: "1", + TERM: "dumb", + LANG: "C.UTF-8", + LC_CTYPE: "C.UTF-8", + LC_ALL: "C.UTF-8", + COLORTERM: "", + PAGER: "cat", + GIT_PAGER: "cat", + GH_PAGER: "cat", + CODEX_CI: "1", +} +``` + +--- + +# Extension Points + +## Adding a New Shell Type + +1. Add variant to `ShellType` enum +2. Implement `derive_exec_args()` for the new shell +3. Add detection in `detect_shell_type()` +4. Add discovery in `get_shell()` + +## Adding a New Sandbox Backend + +1. Add variant to `SandboxType` enum +2. Implement transformation in `SandboxManager.transform()` +3. Add platform detection in `get_platform_sandbox()` +4. Implement denial detection patterns + +## Adding a New Approval Policy + +1. Add variant to `AskForApproval` enum +2. Update `default_exec_approval_requirement()` +3. Update `wants_no_sandbox_approval()` logic +4. Create corresponding prompt template + +## Custom Tool Runtime + +Implement these traits: + +```typescript +interface ToolRuntime { + // From Sandboxable + sandbox_preference(): SandboxablePreference + escalate_on_failure(): boolean + + // From Approvable + approval_keys(req: Request): ApprovalKey[] + start_approval_async(req: Request, ctx: ApprovalCtx): Promise + + // Execution + run(req: Request, attempt: SandboxAttempt, ctx: ToolCtx): Promise +} +``` + +--- + +# Reference Implementation Sketch (Pseudocode) + +``` +// === TYPES === + +enum SandboxType { None, MacosSeatbelt, LinuxSeccomp, WindowsRestricted } +enum ApprovalPolicy { Never, UnlessTrusted, OnFailure, OnRequest } +enum ReviewDecision { Approved, ApprovedForSession, Denied, Abort } + +struct ExecParams { + command: Vec + cwd: Path + timeout: Duration + env: Map + sandbox_permissions: SandboxPermissions +} + +struct ExecEnv { + command: Vec + cwd: Path + env: Map + timeout: Duration + sandbox: SandboxType +} + +struct ExecOutput { + exit_code: i32 + stdout: String + stderr: String + timed_out: bool +} + +// === SHELL DETECTION === + +function detect_user_shell() -> Shell: + path = get_passwd_shell() OR "/bin/sh" + type = match basename(path): + "zsh" -> Zsh + "bash" -> Bash + "pwsh" | "powershell" -> PowerShell + "sh" -> Sh + "cmd" -> Cmd + return Shell { type, path } + +function derive_exec_args(shell: Shell, command: String, login: bool) -> Vec: + match shell.type: + Zsh | Bash | Sh: + flag = login ? "-lc" : "-c" + return [shell.path, flag, command] + PowerShell: + args = [shell.path] + if !login: args.push("-NoProfile") + args.push("-Command", command) + return args + Cmd: + return [shell.path, "/c", command] + +// === SANDBOX TRANSFORMATION === + +function select_sandbox(policy: SandboxPolicy) -> SandboxType: + if policy == DangerFullAccess OR policy == ExternalSandbox: + return None + return get_platform_sandbox() OR None + +function transform_for_sandbox(spec: CommandSpec, sandbox: SandboxType) -> ExecEnv: + env = spec.env.clone() + if !policy.has_network_access(): + env["CODEX_SANDBOX_NETWORK_DISABLED"] = "1" + + command = [spec.program] + spec.args + + match sandbox: + None: + return ExecEnv { command, cwd: spec.cwd, env, sandbox: None } + MacosSeatbelt: + env["CODEX_SANDBOX"] = "seatbelt" + wrapper = ["/usr/bin/sandbox-exec", "-f", profile_path()] + command + return ExecEnv { command: wrapper, cwd: spec.cwd, env, sandbox } + LinuxSeccomp: + wrapper = [sandbox_exe, "--policy", policy_json()] + command + return ExecEnv { command: wrapper, cwd: spec.cwd, env, sandbox } + +// === APPROVAL WORKFLOW === + +async function check_approval( + request: Request, + policy: ApprovalPolicy, + cache: ApprovalCache +) -> ReviewDecision: + + requirement = compute_approval_requirement(request, policy) + + match requirement: + Skip: + return Approved + Forbidden(reason): + throw Rejected(reason) + NeedsApproval: + key = approval_key(request) + if cache.get(key) == ApprovedForSession: + return ApprovedForSession + + decision = await prompt_user(request) + if decision == ApprovedForSession: + cache.put(key, decision) + return decision + +// === PROCESS SPAWNING === + +async function spawn_child(env: ExecEnv) -> Child: + command = Command::new(env.command[0]) + command.args(env.command[1..]) + command.current_dir(env.cwd) + command.env_clear() + command.envs(env.env) + + // Unix: detach from TTY, set parent death signal + command.pre_exec(|| { + setsid() + prctl(PR_SET_PDEATHSIG, SIGTERM) // Linux + }) + + command.stdin(Stdio::null()) // Prevent hanging on stdin + command.stdout(Stdio::piped()) + command.stderr(Stdio::piped()) + command.kill_on_drop(true) + + return command.spawn() + +async function spawn_pty(program: String, args: Vec, env: Map) -> SpawnedProcess: + pty = native_pty_system().openpty(24, 80) + child = pty.slave.spawn_command(CommandBuilder::new(program).args(args).env(env)) + + // Start reader task for PTY output + reader_task = spawn(async || { + loop: + chunk = pty.master.read() + if chunk.empty(): break + output_tx.send(chunk) + }) + + // Start writer task for PTY input + writer_task = spawn(async || { + while input = writer_rx.recv(): + pty.master.write(input) + }) + + return SpawnedProcess { handle, output_rx, exit_rx } + +// === EXECUTION WITH TIMEOUT === + +async function execute_with_timeout(child: Child, timeout: Duration) -> ExecOutput: + stdout_task = spawn(read_capped(child.stdout)) + stderr_task = spawn(read_capped(child.stderr)) + + select: + status = child.wait(): + stdout = await stdout_task + stderr = await stderr_task + return ExecOutput { exit_code: status.code(), stdout, stderr, timed_out: false } + + _ = sleep(timeout): + kill_process_group(child.pid()) + child.kill() + return ExecOutput { exit_code: 124, stdout: "", stderr: "", timed_out: true } + +// === SANDBOX DENIAL DETECTION === + +function is_sandbox_denied(sandbox: SandboxType, output: ExecOutput) -> bool: + if sandbox == None OR output.exit_code == 0: + return false + + keywords = ["operation not permitted", "permission denied", "read-only file system"] + text = (output.stdout + output.stderr).lowercase() + return any(k in text for k in keywords) + +// === MAIN ORCHESTRATION === + +async function run_shell_tool(invocation: ToolInvocation) -> ToolOutput: + params = parse_arguments(invocation.payload) + exec_params = to_exec_params(params, invocation.turn) + + // 1. Approval + decision = await check_approval(exec_params, invocation.turn.approval_policy, cache) + if decision in [Denied, Abort]: + throw Rejected("user denied") + + // 2. First sandbox attempt + sandbox = select_sandbox(invocation.turn.sandbox_policy) + exec_env = transform_for_sandbox(exec_params, sandbox) + child = await spawn_child(exec_env) + output = await execute_with_timeout(child, exec_params.timeout) + + // 3. Retry without sandbox if denied + if is_sandbox_denied(sandbox, output): + if approval_policy != Never: + retry_decision = await prompt_user_for_retry(exec_params) + if retry_decision == Approved: + exec_env = transform_for_sandbox(exec_params, None) + child = await spawn_child(exec_env) + output = await execute_with_timeout(child, exec_params.timeout) + + // 4. Format output + return ToolOutput::Function { + content: format_output(output), + success: output.exit_code == 0 + } +``` + +--- + +# Worked Example + +## Scenario: Execute `grep` Command with Sandbox + +**Agent Request:** + +```json +{ + "type": "function_call", + "name": "shell_command", + "call_id": "call_abc123", + "arguments": "{\"command\": \"grep -r 'TODO' src/\", \"timeout_ms\": 5000}" +} +``` + +**Step 1: Handler Dispatch** + +``` +ShellCommandHandler.handle(invocation) + params = ShellCommandToolCallParams { + command: "grep -r 'TODO' src/", + timeout_ms: 5000, + ...defaults + } +``` + +**Step 2: Shell Command Translation** + +``` +session.user_shell() = Shell { type: Zsh, path: "/bin/zsh" } +derive_exec_args(shell, "grep -r 'TODO' src/", login=true) + → ["/bin/zsh", "-lc", "grep -r 'TODO' src/"] +``` + +**Step 3: Build ExecParams** + +``` +ExecParams { + command: ["/bin/zsh", "-lc", "grep -r 'TODO' src/"], + cwd: "/home/user/project", + expiration: Timeout(5000ms), + env: { PATH: "...", HOME: "...", ... }, + sandbox_permissions: UseDefault +} +``` + +**Step 4: Orchestrator - Approval Check** + +``` +approval_policy = OnRequest +sandbox_policy = WorkspaceWrite +is_known_safe_command(["/bin/zsh", "-lc", "grep ..."]) = true // grep is safe +→ ExecApprovalRequirement::Skip { bypass_sandbox: false } +``` + +**Step 5: Orchestrator - Sandbox Selection** + +``` +sandbox_mode_for_first_attempt(request) = NoOverride +select_initial(WorkspaceWrite, Auto) = MacosSeatbelt // on macOS +``` + +**Step 6: SandboxManager Transform** + +``` +ExecEnv { + command: [ + "/usr/bin/sandbox-exec", + "-f", "/tmp/codex-sandbox-profile.sb", + "-D", "CWD=/home/user/project", + "/bin/zsh", "-lc", "grep -r 'TODO' src/" + ], + cwd: "/home/user/project", + env: { ..., CODEX_SANDBOX: "seatbelt", CODEX_SANDBOX_NETWORK_DISABLED: "1" }, + sandbox: MacosSeatbelt +} +``` + +**Step 7: Process Spawn** + +``` +child = spawn_child_async( + program: "/usr/bin/sandbox-exec", + args: ["-f", "...", "/bin/zsh", "-lc", "grep ..."], + cwd: "/home/user/project", + env: { ... }, + stdio_policy: RedirectForShellTool // stdin=null, stdout/stderr=piped +) +``` + +**Step 8: Output Collection** + +``` +consume_truncated_output(child, Timeout(5000ms)) + → stdout: "src/main.rs:42: // TODO: refactor this\n" + → stderr: "" + → exit_code: 0 + → timed_out: false +``` + +**Step 9: Result Formatting** + +``` +ExecToolCallOutput { + exit_code: 0, + stdout: StreamOutput { text: "src/main.rs:42: // TODO: refactor this\n" }, + stderr: StreamOutput { text: "" }, + aggregated_output: StreamOutput { text: "src/main.rs:42: // TODO: refactor this\n" }, + duration: 127ms, + timed_out: false +} +``` + +**Step 10: Tool Output** + +```json +{ + "type": "function_call_output", + "call_id": "call_abc123", + "output": "src/main.rs:42: // TODO: refactor this\n" +} +``` + +## Scenario: Interactive Session + +**Request 1: Start bash session** + +```json +{ + "name": "exec_command", + "arguments": "{\"command\": [\"bash\", \"-i\"], \"process_id\": \"1001\", \"yield_time_ms\": 2500, \"tty\": true}" +} +``` + +**Processing:** + +1. PTY spawned with bash +2. Output collected for 2500ms +3. Process persisted with ID "1001" +4. Response includes `process_id: "1001"` indicating session is alive + +**Request 2: Send command to session** + +```json +{ + "name": "write_stdin", + "arguments": "{\"process_id\": \"1001\", \"input\": \"export FOO=bar\\n\", \"yield_time_ms\": 1000}" +} +``` + +**Processing:** + +1. Retrieve process "1001" from store +2. Write `export FOO=bar\n` to PTY stdin +3. Wait 100ms for process to react +4. Collect output for remaining yield time +5. Response includes any shell prompt/echo + +**Request 3: Verify variable** + +```json +{ + "name": "write_stdin", + "arguments": "{\"process_id\": \"1001\", \"input\": \"echo $FOO\\n\", \"yield_time_ms\": 1000}" +} +``` + +**Response:** + +```json +{ + "output": "bar\n", + "process_id": "1001", + "exit_code": null +} +``` + +The session maintains state across calls, proving environment variable persistence. diff --git a/src/core/prompts/tools/native-tools/read_command_output.ts b/src/core/prompts/tools/native-tools/read_command_output.ts index 0bab31be9e..b163b46c56 100644 --- a/src/core/prompts/tools/native-tools/read_command_output.ts +++ b/src/core/prompts/tools/native-tools/read_command_output.ts @@ -70,7 +70,7 @@ export default { description: LIMIT_DESCRIPTION, }, }, - required: ["artifact_id", "search", "offset", "limit"], + required: ["artifact_id"], additionalProperties: false, }, }, diff --git a/src/core/tools/ReadCommandOutputTool.ts b/src/core/tools/ReadCommandOutputTool.ts index d81352c30a..7d83c16fba 100644 --- a/src/core/tools/ReadCommandOutputTool.ts +++ b/src/core/tools/ReadCommandOutputTool.ts @@ -223,14 +223,10 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { const { bytesRead } = await fileHandle.read(buffer, 0, buffer.length, offset) const content = buffer.slice(0, bytesRead).toString("utf8") - // Calculate line numbers based on offset + // Calculate line numbers based on offset using chunked reading to avoid large allocations let startLineNumber = 1 if (offset > 0) { - // Count newlines before offset to determine starting line number - const prefixBuffer = Buffer.alloc(offset) - await fileHandle.read(prefixBuffer, 0, offset, 0) - const prefix = prefixBuffer.toString("utf8") - startLineNumber = (prefix.match(/\n/g) || []).length + 1 + startLineNumber = await this.countNewlinesBeforeOffset(fileHandle, offset) } const endOffset = offset + bytesRead @@ -374,6 +370,45 @@ export class ReadCommandOutputTool extends BaseTool<"read_command_output"> { private escapeRegExp(string: string): string { return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") } + + /** + * Count newlines before a given byte offset using fixed-size chunks. + * + * This avoids allocating a buffer of size `offset` which could be huge + * for large files. Instead, we read in 64KB chunks and count newlines. + * + * @param fileHandle - Open file handle for reading + * @param offset - The byte offset to count newlines up to + * @returns The line number at the given offset (1-indexed) + * @private + */ + private async countNewlinesBeforeOffset(fileHandle: fs.FileHandle, offset: number): Promise { + const CHUNK_SIZE = 64 * 1024 // 64KB chunks + let newlineCount = 0 + let bytesRead = 0 + + while (bytesRead < offset) { + const chunkSize = Math.min(CHUNK_SIZE, offset - bytesRead) + const buffer = Buffer.alloc(chunkSize) + const result = await fileHandle.read(buffer, 0, chunkSize, bytesRead) + + if (result.bytesRead === 0) { + break + } + + // Count newlines in this chunk + for (let i = 0; i < result.bytesRead; i++) { + if (buffer[i] === 0x0a) { + // '\n' + newlineCount++ + } + } + + bytesRead += result.bytesRead + } + + return newlineCount + 1 // Line numbers are 1-indexed + } } /** Singleton instance of the ReadCommandOutputTool */