From 6338f3ec57c79db7e268e63f5097dc9fee2ce9d1 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 27 Feb 2026 20:07:28 -0500 Subject: [PATCH] Implement web_fetch tool via exec_command curl Replace the placeholder web_fetch tool with a working implementation that executes curl within the execution environment, respecting Docker/Daytona network sandboxing. - Build curl command with shell-escaped URL, follow redirects, custom user agent, and configurable timeout (default 30s, max 60s) - Validate URL scheme (http/https only) to prevent misuse - Truncate responses exceeding 100KB to protect context window - Register web_fetch in Anthropic and OpenAI profiles (was Gemini-only) - Add web_fetch guidance to all three profile system prompts - Add captured_command to MockExecutionEnvironment for test assertions Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 7 ++ crates/agent/Cargo.toml | 1 + crates/agent/src/profiles/anthropic.rs | 17 ++- crates/agent/src/profiles/openai.rs | 15 ++- crates/agent/src/test_support.rs | 9 +- crates/agent/src/tools.rs | 167 ++++++++++++++++++++++++- 6 files changed, 203 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da513a552..5773bb597 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,6 +22,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", + "shell-escape", "tar", "tempfile", "thiserror 2.0.18", @@ -2826,6 +2827,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shell-escape" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" + [[package]] name = "shell-words" version = "1.1.1" diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index 5e66c538b..b1ed6b8e4 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -40,6 +40,7 @@ reqwest.workspace = true tokio-util.workspace = true dirs = "6" glob = "0.3" +shell-escape = "0.1" bollard = { workspace = true, optional = true } tar = { workspace = true, optional = true } diff --git a/crates/agent/src/profiles/anthropic.rs b/crates/agent/src/profiles/anthropic.rs index 12afadccd..f3add5db7 100644 --- a/crates/agent/src/profiles/anthropic.rs +++ b/crates/agent/src/profiles/anthropic.rs @@ -7,7 +7,7 @@ use crate::skills::Skill; use crate::tool_registry::ToolRegistry; use crate::tools::{ make_edit_file_tool, make_glob_tool, make_grep_tool, make_read_file_tool, - make_shell_tool_with_config, make_web_search_tool, make_write_file_tool, + make_shell_tool_with_config, make_web_fetch_tool, make_web_search_tool, make_write_file_tool, }; use super::EnvContext; @@ -32,6 +32,7 @@ impl AnthropicProfile { registry.register(make_grep_tool()); registry.register(make_glob_tool()); registry.register(make_web_search_tool()); + registry.register(make_web_fetch_tool()); Self { base: BaseProfile { @@ -135,6 +136,9 @@ finding files rather than using shell find or ls commands. ## web_search Search the web using Brave Search. Returns titles, URLs, and descriptions. +## web_fetch +Fetch content from a URL. URLs must start with http:// or https://. + # Coding Best Practices Write clean, maintainable code. Handle errors appropriately. Follow existing code conventions \ @@ -244,6 +248,10 @@ mod tests { prompt.contains("web_search"), "prompt should contain web_search guidance" ); + assert!( + prompt.contains("web_fetch"), + "prompt should contain web_fetch guidance" + ); } #[test] @@ -291,7 +299,7 @@ mod tests { fn anthropic_tools_registered() { let profile = AnthropicProfile::new("claude-sonnet-4-20250514"); let names = profile.tool_registry().names(); - assert_eq!(names.len(), 7); + assert_eq!(names.len(), 8); assert!(names.contains(&"read_file".to_string())); assert!(names.contains(&"write_file".to_string())); assert!(names.contains(&"edit_file".to_string())); @@ -299,6 +307,7 @@ mod tests { assert!(names.contains(&"grep".to_string())); assert!(names.contains(&"glob".to_string())); assert!(names.contains(&"web_search".to_string())); + assert!(names.contains(&"web_fetch".to_string())); } #[test] @@ -348,7 +357,7 @@ mod tests { use std::sync::Arc; let mut profile = AnthropicProfile::new("claude-sonnet-4-20250514"); - assert_eq!(profile.tool_registry().names().len(), 7); + assert_eq!(profile.tool_registry().names().len(), 8); let manager = Arc::new(tokio::sync::Mutex::new(SubAgentManager::new(3))); let factory: SessionFactory = Arc::new(|| { @@ -358,7 +367,7 @@ mod tests { profile.register_subagent_tools(manager, factory, 0); let names = profile.tool_registry().names(); - assert_eq!(names.len(), 11, "should have 7 base + 4 subagent tools"); + assert_eq!(names.len(), 12, "should have 8 base + 4 subagent tools"); assert!(names.contains(&"spawn_agent".to_string())); assert!(names.contains(&"send_input".to_string())); assert!(names.contains(&"wait".to_string())); diff --git a/crates/agent/src/profiles/openai.rs b/crates/agent/src/profiles/openai.rs index 0cd960fc7..8154d1a60 100644 --- a/crates/agent/src/profiles/openai.rs +++ b/crates/agent/src/profiles/openai.rs @@ -6,8 +6,8 @@ use crate::skills::Skill; use crate::tool_registry::{RegisteredTool, ToolRegistry}; use llm::types::ToolDefinition; use crate::tools::{ - make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool, make_web_search_tool, - make_write_file_tool, + make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool, make_web_fetch_tool, + make_web_search_tool, make_write_file_tool, }; use std::sync::Arc; @@ -30,6 +30,7 @@ impl OpenAiProfile { registry.register(make_glob_tool()); registry.register(make_apply_patch_tool()); registry.register(make_web_search_tool()); + registry.register(make_web_fetch_tool()); Self { base: BaseProfile { @@ -149,6 +150,9 @@ Find files by name pattern. ## web_search Search the web using Brave Search. Returns titles, URLs, and descriptions. +## web_fetch +Fetch content from a URL. URLs must start with http:// or https://. + # Coding Best Practices Write clean, maintainable code. Handle errors appropriately. Follow existing code conventions \ @@ -537,21 +541,21 @@ mod tests { use crate::subagent::SessionFactory; let mut profile = OpenAiProfile::new("o3-mini"); - assert_eq!(profile.tool_registry().names().len(), 7); + assert_eq!(profile.tool_registry().names().len(), 8); let manager = Arc::new(tokio::sync::Mutex::new(SubAgentManager::new(3))); let factory: SessionFactory = Arc::new(|| { panic!("should not be called in test") }); profile.register_subagent_tools(manager, factory, 0); - assert_eq!(profile.tool_registry().names().len(), 11); + assert_eq!(profile.tool_registry().names().len(), 12); } #[test] fn openai_tools_registered() { let profile = OpenAiProfile::new("o3-mini"); let names = profile.tool_registry().names(); - assert_eq!(names.len(), 7); + assert_eq!(names.len(), 8); assert!(names.contains(&"read_file".to_string())); assert!(names.contains(&"write_file".to_string())); assert!(names.contains(&"shell".to_string())); @@ -559,6 +563,7 @@ mod tests { assert!(names.contains(&"glob".to_string())); assert!(names.contains(&"apply_patch".to_string())); assert!(names.contains(&"web_search".to_string())); + assert!(names.contains(&"web_fetch".to_string())); } #[test] diff --git a/crates/agent/src/test_support.rs b/crates/agent/src/test_support.rs index 2701c7a33..86243ced7 100644 --- a/crates/agent/src/test_support.rs +++ b/crates/agent/src/test_support.rs @@ -31,6 +31,8 @@ pub struct MockExecutionEnvironment { pub written_files: Mutex>, /// Captures the `timeout_ms` argument from `exec_command` calls. pub captured_timeout: Mutex>, + /// Captures the `command` argument from `exec_command` calls. + pub captured_command: Mutex>, pub event_callback: Option, } @@ -72,6 +74,7 @@ impl Default for MockExecutionEnvironment { apply_read_offset_limit: false, written_files: Mutex::new(Vec::new()), captured_timeout: Mutex::new(None), + captured_command: Mutex::new(None), event_callback: None, } } @@ -128,7 +131,7 @@ impl ExecutionEnvironment for MockExecutionEnvironment { async fn exec_command( &self, - _command: &str, + command: &str, timeout_ms: u64, _working_dir: Option<&str>, _env_vars: Option<&std::collections::HashMap>, @@ -138,6 +141,10 @@ impl ExecutionEnvironment for MockExecutionEnvironment { .captured_timeout .lock() .expect("captured_timeout lock poisoned") = Some(timeout_ms); + *self + .captured_command + .lock() + .expect("captured_command lock poisoned") = Some(command.to_string()); Ok(self.exec_result.clone()) } diff --git a/crates/agent/src/tools.rs b/crates/agent/src/tools.rs index 12bb4569c..3f5feed9f 100644 --- a/crates/agent/src/tools.rs +++ b/crates/agent/src/tools.rs @@ -1,10 +1,13 @@ use crate::config::SessionConfig; use crate::execution_env::GrepOptions; use crate::tool_registry::RegisteredTool; +use std::borrow::Cow; use std::fmt::Write; use std::sync::Arc; use llm::types::ToolDefinition; +const MAX_WEB_FETCH_BYTES: usize = 100 * 1024; + pub(crate) fn required_str<'a>(args: &'a serde_json::Value, key: &str) -> Result<&'a str, String> { args.get(key) .and_then(|v| v.as_str()) @@ -431,14 +434,50 @@ pub(crate) fn make_web_fetch_tool() -> RegisteredTool { parameters: serde_json::json!({ "type": "object", "properties": { - "url": {"type": "string", "description": "URL to fetch"} + "url": {"type": "string", "description": "URL to fetch (must be http:// or https://)"}, + "timeout_ms": {"type": "integer", "description": "Timeout in milliseconds (default 30000, max 60000)"} }, "required": ["url"] }), }, - executor: Arc::new(|_args, _env, _cancel| { + executor: Arc::new(|args, env, cancel| { Box::pin(async move { - Ok("Web fetch is not configured. This is a placeholder tool.".to_string()) + let url = required_str(&args, "url")?; + let timeout_ms = args + .get("timeout_ms") + .and_then(serde_json::Value::as_u64) + .unwrap_or(30_000) + .min(60_000); + + if !url.starts_with("http://") && !url.starts_with("https://") { + return Err("URL must start with http:// or https://".to_string()); + } + + let timeout_secs = timeout_ms.div_ceil(1000); + let escaped_url = shell_escape::escape(Cow::Borrowed(url)); + let command = format!( + "curl -sL --max-time {timeout_secs} -H 'User-Agent: attractor-agent/0.1' {escaped_url}" + ); + + let result = env + .exec_command(&command, timeout_ms, None, None, Some(cancel)) + .await?; + + if result.exit_code != 0 { + return Err(format!( + "curl failed (exit code {}): {}", + result.exit_code, + result.stderr.trim() + )); + } + + let mut output = result.stdout; + if output.len() > MAX_WEB_FETCH_BYTES { + output.truncate(MAX_WEB_FETCH_BYTES); + output.push_str("\n\n[Output truncated at 100KB]"); + } + + Ok(output) }) }), } @@ -742,6 +781,128 @@ mod tests { assert_eq!(format_brave_results(&body), "No results found."); } + #[tokio::test] + async fn web_fetch_builds_curl_command() { + let tool = make_web_fetch_tool(); + let env = Arc::new(MockExecutionEnvironment { + exec_result: ExecResult { + stdout: "hello".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 100, + }, + ..Default::default() + }); + let env_clone: Arc = env.clone(); + let result = (tool.executor)( + serde_json::json!({"url": "https://example.com"}), + env_clone, + CancellationToken::new(), + ) + .await; + assert_eq!(result.unwrap(), "hello"); + let cmd = env.captured_command.lock().unwrap().clone().unwrap(); + assert!(cmd.starts_with("curl -sL --max-time 30 "), "command should start with curl flags, got: {cmd}"); + assert!(cmd.contains("https://example.com"), "command should contain the URL"); + assert!(cmd.contains("User-Agent: attractor-agent/0.1"), "command should set user agent"); + } + + #[tokio::test] + async fn web_fetch_rejects_non_http_url() { + let tool = make_web_fetch_tool(); + let env: Arc = Arc::new(MockExecutionEnvironment::default()); + let result = (tool.executor)( + serde_json::json!({"url": "ftp://example.com/file"}), + env, + CancellationToken::new(), + ) + .await; + let err = result.unwrap_err(); + assert!(err.contains("http://") || err.contains("https://"), "error should mention valid schemes, got: {err}"); + } + + #[tokio::test] + async fn web_fetch_timeout_flows_through() { + let tool = make_web_fetch_tool(); + let env = Arc::new(MockExecutionEnvironment::default()); + let env_clone: Arc = env.clone(); + let _result = (tool.executor)( + serde_json::json!({"url": "https://example.com", "timeout_ms": 15000}), + env_clone, + CancellationToken::new(), + ) + .await; + assert_eq!(*env.captured_timeout.lock().unwrap(), Some(15000)); + let cmd = env.captured_command.lock().unwrap().clone().unwrap(); + assert!(cmd.contains("--max-time 15"), "curl timeout should be 15 seconds, got: {cmd}"); + } + + #[tokio::test] + async fn web_fetch_timeout_capped_at_60s() { + let tool = make_web_fetch_tool(); + let env = Arc::new(MockExecutionEnvironment::default()); + let env_clone: Arc = env.clone(); + let _result = (tool.executor)( + serde_json::json!({"url": "https://example.com", "timeout_ms": 120000}), + env_clone, + CancellationToken::new(), + ) + .await; + assert_eq!(*env.captured_timeout.lock().unwrap(), Some(60000)); + let cmd = env.captured_command.lock().unwrap().clone().unwrap(); + assert!(cmd.contains("--max-time 60"), "curl timeout should be capped at 60 seconds, got: {cmd}"); + } + + #[tokio::test] + async fn web_fetch_truncates_large_output() { + let large_content = "x".repeat(150 * 1024); + let tool = make_web_fetch_tool(); + let env: Arc = Arc::new(MockExecutionEnvironment { + exec_result: ExecResult { + stdout: large_content, + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 100, + }, + ..Default::default() + }); + let result = (tool.executor)( + serde_json::json!({"url": "https://example.com"}), + env, + CancellationToken::new(), + ) + .await; + let output = result.unwrap(); + assert!(output.len() < 110 * 1024, "output should be truncated"); + assert!(output.ends_with("[Output truncated at 100KB]")); + } + + #[tokio::test] + async fn web_fetch_returns_error_on_nonzero_exit() { + let tool = make_web_fetch_tool(); + let env: Arc = Arc::new(MockExecutionEnvironment { + exec_result: ExecResult { + stdout: String::new(), + stderr: "curl: (6) Could not resolve host".into(), + exit_code: 6, + timed_out: false, + duration_ms: 100, + }, + ..Default::default() + }); + let result = (tool.executor)( + serde_json::json!({"url": "https://nonexistent.example.com"}), + env, + CancellationToken::new(), + ) + .await; + let err = result.unwrap_err(); + assert!(err.contains("exit code 6"), "error should contain exit code, got: {err}"); + assert!(err.contains("Could not resolve host"), "error should contain stderr, got: {err}"); + } + #[tokio::test] #[ignore] // Requires BRAVE_SEARCH_API_KEY env var async fn web_search_returns_results() {