From 61ed9a6bdbfe1a6f9cd2d98ead899e21720bfeda Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 27 Feb 2026 22:26:32 -0500 Subject: [PATCH] Add HTML-to-markdown conversion and prompt summarization to web_fetch web_fetch now converts HTML responses to clean markdown using the htmd crate (stripping script/style tags), and supports an optional prompt parameter that makes a secondary LLM call to answer questions about the fetched content. Each provider profile picks a cheap/fast summarizer model (Haiku, gpt-4o-mini, gemini-2.0-flash). Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 230 +++++++++++++++++++++++++ crates/agent/Cargo.toml | 1 + crates/agent/src/cli.rs | 42 +++-- crates/agent/src/lib.rs | 2 +- crates/agent/src/profiles/anthropic.rs | 11 +- crates/agent/src/profiles/gemini.rs | 12 +- crates/agent/src/profiles/openai.rs | 12 +- crates/agent/src/tools.rs | 176 +++++++++++++++++-- crates/agent/tests/parity_matrix.rs | 75 +++++--- 9 files changed, 500 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5773bb597..4b6e48f57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,7 @@ dependencies = [ "dotenvy", "futures", "glob", + "htmd", "jsonschema", "libc", "llm", @@ -954,6 +955,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.32" @@ -1210,6 +1221,28 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "htmd" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60ae59466542f2346e43d4a5e9b4432a1fc915b279c9fc0484e9ed7379121454" +dependencies = [ + "html5ever", + "markup5ever_rcdom", + "phf 0.13.1", +] + +[[package]] +name = "html5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4" +dependencies = [ + "log", + "markup5ever", + "match_token", +] + [[package]] name = "http" version = "1.4.0" @@ -1789,6 +1822,46 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "311fe69c934650f8f19652b3946075f0fc41ad8757dbb68f1ca14e7900ecc1c3" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.35.0+unofficial" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8bcd53df4748257345b8bc156d620340ce0f015ec1c7ef1cff475543888a31d" +dependencies = [ + "html5ever", + "markup5ever", + "tendril", + "xml5ever", +] + +[[package]] +name = "match_token" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "matchit" version = "0.8.4" @@ -1851,6 +1924,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "7.1.3" @@ -2076,6 +2155,87 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared 0.13.1", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -2118,6 +2278,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "predicates" version = "3.1.4" @@ -2861,6 +3027,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + [[package]] name = "slab" version = "0.4.12" @@ -2889,6 +3061,31 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.11.3", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", +] + [[package]] name = "stringmetrics" version = "2.2.2" @@ -2992,6 +3189,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "termtree" version = "0.5.1" @@ -3627,6 +3835,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414" +dependencies = [ + "phf 0.11.3", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.6" @@ -4090,6 +4310,16 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee3f1e41afb31a75aef076563b0ad3ecc24f5bd9d12a72b132222664eb76b494" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index b1ed6b8e4..1d2f25e14 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -41,6 +41,7 @@ tokio-util.workspace = true dirs = "6" glob = "0.3" shell-escape = "0.1" +htmd = "0.5" bollard = { workspace = true, optional = true } tar = { workspace = true, optional = true } diff --git a/crates/agent/src/cli.rs b/crates/agent/src/cli.rs index ad8728a08..324bab607 100644 --- a/crates/agent/src/cli.rs +++ b/crates/agent/src/cli.rs @@ -142,12 +142,27 @@ fn build_tool_approval( }) } -fn build_profile(provider: &str, model: &str) -> Box { - match provider { - "openai" => Box::new(OpenAiProfile::new(model)), - "gemini" => Box::new(GeminiProfile::new(model)), +fn build_summarizer(provider: &str, llm_client: Option) -> Option { + let client = llm_client?; + let model = match provider { + "openai" => "gpt-4o-mini", + "gemini" => "gemini-2.0-flash", // anthropic and unknown providers - _ => Box::new(AnthropicProfile::new(model)), + _ => "claude-haiku-4-5-20251001", + }; + Some(crate::tools::WebFetchSummarizer { + client, + model: model.into(), + }) +} + +fn build_profile(provider: &str, model: &str, llm_client: Option) -> Box { + let summarizer = build_summarizer(provider, llm_client); + match provider { + "openai" => Box::new(OpenAiProfile::with_summarizer(model, summarizer)), + "gemini" => Box::new(GeminiProfile::with_summarizer(model, summarizer)), + // anthropic and unknown providers + _ => Box::new(AnthropicProfile::with_summarizer(model, summarizer)), } } @@ -335,7 +350,7 @@ pub async fn run() -> anyhow::Result<()> { "{}Using model: {model}{}", styles.dim, styles.reset, ); - let mut profile = build_profile(&cli.provider, model); + let mut profile = build_profile(&cli.provider, model, Some(client.clone())); // Build execution environment let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); @@ -363,10 +378,11 @@ pub async fn run() -> anyhow::Result<()> { let factory_env = Arc::clone(&env); let factory_approval = config.tool_approval.clone(); let factory: SessionFactory = Arc::new(move || { + let child_summarizer = build_summarizer(&factory_provider, Some(factory_client.clone())); let child_profile: Arc = match factory_provider.as_str() { - "openai" => Arc::new(OpenAiProfile::new(&factory_model)), - "gemini" => Arc::new(GeminiProfile::new(&factory_model)), - _ => Arc::new(AnthropicProfile::new(&factory_model)), + "openai" => Arc::new(OpenAiProfile::with_summarizer(&factory_model, child_summarizer)), + "gemini" => Arc::new(GeminiProfile::with_summarizer(&factory_model, child_summarizer)), + _ => Arc::new(AnthropicProfile::with_summarizer(&factory_model, child_summarizer)), }; Session::new( factory_client.clone(), @@ -628,19 +644,19 @@ mod tests { #[test] fn build_profile_anthropic() { - let profile = build_profile("anthropic", "model"); + let profile = build_profile("anthropic", "model", None); assert_eq!(profile.id(), "anthropic"); } #[test] fn build_profile_openai() { - let profile = build_profile("openai", "model"); + let profile = build_profile("openai", "model", None); assert_eq!(profile.id(), "openai"); } #[test] fn build_profile_gemini() { - let profile = build_profile("gemini", "model"); + let profile = build_profile("gemini", "model", None); assert_eq!(profile.id(), "gemini"); } @@ -648,7 +664,7 @@ mod tests { #[test] fn build_profile_can_register_subagent_tools() { - let mut profile = build_profile("anthropic", "model"); + let mut profile = build_profile("anthropic", "model", None); let manager = Arc::new(tokio::sync::Mutex::new(SubAgentManager::new(1))); let factory: SessionFactory = Arc::new(|| { panic!("factory should not be called in this test"); diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs index a1acadc90..2c6fc75d2 100644 --- a/crates/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -40,7 +40,7 @@ pub use subagent::{SubAgent, SubAgentEventCallback, SubAgentManager, SubAgentRes pub use tool_registry::ToolRegistry; pub use tools::{ make_edit_file_tool, make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool, - make_shell_tool_with_config, make_write_file_tool, + make_shell_tool_with_config, make_write_file_tool, WebFetchSummarizer, }; pub use truncation::{truncate_lines, truncate_output, truncate_tool_output, TruncationMode}; pub use types::{AgentEvent, SessionEvent, SessionState, Turn}; diff --git a/crates/agent/src/profiles/anthropic.rs b/crates/agent/src/profiles/anthropic.rs index f3add5db7..017c18424 100644 --- a/crates/agent/src/profiles/anthropic.rs +++ b/crates/agent/src/profiles/anthropic.rs @@ -8,6 +8,7 @@ use crate::tool_registry::ToolRegistry; use crate::tools::{ make_edit_file_tool, make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool_with_config, make_web_fetch_tool, make_web_search_tool, make_write_file_tool, + WebFetchSummarizer, }; use super::EnvContext; @@ -19,6 +20,11 @@ pub struct AnthropicProfile { impl AnthropicProfile { #[must_use] pub fn new(model: impl Into) -> Self { + Self::with_summarizer(model, None) + } + + #[must_use] + pub fn with_summarizer(model: impl Into, summarizer: Option) -> Self { let config = SessionConfig { default_command_timeout_ms: 120_000, ..SessionConfig::default() @@ -32,7 +38,7 @@ impl AnthropicProfile { registry.register(make_grep_tool()); registry.register(make_glob_tool()); registry.register(make_web_search_tool()); - registry.register(make_web_fetch_tool()); + registry.register(make_web_fetch_tool(summarizer)); Self { base: BaseProfile { @@ -137,7 +143,8 @@ finding files rather than using shell find or ls commands. Search the web using Brave Search. Returns titles, URLs, and descriptions. ## web_fetch -Fetch content from a URL. URLs must start with http:// or https://. +Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific \ +information instead of returning the full page. URLs must start with http:// or https://. # Coding Best Practices diff --git a/crates/agent/src/profiles/gemini.rs b/crates/agent/src/profiles/gemini.rs index d921a93f9..a702d15b8 100644 --- a/crates/agent/src/profiles/gemini.rs +++ b/crates/agent/src/profiles/gemini.rs @@ -7,7 +7,7 @@ use crate::tool_registry::ToolRegistry; use crate::tools::{ make_edit_file_tool, make_glob_tool, make_grep_tool, make_list_dir_tool, make_read_file_tool, make_read_many_files_tool, make_shell_tool, make_web_fetch_tool, - make_web_search_tool, make_write_file_tool, + make_web_search_tool, make_write_file_tool, WebFetchSummarizer, }; use super::EnvContext; @@ -19,6 +19,11 @@ pub struct GeminiProfile { impl GeminiProfile { #[must_use] pub fn new(model: impl Into) -> Self { + Self::with_summarizer(model, None) + } + + #[must_use] + pub fn with_summarizer(model: impl Into, summarizer: Option) -> Self { let mut registry = ToolRegistry::new(); registry.register(make_read_file_tool()); @@ -30,7 +35,7 @@ impl GeminiProfile { registry.register(make_glob_tool()); registry.register(make_list_dir_tool()); registry.register(make_web_search_tool()); - registry.register(make_web_fetch_tool()); + registry.register(make_web_fetch_tool(summarizer)); Self { base: BaseProfile { @@ -160,7 +165,8 @@ List directory contents with depth control. Search the web for information. ## web_fetch -Fetch content from a URL. +Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific \ +information instead of returning the full page. # Project Docs diff --git a/crates/agent/src/profiles/openai.rs b/crates/agent/src/profiles/openai.rs index 8154d1a60..399f6993e 100644 --- a/crates/agent/src/profiles/openai.rs +++ b/crates/agent/src/profiles/openai.rs @@ -7,7 +7,7 @@ use crate::tool_registry::{RegisteredTool, ToolRegistry}; use llm::types::ToolDefinition; use crate::tools::{ make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool, make_web_fetch_tool, - make_web_search_tool, make_write_file_tool, + make_web_search_tool, make_write_file_tool, WebFetchSummarizer, }; use std::sync::Arc; @@ -21,6 +21,11 @@ pub struct OpenAiProfile { impl OpenAiProfile { #[must_use] pub fn new(model: impl Into) -> Self { + Self::with_summarizer(model, None) + } + + #[must_use] + pub fn with_summarizer(model: impl Into, summarizer: Option) -> Self { let mut registry = ToolRegistry::new(); registry.register(make_read_file_tool()); @@ -30,7 +35,7 @@ impl OpenAiProfile { registry.register(make_glob_tool()); registry.register(make_apply_patch_tool()); registry.register(make_web_search_tool()); - registry.register(make_web_fetch_tool()); + registry.register(make_web_fetch_tool(summarizer)); Self { base: BaseProfile { @@ -151,7 +156,8 @@ Find files by name pattern. Search the web using Brave Search. Returns titles, URLs, and descriptions. ## web_fetch -Fetch content from a URL. URLs must start with http:// or https://. +Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific \ +information instead of returning the full page. URLs must start with http:// or https://. # Coding Best Practices diff --git a/crates/agent/src/tools.rs b/crates/agent/src/tools.rs index 3f5feed9f..69c698250 100644 --- a/crates/agent/src/tools.rs +++ b/crates/agent/src/tools.rs @@ -1,13 +1,40 @@ use crate::config::SessionConfig; use crate::execution_env::GrepOptions; use crate::tool_registry::RegisteredTool; +use llm::client::Client; +use llm::types::{Message, Request, ToolDefinition}; use std::borrow::Cow; use std::fmt::Write; use std::sync::Arc; -use llm::types::ToolDefinition; const MAX_WEB_FETCH_BYTES: usize = 100 * 1024; +/// Configuration for the optional LLM-based summarizer used by `web_fetch`. +#[derive(Clone)] +pub struct WebFetchSummarizer { + pub client: Client, + pub model: String, +} + +/// Returns true if the input looks like it contains HTML markup. +fn looks_like_html(text: &str) -> bool { + let trimmed = text.trim_start(); + trimmed.starts_with("") || trimmed.contains("

") || trimmed.contains("") +} + +/// Converts HTML to Markdown, stripping script/style tags. +/// Non-HTML content (JSON, plain text) passes through unchanged. +fn html_to_markdown(text: &str) -> String { + if !looks_like_html(text) { + return text.to_string(); + } + let converter = htmd::HtmlToMarkdown::builder() + .skip_tags(vec!["script", "style"]) + .build(); + converter.convert(text).unwrap_or_else(|_| text.to_string()) +} + pub(crate) fn required_str<'a>(args: &'a serde_json::Value, key: &str) -> Result<&'a str, String> { args.get(key) .and_then(|v| v.as_str()) @@ -426,23 +453,26 @@ fn make_web_search_tool_with_api_key(api_key: Option) -> RegisteredTool } #[must_use] -pub(crate) fn make_web_fetch_tool() -> RegisteredTool { +pub(crate) fn make_web_fetch_tool(summarizer: Option) -> RegisteredTool { RegisteredTool { definition: ToolDefinition { name: "web_fetch".into(), - description: "Fetch content from a URL".into(), + description: "Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific information instead of returning the full page.".into(), parameters: serde_json::json!({ "type": "object", "properties": { "url": {"type": "string", "description": "URL to fetch (must be http:// or https://)"}, + "prompt": {"type": "string", "description": "A question or instruction about the page content. When provided, returns a concise answer instead of the full page."}, "timeout_ms": {"type": "integer", "description": "Timeout in milliseconds (default 30000, max 60000)"} }, "required": ["url"] }), }, - executor: Arc::new(|args, env, cancel| { + executor: Arc::new(move |args, env, cancel| { + let summarizer = summarizer.clone(); Box::pin(async move { let url = required_str(&args, "url")?; + let prompt = args.get("prompt").and_then(serde_json::Value::as_str); let timeout_ms = args .get("timeout_ms") .and_then(serde_json::Value::as_u64) @@ -471,13 +501,41 @@ pub(crate) fn make_web_fetch_tool() -> RegisteredTool { )); } - let mut output = result.stdout; - if output.len() > MAX_WEB_FETCH_BYTES { - output.truncate(MAX_WEB_FETCH_BYTES); - output.push_str("\n\n[Output truncated at 100KB]"); + let mut content = html_to_markdown(&result.stdout); + if content.len() > MAX_WEB_FETCH_BYTES { + content.truncate(MAX_WEB_FETCH_BYTES); + content.push_str("\n\n[Output truncated at 100KB]"); } - Ok(output) + match (prompt, &summarizer) { + (Some(user_prompt), Some(s)) => { + let summarization_prompt = format!( + "Content from {url}:\n---\n{content}\n---\n\n{user_prompt}\n\nRespond concisely based only on the content above." + ); + let request = Request { + model: s.model.clone(), + messages: vec![Message::user(summarization_prompt)], + provider: None, + tools: None, + tool_choice: None, + response_format: None, + temperature: None, + top_p: None, + max_tokens: None, + stop_sequences: None, + reasoning_effort: None, + metadata: None, + provider_options: None, + }; + let response = s.client.complete(&request).await.map_err(|e| format!("Summarization failed: {e}"))?; + Ok(response.text()) + } + (Some(_), None) => { + // Graceful degradation: return content with a note + Ok(format!("[Note: prompt summarization unavailable, returning full content]\n\n{content}")) + } + (None, _) => Ok(content), + } }) }), } @@ -783,10 +841,10 @@ mod tests { #[tokio::test] async fn web_fetch_builds_curl_command() { - let tool = make_web_fetch_tool(); + let tool = make_web_fetch_tool(None); let env = Arc::new(MockExecutionEnvironment { exec_result: ExecResult { - stdout: "hello".into(), + stdout: "

hello

".into(), stderr: String::new(), exit_code: 0, timed_out: false, @@ -801,7 +859,9 @@ mod tests { CancellationToken::new(), ) .await; - assert_eq!(result.unwrap(), "hello"); + let output = result.unwrap(); + assert!(output.contains("# hello"), "HTML should be converted to markdown, got: {output}"); + assert!(!output.contains(""), "raw HTML tags should be removed, got: {output}"); let cmd = env.captured_command.lock().unwrap().clone().unwrap(); assert!(cmd.starts_with("curl -sL --max-time 30 "), "command should start with curl flags, got: {cmd}"); assert!(cmd.contains("https://example.com"), "command should contain the URL"); @@ -810,7 +870,7 @@ mod tests { #[tokio::test] async fn web_fetch_rejects_non_http_url() { - let tool = make_web_fetch_tool(); + let tool = make_web_fetch_tool(None); let env: Arc = Arc::new(MockExecutionEnvironment::default()); let result = (tool.executor)( serde_json::json!({"url": "ftp://example.com/file"}), @@ -824,7 +884,7 @@ mod tests { #[tokio::test] async fn web_fetch_timeout_flows_through() { - let tool = make_web_fetch_tool(); + let tool = make_web_fetch_tool(None); let env = Arc::new(MockExecutionEnvironment::default()); let env_clone: Arc = env.clone(); let _result = (tool.executor)( @@ -840,7 +900,7 @@ mod tests { #[tokio::test] async fn web_fetch_timeout_capped_at_60s() { - let tool = make_web_fetch_tool(); + let tool = make_web_fetch_tool(None); let env = Arc::new(MockExecutionEnvironment::default()); let env_clone: Arc = env.clone(); let _result = (tool.executor)( @@ -857,7 +917,7 @@ mod tests { #[tokio::test] async fn web_fetch_truncates_large_output() { let large_content = "x".repeat(150 * 1024); - let tool = make_web_fetch_tool(); + let tool = make_web_fetch_tool(None); let env: Arc = Arc::new(MockExecutionEnvironment { exec_result: ExecResult { stdout: large_content, @@ -881,7 +941,7 @@ mod tests { #[tokio::test] async fn web_fetch_returns_error_on_nonzero_exit() { - let tool = make_web_fetch_tool(); + let tool = make_web_fetch_tool(None); let env: Arc = Arc::new(MockExecutionEnvironment { exec_result: ExecResult { stdout: String::new(), @@ -903,6 +963,88 @@ mod tests { assert!(err.contains("Could not resolve host"), "error should contain stderr, got: {err}"); } + #[tokio::test] + async fn web_fetch_prompt_with_summarizer_returns_llm_answer() { + use crate::test_support::{make_client, MockLlmProvider, text_response}; + + let provider = Arc::new(MockLlmProvider::new(vec![ + text_response("Rust is a systems programming language focused on safety and performance."), + ])); + let client = make_client(provider).await; + let summarizer = WebFetchSummarizer { + client, + model: "mock-model".into(), + }; + + let tool = make_web_fetch_tool(Some(summarizer)); + let env: Arc = Arc::new(MockExecutionEnvironment { + exec_result: ExecResult { + stdout: "

Lots of content about Rust...

".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 100, + }, + ..Default::default() + }); + let result = (tool.executor)( + serde_json::json!({"url": "https://example.com", "prompt": "What is Rust?"}), + env, + CancellationToken::new(), + ) + .await; + let output = result.unwrap(); + assert_eq!(output, "Rust is a systems programming language focused on safety and performance."); + } + + #[tokio::test] + async fn web_fetch_prompt_without_summarizer_returns_content_with_note() { + let tool = make_web_fetch_tool(None); + let env: Arc = Arc::new(MockExecutionEnvironment { + exec_result: ExecResult { + stdout: "

Rust is a systems programming language.

".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 100, + }, + ..Default::default() + }); + let result = (tool.executor)( + serde_json::json!({"url": "https://example.com", "prompt": "What is Rust?"}), + env, + CancellationToken::new(), + ) + .await; + let output = result.unwrap(); + assert!(output.contains("summarization unavailable"), "should note unavailability, got: {output}"); + assert!(output.contains("Rust is a systems programming language"), "should contain page content, got: {output}"); + } + + #[test] + fn html_to_markdown_converts_basic_html() { + let result = html_to_markdown("

Hello

World

"); + assert_eq!(result, "# Hello\n\nWorld"); + } + + #[test] + fn html_to_markdown_strips_script_and_style() { + let html = "

Content

"; + let result = html_to_markdown(html); + assert!(!result.contains("alert"), "script content should be stripped"); + assert!(!result.contains("color:red"), "style content should be stripped"); + assert!(result.contains("Content"), "paragraph text should remain"); + } + + #[test] + fn html_to_markdown_passes_through_non_html() { + let json = r#"{"key": "value", "items": [1, 2, 3]}"#; + assert_eq!(html_to_markdown(json), json); + + let plain = "Just some plain text\nwith newlines"; + assert_eq!(html_to_markdown(plain), plain); + } + #[tokio::test] #[ignore] // Requires BRAVE_SEARCH_API_KEY env var async fn web_search_returns_results() { diff --git a/crates/agent/tests/parity_matrix.rs b/crates/agent/tests/parity_matrix.rs index 9c36fdede..f2d897a57 100644 --- a/crates/agent/tests/parity_matrix.rs +++ b/crates/agent/tests/parity_matrix.rs @@ -3,19 +3,36 @@ use std::sync::Arc; use agent::{ AnthropicProfile, GeminiProfile, LocalExecutionEnvironment, OpenAiProfile, ProviderProfile, - Session, SessionConfig, SubAgentManager, + Session, SessionConfig, SubAgentManager, WebFetchSummarizer, }; use llm::client::Client; +fn build_summarizer(provider: &str, client: &Client) -> WebFetchSummarizer { + let summarizer_model = match provider { + "openai" => "gpt-4o-mini", + "gemini" => "gemini-2.0-flash", + _ => "claude-haiku-4-5-20251001", + }; + WebFetchSummarizer { + client: client.clone(), + model: summarizer_model.into(), + } +} + +fn build_profile(provider: &str, model: &str, client: &Client) -> Box { + let summarizer = Some(build_summarizer(provider, client)); + match provider { + "anthropic" => Box::new(AnthropicProfile::with_summarizer(model, summarizer)), + "openai" => Box::new(OpenAiProfile::with_summarizer(model, summarizer)), + "gemini" => Box::new(GeminiProfile::with_summarizer(model, summarizer)), + _ => panic!("unknown provider: {provider}"), + } +} + async fn make_session(provider: &str, model: &str, cwd: &Path) -> Session { dotenvy::dotenv().ok(); let client = Client::from_env().await.expect("Client::from_env failed"); - let mut profile: Box = match provider { - "anthropic" => Box::new(AnthropicProfile::new(model)), - "openai" => Box::new(OpenAiProfile::new(model)), - "gemini" => Box::new(GeminiProfile::new(model)), - _ => panic!("unknown provider: {provider}"), - }; + let mut profile = build_profile(provider, model, &client); let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf())); // Register subagent tools so spawn_agent / wait / send_input / close_agent are available @@ -28,11 +45,14 @@ async fn make_session(provider: &str, model: &str, cwd: &Path) -> Session { let provider = factory_provider.to_string(); let model = factory_model; Arc::new(move || { - let sub_profile: Arc = match provider.as_str() { - "anthropic" => Arc::new(AnthropicProfile::new(&model)), - "openai" => Arc::new(OpenAiProfile::new(&model)), - "gemini" => Arc::new(GeminiProfile::new(&model)), - _ => panic!("unknown provider: {provider}"), + let sub_profile: Arc = { + let summarizer = Some(build_summarizer(&provider, &factory_client)); + match provider.as_str() { + "anthropic" => Arc::new(AnthropicProfile::with_summarizer(&model, summarizer)), + "openai" => Arc::new(OpenAiProfile::with_summarizer(&model, summarizer)), + "gemini" => Arc::new(GeminiProfile::with_summarizer(&model, summarizer)), + _ => panic!("unknown provider: {provider}"), + } }; let sub_env = Arc::new(LocalExecutionEnvironment::new(factory_cwd.clone())); Session::new( @@ -61,12 +81,7 @@ async fn make_session_with_config( ) -> Session { dotenvy::dotenv().ok(); let client = Client::from_env().await.expect("Client::from_env failed"); - let profile: Arc = match provider { - "anthropic" => Arc::new(AnthropicProfile::new(model)), - "openai" => Arc::new(OpenAiProfile::new(model)), - "gemini" => Arc::new(GeminiProfile::new(model)), - _ => panic!("unknown provider: {provider}"), - }; + let profile: Arc = Arc::from(build_profile(provider, model, &client)); let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf())); Session::new(client, profile, env, config) } @@ -395,20 +410,36 @@ async fn scenario_error_recovery(session: &mut Session, dir: &Path) { // Scenario 15: web_fetch // --------------------------------------------------------------------------- async fn scenario_web_fetch(session: &mut Session, dir: &Path) { + // Test basic fetch (HTML-to-markdown conversion) session .process_input( - "Use the web_fetch tool to fetch https://example.com and write its content to a file called fetched.html", + "Use the web_fetch tool to fetch https://example.com and write its content to a file called fetched.txt", ) .await .expect("process_input failed"); - let path = dir.join("fetched.html"); - assert!(path.exists(), "fetched.html should have been created"); - let content = std::fs::read_to_string(&path).expect("failed to read fetched.html"); + let path = dir.join("fetched.txt"); + assert!(path.exists(), "fetched.txt should have been created"); + let content = std::fs::read_to_string(&path).expect("failed to read fetched.txt"); assert!( content.contains("Example Domain"), "Expected 'Example Domain' in fetched content, got first 200 chars: {}", &content[..content.len().min(200)] ); + + // Test fetch with prompt parameter (LLM summarization) + session + .process_input( + "Use the web_fetch tool with the prompt parameter to fetch https://example.com and answer: 'What is the title heading on this page?' Write only the answer to a file called answer.txt", + ) + .await + .expect("process_input failed for prompt test"); + let answer_path = dir.join("answer.txt"); + assert!(answer_path.exists(), "answer.txt should have been created"); + let answer = std::fs::read_to_string(&answer_path).expect("failed to read answer.txt"); + assert!( + answer.to_lowercase().contains("example domain"), + "Expected answer to mention 'example domain', got: {answer}" + ); } // ---------------------------------------------------------------------------