mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-05 08:10:39 +00:00
Add HTML-to-markdown conversion and prompt summarization to web_fetch
web_fetch now converts HTML responses to clean markdown using the htmd crate (stripping script/style tags), and supports an optional prompt parameter that makes a secondary LLM call to answer questions about the fetched content. Each provider profile picks a cheap/fast summarizer model (Haiku, gpt-4o-mini, gemini-2.0-flash). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
35005ca0fe
commit
61ed9a6bdb
9 changed files with 500 additions and 61 deletions
230
Cargo.lock
generated
230
Cargo.lock
generated
|
|
@ -15,6 +15,7 @@ dependencies = [
|
|||
"dotenvy",
|
||||
"futures",
|
||||
"glob",
|
||||
"htmd",
|
||||
"jsonschema",
|
||||
"libc",
|
||||
"llm",
|
||||
|
|
@ -954,6 +955,16 @@ version = "1.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
|
||||
dependencies = [
|
||||
"mac",
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.32"
|
||||
|
|
@ -1210,6 +1221,28 @@ version = "0.4.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "htmd"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60ae59466542f2346e43d4a5e9b4432a1fc915b279c9fc0484e9ed7379121454"
|
||||
dependencies = [
|
||||
"html5ever",
|
||||
"markup5ever_rcdom",
|
||||
"phf 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever",
|
||||
"match_token",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.0"
|
||||
|
|
@ -1789,6 +1822,46 @@ version = "0.1.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "311fe69c934650f8f19652b3946075f0fc41ad8757dbb68f1ca14e7900ecc1c3"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever_rcdom"
|
||||
version = "0.35.0+unofficial"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8bcd53df4748257345b8bc156d620340ce0f015ec1c7ef1cff475543888a31d"
|
||||
dependencies = [
|
||||
"html5ever",
|
||||
"markup5ever",
|
||||
"tendril",
|
||||
"xml5ever",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "match_token"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.8.4"
|
||||
|
|
@ -1851,6 +1924,12 @@ dependencies = [
|
|||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
|
|
@ -2076,6 +2155,87 @@ version = "2.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.13.1",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
|
||||
dependencies = [
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared 0.11.3",
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.16"
|
||||
|
|
@ -2118,6 +2278,12 @@ dependencies = [
|
|||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "predicates"
|
||||
version = "3.1.4"
|
||||
|
|
@ -2861,6 +3027,12 @@ version = "2.7.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
|
|
@ -2889,6 +3061,31 @@ version = "1.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared 0.11.3",
|
||||
"precomputed-hash",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"
|
||||
dependencies = [
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stringmetrics"
|
||||
version = "2.2.2"
|
||||
|
|
@ -2992,6 +3189,17 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
|
||||
dependencies = [
|
||||
"futf",
|
||||
"mac",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termtree"
|
||||
version = "0.5.1"
|
||||
|
|
@ -3627,6 +3835,18 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414"
|
||||
dependencies = [
|
||||
"phf 0.11.3",
|
||||
"phf_codegen",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.6"
|
||||
|
|
@ -4090,6 +4310,16 @@ dependencies = [
|
|||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xml5ever"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee3f1e41afb31a75aef076563b0ad3ecc24f5bd9d12a72b132222664eb76b494"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.1"
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ tokio-util.workspace = true
|
|||
dirs = "6"
|
||||
glob = "0.3"
|
||||
shell-escape = "0.1"
|
||||
htmd = "0.5"
|
||||
bollard = { workspace = true, optional = true }
|
||||
tar = { workspace = true, optional = true }
|
||||
|
||||
|
|
|
|||
|
|
@ -142,12 +142,27 @@ fn build_tool_approval(
|
|||
})
|
||||
}
|
||||
|
||||
fn build_profile(provider: &str, model: &str) -> Box<dyn ProviderProfile> {
|
||||
match provider {
|
||||
"openai" => Box::new(OpenAiProfile::new(model)),
|
||||
"gemini" => Box::new(GeminiProfile::new(model)),
|
||||
fn build_summarizer(provider: &str, llm_client: Option<Client>) -> Option<crate::tools::WebFetchSummarizer> {
|
||||
let client = llm_client?;
|
||||
let model = match provider {
|
||||
"openai" => "gpt-4o-mini",
|
||||
"gemini" => "gemini-2.0-flash",
|
||||
// anthropic and unknown providers
|
||||
_ => Box::new(AnthropicProfile::new(model)),
|
||||
_ => "claude-haiku-4-5-20251001",
|
||||
};
|
||||
Some(crate::tools::WebFetchSummarizer {
|
||||
client,
|
||||
model: model.into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn build_profile(provider: &str, model: &str, llm_client: Option<Client>) -> Box<dyn ProviderProfile> {
|
||||
let summarizer = build_summarizer(provider, llm_client);
|
||||
match provider {
|
||||
"openai" => Box::new(OpenAiProfile::with_summarizer(model, summarizer)),
|
||||
"gemini" => Box::new(GeminiProfile::with_summarizer(model, summarizer)),
|
||||
// anthropic and unknown providers
|
||||
_ => Box::new(AnthropicProfile::with_summarizer(model, summarizer)),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -335,7 +350,7 @@ pub async fn run() -> anyhow::Result<()> {
|
|||
"{}Using model: {model}{}",
|
||||
styles.dim, styles.reset,
|
||||
);
|
||||
let mut profile = build_profile(&cli.provider, model);
|
||||
let mut profile = build_profile(&cli.provider, model, Some(client.clone()));
|
||||
|
||||
// Build execution environment
|
||||
let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
|
||||
|
|
@ -363,10 +378,11 @@ pub async fn run() -> anyhow::Result<()> {
|
|||
let factory_env = Arc::clone(&env);
|
||||
let factory_approval = config.tool_approval.clone();
|
||||
let factory: SessionFactory = Arc::new(move || {
|
||||
let child_summarizer = build_summarizer(&factory_provider, Some(factory_client.clone()));
|
||||
let child_profile: Arc<dyn ProviderProfile> = match factory_provider.as_str() {
|
||||
"openai" => Arc::new(OpenAiProfile::new(&factory_model)),
|
||||
"gemini" => Arc::new(GeminiProfile::new(&factory_model)),
|
||||
_ => Arc::new(AnthropicProfile::new(&factory_model)),
|
||||
"openai" => Arc::new(OpenAiProfile::with_summarizer(&factory_model, child_summarizer)),
|
||||
"gemini" => Arc::new(GeminiProfile::with_summarizer(&factory_model, child_summarizer)),
|
||||
_ => Arc::new(AnthropicProfile::with_summarizer(&factory_model, child_summarizer)),
|
||||
};
|
||||
Session::new(
|
||||
factory_client.clone(),
|
||||
|
|
@ -628,19 +644,19 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn build_profile_anthropic() {
|
||||
let profile = build_profile("anthropic", "model");
|
||||
let profile = build_profile("anthropic", "model", None);
|
||||
assert_eq!(profile.id(), "anthropic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_profile_openai() {
|
||||
let profile = build_profile("openai", "model");
|
||||
let profile = build_profile("openai", "model", None);
|
||||
assert_eq!(profile.id(), "openai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_profile_gemini() {
|
||||
let profile = build_profile("gemini", "model");
|
||||
let profile = build_profile("gemini", "model", None);
|
||||
assert_eq!(profile.id(), "gemini");
|
||||
}
|
||||
|
||||
|
|
@ -648,7 +664,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn build_profile_can_register_subagent_tools() {
|
||||
let mut profile = build_profile("anthropic", "model");
|
||||
let mut profile = build_profile("anthropic", "model", None);
|
||||
let manager = Arc::new(tokio::sync::Mutex::new(SubAgentManager::new(1)));
|
||||
let factory: SessionFactory = Arc::new(|| {
|
||||
panic!("factory should not be called in this test");
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ pub use subagent::{SubAgent, SubAgentEventCallback, SubAgentManager, SubAgentRes
|
|||
pub use tool_registry::ToolRegistry;
|
||||
pub use tools::{
|
||||
make_edit_file_tool, make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool,
|
||||
make_shell_tool_with_config, make_write_file_tool,
|
||||
make_shell_tool_with_config, make_write_file_tool, WebFetchSummarizer,
|
||||
};
|
||||
pub use truncation::{truncate_lines, truncate_output, truncate_tool_output, TruncationMode};
|
||||
pub use types::{AgentEvent, SessionEvent, SessionState, Turn};
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use crate::tool_registry::ToolRegistry;
|
|||
use crate::tools::{
|
||||
make_edit_file_tool, make_glob_tool, make_grep_tool, make_read_file_tool,
|
||||
make_shell_tool_with_config, make_web_fetch_tool, make_web_search_tool, make_write_file_tool,
|
||||
WebFetchSummarizer,
|
||||
};
|
||||
|
||||
use super::EnvContext;
|
||||
|
|
@ -19,6 +20,11 @@ pub struct AnthropicProfile {
|
|||
impl AnthropicProfile {
|
||||
#[must_use]
|
||||
pub fn new(model: impl Into<String>) -> Self {
|
||||
Self::with_summarizer(model, None)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_summarizer(model: impl Into<String>, summarizer: Option<WebFetchSummarizer>) -> Self {
|
||||
let config = SessionConfig {
|
||||
default_command_timeout_ms: 120_000,
|
||||
..SessionConfig::default()
|
||||
|
|
@ -32,7 +38,7 @@ impl AnthropicProfile {
|
|||
registry.register(make_grep_tool());
|
||||
registry.register(make_glob_tool());
|
||||
registry.register(make_web_search_tool());
|
||||
registry.register(make_web_fetch_tool());
|
||||
registry.register(make_web_fetch_tool(summarizer));
|
||||
|
||||
Self {
|
||||
base: BaseProfile {
|
||||
|
|
@ -137,7 +143,8 @@ finding files rather than using shell find or ls commands.
|
|||
Search the web using Brave Search. Returns titles, URLs, and descriptions.
|
||||
|
||||
## web_fetch
|
||||
Fetch content from a URL. URLs must start with http:// or https://.
|
||||
Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific \
|
||||
information instead of returning the full page. URLs must start with http:// or https://.
|
||||
|
||||
# Coding Best Practices
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use crate::tool_registry::ToolRegistry;
|
|||
use crate::tools::{
|
||||
make_edit_file_tool, make_glob_tool, make_grep_tool, make_list_dir_tool,
|
||||
make_read_file_tool, make_read_many_files_tool, make_shell_tool, make_web_fetch_tool,
|
||||
make_web_search_tool, make_write_file_tool,
|
||||
make_web_search_tool, make_write_file_tool, WebFetchSummarizer,
|
||||
};
|
||||
|
||||
use super::EnvContext;
|
||||
|
|
@ -19,6 +19,11 @@ pub struct GeminiProfile {
|
|||
impl GeminiProfile {
|
||||
#[must_use]
|
||||
pub fn new(model: impl Into<String>) -> Self {
|
||||
Self::with_summarizer(model, None)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_summarizer(model: impl Into<String>, summarizer: Option<WebFetchSummarizer>) -> Self {
|
||||
let mut registry = ToolRegistry::new();
|
||||
|
||||
registry.register(make_read_file_tool());
|
||||
|
|
@ -30,7 +35,7 @@ impl GeminiProfile {
|
|||
registry.register(make_glob_tool());
|
||||
registry.register(make_list_dir_tool());
|
||||
registry.register(make_web_search_tool());
|
||||
registry.register(make_web_fetch_tool());
|
||||
registry.register(make_web_fetch_tool(summarizer));
|
||||
|
||||
Self {
|
||||
base: BaseProfile {
|
||||
|
|
@ -160,7 +165,8 @@ List directory contents with depth control.
|
|||
Search the web for information.
|
||||
|
||||
## web_fetch
|
||||
Fetch content from a URL.
|
||||
Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific \
|
||||
information instead of returning the full page.
|
||||
|
||||
# Project Docs
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use crate::tool_registry::{RegisteredTool, ToolRegistry};
|
|||
use llm::types::ToolDefinition;
|
||||
use crate::tools::{
|
||||
make_glob_tool, make_grep_tool, make_read_file_tool, make_shell_tool, make_web_fetch_tool,
|
||||
make_web_search_tool, make_write_file_tool,
|
||||
make_web_search_tool, make_write_file_tool, WebFetchSummarizer,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -21,6 +21,11 @@ pub struct OpenAiProfile {
|
|||
impl OpenAiProfile {
|
||||
#[must_use]
|
||||
pub fn new(model: impl Into<String>) -> Self {
|
||||
Self::with_summarizer(model, None)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_summarizer(model: impl Into<String>, summarizer: Option<WebFetchSummarizer>) -> Self {
|
||||
let mut registry = ToolRegistry::new();
|
||||
|
||||
registry.register(make_read_file_tool());
|
||||
|
|
@ -30,7 +35,7 @@ impl OpenAiProfile {
|
|||
registry.register(make_glob_tool());
|
||||
registry.register(make_apply_patch_tool());
|
||||
registry.register(make_web_search_tool());
|
||||
registry.register(make_web_fetch_tool());
|
||||
registry.register(make_web_fetch_tool(summarizer));
|
||||
|
||||
Self {
|
||||
base: BaseProfile {
|
||||
|
|
@ -151,7 +156,8 @@ Find files by name pattern.
|
|||
Search the web using Brave Search. Returns titles, URLs, and descriptions.
|
||||
|
||||
## web_fetch
|
||||
Fetch content from a URL. URLs must start with http:// or https://.
|
||||
Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific \
|
||||
information instead of returning the full page. URLs must start with http:// or https://.
|
||||
|
||||
# Coding Best Practices
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,40 @@
|
|||
use crate::config::SessionConfig;
|
||||
use crate::execution_env::GrepOptions;
|
||||
use crate::tool_registry::RegisteredTool;
|
||||
use llm::client::Client;
|
||||
use llm::types::{Message, Request, ToolDefinition};
|
||||
use std::borrow::Cow;
|
||||
use std::fmt::Write;
|
||||
use std::sync::Arc;
|
||||
use llm::types::ToolDefinition;
|
||||
|
||||
const MAX_WEB_FETCH_BYTES: usize = 100 * 1024;
|
||||
|
||||
/// Configuration for the optional LLM-based summarizer used by `web_fetch`.
|
||||
#[derive(Clone)]
|
||||
pub struct WebFetchSummarizer {
|
||||
pub client: Client,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
/// Returns true if the input looks like it contains HTML markup.
|
||||
fn looks_like_html(text: &str) -> bool {
|
||||
let trimmed = text.trim_start();
|
||||
trimmed.starts_with("<!") || trimmed.starts_with("<html") || trimmed.starts_with("<HTML")
|
||||
|| trimmed.contains("</div>") || trimmed.contains("</p>") || trimmed.contains("</body>")
|
||||
}
|
||||
|
||||
/// Converts HTML to Markdown, stripping script/style tags.
|
||||
/// Non-HTML content (JSON, plain text) passes through unchanged.
|
||||
fn html_to_markdown(text: &str) -> String {
|
||||
if !looks_like_html(text) {
|
||||
return text.to_string();
|
||||
}
|
||||
let converter = htmd::HtmlToMarkdown::builder()
|
||||
.skip_tags(vec!["script", "style"])
|
||||
.build();
|
||||
converter.convert(text).unwrap_or_else(|_| text.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn required_str<'a>(args: &'a serde_json::Value, key: &str) -> Result<&'a str, String> {
|
||||
args.get(key)
|
||||
.and_then(|v| v.as_str())
|
||||
|
|
@ -426,23 +453,26 @@ fn make_web_search_tool_with_api_key(api_key: Option<String>) -> RegisteredTool
|
|||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn make_web_fetch_tool() -> RegisteredTool {
|
||||
pub(crate) fn make_web_fetch_tool(summarizer: Option<WebFetchSummarizer>) -> RegisteredTool {
|
||||
RegisteredTool {
|
||||
definition: ToolDefinition {
|
||||
name: "web_fetch".into(),
|
||||
description: "Fetch content from a URL".into(),
|
||||
description: "Fetch content from a URL and optionally summarize it. Pass a prompt to extract specific information instead of returning the full page.".into(),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {"type": "string", "description": "URL to fetch (must be http:// or https://)"},
|
||||
"prompt": {"type": "string", "description": "A question or instruction about the page content. When provided, returns a concise answer instead of the full page."},
|
||||
"timeout_ms": {"type": "integer", "description": "Timeout in milliseconds (default 30000, max 60000)"}
|
||||
},
|
||||
"required": ["url"]
|
||||
}),
|
||||
},
|
||||
executor: Arc::new(|args, env, cancel| {
|
||||
executor: Arc::new(move |args, env, cancel| {
|
||||
let summarizer = summarizer.clone();
|
||||
Box::pin(async move {
|
||||
let url = required_str(&args, "url")?;
|
||||
let prompt = args.get("prompt").and_then(serde_json::Value::as_str);
|
||||
let timeout_ms = args
|
||||
.get("timeout_ms")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
|
|
@ -471,13 +501,41 @@ pub(crate) fn make_web_fetch_tool() -> RegisteredTool {
|
|||
));
|
||||
}
|
||||
|
||||
let mut output = result.stdout;
|
||||
if output.len() > MAX_WEB_FETCH_BYTES {
|
||||
output.truncate(MAX_WEB_FETCH_BYTES);
|
||||
output.push_str("\n\n[Output truncated at 100KB]");
|
||||
let mut content = html_to_markdown(&result.stdout);
|
||||
if content.len() > MAX_WEB_FETCH_BYTES {
|
||||
content.truncate(MAX_WEB_FETCH_BYTES);
|
||||
content.push_str("\n\n[Output truncated at 100KB]");
|
||||
}
|
||||
|
||||
Ok(output)
|
||||
match (prompt, &summarizer) {
|
||||
(Some(user_prompt), Some(s)) => {
|
||||
let summarization_prompt = format!(
|
||||
"Content from {url}:\n---\n{content}\n---\n\n{user_prompt}\n\nRespond concisely based only on the content above."
|
||||
);
|
||||
let request = Request {
|
||||
model: s.model.clone(),
|
||||
messages: vec![Message::user(summarization_prompt)],
|
||||
provider: None,
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
response_format: None,
|
||||
temperature: None,
|
||||
top_p: None,
|
||||
max_tokens: None,
|
||||
stop_sequences: None,
|
||||
reasoning_effort: None,
|
||||
metadata: None,
|
||||
provider_options: None,
|
||||
};
|
||||
let response = s.client.complete(&request).await.map_err(|e| format!("Summarization failed: {e}"))?;
|
||||
Ok(response.text())
|
||||
}
|
||||
(Some(_), None) => {
|
||||
// Graceful degradation: return content with a note
|
||||
Ok(format!("[Note: prompt summarization unavailable, returning full content]\n\n{content}"))
|
||||
}
|
||||
(None, _) => Ok(content),
|
||||
}
|
||||
})
|
||||
}),
|
||||
}
|
||||
|
|
@ -783,10 +841,10 @@ mod tests {
|
|||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_builds_curl_command() {
|
||||
let tool = make_web_fetch_tool();
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env = Arc::new(MockExecutionEnvironment {
|
||||
exec_result: ExecResult {
|
||||
stdout: "<html>hello</html>".into(),
|
||||
stdout: "<html><body><h1>hello</h1></body></html>".into(),
|
||||
stderr: String::new(),
|
||||
exit_code: 0,
|
||||
timed_out: false,
|
||||
|
|
@ -801,7 +859,9 @@ mod tests {
|
|||
CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.unwrap(), "<html>hello</html>");
|
||||
let output = result.unwrap();
|
||||
assert!(output.contains("# hello"), "HTML should be converted to markdown, got: {output}");
|
||||
assert!(!output.contains("<html>"), "raw HTML tags should be removed, got: {output}");
|
||||
let cmd = env.captured_command.lock().unwrap().clone().unwrap();
|
||||
assert!(cmd.starts_with("curl -sL --max-time 30 "), "command should start with curl flags, got: {cmd}");
|
||||
assert!(cmd.contains("https://example.com"), "command should contain the URL");
|
||||
|
|
@ -810,7 +870,7 @@ mod tests {
|
|||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_rejects_non_http_url() {
|
||||
let tool = make_web_fetch_tool();
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env: Arc<dyn ExecutionEnvironment> = Arc::new(MockExecutionEnvironment::default());
|
||||
let result = (tool.executor)(
|
||||
serde_json::json!({"url": "ftp://example.com/file"}),
|
||||
|
|
@ -824,7 +884,7 @@ mod tests {
|
|||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_timeout_flows_through() {
|
||||
let tool = make_web_fetch_tool();
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env = Arc::new(MockExecutionEnvironment::default());
|
||||
let env_clone: Arc<dyn ExecutionEnvironment> = env.clone();
|
||||
let _result = (tool.executor)(
|
||||
|
|
@ -840,7 +900,7 @@ mod tests {
|
|||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_timeout_capped_at_60s() {
|
||||
let tool = make_web_fetch_tool();
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env = Arc::new(MockExecutionEnvironment::default());
|
||||
let env_clone: Arc<dyn ExecutionEnvironment> = env.clone();
|
||||
let _result = (tool.executor)(
|
||||
|
|
@ -857,7 +917,7 @@ mod tests {
|
|||
#[tokio::test]
|
||||
async fn web_fetch_truncates_large_output() {
|
||||
let large_content = "x".repeat(150 * 1024);
|
||||
let tool = make_web_fetch_tool();
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env: Arc<dyn ExecutionEnvironment> = Arc::new(MockExecutionEnvironment {
|
||||
exec_result: ExecResult {
|
||||
stdout: large_content,
|
||||
|
|
@ -881,7 +941,7 @@ mod tests {
|
|||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_returns_error_on_nonzero_exit() {
|
||||
let tool = make_web_fetch_tool();
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env: Arc<dyn ExecutionEnvironment> = Arc::new(MockExecutionEnvironment {
|
||||
exec_result: ExecResult {
|
||||
stdout: String::new(),
|
||||
|
|
@ -903,6 +963,88 @@ mod tests {
|
|||
assert!(err.contains("Could not resolve host"), "error should contain stderr, got: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_prompt_with_summarizer_returns_llm_answer() {
|
||||
use crate::test_support::{make_client, MockLlmProvider, text_response};
|
||||
|
||||
let provider = Arc::new(MockLlmProvider::new(vec![
|
||||
text_response("Rust is a systems programming language focused on safety and performance."),
|
||||
]));
|
||||
let client = make_client(provider).await;
|
||||
let summarizer = WebFetchSummarizer {
|
||||
client,
|
||||
model: "mock-model".into(),
|
||||
};
|
||||
|
||||
let tool = make_web_fetch_tool(Some(summarizer));
|
||||
let env: Arc<dyn ExecutionEnvironment> = Arc::new(MockExecutionEnvironment {
|
||||
exec_result: ExecResult {
|
||||
stdout: "<html><body><p>Lots of content about Rust...</p></body></html>".into(),
|
||||
stderr: String::new(),
|
||||
exit_code: 0,
|
||||
timed_out: false,
|
||||
duration_ms: 100,
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
let result = (tool.executor)(
|
||||
serde_json::json!({"url": "https://example.com", "prompt": "What is Rust?"}),
|
||||
env,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let output = result.unwrap();
|
||||
assert_eq!(output, "Rust is a systems programming language focused on safety and performance.");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn web_fetch_prompt_without_summarizer_returns_content_with_note() {
|
||||
let tool = make_web_fetch_tool(None);
|
||||
let env: Arc<dyn ExecutionEnvironment> = Arc::new(MockExecutionEnvironment {
|
||||
exec_result: ExecResult {
|
||||
stdout: "<html><body><p>Rust is a systems programming language.</p></body></html>".into(),
|
||||
stderr: String::new(),
|
||||
exit_code: 0,
|
||||
timed_out: false,
|
||||
duration_ms: 100,
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
let result = (tool.executor)(
|
||||
serde_json::json!({"url": "https://example.com", "prompt": "What is Rust?"}),
|
||||
env,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let output = result.unwrap();
|
||||
assert!(output.contains("summarization unavailable"), "should note unavailability, got: {output}");
|
||||
assert!(output.contains("Rust is a systems programming language"), "should contain page content, got: {output}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_to_markdown_converts_basic_html() {
|
||||
let result = html_to_markdown("<h1>Hello</h1><p>World</p>");
|
||||
assert_eq!(result, "# Hello\n\nWorld");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_to_markdown_strips_script_and_style() {
|
||||
let html = "<html><head><style>body{color:red}</style></head><body><script>alert(1)</script><p>Content</p></body></html>";
|
||||
let result = html_to_markdown(html);
|
||||
assert!(!result.contains("alert"), "script content should be stripped");
|
||||
assert!(!result.contains("color:red"), "style content should be stripped");
|
||||
assert!(result.contains("Content"), "paragraph text should remain");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_to_markdown_passes_through_non_html() {
|
||||
let json = r#"{"key": "value", "items": [1, 2, 3]}"#;
|
||||
assert_eq!(html_to_markdown(json), json);
|
||||
|
||||
let plain = "Just some plain text\nwith newlines";
|
||||
assert_eq!(html_to_markdown(plain), plain);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // Requires BRAVE_SEARCH_API_KEY env var
|
||||
async fn web_search_returns_results() {
|
||||
|
|
|
|||
|
|
@ -3,19 +3,36 @@ use std::sync::Arc;
|
|||
|
||||
use agent::{
|
||||
AnthropicProfile, GeminiProfile, LocalExecutionEnvironment, OpenAiProfile, ProviderProfile,
|
||||
Session, SessionConfig, SubAgentManager,
|
||||
Session, SessionConfig, SubAgentManager, WebFetchSummarizer,
|
||||
};
|
||||
use llm::client::Client;
|
||||
|
||||
fn build_summarizer(provider: &str, client: &Client) -> WebFetchSummarizer {
|
||||
let summarizer_model = match provider {
|
||||
"openai" => "gpt-4o-mini",
|
||||
"gemini" => "gemini-2.0-flash",
|
||||
_ => "claude-haiku-4-5-20251001",
|
||||
};
|
||||
WebFetchSummarizer {
|
||||
client: client.clone(),
|
||||
model: summarizer_model.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_profile(provider: &str, model: &str, client: &Client) -> Box<dyn ProviderProfile> {
|
||||
let summarizer = Some(build_summarizer(provider, client));
|
||||
match provider {
|
||||
"anthropic" => Box::new(AnthropicProfile::with_summarizer(model, summarizer)),
|
||||
"openai" => Box::new(OpenAiProfile::with_summarizer(model, summarizer)),
|
||||
"gemini" => Box::new(GeminiProfile::with_summarizer(model, summarizer)),
|
||||
_ => panic!("unknown provider: {provider}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_session(provider: &str, model: &str, cwd: &Path) -> Session {
|
||||
dotenvy::dotenv().ok();
|
||||
let client = Client::from_env().await.expect("Client::from_env failed");
|
||||
let mut profile: Box<dyn ProviderProfile> = match provider {
|
||||
"anthropic" => Box::new(AnthropicProfile::new(model)),
|
||||
"openai" => Box::new(OpenAiProfile::new(model)),
|
||||
"gemini" => Box::new(GeminiProfile::new(model)),
|
||||
_ => panic!("unknown provider: {provider}"),
|
||||
};
|
||||
let mut profile = build_profile(provider, model, &client);
|
||||
let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf()));
|
||||
|
||||
// Register subagent tools so spawn_agent / wait / send_input / close_agent are available
|
||||
|
|
@ -28,11 +45,14 @@ async fn make_session(provider: &str, model: &str, cwd: &Path) -> Session {
|
|||
let provider = factory_provider.to_string();
|
||||
let model = factory_model;
|
||||
Arc::new(move || {
|
||||
let sub_profile: Arc<dyn ProviderProfile> = match provider.as_str() {
|
||||
"anthropic" => Arc::new(AnthropicProfile::new(&model)),
|
||||
"openai" => Arc::new(OpenAiProfile::new(&model)),
|
||||
"gemini" => Arc::new(GeminiProfile::new(&model)),
|
||||
_ => panic!("unknown provider: {provider}"),
|
||||
let sub_profile: Arc<dyn ProviderProfile> = {
|
||||
let summarizer = Some(build_summarizer(&provider, &factory_client));
|
||||
match provider.as_str() {
|
||||
"anthropic" => Arc::new(AnthropicProfile::with_summarizer(&model, summarizer)),
|
||||
"openai" => Arc::new(OpenAiProfile::with_summarizer(&model, summarizer)),
|
||||
"gemini" => Arc::new(GeminiProfile::with_summarizer(&model, summarizer)),
|
||||
_ => panic!("unknown provider: {provider}"),
|
||||
}
|
||||
};
|
||||
let sub_env = Arc::new(LocalExecutionEnvironment::new(factory_cwd.clone()));
|
||||
Session::new(
|
||||
|
|
@ -61,12 +81,7 @@ async fn make_session_with_config(
|
|||
) -> Session {
|
||||
dotenvy::dotenv().ok();
|
||||
let client = Client::from_env().await.expect("Client::from_env failed");
|
||||
let profile: Arc<dyn ProviderProfile> = match provider {
|
||||
"anthropic" => Arc::new(AnthropicProfile::new(model)),
|
||||
"openai" => Arc::new(OpenAiProfile::new(model)),
|
||||
"gemini" => Arc::new(GeminiProfile::new(model)),
|
||||
_ => panic!("unknown provider: {provider}"),
|
||||
};
|
||||
let profile: Arc<dyn ProviderProfile> = Arc::from(build_profile(provider, model, &client));
|
||||
let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf()));
|
||||
Session::new(client, profile, env, config)
|
||||
}
|
||||
|
|
@ -395,20 +410,36 @@ async fn scenario_error_recovery(session: &mut Session, dir: &Path) {
|
|||
// Scenario 15: web_fetch
|
||||
// ---------------------------------------------------------------------------
|
||||
async fn scenario_web_fetch(session: &mut Session, dir: &Path) {
|
||||
// Test basic fetch (HTML-to-markdown conversion)
|
||||
session
|
||||
.process_input(
|
||||
"Use the web_fetch tool to fetch https://example.com and write its content to a file called fetched.html",
|
||||
"Use the web_fetch tool to fetch https://example.com and write its content to a file called fetched.txt",
|
||||
)
|
||||
.await
|
||||
.expect("process_input failed");
|
||||
let path = dir.join("fetched.html");
|
||||
assert!(path.exists(), "fetched.html should have been created");
|
||||
let content = std::fs::read_to_string(&path).expect("failed to read fetched.html");
|
||||
let path = dir.join("fetched.txt");
|
||||
assert!(path.exists(), "fetched.txt should have been created");
|
||||
let content = std::fs::read_to_string(&path).expect("failed to read fetched.txt");
|
||||
assert!(
|
||||
content.contains("Example Domain"),
|
||||
"Expected 'Example Domain' in fetched content, got first 200 chars: {}",
|
||||
&content[..content.len().min(200)]
|
||||
);
|
||||
|
||||
// Test fetch with prompt parameter (LLM summarization)
|
||||
session
|
||||
.process_input(
|
||||
"Use the web_fetch tool with the prompt parameter to fetch https://example.com and answer: 'What is the title heading on this page?' Write only the answer to a file called answer.txt",
|
||||
)
|
||||
.await
|
||||
.expect("process_input failed for prompt test");
|
||||
let answer_path = dir.join("answer.txt");
|
||||
assert!(answer_path.exists(), "answer.txt should have been created");
|
||||
let answer = std::fs::read_to_string(&answer_path).expect("failed to read answer.txt");
|
||||
assert!(
|
||||
answer.to_lowercase().contains("example domain"),
|
||||
"Expected answer to mention 'example domain', got: {answer}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue