scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-10 03:30:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Add e2e parity tests for web_fetch and web_search tools

Browse source 
Test both tools across all three providers (Anthropic, OpenAI, Gemini).
web_fetch fetches example.com and asserts content is written to a file.
web_search searches for "Rust programming language" and asserts results
are saved.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-02-27 20:09:07 -05:00
parent 6338f3ec57
commit 8c90945ab6
1 changed files with 43 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
crates/agent/tests/parity_matrix.rs
View file

@ -115,6 +115,9 @@ provider_tests!(parallel_tool_calls);
provider_tests!(steering);
provider_tests!(subagent_spawn);
provider_tests!(web_fetch);
provider_tests!(web_search);
// Scenarios below are only generated for providers where they are supported.
// - multi_step_read_analyze_edit / provider_specific_editing: gpt-4o-mini is too
// weak to reliably apply precise file edits (uses apply_patch, not edit_file).
@ -389,7 +392,46 @@ async fn scenario_error_recovery(session: &mut Session, dir: &Path) {
}
// ---------------------------------------------------------------------------
// Scenario 15: provider_specific_editing
// Scenario 15: web_fetch
// ---------------------------------------------------------------------------
async fn scenario_web_fetch(session: &mut Session, dir: &Path) {
session
.process_input(
"Use the web_fetch tool to fetch https://example.com and write its content to a file called fetched.html",
)
.await
.expect("process_input failed");
let path = dir.join("fetched.html");
assert!(path.exists(), "fetched.html should have been created");
let content = std::fs::read_to_string(&path).expect("failed to read fetched.html");
assert!(
content.contains("Example Domain"),
"Expected 'Example Domain' in fetched content, got first 200 chars: {}",
&content[..content.len().min(200)]
);
}
// ---------------------------------------------------------------------------
// Scenario 16: web_search
// ---------------------------------------------------------------------------
async fn scenario_web_search(session: &mut Session, dir: &Path) {
session
.process_input(
"Use the web_search tool to search for 'Rust programming language' and write the first result's title and URL to a file called search_results.txt",
)
.await
.expect("process_input failed");
let path = dir.join("search_results.txt");
assert!(path.exists(), "search_results.txt should have been created");
let content = std::fs::read_to_string(&path).expect("failed to read search_results.txt");
assert!(
!content.is_empty(),
"search_results.txt should not be empty"
);
}
// ---------------------------------------------------------------------------
// Scenario 17: provider_specific_editing
// ---------------------------------------------------------------------------
async fn scenario_provider_specific_editing(session: &mut Session, dir: &Path) {
std::fs::write(