diff --git a/lib/crates/fabro-llm/tests/it/main.rs b/lib/crates/fabro-llm/tests/it/main.rs new file mode 100644 index 000000000..66dc027b4 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/main.rs @@ -0,0 +1,7 @@ +#![allow( + clippy::absolute_paths, + reason = "This test module prefers explicit type paths over extra imports." +)] + +mod support; +mod wire; diff --git a/lib/crates/fabro-llm/tests/it/support.rs b/lib/crates/fabro-llm/tests/it/support.rs new file mode 100644 index 000000000..c2d30c58c --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/support.rs @@ -0,0 +1,470 @@ +//! Shared helpers for capturing the wire requests adapters send, plus the +//! canonical request corpus pinned across all four provider dialects. + +use std::sync::{Arc, Mutex}; + +use fabro_llm::provider::ProviderAdapter; +use fabro_llm::types::{ + AudioData, ContentPart, DocumentData, ImageData, Message, Request, ResponseFormat, Role, + ThinkingData, ToolCall, ToolChoice, ToolDefinition, ToolResult, +}; +use fabro_model::Catalog; +use fabro_model::catalog::LlmCatalogSettings; +use httpmock::prelude::*; + +// --------------------------------------------------------------------------- +// Wire capture +// --------------------------------------------------------------------------- + +/// One captured wire request, normalized for snapshot stability. +#[derive(Debug, Clone, serde::Serialize)] +pub(crate) struct WireCapture { + pub(crate) method: String, + pub(crate) path: String, + pub(crate) headers: Vec<(String, String)>, + pub(crate) body: serde_json::Value, +} + +/// Shared slot the matcher closure writes the captured request into. +pub(crate) type CaptureSlot = Arc>>; + +fn capture_request(req: &HttpMockRequest) -> WireCapture { + let mut headers: Vec<(String, String)> = req + .headers_vec() + .iter() + .map(|(name, value)| { + let name = name.to_ascii_lowercase(); + let value = match name.as_str() { + // The mock server binds a random port. + "host" => "[host]".to_string(), + // Carries a client version that would churn snapshots. + "user-agent" => "[user-agent]".to_string(), + _ => value.clone(), + }; + (name, value) + }) + .collect(); + headers.sort(); + + let path = match req.uri().query() { + Some(query) => format!("{}?{}", req.uri().path(), query), + None => req.uri().path().to_string(), + }; + + WireCapture { + method: req.method_str().to_string(), + path, + headers, + body: serde_json::from_str(&req.body_string()).expect("request body should be JSON"), + } +} + +/// Mounts a mock on `path` that captures the full request into the returned +/// slot and responds with the JSON `response_body`. +pub(crate) fn mount_capture<'a>( + server: &'a MockServer, + path: &'static str, + response_body: serde_json::Value, +) -> (httpmock::Mock<'a>, CaptureSlot) { + let slot: CaptureSlot = Arc::new(Mutex::new(None)); + let writer = Arc::clone(&slot); + let mock = server.mock(move |when, then| { + when.method(POST) + .path(path) + .is_true(move |req: &HttpMockRequest| { + *writer.lock().unwrap() = Some(capture_request(req)); + true + }); + then.status(200) + .header("content-type", "application/json") + .json_body(response_body); + }); + (mock, slot) +} + +/// Like [`mount_capture`] but responds with a raw SSE transcript. +pub(crate) fn mount_capture_sse<'a>( + server: &'a MockServer, + path: &'static str, + sse_body: &str, +) -> (httpmock::Mock<'a>, CaptureSlot) { + let slot: CaptureSlot = Arc::new(Mutex::new(None)); + let writer = Arc::clone(&slot); + let body = sse_body.to_string(); + let mock = server.mock(move |when, then| { + when.method(POST) + .path(path) + .is_true(move |req: &HttpMockRequest| { + *writer.lock().unwrap() = Some(capture_request(req)); + true + }); + then.status(200) + .header("content-type", "text/event-stream") + .body(body.clone()); + }); + (mock, slot) +} + +pub(crate) fn take_capture(slot: &CaptureSlot) -> WireCapture { + slot.lock() + .unwrap() + .take() + .expect("matcher should have captured the request") +} + +/// Drives `adapter.stream(request)` to completion and returns every emitted +/// item as JSON: `Ok` events serialize verbatim (the public SSE wire shape); +/// `Err` items pin the message plus the failover/retry flags consumers key on. +pub(crate) async fn collect_stream_events( + adapter: &dyn ProviderAdapter, + request: &Request, +) -> Vec { + use futures::StreamExt; + + let mut stream = adapter.stream(request).await.expect("stream should start"); + let mut events = Vec::new(); + while let Some(item) = stream.next().await { + events.push(match item { + Ok(event) => serde_json::to_value(&event).expect("event should serialize"), + Err(error) => serde_json::json!({ + "stream_item_error": error.to_string(), + "retryable": error.retryable(), + "failover_eligible": error.failover_eligible(), + }), + }); + } + events +} + +/// Builds a catalog from inline TOML (same `LlmCatalogSettings` schema as the +/// shipped catalog files). +pub(crate) fn catalog_from_toml(source: &str) -> Arc { + let settings: LlmCatalogSettings = toml::from_str(source).expect("catalog TOML should parse"); + Arc::new(Catalog::from_settings(&settings).expect("catalog should build")) +} + +fn is_uuid(s: &str) -> bool { + s.len() == 36 + && s.bytes().enumerate().all(|(i, b)| match i { + 8 | 13 | 18 | 23 => b == b'-', + _ => b.is_ascii_hexdigit(), + }) +} + +/// Replaces UUID-shaped strings with `[UUID]` for snapshot stability — the +/// Gemini decoder mints synthetic `Uuid::new_v4()` tool-call ids. +pub(crate) fn normalize_uuids(value: &mut serde_json::Value) { + match value { + serde_json::Value::String(s) if is_uuid(s) => "[UUID]".clone_into(s), + serde_json::Value::Array(items) => items.iter_mut().for_each(normalize_uuids), + serde_json::Value::Object(map) => map.values_mut().for_each(normalize_uuids), + _ => {} + } +} + +/// Renders `(event, data)` pairs as an SSE transcript with `event:` lines +/// (the Anthropic framing). +pub(crate) fn sse_transcript(events: &[(&str, &str)]) -> String { + use std::fmt::Write; + + events.iter().fold(String::new(), |mut out, (event, data)| { + let _ = writeln!(out, "event: {event}\ndata: {data}\n"); + out + }) +} + +/// Renders data-only SSE lines (the OpenAI/Gemini framing). +pub(crate) fn sse_data_transcript(lines: &[&str]) -> String { + use std::fmt::Write; + + lines.iter().fold(String::new(), |mut out, data| { + let _ = writeln!(out, "data: {data}\n"); + out + }) +} + +// --------------------------------------------------------------------------- +// Canonical request corpus +// +// Each constructor returns one canonical `Request` shape that every dialect +// file pins through its own adapter. Keep these stable: editing a corpus +// request invalidates the pinned wire snapshots in all four dialect files. +// --------------------------------------------------------------------------- + +pub(crate) fn base_request(model: &str) -> Request { + Request { + model: model.to_string(), + messages: vec![Message::user("Hello")], + provider: None, + tools: None, + tool_choice: None, + response_format: None, + temperature: None, + top_p: None, + max_tokens: Some(128), + stop_sequences: None, + reasoning_effort: None, + speed: None, + metadata: None, + provider_options: None, + } +} + +/// Multi-turn conversation: system + user/assistant/user. +pub(crate) fn corpus_multi_turn(model: &str) -> Request { + Request { + messages: vec![ + Message::system("You are a terse assistant."), + Message::user("What is the capital of France?"), + Message::assistant("Paris."), + Message::user("And of Spain?"), + ], + ..base_request(model) + } +} + +/// Two tools plus an optional tool choice. +pub(crate) fn corpus_tools(model: &str, tool_choice: Option) -> Request { + Request { + tools: Some(vec![ + ToolDefinition::function( + "search", + "Search files", + serde_json::json!({ + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"] + }), + ), + ToolDefinition::function( + "read_file", + "Read a file by path", + serde_json::json!({ + "type": "object", + "properties": {"path": {"type": "string"}} + }), + ), + ]), + tool_choice, + ..base_request(model) + } +} + +/// A full tool round trip: assistant emits two tool calls, the tool turn +/// returns one success carrying an image and one error result. +pub(crate) fn corpus_tool_round_trip(model: &str) -> Request { + let mut image_result = ToolResult::success("call_1", serde_json::json!({"matches": 2})); + image_result.image_data = Some(b"fake-screenshot-bytes".to_vec()); + image_result.image_media_type = Some("image/png".to_string()); + + let mut request = corpus_tools(model, None); + request.messages = vec![ + Message::user("Find foo and read /tmp/x"), + Message { + role: Role::Assistant, + content: vec![ + ContentPart::text("Let me check."), + ContentPart::ToolCall(ToolCall::new( + "call_1", + "search", + serde_json::json!({"query": "foo"}), + )), + ContentPart::ToolCall(ToolCall::new( + "call_2", + "read_file", + serde_json::json!({"path": "/tmp/x"}), + )), + ], + name: None, + tool_call_id: None, + }, + Message { + role: Role::Tool, + content: vec![ContentPart::ToolResult(image_result)], + name: None, + tool_call_id: Some("call_1".to_string()), + }, + Message::tool_result( + "call_2", + serde_json::Value::String("file not found".to_string()), + true, + ), + ]; + request +} + +/// Assistant thinking block with a signature, round-tripped back as history. +pub(crate) fn corpus_thinking_round_trip(model: &str) -> Request { + Request { + messages: vec![ + Message::user("Think step by step: what is 2+2?"), + Message { + role: Role::Assistant, + content: vec![ + ContentPart::Thinking(ThinkingData { + text: "The user wants 2+2, which is 4.".to_string(), + signature: Some("sig_test_abc123".to_string()), + redacted: false, + }), + ContentPart::text("4."), + ], + name: None, + tool_call_id: None, + }, + Message::user("Now 3+3?"), + ], + ..base_request(model) + } +} + +/// Image and document attachments as inline bytes (no file I/O involved). +pub(crate) fn corpus_inline_attachments(model: &str) -> Request { + Request { + messages: vec![Message { + role: Role::User, + content: vec![ + ContentPart::text("Describe these attachments."), + ContentPart::Image(ImageData { + url: None, + data: Some(b"fake-png-bytes".to_vec()), + media_type: Some("image/png".to_string()), + detail: None, + }), + ContentPart::Document(DocumentData { + url: None, + data: Some(b"fake-pdf-bytes".to_vec()), + media_type: Some("application/pdf".to_string()), + file_name: Some("report.pdf".to_string()), + }), + ], + name: None, + tool_call_id: None, + }], + ..base_request(model) + } +} + +/// Image and document attachments as non-file https URLs. Each dialect has +/// its own URL-passthrough wire shape; resolving these to inline data would +/// be a wire change. +pub(crate) fn corpus_url_attachments(model: &str) -> Request { + Request { + messages: vec![Message { + role: Role::User, + content: vec![ + ContentPart::text("Describe these attachments."), + ContentPart::Image(ImageData { + url: Some("https://example.com/picture.png".to_string()), + data: None, + media_type: Some("image/png".to_string()), + detail: None, + }), + ContentPart::Document(DocumentData { + url: Some("https://example.com/report.pdf".to_string()), + data: None, + media_type: Some("application/pdf".to_string()), + file_name: Some("report.pdf".to_string()), + }), + ], + name: None, + tool_call_id: None, + }], + ..base_request(model) + } +} + +/// Attachments referencing file paths that do not exist. Today every adapter +/// silently drops the part on load failure (`Err(_) => None`) and sends the +/// rest of the request; these requests pin that contract. +pub(crate) fn corpus_bad_file_path_attachments(model: &str) -> Request { + Request { + messages: vec![Message { + role: Role::User, + content: vec![ + ContentPart::text("Describe these attachments."), + ContentPart::Image(ImageData { + url: Some("/nonexistent/fabro-wire-pin.png".to_string()), + data: None, + media_type: Some("image/png".to_string()), + detail: None, + }), + ContentPart::Document(DocumentData { + url: Some("/nonexistent/fabro-wire-pin.pdf".to_string()), + data: None, + media_type: Some("application/pdf".to_string()), + file_name: Some("missing.pdf".to_string()), + }), + ], + name: None, + tool_call_id: None, + }], + ..base_request(model) + } +} + +/// Inline audio attachment (support differs per dialect: gemini sends it, +/// openai-responses falls back to text, anthropic/compat drop or warn). +pub(crate) fn corpus_audio_attachment(model: &str) -> Request { + Request { + messages: vec![Message { + role: Role::User, + content: vec![ + ContentPart::text("Transcribe this."), + ContentPart::Audio(AudioData { + url: None, + data: Some(b"fake-wav-bytes".to_vec()), + media_type: Some("audio/wav".to_string()), + }), + ], + name: None, + tool_call_id: None, + }], + ..base_request(model) + } +} + +/// Response-format request (callers pass each of the three kinds). +pub(crate) fn corpus_response_format(model: &str, format: ResponseFormat) -> Request { + Request { + response_format: Some(format), + ..base_request(model) + } +} + +/// A JSON-schema response format with `strict` set. The schema is passed raw +/// (no name/schema wrapper) — the shape `generate_object` produces. +pub(crate) fn json_schema_format() -> ResponseFormat { + ResponseFormat { + kind: fabro_llm::types::ResponseFormatType::JsonSchema, + json_schema: Some(serde_json::json!({ + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"] + })), + strict: true, + } +} + +/// Sampling parameters: temperature, top_p, stop sequences, and metadata. +/// Metadata deliberately holds a single key — `HashMap` iteration order would +/// make multi-key snapshots nondeterministic. +pub(crate) fn corpus_sampling_params(model: &str) -> Request { + Request { + temperature: Some(0.7), + top_p: Some(0.9), + stop_sequences: Some(vec!["END".to_string()]), + metadata: Some(std::collections::HashMap::from([( + "trace_id".to_string(), + "trace-123".to_string(), + )])), + ..base_request(model) + } +} + +/// Provider-options escape hatch (callers pass the dialect's namespace key). +pub(crate) fn corpus_provider_options(model: &str, options: serde_json::Value) -> Request { + Request { + provider_options: Some(options), + ..base_request(model) + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/anthropic.rs b/lib/crates/fabro-llm/tests/it/wire/anthropic.rs new file mode 100644 index 000000000..bbeec415d --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/anthropic.rs @@ -0,0 +1,593 @@ +//! Wire snapshots for the Anthropic Messages dialect. + +use fabro_llm::provider::ProviderAdapter; +use fabro_llm::providers::AnthropicAdapter; +use fabro_llm::types::{ + Message, Request, ResponseFormat, ResponseFormatType, ToolChoice, ToolDefinition, +}; +use httpmock::prelude::*; + +use crate::support::{ + self, WireCapture, base_request, corpus_audio_attachment, corpus_bad_file_path_attachments, + corpus_inline_attachments, corpus_multi_turn, corpus_provider_options, corpus_response_format, + corpus_sampling_params, corpus_thinking_round_trip, corpus_tool_round_trip, corpus_tools, + corpus_url_attachments, json_schema_format, mount_capture, mount_capture_sse, take_capture, +}; + +const MODEL: &str = "claude-sonnet-4-20250514"; + +/// Minimal valid Messages API body for encode-side tests that only assert on +/// the captured request. +fn minimal_body() -> serde_json::Value { + serde_json::json!({ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 1, "output_tokens": 1} + }) +} + +/// Runs `complete()` against a capture mock and returns the captured wire +/// request. +async fn encode_capture(adapter: AnthropicAdapter, request: &Request) -> WireCapture { + let server = MockServer::start(); + let (mock, slot) = mount_capture(&server, "/messages", minimal_body()); + let adapter = adapter.with_base_url(server.base_url()); + adapter + .complete(request) + .await + .expect("complete should succeed"); + mock.assert(); + take_capture(&slot) +} + +/// Runs `stream()` against an SSE transcript and returns the captured wire +/// request plus every emitted stream item as JSON. +async fn stream_capture( + adapter: AnthropicAdapter, + request: &Request, + sse_body: &str, +) -> (WireCapture, Vec) { + let server = MockServer::start(); + let (mock, slot) = mount_capture_sse(&server, "/messages", sse_body); + let adapter = adapter.with_base_url(server.base_url()); + let events = support::collect_stream_events(&adapter, request).await; + mock.assert(); + (take_capture(&slot), events) +} + +fn adapter() -> AnthropicAdapter { + AnthropicAdapter::new("test-key") +} + +// --------------------------------------------------------------------------- +// Round trip (encode + decode) +// --------------------------------------------------------------------------- + +/// Shared setup for the system+tools round trip: runs `complete()` against a +/// canned response and returns both the captured request and decoded response +/// so the encode and decode halves can be pinned by separate tests. +async fn system_and_tools_roundtrip() -> (WireCapture, fabro_llm::types::Response) { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + "/messages", + serde_json::json!({ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [{"type": "text", "text": "Hello back"}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 42, + "output_tokens": 7, + "cache_read_input_tokens": 10, + "cache_creation_input_tokens": 3 + } + }), + ); + + let adapter = AnthropicAdapter::new("test-key").with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + tools: Some(vec![ToolDefinition::function( + "search", + "Search files", + serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}}), + )]), + temperature: Some(0.5), + ..base_request("claude-sonnet-4-20250514") + }; + + let response = adapter + .complete(&request) + .await + .expect("complete should succeed"); + mock.assert(); + (take_capture(&slot), response) +} + +#[tokio::test] +async fn system_and_tools_encode() { + let (capture, _) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn system_and_tools_decode() { + let (_, response) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Encode +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn encode_multi_turn() { + let capture = encode_capture(adapter(), &corpus_multi_turn(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn encode_tool_choice_auto() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::Auto))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_required() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::Required))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_named() { + let capture = encode_capture( + adapter(), + &corpus_tools(MODEL, Some(ToolChoice::named("search"))), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_none() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::None))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_round_trip() { + let capture = encode_capture(adapter(), &corpus_tool_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_thinking_round_trip() { + let capture = encode_capture(adapter(), &corpus_thinking_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_inline_attachments() { + let capture = encode_capture(adapter(), &corpus_inline_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_url_attachments() { + let capture = encode_capture(adapter(), &corpus_url_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_bad_file_path_attachments_dropped() { + let capture = encode_capture(adapter(), &corpus_bad_file_path_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_audio_attachment() { + let capture = encode_capture(adapter(), &corpus_audio_attachment(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_object() { + let format = ResponseFormat { + kind: ResponseFormatType::JsonObject, + json_schema: None, + strict: false, + }; + let capture = encode_capture(adapter(), &corpus_response_format(MODEL, format)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_schema() { + let capture = encode_capture( + adapter(), + &corpus_response_format(MODEL, json_schema_format()), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_sampling_params() { + let capture = encode_capture(adapter(), &corpus_sampling_params(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_provider_options_anthropic_namespace() { + let capture = encode_capture( + adapter(), + &corpus_provider_options(MODEL, serde_json::json!({"anthropic": {"top_k": 5}})), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_reasoning_effort_with_levels_catalog() { + let catalog = support::catalog_from_toml( + r#" +[providers.anthropic] +display_name = "Anthropic" +adapter = "anthropic" +agent_profile = "anthropic" + +[models."test-claude"] +provider = "anthropic" +display_name = "Test Claude" +family = "claude" +default = true + +[models."test-claude".limits] +context_window = 200000 +max_output = 4096 + +[models."test-claude".features] +tools = true +vision = true +reasoning = true +reasoning_effort = "levels" +prompt_cache = false +"#, + ); + let request = Request { + reasoning_effort: Some(fabro_llm::types::ReasoningEffort::High), + ..base_request("test-claude") + }; + let capture = encode_capture(adapter().with_catalog(catalog), &request).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_prompt_cache_with_catalog() { + let catalog = support::catalog_from_toml( + r#" +[providers.anthropic] +display_name = "Anthropic" +adapter = "anthropic" +agent_profile = "anthropic" + +[models."test-claude"] +provider = "anthropic" +display_name = "Test Claude" +family = "claude" +default = true + +[models."test-claude".limits] +context_window = 200000 +max_output = 4096 + +[models."test-claude".features] +tools = true +vision = true +reasoning = true +prompt_cache = true +"#, + ); + let request = Request { + messages: vec![ + Message::system("You are a careful reviewer."), + Message::user("Review this."), + ], + ..corpus_tools("test-claude", None) + }; + // Full capture: the prompt-cache path also controls the beta header. + let capture = encode_capture(adapter().with_catalog(catalog), &request).await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn count_tokens_wire_shape() { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + "/messages/count_tokens", + serde_json::json!({"input_tokens": 123}), + ); + + let adapter = adapter().with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + ..corpus_tools(MODEL, None) + }; + let count = adapter + .count_input_tokens(&request) + .await + .unwrap() + .expect("anthropic should count tokens"); + + mock.assert(); + assert_eq!(count.input_tokens, 123); + fabro_test::fabro_json_snapshot!(take_capture(&slot)); +} + +// --------------------------------------------------------------------------- +// Decode +// --------------------------------------------------------------------------- + +/// Runs `complete()` against a canned body and returns the decoded response. +async fn decode_response(body: serde_json::Value) -> fabro_llm::types::Response { + let server = MockServer::start(); + let (mock, _slot) = mount_capture(&server, "/messages", body); + let adapter = adapter().with_base_url(server.base_url()); + let response = adapter + .complete(&base_request(MODEL)) + .await + .expect("complete should succeed"); + mock.assert(); + response +} + +#[tokio::test] +async fn decode_tool_use_stop_reason() { + let response = decode_response(serde_json::json!({ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [ + {"type": "text", "text": "Let me search."}, + { + "type": "tool_use", + "id": "toolu_01", + "name": "search", + "input": {"query": "foo"} + } + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": {"input_tokens": 30, "output_tokens": 12} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +#[tokio::test] +async fn decode_thinking_and_redacted_thinking() { + let response = decode_response(serde_json::json!({ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [ + {"type": "thinking", "thinking": "Step one.", "signature": "sig_decode_abc"}, + {"type": "redacted_thinking", "data": "opaque-blob"}, + {"type": "text", "text": "Done."} + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 25, "output_tokens": 40} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +#[tokio::test] +async fn decode_max_tokens_stop_reason() { + let response = decode_response(serde_json::json!({ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "Truncated answe"}], + "stop_reason": "max_tokens", + "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 128} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Stream +// --------------------------------------------------------------------------- + +/// Shared setup for the happy-path text stream; the request and event halves +/// are pinned by separate tests. +async fn stream_text_happy_path_capture() -> (WireCapture, Vec) { + let sse = support::sse_transcript(&[ + ( + "message_start", + r#"{"type":"message_start","message":{"id":"msg_stream_test","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"usage":{"input_tokens":11,"cache_read_input_tokens":2,"cache_creation_input_tokens":1,"output_tokens":0}}}"#, + ), + ("ping", r#"{"type":"ping"}"#), + ( + "content_block_start", + r#"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel"}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"lo"}}"#, + ), + ( + "content_block_stop", + r#"{"type":"content_block_stop","index":0}"#, + ), + ( + "message_delta", + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5}}"#, + ), + ("message_stop", r#"{"type":"message_stop"}"#), + ]); + stream_capture(adapter(), &base_request(MODEL), &sse).await +} + +/// The captured request pins the stream flag on the wire. +#[tokio::test] +async fn stream_text_happy_path_request() { + let (capture, _) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn stream_text_happy_path_events() { + let (_, events) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_tool_call_deltas() { + let sse = support::sse_transcript(&[ + ( + "message_start", + r#"{"type":"message_start","message":{"id":"msg_stream_tool","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"usage":{"input_tokens":20,"output_tokens":0}}}"#, + ), + ( + "content_block_start", + r#"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01","name":"search","input":{}}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"qu"}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"ery\":\"foo\"}"}}"#, + ), + ( + "content_block_stop", + r#"{"type":"content_block_stop","index":0}"#, + ), + ( + "message_delta", + r#"{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":9}}"#, + ), + ("message_stop", r#"{"type":"message_stop"}"#), + ]); + let (_capture, events) = stream_capture( + adapter(), + &corpus_tools(MODEL, Some(ToolChoice::Auto)), + &sse, + ) + .await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_thinking_with_signature_delta() { + let sse = support::sse_transcript(&[ + ( + "message_start", + r#"{"type":"message_start","message":{"id":"msg_stream_think","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"usage":{"input_tokens":15,"output_tokens":0}}}"#, + ), + ( + "content_block_start", + r#"{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me think"}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig_stream_xyz"}}"#, + ), + ( + "content_block_stop", + r#"{"type":"content_block_stop","index":0}"#, + ), + ( + "content_block_start", + r#"{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"4."}}"#, + ), + ( + "content_block_stop", + r#"{"type":"content_block_stop","index":1}"#, + ), + ( + "message_delta", + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":12}}"#, + ), + ("message_stop", r#"{"type":"message_stop"}"#), + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_error_event_mid_stream() { + let sse = support::sse_transcript(&[ + ( + "message_start", + r#"{"type":"message_start","message":{"id":"msg_stream_err","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"usage":{"input_tokens":9,"output_tokens":0}}}"#, + ), + ( + "error", + r#"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#, + ), + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +/// The Anthropic decoder never synthesizes a `Finish` on byte-stream end: +/// `message_stop` is the only finisher. A transcript that ends without it +/// must produce no `Finish` event. +#[tokio::test] +async fn stream_without_message_stop_emits_no_finish() { + let sse = support::sse_transcript(&[ + ( + "message_start", + r#"{"type":"message_start","message":{"id":"msg_stream_cut","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"usage":{"input_tokens":11,"output_tokens":0}}}"#, + ), + ( + "content_block_start", + r#"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#, + ), + ( + "content_block_delta", + r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}"#, + ), + ( + "content_block_stop", + r#"{"type":"content_block_stop","index":0}"#, + ), + ( + "message_delta", + r#"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":5}}"#, + ), + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} diff --git a/lib/crates/fabro-llm/tests/it/wire/gemini.rs b/lib/crates/fabro-llm/tests/it/wire/gemini.rs new file mode 100644 index 000000000..0471b585b --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/gemini.rs @@ -0,0 +1,487 @@ +//! Wire snapshots for the Gemini `generateContent` dialect. The model is +//! part of the URL path, auth is the `x-goog-api-key` header, and the +//! decoder mints synthetic UUID tool-call ids (normalized to `[UUID]` in +//! these snapshots). + +use fabro_llm::provider::ProviderAdapter; +use fabro_llm::providers::GeminiAdapter; +use fabro_llm::types::{ + Message, Request, ResponseFormat, ResponseFormatType, ToolChoice, ToolDefinition, +}; +use httpmock::prelude::*; + +use crate::support::{ + self, WireCapture, base_request, corpus_audio_attachment, corpus_bad_file_path_attachments, + corpus_inline_attachments, corpus_multi_turn, corpus_provider_options, corpus_response_format, + corpus_sampling_params, corpus_thinking_round_trip, corpus_tool_round_trip, corpus_tools, + corpus_url_attachments, json_schema_format, mount_capture, mount_capture_sse, take_capture, +}; + +const MODEL: &str = "gemini-test"; +const COMPLETE_PATH: &str = "/models/gemini-test:generateContent"; +const STREAM_PATH: &str = "/models/gemini-test:streamGenerateContent"; + +/// Minimal valid generateContent body for encode-side tests. +fn minimal_body() -> serde_json::Value { + serde_json::json!({ + "candidates": [{ + "content": {"role": "model", "parts": [{"text": "ok"}]}, + "finishReason": "STOP" + }], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1} + }) +} + +fn adapter() -> GeminiAdapter { + GeminiAdapter::new("test-key") +} + +/// Runs `complete()` against a capture mock and returns the captured wire +/// request. +async fn encode_capture(adapter: GeminiAdapter, request: &Request) -> WireCapture { + let server = MockServer::start(); + let (mock, slot) = mount_capture(&server, COMPLETE_PATH, minimal_body()); + let adapter = adapter.with_base_url(server.base_url()); + adapter + .complete(request) + .await + .expect("complete should succeed"); + mock.assert(); + take_capture(&slot) +} + +/// Runs `stream()` against an SSE transcript and returns the captured wire +/// request plus every emitted stream item as JSON (UUIDs normalized). +async fn stream_capture( + adapter: GeminiAdapter, + request: &Request, + sse_body: &str, +) -> (WireCapture, Vec) { + let server = MockServer::start(); + let (mock, slot) = mount_capture_sse(&server, STREAM_PATH, sse_body); + let adapter = adapter.with_base_url(server.base_url()); + let mut events = support::collect_stream_events(&adapter, request).await; + mock.assert(); + events.iter_mut().for_each(support::normalize_uuids); + (take_capture(&slot), events) +} + +// --------------------------------------------------------------------------- +// Round trip (encode + decode) +// --------------------------------------------------------------------------- + +/// Shared setup for the system+tools round trip. The decoded response is +/// returned as a UUID-normalized JSON value (gemini mints a synthetic UUID +/// for the response id); the encode and decode halves are pinned separately. +async fn system_and_tools_roundtrip() -> (WireCapture, serde_json::Value) { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + COMPLETE_PATH, + serde_json::json!({ + "candidates": [{ + "content": {"role": "model", "parts": [{"text": "Hello back"}]}, + "finishReason": "STOP" + }], + "usageMetadata": { + "promptTokenCount": 42, + "candidatesTokenCount": 7, + "cachedContentTokenCount": 10 + } + }), + ); + + let adapter = adapter().with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + tools: Some(vec![ToolDefinition::function( + "search", + "Search files", + serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}}), + )]), + temperature: Some(0.5), + ..base_request(MODEL) + }; + + let response = adapter + .complete(&request) + .await + .expect("complete should succeed"); + mock.assert(); + let mut response_value = serde_json::to_value(&response).expect("response should serialize"); + support::normalize_uuids(&mut response_value); + (take_capture(&slot), response_value) +} + +#[tokio::test] +async fn system_and_tools_encode() { + let (capture, _) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn system_and_tools_decode() { + let (_, response) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Encode +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn encode_multi_turn() { + let capture = encode_capture(adapter(), &corpus_multi_turn(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_auto() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::Auto))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_required() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::Required))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_named() { + let capture = encode_capture( + adapter(), + &corpus_tools(MODEL, Some(ToolChoice::named("search"))), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_none() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::None))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_round_trip() { + let capture = encode_capture(adapter(), &corpus_tool_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_thinking_round_trip() { + let capture = encode_capture(adapter(), &corpus_thinking_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_inline_attachments() { + let capture = encode_capture(adapter(), &corpus_inline_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_url_attachments() { + let capture = encode_capture(adapter(), &corpus_url_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_bad_file_path_attachments_dropped() { + let capture = encode_capture(adapter(), &corpus_bad_file_path_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// Gemini sends inline audio (the only dialect that does). +#[tokio::test] +async fn encode_audio_attachment() { + let capture = encode_capture(adapter(), &corpus_audio_attachment(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_object() { + let format = ResponseFormat { + kind: ResponseFormatType::JsonObject, + json_schema: None, + strict: false, + }; + let capture = encode_capture(adapter(), &corpus_response_format(MODEL, format)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_schema() { + let capture = encode_capture( + adapter(), + &corpus_response_format(MODEL, json_schema_format()), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_sampling_params() { + let capture = encode_capture(adapter(), &corpus_sampling_params(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// The "gemini"-namespaced provider_options merge — and the default +/// safety_settings injection it can override. +#[tokio::test] +async fn encode_provider_options_gemini_namespace() { + let capture = encode_capture( + adapter(), + &corpus_provider_options( + MODEL, + serde_json::json!({"gemini": {"cached_content": "cachedContents/abc"}}), + ), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_provider_options_can_override_safety_settings() { + let capture = encode_capture( + adapter(), + &corpus_provider_options( + MODEL, + serde_json::json!({"gemini": {"safety_settings": []}}), + ), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_reasoning_effort_with_levels_catalog() { + let catalog = support::catalog_from_toml( + r#" +[providers.gemini] +display_name = "Gemini" +adapter = "gemini" +agent_profile = "gemini" + +[models."gemini-test"] +provider = "gemini" +display_name = "Test Gemini" +family = "gemini" +default = true + +[models."gemini-test".limits] +context_window = 200000 +max_output = 4096 + +[models."gemini-test".features] +tools = true +vision = true +reasoning = true +reasoning_effort = "levels" +"#, + ); + let request = Request { + reasoning_effort: Some(fabro_llm::types::ReasoningEffort::High), + ..base_request(MODEL) + }; + let capture = encode_capture(adapter().with_catalog(catalog), &request).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn count_tokens_wire_shape() { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + "/models/gemini-test:countTokens", + serde_json::json!({"totalTokens": 123}), + ); + let adapter = adapter().with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + ..corpus_tools(MODEL, None) + }; + let count = adapter + .count_input_tokens(&request) + .await + .unwrap() + .expect("gemini should count tokens"); + + mock.assert(); + assert_eq!(count.input_tokens, 123); + fabro_test::fabro_json_snapshot!(take_capture(&slot)); +} + +// --------------------------------------------------------------------------- +// Decode +// --------------------------------------------------------------------------- + +/// Runs `complete()` against a canned body and returns the decoded response +/// as JSON with synthetic UUIDs normalized. +async fn decode_response(body: serde_json::Value) -> serde_json::Value { + let server = MockServer::start(); + let (mock, _slot) = mount_capture(&server, COMPLETE_PATH, body); + let adapter = adapter().with_base_url(server.base_url()); + let response = adapter + .complete(&base_request(MODEL)) + .await + .expect("complete should succeed"); + mock.assert(); + let mut value = serde_json::to_value(&response).expect("response should serialize"); + support::normalize_uuids(&mut value); + value +} + +/// functionCall parts get synthetic UUID ids, preserve `thoughtSignature`, +/// and force the finish reason to ToolCalls regardless of `finishReason`. +#[tokio::test] +async fn decode_function_call_with_thought_signature() { + let response = decode_response(serde_json::json!({ + "candidates": [{ + "content": { + "role": "model", + "parts": [ + {"text": "Let me search."}, + { + "functionCall": {"name": "search", "args": {"query": "foo"}}, + "thoughtSignature": "sig_gemini_xyz" + } + ] + }, + "finishReason": "STOP" + }], + "usageMetadata": {"promptTokenCount": 30, "candidatesTokenCount": 12} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +/// The Gemini usage arithmetic: input = (prompt - cached) + tool_use_prompt; +/// thoughts become reasoning tokens. +#[tokio::test] +async fn decode_usage_arithmetic() { + let response = decode_response(serde_json::json!({ + "candidates": [{ + "content": {"role": "model", "parts": [{"text": "ok"}]}, + "finishReason": "STOP" + }], + "usageMetadata": { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "thoughtsTokenCount": 8, + "cachedContentTokenCount": 30, + "toolUsePromptTokenCount": 5 + } + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +/// `thought: true` text parts decode as Thinking content. +#[tokio::test] +async fn decode_thought_parts() { + let response = decode_response(serde_json::json!({ + "candidates": [{ + "content": { + "role": "model", + "parts": [ + {"text": "Adding the numbers.", "thought": true}, + {"text": "4."} + ] + }, + "finishReason": "STOP" + }], + "usageMetadata": {"promptTokenCount": 25, "candidatesTokenCount": 40} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +#[tokio::test] +async fn decode_max_tokens_finish_reason() { + let length = decode_response(serde_json::json!({ + "candidates": [{ + "content": {"role": "model", "parts": [{"text": "Trunc"}]}, + "finishReason": "MAX_TOKENS" + }], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 128} + })) + .await; + fabro_test::fabro_json_snapshot!(length["finish_reason"]); +} + +#[tokio::test] +async fn decode_safety_finish_reason() { + let safety = decode_response(serde_json::json!({ + "candidates": [{ + "content": {"role": "model", "parts": [{"text": ""}]}, + "finishReason": "SAFETY" + }], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 0} + })) + .await; + fabro_test::fabro_json_snapshot!(safety["finish_reason"]); +} + +// --------------------------------------------------------------------------- +// Stream +// --------------------------------------------------------------------------- + +/// Shared setup for the happy-path text stream; the request and event halves +/// are pinned by separate tests. +async fn stream_text_happy_path_capture() -> (WireCapture, Vec) { + let sse = support::sse_data_transcript(&[ + r#"{"candidates":[{"content":{"role":"model","parts":[{"text":"Hel"}]}}]}"#, + r#"{"candidates":[{"content":{"role":"model","parts":[{"text":"lo"}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":11,"candidatesTokenCount":5}}"#, + ]); + stream_capture(adapter(), &base_request(MODEL), &sse).await +} + +/// The captured request pins model-in-URL and `?alt=sse` on the wire. +#[tokio::test] +async fn stream_text_happy_path_request() { + let (capture, _) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn stream_text_happy_path_events() { + let (_, events) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_function_call() { + let sse = support::sse_data_transcript(&[ + r#"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"search","args":{"query":"foo"}},"thoughtSignature":"sig_stream_g"}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":20,"candidatesTokenCount":9}}"#, + ]); + let (_capture, events) = stream_capture( + adapter(), + &corpus_tools(MODEL, Some(ToolChoice::Auto)), + &sse, + ) + .await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_thought_parts() { + let sse = support::sse_data_transcript(&[ + r#"{"candidates":[{"content":{"role":"model","parts":[{"text":"Let me think","thought":true}]}}]}"#, + r#"{"candidates":[{"content":{"role":"model","parts":[{"text":"4."}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":15,"candidatesTokenCount":12,"thoughtsTokenCount":6}}"#, + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +/// The Gemini decoder synthesizes a `Finish` on byte-stream end +/// unconditionally — even when no chunk carried a `finishReason`. +#[tokio::test] +async fn stream_end_synthesizes_finish_without_finish_reason() { + let sse = support::sse_data_transcript(&[ + r#"{"candidates":[{"content":{"role":"model","parts":[{"text":"Hello"}]}}]}"#, + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} diff --git a/lib/crates/fabro-llm/tests/it/wire/mod.rs b/lib/crates/fabro-llm/tests/it/wire/mod.rs new file mode 100644 index 000000000..16ad18987 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/mod.rs @@ -0,0 +1,22 @@ +//! Wire snapshot tests pinning per-dialect encode/decode behavior. +//! +//! Each test points a real adapter at a local httpmock server, side-channels +//! the full received request (method, path, headers, body) out of an +//! `is_true` matcher closure, responds with a canned provider body, and +//! snapshots both the captured wire request (encode) and the decoded +//! canonical `Response` (decode). The codec extraction PRs must keep these +//! snapshot values identical. +//! +//! The anthropic/gemini dialects have no twin coverage, so these snapshots +//! are the only behavior net for those extractions. +//! +//! Snapshots are stored externally under `snapshots/` (via +//! `fabro_test::fabro_json_snapshot!(value)` with no inline literal) to keep +//! these source files small. Review and accept with `cargo insta` +//! (`pending-snapshots` then `accept`), per CLAUDE.md. A few tests assert two +//! snapshots in one function; insta names the second `-2.snap`. + +mod anthropic; +mod gemini; +mod openai_compatible; +mod openai_responses; diff --git a/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs b/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs new file mode 100644 index 000000000..661141eb4 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs @@ -0,0 +1,439 @@ +//! Wire snapshots for the OpenAI Chat Completions dialect served by +//! `OpenAiCompatibleAdapter` (kimi, zai, minimax, venice, inception, ollama, +//! litellm — all config-only routes over this adapter). + +use fabro_llm::provider::ProviderAdapter; +use fabro_llm::providers::OpenAiCompatibleAdapter; +use fabro_llm::types::{ + Message, Request, ResponseFormat, ResponseFormatType, ToolChoice, ToolDefinition, +}; +use httpmock::prelude::*; + +use crate::support::{ + self, WireCapture, base_request, corpus_audio_attachment, corpus_bad_file_path_attachments, + corpus_inline_attachments, corpus_multi_turn, corpus_provider_options, corpus_response_format, + corpus_sampling_params, corpus_thinking_round_trip, corpus_tool_round_trip, corpus_tools, + corpus_url_attachments, json_schema_format, mount_capture, mount_capture_sse, take_capture, +}; + +const MODEL: &str = "test-model"; + +/// Fixed `created` timestamp for canned bodies (named to satisfy clippy's +/// unreadable-literal lint without touching the JSON wire value). +const CREATED_TS: i64 = 1_700_000_000; + +/// Minimal valid Chat Completions body for encode-side tests. +fn minimal_body() -> serde_json::Value { + serde_json::json!({ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + }) +} + +fn adapter(server: &MockServer) -> OpenAiCompatibleAdapter { + OpenAiCompatibleAdapter::new("test-key", server.base_url()) +} + +/// Runs `complete()` against a capture mock and returns the captured wire +/// request. +async fn encode_capture_with( + request: &Request, + configure: impl FnOnce(OpenAiCompatibleAdapter) -> OpenAiCompatibleAdapter, +) -> WireCapture { + let server = MockServer::start(); + let (mock, slot) = mount_capture(&server, "/chat/completions", minimal_body()); + let adapter = configure(adapter(&server)); + adapter + .complete(request) + .await + .expect("complete should succeed"); + mock.assert(); + take_capture(&slot) +} + +async fn encode_capture(request: &Request) -> WireCapture { + encode_capture_with(request, |adapter| adapter).await +} + +/// Runs `stream()` against an SSE transcript and returns the captured wire +/// request plus every emitted stream item as JSON. +async fn stream_capture( + request: &Request, + sse_body: &str, +) -> (WireCapture, Vec) { + let server = MockServer::start(); + let (mock, slot) = mount_capture_sse(&server, "/chat/completions", sse_body); + let adapter = adapter(&server); + let events = support::collect_stream_events(&adapter, request).await; + mock.assert(); + (take_capture(&slot), events) +} + +// --------------------------------------------------------------------------- +// Round trip (encode + decode) +// --------------------------------------------------------------------------- + +/// Shared setup for the system+tools round trip; the encode and decode halves +/// are pinned by separate tests. +async fn system_and_tools_roundtrip() -> (WireCapture, fabro_llm::types::Response) { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + "/chat/completions", + serde_json::json!({ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Hello back"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 42, "completion_tokens": 7, "total_tokens": 49} + }), + ); + + let adapter = adapter(&server); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + tools: Some(vec![ToolDefinition::function( + "search", + "Search files", + serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}}), + )]), + temperature: Some(0.5), + ..base_request(MODEL) + }; + + let response = adapter + .complete(&request) + .await + .expect("complete should succeed"); + mock.assert(); + (take_capture(&slot), response) +} + +#[tokio::test] +async fn system_and_tools_encode() { + let (capture, _) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn system_and_tools_decode() { + let (_, response) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Encode +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn encode_multi_turn() { + let capture = encode_capture(&corpus_multi_turn(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_auto() { + let capture = encode_capture(&corpus_tools(MODEL, Some(ToolChoice::Auto))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_required() { + let capture = encode_capture(&corpus_tools(MODEL, Some(ToolChoice::Required))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_named() { + let capture = encode_capture(&corpus_tools(MODEL, Some(ToolChoice::named("search")))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_none() { + let capture = encode_capture(&corpus_tools(MODEL, Some(ToolChoice::None))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_round_trip() { + let capture = encode_capture(&corpus_tool_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// Assistant thinking parts echo back as `reasoning_content` (Kimi-motivated, +/// applies to every compat assistant message). +#[tokio::test] +async fn encode_thinking_round_trip_as_reasoning_content() { + let capture = encode_capture(&corpus_thinking_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// The compat encoder performs no attachment I/O: images are dropped +/// outright, documents become fallback text. +#[tokio::test] +async fn encode_inline_attachments() { + let capture = encode_capture(&corpus_inline_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_url_attachments() { + let capture = encode_capture(&corpus_url_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_bad_file_path_attachments() { + let capture = encode_capture(&corpus_bad_file_path_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_audio_attachment() { + let capture = encode_capture(&corpus_audio_attachment(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_object() { + let format = ResponseFormat { + kind: ResponseFormatType::JsonObject, + json_schema: None, + strict: false, + }; + let capture = encode_capture(&corpus_response_format(MODEL, format)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_schema() { + let capture = encode_capture(&corpus_response_format(MODEL, json_schema_format())).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_sampling_params() { + let capture = encode_capture(&corpus_sampling_params(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// The provider_options namespace key is the runtime adapter NAME, not a +/// static "openai_compatible" key (pinned in-module by +/// `provider_options_uses_adapter_name`; this pins it from outside). +#[tokio::test] +async fn encode_provider_options_keyed_by_adapter_name() { + let request = corpus_provider_options( + MODEL, + serde_json::json!({"kimi": {"repetition_penalty": 1.2}}), + ); + let capture = encode_capture_with(&request, |adapter| adapter.with_name("kimi")).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// Options under a key that does not match the adapter name must not merge. +#[tokio::test] +async fn encode_provider_options_other_namespace_ignored() { + let request = corpus_provider_options( + MODEL, + serde_json::json!({"openai": {"repetition_penalty": 1.2}}), + ); + let capture = encode_capture_with(&request, |adapter| adapter.with_name("kimi")).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// The compat adapter has no count-tokens wire route. +#[tokio::test] +async fn count_input_tokens_unavailable() { + let server = MockServer::start(); + let adapter = adapter(&server); + let count = adapter + .count_input_tokens(&base_request(MODEL)) + .await + .unwrap(); + assert!(count.is_none()); +} + +// --------------------------------------------------------------------------- +// Decode +// --------------------------------------------------------------------------- + +async fn decode_response(body: serde_json::Value) -> fabro_llm::types::Response { + let server = MockServer::start(); + let (mock, _slot) = mount_capture(&server, "/chat/completions", body); + let adapter = adapter(&server); + let response = adapter + .complete(&base_request(MODEL)) + .await + .expect("complete should succeed"); + mock.assert(); + response +} + +#[tokio::test] +async fn decode_tool_calls_with_string_arguments() { + let response = decode_response(serde_json::json!({ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call_abc", + "type": "function", + "function": {"name": "search", "arguments": "{\"query\":\"foo\"}"} + }] + }, + "finish_reason": "tool_calls" + }], + "usage": {"prompt_tokens": 30, "completion_tokens": 12, "total_tokens": 42} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +#[tokio::test] +async fn decode_reasoning_content_as_thinking() { + let response = decode_response(serde_json::json!({ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "4.", + "reasoning_content": "The user wants 2+2." + }, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 25, "completion_tokens": 40, "total_tokens": 65} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +/// Compat usage reads only prompt/completion tokens; cached-token details are +/// ignored today (parsing them is a 438-redo behavior change). +#[tokio::test] +async fn decode_usage_ignores_token_details() { + let response = decode_response(serde_json::json!({ + "id": "chatcmpl_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "length" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "prompt_tokens_details": {"cached_tokens": 80}, + "completion_tokens_details": {"reasoning_tokens": 20} + } + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Stream +// --------------------------------------------------------------------------- + +/// Shared setup for the happy-path text stream; the request and event halves +/// are pinned by separate tests. +async fn stream_text_happy_path_capture() -> (WireCapture, Vec) { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hel"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[],"usage":{"prompt_tokens":11,"completion_tokens":5,"total_tokens":16}}"#, + "[DONE]", + ]); + stream_capture(&base_request(MODEL), &sse).await +} + +/// The captured request pins the stream flag on the wire. +#[tokio::test] +async fn stream_text_happy_path_request() { + let (capture, _) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn stream_text_happy_path_events() { + let (_, events) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_tool_call_deltas() { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_abc","type":"function","function":{"name":"search","arguments":""}}]},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"qu"}}]},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ery\":\"foo\"}"}}]},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[],"usage":{"prompt_tokens":20,"completion_tokens":9,"total_tokens":29}}"#, + "[DONE]", + ]); + let (_capture, events) = + stream_capture(&corpus_tools(MODEL, Some(ToolChoice::Auto)), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_reasoning_content_deltas() { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","reasoning_content":"Let me "},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"reasoning_content":"think"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"content":"4."},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + "[DONE]", + ]); + let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +/// Minimax tolerance: a stream that ends without `[DONE]` still synthesizes +/// the finish — but only because content was started. +#[tokio::test] +async fn stream_without_done_synthesizes_finish_when_content_started() { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + ]); + let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +/// The other half of the minimax contract: no content started and no +/// `[DONE]` — nothing is synthesized. +#[tokio::test] +async fn stream_without_done_or_content_synthesizes_nothing() { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}"#, + ]); + let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} diff --git a/lib/crates/fabro-llm/tests/it/wire/openai_responses.rs b/lib/crates/fabro-llm/tests/it/wire/openai_responses.rs new file mode 100644 index 000000000..d3434f018 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/openai_responses.rs @@ -0,0 +1,563 @@ +//! Wire snapshots for the OpenAI Responses API dialect (`POST /responses`). + +use fabro_llm::provider::ProviderAdapter; +use fabro_llm::providers::OpenAiAdapter; +use fabro_llm::types::{ + ContentPart, Message, Request, ResponseFormat, ResponseFormatType, Role, ToolCall, ToolChoice, + ToolDefinition, +}; +use httpmock::prelude::*; + +use crate::support::{ + self, WireCapture, base_request, corpus_audio_attachment, corpus_bad_file_path_attachments, + corpus_inline_attachments, corpus_multi_turn, corpus_provider_options, corpus_response_format, + corpus_sampling_params, corpus_thinking_round_trip, corpus_tool_round_trip, corpus_tools, + corpus_url_attachments, json_schema_format, mount_capture, mount_capture_sse, take_capture, +}; + +const MODEL: &str = "gpt-test"; + +/// Minimal valid Responses API body for encode-side tests. +fn minimal_body() -> serde_json::Value { + serde_json::json!({ + "id": "resp_test", + "object": "response", + "model": MODEL, + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1} + }) +} + +fn adapter() -> OpenAiAdapter { + OpenAiAdapter::new("test-key") +} + +/// Runs `complete()` against a capture mock and returns the captured wire +/// request. +async fn encode_capture(adapter: OpenAiAdapter, request: &Request) -> WireCapture { + let server = MockServer::start(); + let (mock, slot) = mount_capture(&server, "/responses", minimal_body()); + let adapter = adapter.with_base_url(server.base_url()); + adapter + .complete(request) + .await + .expect("complete should succeed"); + mock.assert(); + take_capture(&slot) +} + +/// Runs `stream()` against an SSE transcript and returns the captured wire +/// request plus every emitted stream item as JSON. +async fn stream_capture( + adapter: OpenAiAdapter, + request: &Request, + sse_body: &str, +) -> (WireCapture, Vec) { + let server = MockServer::start(); + let (mock, slot) = mount_capture_sse(&server, "/responses", sse_body); + let adapter = adapter.with_base_url(server.base_url()); + let events = support::collect_stream_events(&adapter, request).await; + mock.assert(); + (take_capture(&slot), events) +} + +// --------------------------------------------------------------------------- +// Round trip (encode + decode) +// --------------------------------------------------------------------------- + +/// Shared setup for the system+tools round trip; the encode and decode halves +/// are pinned by separate tests. +async fn system_and_tools_roundtrip() -> (WireCapture, fabro_llm::types::Response) { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + "/responses", + serde_json::json!({ + "id": "resp_test", + "object": "response", + "model": MODEL, + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [{"type": "output_text", "text": "Hello back"}] + }], + "usage": { + "input_tokens": 42, + "output_tokens": 7, + "input_tokens_details": {"cached_tokens": 10}, + "output_tokens_details": {"reasoning_tokens": 3} + } + }), + ); + + let adapter = adapter().with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + tools: Some(vec![ToolDefinition::function( + "search", + "Search files", + serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}}), + )]), + temperature: Some(0.5), + ..base_request(MODEL) + }; + + let response = adapter + .complete(&request) + .await + .expect("complete should succeed"); + mock.assert(); + (take_capture(&slot), response) +} + +#[tokio::test] +async fn system_and_tools_encode() { + let (capture, _) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(capture); +} + +#[tokio::test] +async fn system_and_tools_decode() { + let (_, response) = system_and_tools_roundtrip().await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Encode +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn encode_multi_turn() { + let capture = encode_capture(adapter(), &corpus_multi_turn(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_auto() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::Auto))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_required() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::Required))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_named() { + let capture = encode_capture( + adapter(), + &corpus_tools(MODEL, Some(ToolChoice::named("search"))), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_choice_none() { + let capture = encode_capture(adapter(), &corpus_tools(MODEL, Some(ToolChoice::None))).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_tool_round_trip() { + let capture = encode_capture(adapter(), &corpus_tool_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// A tool call that decoded with an item-level id (`fc_…`) in +/// provider_metadata re-encodes with the dual ids split correctly. +#[tokio::test] +async fn encode_dual_id_tool_round_trip() { + let mut tool_call = ToolCall::new("call_abc", "search", serde_json::json!({"query": "foo"})); + tool_call.provider_metadata = Some(serde_json::json!({"id": "fc_123"})); + let mut request = corpus_tools(MODEL, None); + request.messages = vec![ + Message::user("Find foo"), + Message { + role: Role::Assistant, + content: vec![ContentPart::ToolCall(tool_call)], + name: None, + tool_call_id: None, + }, + Message::tool_result( + "call_abc", + serde_json::Value::String("2 matches".to_string()), + false, + ), + ]; + let capture = encode_capture(adapter(), &request).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// Opaque OpenAI items (reasoning / message) round-trip verbatim into the +/// input array. +#[tokio::test] +async fn encode_opaque_items_round_trip() { + let request = Request { + messages: vec![ + Message::user("Think about 2+2."), + Message { + role: Role::Assistant, + content: vec![ + ContentPart::Other { + kind: ContentPart::OPENAI_REASONING.to_string(), + data: serde_json::json!({ + "type": "reasoning", + "id": "rs_1", + "summary": [{"type": "summary_text", "text": "Adding."}] + }), + }, + ContentPart::Other { + kind: ContentPart::OPENAI_MESSAGE.to_string(), + data: serde_json::json!({ + "type": "message", + "role": "assistant", + "id": "msg_1", + "content": [{"type": "output_text", "text": "4."}] + }), + }, + ], + name: None, + tool_call_id: None, + }, + Message::user("Now 3+3?"), + ], + ..base_request(MODEL) + }; + let capture = encode_capture(adapter(), &request).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// Canonical Thinking parts (anthropic-style) — distinct from the opaque +/// reasoning round-trip above. +#[tokio::test] +async fn encode_thinking_round_trip() { + let capture = encode_capture(adapter(), &corpus_thinking_round_trip(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_inline_attachments() { + let capture = encode_capture(adapter(), &corpus_inline_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_url_attachments() { + let capture = encode_capture(adapter(), &corpus_url_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_bad_file_path_attachments_dropped() { + let capture = encode_capture(adapter(), &corpus_bad_file_path_attachments(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_audio_attachment() { + let capture = encode_capture(adapter(), &corpus_audio_attachment(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_object() { + let format = ResponseFormat { + kind: ResponseFormatType::JsonObject, + json_schema: None, + strict: false, + }; + let capture = encode_capture(adapter(), &corpus_response_format(MODEL, format)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_response_format_json_schema() { + let capture = encode_capture( + adapter(), + &corpus_response_format(MODEL, json_schema_format()), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_sampling_params() { + let capture = encode_capture(adapter(), &corpus_sampling_params(MODEL)).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_provider_options_openai_namespace() { + let capture = encode_capture( + adapter(), + &corpus_provider_options(MODEL, serde_json::json!({"openai": {"seed": 42}})), + ) + .await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn encode_reasoning_effort_with_levels_catalog() { + let catalog = support::catalog_from_toml( + r#" +[providers.openai] +display_name = "OpenAI" +adapter = "openai" +agent_profile = "openai" + +[models."test-gpt"] +provider = "openai" +display_name = "Test GPT" +family = "gpt" +default = true + +[models."test-gpt".limits] +context_window = 200000 +max_output = 4096 + +[models."test-gpt".features] +tools = true +vision = true +reasoning = true +reasoning_effort = "levels" +"#, + ); + let request = Request { + reasoning_effort: Some(fabro_llm::types::ReasoningEffort::High), + ..base_request("test-gpt") + }; + let capture = encode_capture(adapter().with_catalog(catalog), &request).await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +/// Codex mode forces streaming for `complete()` and omits sampling params +/// from the encoded body. +#[tokio::test] +async fn encode_codex_mode_forces_streaming_and_omits_params() { + let sse = support::sse_data_transcript(&[ + r#"{"type":"response.created","response":{"id":"resp_codex","model":"gpt-test"}}"#, + r#"{"type":"response.output_text.delta","delta":"ok"}"#, + r#"{"type":"response.completed","response":{"id":"resp_codex","model":"gpt-test","status":"completed","output":[],"usage":{"input_tokens":5,"output_tokens":2}}}"#, + ]); + let server = MockServer::start(); + let (mock, slot) = mount_capture_sse(&server, "/responses", &sse); + let adapter = OpenAiAdapter::new("test-key") + .with_codex_mode() + .with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + temperature: Some(0.5), + top_p: Some(0.9), + ..base_request(MODEL) + }; + adapter.complete(&request).await.unwrap(); + mock.assert(); + fabro_test::fabro_json_snapshot!(take_capture(&slot)); +} + +#[tokio::test] +async fn count_tokens_wire_shape() { + let server = MockServer::start(); + let (mock, slot) = mount_capture( + &server, + "/responses/input_tokens", + serde_json::json!({"input_tokens": 123, "object": "response.input_tokens"}), + ); + let adapter = adapter().with_base_url(server.base_url()); + let request = Request { + messages: vec![Message::system("Be concise"), Message::user("Hello")], + ..corpus_tools(MODEL, None) + }; + let count = adapter + .count_input_tokens(&request) + .await + .unwrap() + .expect("openai should count tokens"); + + mock.assert(); + assert_eq!(count.input_tokens, 123); + fabro_test::fabro_json_snapshot!(take_capture(&slot)); +} + +// --------------------------------------------------------------------------- +// Decode +// --------------------------------------------------------------------------- + +async fn decode_response(body: serde_json::Value) -> fabro_llm::types::Response { + let server = MockServer::start(); + let (mock, _slot) = mount_capture(&server, "/responses", body); + let adapter = adapter().with_base_url(server.base_url()); + let response = adapter + .complete(&base_request(MODEL)) + .await + .expect("complete should succeed"); + mock.assert(); + response +} + +/// The Responses usage arithmetic: cached tokens are subtracted from input, +/// reasoning tokens from output. +#[tokio::test] +async fn decode_usage_subtracts_cached_and_reasoning() { + let response = decode_response(serde_json::json!({ + "id": "resp_test", + "object": "response", + "model": MODEL, + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "input_tokens_details": {"cached_tokens": 80}, + "output_tokens_details": {"reasoning_tokens": 20} + } + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +/// Reasoning and function_call output items: reasoning becomes an opaque +/// round-trip part, function_call splits dual ids into id + metadata. +#[tokio::test] +async fn decode_reasoning_and_function_call_items() { + let response = decode_response(serde_json::json!({ + "id": "resp_test", + "object": "response", + "model": MODEL, + "status": "completed", + "output": [ + { + "type": "reasoning", + "id": "rs_1", + "summary": [{"type": "summary_text", "text": "Searching."}] + }, + { + "type": "function_call", + "id": "fc_123", + "call_id": "call_abc", + "name": "search", + "arguments": "{\"query\":\"foo\"}" + } + ], + "usage": {"input_tokens": 30, "output_tokens": 12} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +#[tokio::test] +async fn decode_incomplete_status_maps_to_length() { + let response = decode_response(serde_json::json!({ + "id": "resp_test", + "object": "response", + "model": MODEL, + "status": "incomplete", + "output": [{ + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [{"type": "output_text", "text": "Truncated"}] + }], + "usage": {"input_tokens": 10, "output_tokens": 128} + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + +// --------------------------------------------------------------------------- +// Stream +// --------------------------------------------------------------------------- + +/// Shared setup for the happy-path text stream; the request and event halves +/// are pinned by separate tests. +async fn stream_text_happy_path_capture() -> (WireCapture, Vec) { + let sse = support::sse_data_transcript(&[ + r#"{"type":"response.created","response":{"id":"resp_stream","model":"gpt-test"}}"#, + r#"{"type":"response.output_text.delta","delta":"Hel"}"#, + r#"{"type":"response.output_text.delta","delta":"lo"}"#, + r#"{"type":"response.completed","response":{"id":"resp_stream","model":"gpt-test","status":"completed","output":[],"usage":{"input_tokens":11,"output_tokens":5,"input_tokens_details":{"cached_tokens":2},"output_tokens_details":{"reasoning_tokens":1}}}}"#, + ]); + stream_capture(adapter(), &base_request(MODEL), &sse).await +} + +/// The captured request pins the stream flag (and `include`) on the wire. +#[tokio::test] +async fn stream_text_happy_path_request() { + let (capture, _) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(capture.body); +} + +#[tokio::test] +async fn stream_text_happy_path_events() { + let (_, events) = stream_text_happy_path_capture().await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_tool_call_deltas() { + let sse = support::sse_data_transcript(&[ + r#"{"type":"response.created","response":{"id":"resp_stream","model":"gpt-test"}}"#, + r#"{"type":"response.function_call_arguments.delta","item_id":"fc_123","call_id":"call_abc","name":"search","delta":"{\"qu"}"#, + r#"{"type":"response.function_call_arguments.delta","item_id":"fc_123","call_id":"call_abc","delta":"ery\":\"foo\"}"}"#, + r#"{"type":"response.output_item.done","item":{"type":"function_call","id":"fc_123","call_id":"call_abc","name":"search","arguments":"{\"query\":\"foo\"}"}}"#, + r#"{"type":"response.completed","response":{"id":"resp_stream","model":"gpt-test","status":"completed","output":[],"usage":{"input_tokens":20,"output_tokens":9}}}"#, + ]); + let (_capture, events) = stream_capture( + adapter(), + &corpus_tools(MODEL, Some(ToolChoice::Auto)), + &sse, + ) + .await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_reasoning_summary_deltas() { + let sse = support::sse_data_transcript(&[ + r#"{"type":"response.created","response":{"id":"resp_stream","model":"gpt-test"}}"#, + r#"{"type":"response.reasoning_summary_text.delta","delta":"Let me "}"#, + r#"{"type":"response.reasoning_summary_text.delta","delta":"think"}"#, + r#"{"type":"response.output_text.delta","delta":"4."}"#, + r#"{"type":"response.completed","response":{"id":"resp_stream","model":"gpt-test","status":"completed","output":[],"usage":{"input_tokens":15,"output_tokens":12,"output_tokens_details":{"reasoning_tokens":8}}}}"#, + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +#[tokio::test] +async fn stream_failed_event_maps_to_error() { + let sse = support::sse_data_transcript(&[ + r#"{"type":"response.created","response":{"id":"resp_stream","model":"gpt-test"}}"#, + r#"{"type":"response.failed","response":{"id":"resp_stream","error":{"code":"server_error","message":"boom"}}}"#, + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + +/// `response.incomplete` finishes the stream with `Length`. +#[tokio::test] +async fn stream_incomplete_maps_to_length() { + let sse = support::sse_data_transcript(&[ + r#"{"type":"response.created","response":{"id":"resp_stream","model":"gpt-test"}}"#, + r#"{"type":"response.output_text.delta","delta":"Trunc"}"#, + r#"{"type":"response.incomplete","response":{"id":"resp_stream","model":"gpt-test","status":"incomplete","output":[],"usage":{"input_tokens":10,"output_tokens":128}}}"#, + ]); + let (_capture, events) = stream_capture(adapter(), &base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__count_tokens_wire_shape.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__count_tokens_wire_shape.snap new file mode 100644 index 000000000..566a6a703 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__count_tokens_wire_shape.snap @@ -0,0 +1,78 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/messages/count_tokens", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "anthropic-version", + "2023-06-01" + ], + [ + "content-length", + "412" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-api-key", + "test-key" + ] + ], + "body": { + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "system": "Be concise", + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_max_tokens_stop_reason.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_max_tokens_stop_reason.snap new file mode 100644 index 000000000..7c2a3b747 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_max_tokens_stop_reason.snap @@ -0,0 +1,48 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "id": "msg_test", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Truncated answe" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "length", + "usage": { + "input_tokens": 10, + "output_tokens": 128, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + { + "type": "text", + "text": "Truncated answe" + } + ], + "stop_reason": "max_tokens", + "stop_sequence": null, + "usage": { + "input_tokens": 10, + "output_tokens": 128 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_thinking_and_redacted_thinking.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_thinking_and_redacted_thinking.snap new file mode 100644 index 000000000..9e3ef7bfd --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_thinking_and_redacted_thinking.snap @@ -0,0 +1,73 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "id": "msg_test", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "thinking", + "data": { + "text": "Step one.", + "signature": "sig_decode_abc", + "redacted": false + } + }, + { + "kind": "redacted_thinking", + "data": { + "text": "opaque-blob", + "signature": null, + "redacted": true + } + }, + { + "kind": "text", + "data": "Done." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 25, + "output_tokens": 40, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + { + "type": "thinking", + "thinking": "Step one.", + "signature": "sig_decode_abc" + }, + { + "type": "redacted_thinking", + "data": "opaque-blob" + }, + { + "type": "text", + "text": "Done." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 25, + "output_tokens": 40 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_tool_use_stop_reason.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_tool_use_stop_reason.snap new file mode 100644 index 000000000..dac531d55 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__decode_tool_use_stop_reason.snap @@ -0,0 +1,68 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "id": "msg_test", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Let me search." + }, + { + "kind": "tool_call", + "data": { + "id": "toolu_01", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": null + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 30, + "output_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + { + "type": "text", + "text": "Let me search." + }, + { + "type": "tool_use", + "id": "toolu_01", + "name": "search", + "input": { + "query": "foo" + } + } + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": { + "input_tokens": 30, + "output_tokens": 12 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_audio_attachment.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_audio_attachment.snap new file mode 100644 index 000000000..69a05af0e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_audio_attachment.snap @@ -0,0 +1,24 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Transcribe this." + }, + { + "type": "text", + "text": "[Audio content not supported by this provider]" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_bad_file_path_attachments_dropped.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_bad_file_path_attachments_dropped.snap new file mode 100644 index 000000000..498b4748b --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_bad_file_path_attachments_dropped.snap @@ -0,0 +1,20 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these attachments." + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_inline_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_inline_attachments.snap new file mode 100644 index 000000000..443e1e221 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_inline_attachments.snap @@ -0,0 +1,36 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these attachments." + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "ZmFrZS1wbmctYnl0ZXM=" + } + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "ZmFrZS1wZGYtYnl0ZXM=" + } + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_multi_turn.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_multi_turn.snap new file mode 100644 index 000000000..bc576585a --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_multi_turn.snap @@ -0,0 +1,69 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/messages", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "anthropic-version", + "2023-06-01" + ], + [ + "content-length", + "340" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-api-key", + "test-key" + ] + ], + "body": { + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is the capital of France?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Paris." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "And of Spain?" + } + ] + } + ], + "max_tokens": 128, + "system": "You are a terse assistant.", + "stop_sequences": [] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_prompt_cache_with_catalog.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_prompt_cache_with_catalog.snap new file mode 100644 index 000000000..f29eca145 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_prompt_cache_with_catalog.snap @@ -0,0 +1,95 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/messages", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "anthropic-beta", + "prompt-caching-2024-07-31" + ], + [ + "anthropic-version", + "2023-06-01" + ], + [ + "content-length", + "559" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-api-key", + "test-key" + ] + ], + "body": { + "model": "test-claude", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Review this." + } + ] + } + ], + "max_tokens": 128, + "system": [ + { + "type": "text", + "text": "You are a careful reviewer.", + "cache_control": { + "type": "ephemeral" + } + } + ], + "stop_sequences": [], + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + }, + "cache_control": { + "type": "ephemeral" + } + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_provider_options_anthropic_namespace.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_provider_options_anthropic_namespace.snap new file mode 100644 index 000000000..1475f5820 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_provider_options_anthropic_namespace.snap @@ -0,0 +1,21 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "top_k": 5 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_reasoning_effort_with_levels_catalog.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_reasoning_effort_with_levels_catalog.snap new file mode 100644 index 000000000..92b9f5904 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_reasoning_effort_with_levels_catalog.snap @@ -0,0 +1,23 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "test-claude", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "output_config": { + "effort": "high" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_response_format_json_object.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_response_format_json_object.snap new file mode 100644 index 000000000..fe60412fb --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_response_format_json_object.snap @@ -0,0 +1,21 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "system": "You must respond with valid JSON only, no other text.", + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_response_format_json_schema.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_response_format_json_schema.snap new file mode 100644 index 000000000..e757af0a9 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_response_format_json_schema.snap @@ -0,0 +1,41 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "tools": [ + { + "name": "json_output", + "description": "Output the requested structured data", + "input_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + } + } + ], + "tool_choice": { + "type": "tool", + "name": "json_output" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_sampling_params.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_sampling_params.snap new file mode 100644 index 000000000..b50a93df6 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_sampling_params.snap @@ -0,0 +1,27 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "temperature": 0.7, + "top_p": 0.9, + "stop_sequences": [ + "END" + ], + "metadata": { + "trace_id": "trace-123" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_thinking_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_thinking_round_trip.snap new file mode 100644 index 000000000..5fedc6ff1 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_thinking_round_trip.snap @@ -0,0 +1,43 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Think step by step: what is 2+2?" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "The user wants 2+2, which is 4.", + "signature": "sig_test_abc123" + }, + { + "type": "text", + "text": "4." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Now 3+3?" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_auto.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_auto.snap new file mode 100644 index 000000000..d181d4796 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_auto.snap @@ -0,0 +1,52 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": { + "type": "auto" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_named.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_named.snap new file mode 100644 index 000000000..5d7d5214b --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_named.snap @@ -0,0 +1,53 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": { + "type": "tool", + "name": "search" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_none.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_none.snap new file mode 100644 index 000000000..4fbbbaa5f --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_none.snap @@ -0,0 +1,20 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_required.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_required.snap new file mode 100644 index 000000000..b114a2048 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_choice_required.snap @@ -0,0 +1,52 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": { + "type": "any" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_round_trip.snap new file mode 100644 index 000000000..f744b1d64 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_tool_round_trip.snap @@ -0,0 +1,91 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Find foo and read /tmp/x" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Let me check." + }, + { + "type": "tool_use", + "id": "call_1", + "name": "search", + "input": { + "query": "foo" + } + }, + { + "type": "tool_use", + "id": "call_2", + "name": "read_file", + "input": { + "path": "/tmp/x" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": "{\"matches\":2}", + "is_error": false + }, + { + "type": "tool_result", + "tool_use_id": "call_2", + "content": "file not found", + "is_error": true + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_url_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_url_attachments.snap new file mode 100644 index 000000000..f4abc02be --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__encode_url_attachments.snap @@ -0,0 +1,34 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe these attachments." + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/picture.png" + } + }, + { + "type": "document", + "source": { + "type": "url", + "url": "https://example.com/report.pdf" + } + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_error_event_mid_stream.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_error_event_mid_stream.snap new file mode 100644 index 000000000..d2b78a3a2 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_error_event_mid_stream.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "stream_item_error": "Server error from anthropic: Overloaded", + "retryable": true, + "failover_eligible": true + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_text_happy_path_events.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_text_happy_path_events.snap new file mode 100644 index 000000000..a8ad9d9d1 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_text_happy_path_events.snap @@ -0,0 +1,65 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": "block_0" + }, + { + "type": "text_delta", + "delta": "Hel", + "text_id": "block_0" + }, + { + "type": "text_delta", + "delta": "lo", + "text_id": "block_0" + }, + { + "type": "text_end", + "text_id": "block_0" + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "reasoning_tokens": 0, + "cache_read_tokens": 2, + "cache_write_tokens": 1 + }, + "response": { + "id": "msg_stream_test", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "reasoning_tokens": 0, + "cache_read_tokens": 2, + "cache_write_tokens": 1 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_text_happy_path_request.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_text_happy_path_request.snap new file mode 100644 index 000000000..8b641b028 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_text_happy_path_request.snap @@ -0,0 +1,21 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "stop_sequences": [], + "stream": true +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_thinking_with_signature_delta.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_thinking_with_signature_delta.snap new file mode 100644 index 000000000..2dc7b67fb --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_thinking_with_signature_delta.snap @@ -0,0 +1,78 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "reasoning_start" + }, + { + "type": "reasoning_delta", + "delta": "Let me think" + }, + { + "type": "reasoning_end" + }, + { + "type": "text_start", + "text_id": "block_1" + }, + { + "type": "text_delta", + "delta": "4.", + "text_id": "block_1" + }, + { + "type": "text_end", + "text_id": "block_1" + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 15, + "output_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "msg_stream_think", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "thinking", + "data": { + "text": "Let me think", + "signature": "sig_stream_xyz", + "redacted": false + } + }, + { + "kind": "text", + "data": "4." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 15, + "output_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_tool_call_deltas.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_tool_call_deltas.snap new file mode 100644 index 000000000..d2dc68988 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_tool_call_deltas.snap @@ -0,0 +1,97 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "tool_call_start", + "tool_call": { + "id": "toolu_01", + "name": "search", + "type": "function", + "arguments": {}, + "raw_arguments": null + } + }, + { + "type": "tool_call_delta", + "tool_call": { + "id": "toolu_01", + "name": "search", + "type": "function", + "arguments": "{\"qu", + "raw_arguments": null + } + }, + { + "type": "tool_call_delta", + "tool_call": { + "id": "toolu_01", + "name": "search", + "type": "function", + "arguments": "ery\":\"foo\"}", + "raw_arguments": null + } + }, + { + "type": "tool_call_end", + "tool_call": { + "id": "toolu_01", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}" + } + }, + { + "type": "finish", + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "msg_stream_tool", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "tool_call", + "data": { + "id": "toolu_01", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}" + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_without_message_stop_emits_no_finish.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_without_message_stop_emits_no_finish.snap new file mode 100644 index 000000000..a12eec3d6 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__stream_without_message_stop_emits_no_finish.snap @@ -0,0 +1,22 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": "block_0" + }, + { + "type": "text_delta", + "delta": "Hello", + "text_id": "block_0" + }, + { + "type": "text_end", + "text_id": "block_0" + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__system_and_tools_decode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__system_and_tools_decode.snap new file mode 100644 index 000000000..44e29f38e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__system_and_tools_decode.snap @@ -0,0 +1,50 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "id": "msg_test", + "model": "claude-sonnet-4-20250514", + "provider": "anthropic", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello back" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 42, + "output_tokens": 7, + "reasoning_tokens": 0, + "cache_read_tokens": 10, + "cache_write_tokens": 3 + }, + "raw": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + { + "type": "text", + "text": "Hello back" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 42, + "output_tokens": 7, + "cache_read_input_tokens": 10, + "cache_creation_input_tokens": 3 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__system_and_tools_encode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__system_and_tools_encode.snap new file mode 100644 index 000000000..63678178e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__anthropic__system_and_tools_encode.snap @@ -0,0 +1,66 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/anthropic.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/messages", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "anthropic-version", + "2023-06-01" + ], + [ + "content-length", + "316" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-api-key", + "test-key" + ] + ], + "body": { + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello" + } + ] + } + ], + "max_tokens": 128, + "system": "Be concise", + "temperature": 0.5, + "stop_sequences": [], + "tools": [ + { + "name": "search", + "description": "Search files", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + } + } + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__count_tokens_wire_shape.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__count_tokens_wire_shape.snap new file mode 100644 index 000000000..4056e3979 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__count_tokens_wire_shape.snap @@ -0,0 +1,93 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/models/gemini-test:countTokens", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "content-length", + "583" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-goog-api-key", + "test-key" + ] + ], + "body": { + "generateContentRequest": { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "systemInstruction": { + "parts": [ + { + "text": "Be concise" + } + ] + }, + "generationConfig": { + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } + ], + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] + } + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_function_call_with_thought_signature.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_function_call_with_thought_signature.snap new file mode 100644 index 000000000..bbe899144 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_function_call_with_thought_signature.snap @@ -0,0 +1,73 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Let me search." + }, + { + "kind": "tool_call", + "data": { + "id": "[UUID]", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": null, + "provider_metadata": { + "thoughtSignature": "sig_gemini_xyz" + } + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 30, + "output_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "Let me search." + }, + { + "functionCall": { + "name": "search", + "args": { + "query": "foo" + } + }, + "thoughtSignature": "sig_gemini_xyz" + } + ] + }, + "finishReason": "STOP" + } + ], + "usageMetadata": { + "promptTokenCount": 30, + "candidatesTokenCount": 12 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_max_tokens_finish_reason.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_max_tokens_finish_reason.snap new file mode 100644 index 000000000..f80ff0a94 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_max_tokens_finish_reason.snap @@ -0,0 +1,5 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +"length" diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_safety_finish_reason.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_safety_finish_reason.snap new file mode 100644 index 000000000..aedfd4696 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_safety_finish_reason.snap @@ -0,0 +1,5 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +"content_filter" diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_thought_parts.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_thought_parts.snap new file mode 100644 index 000000000..2eeb7b0af --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_thought_parts.snap @@ -0,0 +1,61 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "thinking", + "data": { + "text": "Adding the numbers.", + "signature": null, + "redacted": false + } + }, + { + "kind": "text", + "data": "4." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 25, + "output_tokens": 40, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "Adding the numbers.", + "thought": true + }, + { + "text": "4." + } + ] + }, + "finishReason": "STOP" + } + ], + "usageMetadata": { + "promptTokenCount": 25, + "candidatesTokenCount": 40 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_usage_arithmetic.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_usage_arithmetic.snap new file mode 100644 index 000000000..f332140ab --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__decode_usage_arithmetic.snap @@ -0,0 +1,52 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "ok" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 75, + "output_tokens": 50, + "reasoning_tokens": 8, + "cache_read_tokens": 30, + "cache_write_tokens": 0 + }, + "raw": { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "ok" + } + ] + }, + "finishReason": "STOP" + } + ], + "usageMetadata": { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "thoughtsTokenCount": 8, + "cachedContentTokenCount": 30, + "toolUsePromptTokenCount": 5 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_audio_attachment.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_audio_attachment.snap new file mode 100644 index 000000000..d3f6c4a7f --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_audio_attachment.snap @@ -0,0 +1,31 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Transcribe this." + }, + { + "inlineData": { + "mimeType": "audio/wav", + "data": "ZmFrZS13YXYtYnl0ZXM=" + } + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_bad_file_path_attachments_dropped.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_bad_file_path_attachments_dropped.snap new file mode 100644 index 000000000..b4605fdbb --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_bad_file_path_attachments_dropped.snap @@ -0,0 +1,25 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Describe these attachments." + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_inline_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_inline_attachments.snap new file mode 100644 index 000000000..130c517ee --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_inline_attachments.snap @@ -0,0 +1,37 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Describe these attachments." + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "ZmFrZS1wbmctYnl0ZXM=" + } + }, + { + "inlineData": { + "mimeType": "application/pdf", + "data": "ZmFrZS1wZGYtYnl0ZXM=" + } + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_multi_turn.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_multi_turn.snap new file mode 100644 index 000000000..af95759cb --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_multi_turn.snap @@ -0,0 +1,48 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "What is the capital of France?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "Paris." + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "And of Spain?" + } + ] + } + ], + "systemInstruction": { + "parts": [ + { + "text": "You are a terse assistant." + } + ] + }, + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_provider_options_can_override_safety_settings.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_provider_options_can_override_safety_settings.snap new file mode 100644 index 000000000..105e76111 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_provider_options_can_override_safety_settings.snap @@ -0,0 +1,20 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_provider_options_gemini_namespace.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_provider_options_gemini_namespace.snap new file mode 100644 index 000000000..e6b616613 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_provider_options_gemini_namespace.snap @@ -0,0 +1,26 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "cached_content": "cachedContents/abc", + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_reasoning_effort_with_levels_catalog.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_reasoning_effort_with_levels_catalog.snap new file mode 100644 index 000000000..83f70e81f --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_reasoning_effort_with_levels_catalog.snap @@ -0,0 +1,25 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_response_format_json_object.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_response_format_json_object.snap new file mode 100644 index 000000000..d16ceda4c --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_response_format_json_object.snap @@ -0,0 +1,26 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128, + "responseMimeType": "application/json" + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_response_format_json_schema.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_response_format_json_schema.snap new file mode 100644 index 000000000..7a6a39742 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_response_format_json_schema.snap @@ -0,0 +1,37 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128, + "responseMimeType": "application/json", + "responseSchema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + } + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_sampling_params.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_sampling_params.snap new file mode 100644 index 000000000..1e9350fcd --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_sampling_params.snap @@ -0,0 +1,30 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "temperature": 0.7, + "maxOutputTokens": 128, + "topP": 0.9, + "stopSequences": [ + "END" + ] + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_thinking_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_thinking_round_trip.snap new file mode 100644 index 000000000..c0cdbd040 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_thinking_round_trip.snap @@ -0,0 +1,41 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Think step by step: what is 2+2?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "4." + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "Now 3+3?" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_auto.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_auto.snap new file mode 100644 index 000000000..186a9afce --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_auto.snap @@ -0,0 +1,63 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } + ], + "toolConfig": { + "functionCallingConfig": { + "mode": "AUTO" + } + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_named.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_named.snap new file mode 100644 index 000000000..d31af7c7d --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_named.snap @@ -0,0 +1,66 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } + ], + "toolConfig": { + "functionCallingConfig": { + "mode": "ANY", + "allowedFunctionNames": [ + "search" + ] + } + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_none.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_none.snap new file mode 100644 index 000000000..7bb3b0bfa --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_none.snap @@ -0,0 +1,63 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } + ], + "toolConfig": { + "functionCallingConfig": { + "mode": "NONE" + } + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_required.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_required.snap new file mode 100644 index 000000000..7832bc141 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_choice_required.snap @@ -0,0 +1,63 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } + ], + "toolConfig": { + "functionCallingConfig": { + "mode": "ANY" + } + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_round_trip.snap new file mode 100644 index 000000000..d0fdee6ce --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_tool_round_trip.snap @@ -0,0 +1,108 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Find foo and read /tmp/x" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "Let me check." + }, + { + "functionCall": { + "name": "search", + "args": { + "query": "foo" + } + } + }, + { + "functionCall": { + "name": "read_file", + "args": { + "path": "/tmp/x" + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "search", + "response": { + "matches": 2 + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "read_file", + "response": { + "result": "file not found" + } + } + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } + ], + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_url_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_url_attachments.snap new file mode 100644 index 000000000..dcf7ef4e2 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__encode_url_attachments.snap @@ -0,0 +1,37 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Describe these attachments." + }, + { + "fileData": { + "mimeType": "image/png", + "fileUri": "https://example.com/picture.png" + } + }, + { + "fileData": { + "mimeType": "application/pdf", + "fileUri": "https://example.com/report.pdf" + } + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_end_synthesizes_finish_without_finish_reason.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_end_synthesizes_finish_without_finish_reason.snap new file mode 100644 index 000000000..4fd8569ca --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_end_synthesizes_finish_without_finish_reason.snap @@ -0,0 +1,56 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": "[UUID]" + }, + { + "type": "text_delta", + "delta": "Hello", + "text_id": "[UUID]" + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_function_call.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_function_call.snap new file mode 100644 index 000000000..d006bcbf7 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_function_call.snap @@ -0,0 +1,88 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "tool_call_start", + "tool_call": { + "id": "[UUID]", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": null, + "provider_metadata": { + "thoughtSignature": "sig_stream_g" + } + } + }, + { + "type": "tool_call_end", + "tool_call": { + "id": "[UUID]", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": null, + "provider_metadata": { + "thoughtSignature": "sig_stream_g" + } + } + }, + { + "type": "finish", + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "tool_call", + "data": { + "id": "[UUID]", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": null, + "provider_metadata": { + "thoughtSignature": "sig_stream_g" + } + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_text_happy_path_events.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_text_happy_path_events.snap new file mode 100644 index 000000000..2518f784a --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_text_happy_path_events.snap @@ -0,0 +1,65 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": "[UUID]" + }, + { + "type": "text_delta", + "delta": "Hel", + "text_id": "[UUID]" + }, + { + "type": "text_delta", + "delta": "lo", + "text_id": "[UUID]" + }, + { + "type": "text_end", + "text_id": "[UUID]" + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_text_happy_path_request.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_text_happy_path_request.snap new file mode 100644 index 000000000..cb853aaf9 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_text_happy_path_request.snap @@ -0,0 +1,51 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/models/gemini-test:streamGenerateContent?alt=sse", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "content-length", + "197" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-goog-api-key", + "test-key" + ] + ], + "body": { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "generationConfig": { + "maxOutputTokens": 128 + }, + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_thought_parts.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_thought_parts.snap new file mode 100644 index 000000000..6067c0cba --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__stream_thought_parts.snap @@ -0,0 +1,78 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "reasoning_start" + }, + { + "type": "reasoning_delta", + "delta": "Let me think" + }, + { + "type": "reasoning_end" + }, + { + "type": "text_start", + "text_id": "[UUID]" + }, + { + "type": "text_delta", + "delta": "4.", + "text_id": "[UUID]" + }, + { + "type": "text_end", + "text_id": "[UUID]" + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 15, + "output_tokens": 12, + "reasoning_tokens": 6, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "thinking", + "data": { + "text": "Let me think", + "signature": null, + "redacted": false + } + }, + { + "kind": "text", + "data": "4." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 15, + "output_tokens": 12, + "reasoning_tokens": 6, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__system_and_tools_decode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__system_and_tools_decode.snap new file mode 100644 index 000000000..0ed032bbc --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__system_and_tools_decode.snap @@ -0,0 +1,50 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "id": "[UUID]", + "model": "gemini-test", + "provider": "gemini", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello back" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 32, + "output_tokens": 7, + "reasoning_tokens": 0, + "cache_read_tokens": 10, + "cache_write_tokens": 0 + }, + "raw": { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "Hello back" + } + ] + }, + "finishReason": "STOP" + } + ], + "usageMetadata": { + "promptTokenCount": 42, + "candidatesTokenCount": 7, + "cachedContentTokenCount": 10 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__system_and_tools_encode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__system_and_tools_encode.snap new file mode 100644 index 000000000..c6d3c6a48 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__gemini__system_and_tools_encode.snap @@ -0,0 +1,77 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/gemini.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/models/gemini-test:generateContent", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "content-length", + "425" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ], + [ + "x-goog-api-key", + "test-key" + ] + ], + "body": { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Hello" + } + ] + } + ], + "systemInstruction": { + "parts": [ + { + "text": "Be concise" + } + ] + }, + "generationConfig": { + "temperature": 0.5, + "maxOutputTokens": 128 + }, + "tools": [ + { + "functionDeclarations": [ + { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + } + } + } + ] + } + ], + "safety_settings": [ + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_ONLY_HIGH" + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_reasoning_content_as_thinking.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_reasoning_content_as_thinking.snap new file mode 100644 index 000000000..5ed2b7de1 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_reasoning_content_as_thinking.snap @@ -0,0 +1,60 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "id": "chatcmpl_test", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "thinking", + "data": { + "text": "The user wants 2+2.", + "signature": null, + "redacted": false + } + }, + { + "kind": "text", + "data": "4." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 25, + "output_tokens": 40, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1700000000, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "4.", + "reasoning_content": "The user wants 2+2." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 40, + "total_tokens": 65 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_tool_calls_with_string_arguments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_tool_calls_with_string_arguments.snap new file mode 100644 index 000000000..768e9aaa8 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_tool_calls_with_string_arguments.snap @@ -0,0 +1,69 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "id": "chatcmpl_test", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "tool_call", + "data": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}" + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 30, + "output_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1700000000, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "search", + "arguments": "{\"query\":\"foo\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 30, + "completion_tokens": 12, + "total_tokens": 42 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_ignores_token_details.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_ignores_token_details.snap new file mode 100644 index 000000000..35cbb5f1e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_ignores_token_details.snap @@ -0,0 +1,57 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "id": "chatcmpl_test", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "ok" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "length", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1700000000, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "ok" + }, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "prompt_tokens_details": { + "cached_tokens": 80 + }, + "completion_tokens_details": { + "reasoning_tokens": 20 + } + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_audio_attachment.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_audio_attachment.snap new file mode 100644 index 000000000..f647c24c0 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_audio_attachment.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Transcribe this.[Audio content not supported by this provider]" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_bad_file_path_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_bad_file_path_attachments.snap new file mode 100644 index 000000000..7b3833469 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_bad_file_path_attachments.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Describe these attachments.[Document 'missing.pdf': content type not supported by this provider]" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_inline_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_inline_attachments.snap new file mode 100644 index 000000000..fcc58e5cf --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_inline_attachments.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Describe these attachments.[Document 'report.pdf': content type not supported by this provider]" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_multi_turn.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_multi_turn.snap new file mode 100644 index 000000000..6e4c35fa0 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_multi_turn.snap @@ -0,0 +1,26 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "system", + "content": "You are a terse assistant." + }, + { + "role": "user", + "content": "What is the capital of France?" + }, + { + "role": "assistant", + "content": "Paris." + }, + { + "role": "user", + "content": "And of Spain?" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_provider_options_keyed_by_adapter_name.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_provider_options_keyed_by_adapter_name.snap new file mode 100644 index 000000000..2b0718398 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_provider_options_keyed_by_adapter_name.snap @@ -0,0 +1,15 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "repetition_penalty": 1.2 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_provider_options_other_namespace_ignored.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_provider_options_other_namespace_ignored.snap new file mode 100644 index 000000000..bb8c27f58 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_provider_options_other_namespace_ignored.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_response_format_json_object.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_response_format_json_object.snap new file mode 100644 index 000000000..84f716422 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_response_format_json_object.snap @@ -0,0 +1,17 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "response_format": { + "type": "json_object" + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_response_format_json_schema.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_response_format_json_schema.snap new file mode 100644 index 000000000..13961444e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_response_format_json_schema.snap @@ -0,0 +1,32 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "response", + "strict": true, + "schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + } + } + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_sampling_params.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_sampling_params.snap new file mode 100644 index 000000000..6a345a9bf --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_sampling_params.snap @@ -0,0 +1,19 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "temperature": 0.7, + "max_tokens": 128, + "top_p": 0.9, + "stop": [ + "END" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_thinking_round_trip_as_reasoning_content.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_thinking_round_trip_as_reasoning_content.snap new file mode 100644 index 000000000..210eaa446 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_thinking_round_trip_as_reasoning_content.snap @@ -0,0 +1,23 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Think step by step: what is 2+2?" + }, + { + "role": "assistant", + "content": "4.", + "reasoning_content": "The user wants 2+2, which is 4." + }, + { + "role": "user", + "content": "Now 3+3?" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_auto.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_auto.snap new file mode 100644 index 000000000..7b9bdab82 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_auto.snap @@ -0,0 +1,50 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + } + ], + "tool_choice": "auto" +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_named.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_named.snap new file mode 100644 index 000000000..5da2b548a --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_named.snap @@ -0,0 +1,55 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + } + ], + "tool_choice": { + "type": "function", + "function": { + "name": "search" + } + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_none.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_none.snap new file mode 100644 index 000000000..a7ace0225 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_none.snap @@ -0,0 +1,50 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + } + ], + "tool_choice": "none" +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_required.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_required.snap new file mode 100644 index 000000000..d77d00bef --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_choice_required.snap @@ -0,0 +1,50 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + } + ], + "tool_choice": "required" +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_round_trip.snap new file mode 100644 index 000000000..43e81924f --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_tool_round_trip.snap @@ -0,0 +1,81 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Find foo and read /tmp/x" + }, + { + "role": "assistant", + "content": "Let me check.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "search", + "arguments": "{\"query\":\"foo\"}" + } + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"path\":\"/tmp/x\"}" + } + } + ] + }, + { + "role": "tool", + "content": "{\"matches\":2}", + "tool_call_id": "call_1" + }, + { + "role": "tool", + "content": "file not found", + "tool_call_id": "call_2" + } + ], + "max_tokens": 128, + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + } + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_url_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_url_attachments.snap new file mode 100644 index 000000000..fcc58e5cf --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__encode_url_attachments.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Describe these attachments.[Document 'report.pdf': content type not supported by this provider]" + } + ], + "max_tokens": 128 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_reasoning_content_deltas.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_reasoning_content_deltas.snap new file mode 100644 index 000000000..bd4c6dcf3 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_reasoning_content_deltas.snap @@ -0,0 +1,65 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[ + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "4.", + "text_id": null + }, + { + "type": "text_end", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "chatcmpl_stream", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "thinking", + "data": { + "text": "Let me think", + "signature": null, + "redacted": false + } + }, + { + "kind": "text", + "data": "4." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_text_happy_path_events.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_text_happy_path_events.snap new file mode 100644 index 000000000..e7b975533 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_text_happy_path_events.snap @@ -0,0 +1,62 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[ + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "Hel", + "text_id": null + }, + { + "type": "text_delta", + "delta": "lo", + "text_id": null + }, + { + "type": "text_end", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "chatcmpl_stream", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_text_happy_path_request.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_text_happy_path_request.snap new file mode 100644 index 000000000..502647610 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_text_happy_path_request.snap @@ -0,0 +1,15 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "model": "test-model", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "max_tokens": 128, + "stream": true +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_tool_call_deltas.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_tool_call_deltas.snap new file mode 100644 index 000000000..6ab800ec8 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_tool_call_deltas.snap @@ -0,0 +1,94 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[ + { + "type": "tool_call_start", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": null, + "raw_arguments": null + } + }, + { + "type": "tool_call_delta", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": null, + "raw_arguments": null + } + }, + { + "type": "tool_call_delta", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": null, + "raw_arguments": null + } + }, + { + "type": "tool_call_end", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}" + } + }, + { + "type": "finish", + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "chatcmpl_stream", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "tool_call", + "data": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}" + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_without_done_or_content_synthesizes_nothing.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_without_done_or_content_synthesizes_nothing.snap new file mode 100644 index 000000000..4c107d93b --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_without_done_or_content_synthesizes_nothing.snap @@ -0,0 +1,5 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_without_done_synthesizes_finish_when_content_started.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_without_done_synthesizes_finish_when_content_started.snap new file mode 100644 index 000000000..858a68b66 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_without_done_synthesizes_finish_when_content_started.snap @@ -0,0 +1,57 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[ + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "Hello", + "text_id": null + }, + { + "type": "text_end", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "chatcmpl_stream", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__system_and_tools_decode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__system_and_tools_decode.snap new file mode 100644 index 000000000..fd4a0a4d6 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__system_and_tools_decode.snap @@ -0,0 +1,51 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "id": "chatcmpl_test", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello back" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 42, + "output_tokens": 7, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "chatcmpl_test", + "object": "chat.completion", + "created": 1700000000, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello back" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 42, + "completion_tokens": 7, + "total_tokens": 49 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__system_and_tools_encode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__system_and_tools_encode.snap new file mode 100644 index 000000000..a732b5d44 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__system_and_tools_encode.snap @@ -0,0 +1,62 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/chat/completions", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "authorization", + "Bearer test-key" + ], + [ + "content-length", + "305" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ] + ], + "body": { + "model": "test-model", + "messages": [ + { + "role": "system", + "content": "Be concise" + }, + { + "role": "user", + "content": "Hello" + } + ], + "temperature": 0.5, + "max_tokens": 128, + "tools": [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + } + } + } + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__count_tokens_wire_shape.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__count_tokens_wire_shape.snap new file mode 100644 index 000000000..4d98746a8 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__count_tokens_wire_shape.snap @@ -0,0 +1,77 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/responses/input_tokens", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "authorization", + "Bearer test-key" + ], + [ + "content-length", + "454" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ] + ], + "body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "instructions": "Be concise", + "model": "gpt-test", + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ] + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_incomplete_status_maps_to_length.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_incomplete_status_maps_to_length.snap new file mode 100644 index 000000000..070663013 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_incomplete_status_maps_to_length.snap @@ -0,0 +1,67 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "id": "resp_test", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "openai_message", + "data": { + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [ + { + "type": "output_text", + "text": "Truncated" + } + ] + } + }, + { + "kind": "text", + "data": "Truncated" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "length", + "usage": { + "input_tokens": 10, + "output_tokens": 128, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_test", + "object": "response", + "model": "gpt-test", + "status": "incomplete", + "output": [ + { + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [ + { + "type": "output_text", + "text": "Truncated" + } + ] + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 128 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_reasoning_and_function_call_items.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_reasoning_and_function_call_items.snap new file mode 100644 index 000000000..a453f0698 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_reasoning_and_function_call_items.snap @@ -0,0 +1,83 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "id": "resp_test", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "openai_reasoning", + "data": { + "type": "reasoning", + "id": "rs_1", + "summary": [ + { + "type": "summary_text", + "text": "Searching." + } + ] + } + }, + { + "kind": "tool_call", + "data": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}", + "provider_metadata": { + "id": "fc_123" + } + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 30, + "output_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_test", + "object": "response", + "model": "gpt-test", + "status": "completed", + "output": [ + { + "type": "reasoning", + "id": "rs_1", + "summary": [ + { + "type": "summary_text", + "text": "Searching." + } + ] + }, + { + "type": "function_call", + "id": "fc_123", + "call_id": "call_abc", + "name": "search", + "arguments": "{\"query\":\"foo\"}" + } + ], + "usage": { + "input_tokens": 30, + "output_tokens": 12 + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_usage_subtracts_cached_and_reasoning.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_usage_subtracts_cached_and_reasoning.snap new file mode 100644 index 000000000..846589d95 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__decode_usage_subtracts_cached_and_reasoning.snap @@ -0,0 +1,73 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "id": "resp_test", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "openai_message", + "data": { + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [ + { + "type": "output_text", + "text": "ok" + } + ] + } + }, + { + "kind": "text", + "data": "ok" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 20, + "output_tokens": 30, + "reasoning_tokens": 20, + "cache_read_tokens": 80, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_test", + "object": "response", + "model": "gpt-test", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [ + { + "type": "output_text", + "text": "ok" + } + ] + } + ], + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "input_tokens_details": { + "cached_tokens": 80 + }, + "output_tokens_details": { + "reasoning_tokens": 20 + } + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_audio_attachment.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_audio_attachment.snap new file mode 100644 index 000000000..5e12b469e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_audio_attachment.snap @@ -0,0 +1,28 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Transcribe this." + }, + { + "type": "input_text", + "text": "[Audio content not supported by this provider]" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_bad_file_path_attachments_dropped.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_bad_file_path_attachments_dropped.snap new file mode 100644 index 000000000..93b5f16a3 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_bad_file_path_attachments_dropped.snap @@ -0,0 +1,28 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Describe these attachments." + }, + { + "type": "input_text", + "text": "[Document 'missing.pdf': content type not supported by this provider]" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_codex_mode_forces_streaming_and_omits_params.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_codex_mode_forces_streaming_and_omits_params.snap new file mode 100644 index 000000000..655b2e204 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_codex_mode_forces_streaming_and_omits_params.snap @@ -0,0 +1,51 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/responses", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "authorization", + "Bearer test-key" + ], + [ + "content-length", + "210" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ] + ], + "body": { + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "instructions": "Be concise", + "store": false, + "include": [ + "reasoning.encrypted_content" + ], + "stream": true + } +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_dual_id_tool_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_dual_id_tool_round_trip.snap new file mode 100644 index 000000000..990bf91a8 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_dual_id_tool_round_trip.snap @@ -0,0 +1,67 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Find foo" + } + ] + }, + { + "type": "function_call", + "id": "fc_123", + "call_id": "call_abc", + "name": "search", + "arguments": "{\"query\":\"foo\"}" + }, + { + "type": "function_call_output", + "call_id": "call_abc", + "output": "2 matches" + } + ], + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_inline_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_inline_attachments.snap new file mode 100644 index 000000000..3d74e1cdd --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_inline_attachments.snap @@ -0,0 +1,32 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Describe these attachments." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,ZmFrZS1wbmctYnl0ZXM=" + }, + { + "type": "input_text", + "text": "[Document 'report.pdf': content type not supported by this provider]" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_multi_turn.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_multi_turn.snap new file mode 100644 index 000000000..3e049c7fe --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_multi_turn.snap @@ -0,0 +1,45 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "What is the capital of France?" + } + ] + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Paris." + } + ] + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "And of Spain?" + } + ] + } + ], + "instructions": "You are a terse assistant.", + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_opaque_items_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_opaque_items_round_trip.snap new file mode 100644 index 000000000..acfb705af --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_opaque_items_round_trip.snap @@ -0,0 +1,55 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Think about 2+2." + } + ] + }, + { + "type": "reasoning", + "id": "rs_1", + "summary": [ + { + "type": "summary_text", + "text": "Adding." + } + ] + }, + { + "type": "message", + "role": "assistant", + "id": "msg_1", + "content": [ + { + "type": "output_text", + "text": "4." + } + ] + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Now 3+3?" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_provider_options_openai_namespace.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_provider_options_openai_namespace.snap new file mode 100644 index 000000000..cd69aff6a --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_provider_options_openai_namespace.snap @@ -0,0 +1,25 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ], + "seed": 42 +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_reasoning_effort_with_levels_catalog.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_reasoning_effort_with_levels_catalog.snap new file mode 100644 index 000000000..c9efcd60f --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_reasoning_effort_with_levels_catalog.snap @@ -0,0 +1,27 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "test-gpt", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "reasoning": { + "effort": "high" + }, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_response_format_json_object.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_response_format_json_object.snap new file mode 100644 index 000000000..f4ccc269d --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_response_format_json_object.snap @@ -0,0 +1,29 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "text": { + "format": { + "type": "json_object" + } + }, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_response_format_json_schema.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_response_format_json_schema.snap new file mode 100644 index 000000000..3fa0b83e2 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_response_format_json_schema.snap @@ -0,0 +1,42 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "text": { + "format": { + "type": "json_schema", + "name": "response", + "strict": true, + "schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + } + } + }, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_sampling_params.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_sampling_params.snap new file mode 100644 index 000000000..3634e5fb3 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_sampling_params.snap @@ -0,0 +1,32 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "temperature": 0.7, + "max_output_tokens": 128, + "top_p": 0.9, + "stop": [ + "END" + ], + "metadata": { + "trace_id": "trace-123" + }, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_thinking_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_thinking_round_trip.snap new file mode 100644 index 000000000..e4b3593b0 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_thinking_round_trip.snap @@ -0,0 +1,44 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Think step by step: what is 2+2?" + } + ] + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "4." + } + ] + }, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Now 3+3?" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_auto.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_auto.snap new file mode 100644 index 000000000..af2dfe6a9 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_auto.snap @@ -0,0 +1,56 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": "auto", + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_named.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_named.snap new file mode 100644 index 000000000..cc00b7891 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_named.snap @@ -0,0 +1,59 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": { + "type": "function", + "name": "search" + }, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_none.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_none.snap new file mode 100644 index 000000000..fda379759 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_none.snap @@ -0,0 +1,56 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": "none", + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_required.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_required.snap new file mode 100644 index 000000000..b9e9d82e0 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_choice_required.snap @@ -0,0 +1,56 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "tool_choice": "required", + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_round_trip.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_round_trip.snap new file mode 100644 index 000000000..97c1654d9 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_tool_round_trip.snap @@ -0,0 +1,90 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Find foo and read /tmp/x" + } + ] + }, + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Let me check." + } + ] + }, + { + "type": "function_call", + "id": "call_1", + "call_id": "call_1", + "name": "search", + "arguments": "{\"query\":\"foo\"}" + }, + { + "type": "function_call", + "id": "call_2", + "call_id": "call_2", + "name": "read_file", + "arguments": "{\"path\":\"/tmp/x\"}" + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": "{\"matches\":2}" + }, + { + "type": "function_call_output", + "call_id": "call_2", + "output": "file not found", + "status": "incomplete" + } + ], + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ] + } + }, + { + "type": "function", + "name": "read_file", + "description": "Read a file by path", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + ], + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_url_attachments.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_url_attachments.snap new file mode 100644 index 000000000..9b67b8abc --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__encode_url_attachments.snap @@ -0,0 +1,32 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Describe these attachments." + }, + { + "type": "input_image", + "image_url": "https://example.com/picture.png" + }, + { + "type": "input_text", + "text": "[Document 'report.pdf': content type not supported by this provider]" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ] +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_failed_event_maps_to_error.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_failed_event_maps_to_error.snap new file mode 100644 index 000000000..68057a6b7 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_failed_event_maps_to_error.snap @@ -0,0 +1,14 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "stream_item_error": "Server error from openai: boom", + "retryable": true, + "failover_eligible": true + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_incomplete_maps_to_length.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_incomplete_maps_to_length.snap new file mode 100644 index 000000000..787a6f4df --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_incomplete_maps_to_length.snap @@ -0,0 +1,65 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "Trunc", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "length", + "usage": { + "input_tokens": 10, + "output_tokens": 128, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "resp_stream", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Trunc" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "length", + "usage": { + "input_tokens": 10, + "output_tokens": 128, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_stream", + "model": "gpt-test", + "status": "incomplete", + "output": [], + "usage": { + "input_tokens": 10, + "output_tokens": 128 + } + }, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_reasoning_summary_deltas.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_reasoning_summary_deltas.snap new file mode 100644 index 000000000..c38adf4e4 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_reasoning_summary_deltas.snap @@ -0,0 +1,79 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "reasoning_start" + }, + { + "type": "reasoning_delta", + "delta": "Let me " + }, + { + "type": "reasoning_delta", + "delta": "think" + }, + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "4.", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 15, + "output_tokens": 4, + "reasoning_tokens": 8, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "resp_stream", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "4." + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 15, + "output_tokens": 4, + "reasoning_tokens": 8, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_stream", + "model": "gpt-test", + "status": "completed", + "output": [], + "usage": { + "input_tokens": 15, + "output_tokens": 12, + "output_tokens_details": { + "reasoning_tokens": 8 + } + } + }, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_text_happy_path_events.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_text_happy_path_events.snap new file mode 100644 index 000000000..c0b41b21e --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_text_happy_path_events.snap @@ -0,0 +1,76 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "Hel", + "text_id": null + }, + { + "type": "text_delta", + "delta": "lo", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 9, + "output_tokens": 4, + "reasoning_tokens": 1, + "cache_read_tokens": 2, + "cache_write_tokens": 0 + }, + "response": { + "id": "resp_stream", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hello" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 9, + "output_tokens": 4, + "reasoning_tokens": 1, + "cache_read_tokens": 2, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_stream", + "model": "gpt-test", + "status": "completed", + "output": [], + "usage": { + "input_tokens": 11, + "output_tokens": 5, + "input_tokens_details": { + "cached_tokens": 2 + }, + "output_tokens_details": { + "reasoning_tokens": 1 + } + } + }, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_text_happy_path_request.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_text_happy_path_request.snap new file mode 100644 index 000000000..c5f2d85eb --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_text_happy_path_request.snap @@ -0,0 +1,25 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "max_output_tokens": 128, + "store": false, + "include": [ + "reasoning.encrypted_content" + ], + "stream": true +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_tool_call_deltas.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_tool_call_deltas.snap new file mode 100644 index 000000000..66c3b5d91 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__stream_tool_call_deltas.snap @@ -0,0 +1,121 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "tool_call_start", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": {}, + "raw_arguments": "{\"qu", + "provider_metadata": { + "id": "fc_123" + } + } + }, + { + "type": "tool_call_delta", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": {}, + "raw_arguments": "{\"qu", + "provider_metadata": { + "id": "fc_123" + } + } + }, + { + "type": "tool_call_delta", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": {}, + "raw_arguments": "ery\":\"foo\"}", + "provider_metadata": { + "id": "fc_123" + } + } + }, + { + "type": "tool_call_end", + "tool_call": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}", + "provider_metadata": { + "id": "fc_123" + } + } + }, + { + "type": "finish", + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "resp_stream", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "tool_call", + "data": { + "id": "call_abc", + "name": "search", + "type": "function", + "arguments": { + "query": "foo" + }, + "raw_arguments": "{\"query\":\"foo\"}", + "provider_metadata": { + "id": "fc_123" + } + } + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "tool_calls", + "usage": { + "input_tokens": 20, + "output_tokens": 9, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_stream", + "model": "gpt-test", + "status": "completed", + "output": [], + "usage": { + "input_tokens": 20, + "output_tokens": 9 + } + }, + "warnings": [], + "rate_limit": null + } + } +] diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__system_and_tools_decode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__system_and_tools_decode.snap new file mode 100644 index 000000000..1ce06379d --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__system_and_tools_decode.snap @@ -0,0 +1,73 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "id": "resp_test", + "model": "gpt-test", + "provider": "openai", + "message": { + "role": "assistant", + "content": [ + { + "kind": "openai_message", + "data": { + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [ + { + "type": "output_text", + "text": "Hello back" + } + ] + } + }, + { + "kind": "text", + "data": "Hello back" + } + ], + "name": null, + "tool_call_id": null + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 32, + "output_tokens": 4, + "reasoning_tokens": 3, + "cache_read_tokens": 10, + "cache_write_tokens": 0 + }, + "raw": { + "id": "resp_test", + "object": "response", + "model": "gpt-test", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "id": "msg_out", + "content": [ + { + "type": "output_text", + "text": "Hello back" + } + ] + } + ], + "usage": { + "input_tokens": 42, + "output_tokens": 7, + "input_tokens_details": { + "cached_tokens": 10 + }, + "output_tokens_details": { + "reasoning_tokens": 3 + } + } + }, + "warnings": [], + "rate_limit": null +} diff --git a/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__system_and_tools_encode.snap b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__system_and_tools_encode.snap new file mode 100644 index 000000000..5c7aca225 --- /dev/null +++ b/lib/crates/fabro-llm/tests/it/wire/snapshots/it__wire__openai_responses__system_and_tools_encode.snap @@ -0,0 +1,67 @@ +--- +source: lib/crates/fabro-llm/tests/it/wire/openai_responses.rs +expression: rendered +--- +{ + "method": "POST", + "path": "/responses", + "headers": [ + [ + "accept", + "*/*" + ], + [ + "authorization", + "Bearer test-key" + ], + [ + "content-length", + "385" + ], + [ + "content-type", + "application/json" + ], + [ + "host", + "[host]" + ] + ], + "body": { + "model": "gpt-test", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Hello" + } + ] + } + ], + "instructions": "Be concise", + "temperature": 0.5, + "max_output_tokens": 128, + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search files", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + } + } + } + ], + "store": false, + "include": [ + "reasoning.encrypted_content" + ] + } +} diff --git a/lib/crates/fabro-test/src/lib.rs b/lib/crates/fabro-test/src/lib.rs index 8db94500c..486ebdf2f 100644 --- a/lib/crates/fabro-test/src/lib.rs +++ b/lib/crates/fabro-test/src/lib.rs @@ -2004,6 +2004,32 @@ macro_rules! fabro_json_snapshot { insta::assert_snapshot!(rendered, @$snapshot); }); }}; + // External-snapshot forms (no inline `@"..."`): insta writes the snapshot + // to a `.snap` file under the test's `snapshots/` directory. Use these to + // keep large snapshots out of the `.rs` source. + ($filter_source:expr, $value:expr $(,)?) => {{ + let filters = + $crate::json_snapshot_filters($crate::snapshot_filters_from(&$filter_source)); + let filters: Vec<(&str, &str)> = filters + .iter() + .map(|(pattern, replacement)| (pattern.as_str(), replacement.as_str())) + .collect(); + let rendered = serde_json::to_string_pretty(&$value).unwrap(); + insta::with_settings!({ filters => filters }, { + insta::assert_snapshot!(rendered); + }); + }}; + ($value:expr $(,)?) => {{ + let filters = $crate::json_snapshot_filters($crate::TestContext::default_filters()); + let filters: Vec<(&str, &str)> = filters + .iter() + .map(|(pattern, replacement)| (pattern.as_str(), replacement.as_str())) + .collect(); + let rendered = serde_json::to_string_pretty(&$value).unwrap(); + insta::with_settings!({ filters => filters }, { + insta::assert_snapshot!(rendered); + }); + }}; } impl TestContext {