From 8ed47d31ba379eed9536fbc8388a9d6504e995e7 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Wed, 10 Jun 2026 13:13:25 -0400 Subject: [PATCH] refactor(llm): add attachment-resolution infra (#485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Next step of the gateway refactor (after #481): a small, codec-agnostic step for resolving file-backed attachments to inline data, shared by the per-dialect codec extractions that follow (anthropic, openai_responses, gemini). Codec `encode` is sync and never touches the filesystem. Today each adapter loads file-path `Image`/`Document`/`Audio` parts inline via `common::load_file_as_base64` mid-translation; the codec split needs that I/O hoisted out so encode can stay pure. `attachments::resolve` does it: clone the request, load each file-path part (per the caller's `AttachmentPolicy`) into inline bytes + MIME, drop the part on load error (the long-standing contract), and leave non-file URLs and already-inline data untouched. - `AttachmentPolicy { images, documents, audio }` — each dialect adapter constructs the policy it wants when it wires this in (anthropic: images+documents; openai: images only; gemini: all three). - `common::load_file_bytes` (raw bytes + MIME) factored out of `load_file_as_base64`, which now delegates to it. Splitting this out of the anthropic extraction makes the three dialect-codec PRs independent siblings — they can go up and land in parallel once this merges. Added ahead of its consumers, so the module sits behind a justified `dead_code` allow until the first dialect codec calls it (the anthropic PR drops the allow). No behavior change. ## Testing - `cargo nextest run -p fabro-llm` — 515 passed (including the 126 wire snapshots; byte-identical, nothing reachable changes) - `cargo check --workspace` - `cargo +nightly-2026-04-14 clippy -p fabro-llm --all-targets -- -D warnings` - `cargo +nightly-2026-04-14 fmt --check` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 (1M context) --- lib/crates/fabro-llm/src/attachments.rs | 96 ++++++++++++++++++++ lib/crates/fabro-llm/src/lib.rs | 1 + lib/crates/fabro-llm/src/providers/common.rs | 15 ++- 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 lib/crates/fabro-llm/src/attachments.rs diff --git a/lib/crates/fabro-llm/src/attachments.rs b/lib/crates/fabro-llm/src/attachments.rs new file mode 100644 index 000000000..0d019051f --- /dev/null +++ b/lib/crates/fabro-llm/src/attachments.rs @@ -0,0 +1,96 @@ +//! Resolve file-backed attachments to inline data before a codec encodes. +//! +//! Codec `encode` is sync and never touches the filesystem, so any +//! `Image`/`Document`/`Audio` part whose `url` is a local file path is loaded +//! here (async) and rewritten to inline bytes + MIME, per the codec's policy. +//! Loads that fail drop the part silently — the long-standing contract — and +//! non-file URLs and already-inline data pass through untouched. +//! +//! Shared infra introduced ahead of its consumers: the per-dialect codecs +//! (anthropic/openai_responses/gemini) each construct their own +//! [`AttachmentPolicy`] and call [`resolve`] from their adapter shells when +//! they are wired. Until the first of those lands, nothing here is reachable. + +// Each dialect adapter that wires this in removes the allow as part of its +// rewire; harmless if it lingers when several land in parallel. +#![allow( + dead_code, + reason = "Attachment-resolution infra added ahead of the dialect codecs (PRs 3-5) that \ + construct an AttachmentPolicy and call resolve from their adapter shells." +)] + +use crate::providers::common; +use crate::types::{AudioData, ContentPart, DocumentData, ImageData, Request}; + +/// Which attachment kinds a codec loads from local file paths. Each dialect +/// adapter constructs the policy it wants (e.g. images + documents but not +/// audio for Anthropic, which renders audio as a text placeholder). +#[derive(Clone, Copy)] +pub(crate) struct AttachmentPolicy { + pub images: bool, + pub documents: bool, + pub audio: bool, +} + +/// Return a copy of `request` with file-path attachments (per `policy`) +/// resolved to inline data. Parts whose file fails to load are dropped. +pub(crate) async fn resolve(request: &Request, policy: AttachmentPolicy) -> Request { + let mut resolved = request.clone(); + for message in &mut resolved.messages { + let mut new_content = Vec::with_capacity(message.content.len()); + for part in std::mem::take(&mut message.content) { + if let Some(part) = resolve_part(part, policy).await { + new_content.push(part); + } + } + message.content = new_content; + } + resolved +} + +/// Resolve a single part. `None` means the part was dropped (load error). +async fn resolve_part(part: ContentPart, policy: AttachmentPolicy) -> Option { + match part { + ContentPart::Image(img) if policy.images && is_local_file(img.url.as_deref()) => { + // `is_local_file` guarantees `url` is `Some`. + let url = img.url.as_deref().unwrap_or_default(); + match common::load_file_bytes(url).await { + Ok((data, mime)) => Some(ContentPart::Image(ImageData { + url: None, + data: Some(data), + media_type: Some(mime), + detail: img.detail, + })), + Err(_) => None, + } + } + ContentPart::Document(doc) if policy.documents && is_local_file(doc.url.as_deref()) => { + let url = doc.url.as_deref().unwrap_or_default(); + match common::load_file_bytes(url).await { + Ok((data, mime)) => Some(ContentPart::Document(DocumentData { + url: None, + data: Some(data), + media_type: Some(mime), + file_name: doc.file_name, + })), + Err(_) => None, + } + } + ContentPart::Audio(audio) if policy.audio && is_local_file(audio.url.as_deref()) => { + let url = audio.url.as_deref().unwrap_or_default(); + match common::load_file_bytes(url).await { + Ok((data, mime)) => Some(ContentPart::Audio(AudioData { + url: None, + data: Some(data), + media_type: Some(mime), + })), + Err(_) => None, + } + } + other => Some(other), + } +} + +fn is_local_file(url: Option<&str>) -> bool { + url.is_some_and(common::is_file_path) +} diff --git a/lib/crates/fabro-llm/src/lib.rs b/lib/crates/fabro-llm/src/lib.rs index e1f1c9b01..afe1bc306 100644 --- a/lib/crates/fabro-llm/src/lib.rs +++ b/lib/crates/fabro-llm/src/lib.rs @@ -1,4 +1,5 @@ pub mod adapter_registry; +mod attachments; pub mod client; mod codec; pub mod error; diff --git a/lib/crates/fabro-llm/src/providers/common.rs b/lib/crates/fabro-llm/src/providers/common.rs index c865c0859..883a4634d 100644 --- a/lib/crates/fabro-llm/src/providers/common.rs +++ b/lib/crates/fabro-llm/src/providers/common.rs @@ -110,7 +110,7 @@ pub fn mime_from_extension(path: &str) -> &str { clippy::disallowed_methods, reason = "Attachment path expansion supports the conventional HOME env var." )] -pub async fn load_file_as_base64(path: &str) -> Result<(String, String), std::io::Error> { +pub async fn load_file_bytes(path: &str) -> Result<(Vec, String), std::io::Error> { let expanded = path.strip_prefix("~/").map_or_else( || path.to_string(), |rest| { @@ -122,8 +122,17 @@ pub async fn load_file_as_base64(path: &str) -> Result<(String, String), std::io std::io::Error::new(err.kind(), format!("read attachment {expanded}: {err}")) })?; let mime = mime_from_extension(&expanded).to_string(); - let b64 = BASE64_STANDARD.encode(&data); - Ok((b64, mime)) + Ok((data, mime)) +} + +/// Read a file and return base64-encoded contents plus the inferred MIME type. +/// +/// # Errors +/// +/// Returns an error if the file cannot be read. +pub async fn load_file_as_base64(path: &str) -> Result<(String, String), std::io::Error> { + let (data, mime) = load_file_bytes(path).await?; + Ok((BASE64_STANDARD.encode(&data), mime)) } /// Extract the `Retry-After` header value from an HTTP response as seconds.