diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 2d96efa6077..e7d911f5fd9 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -73,6 +73,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -1470,6 +1479,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -1643,6 +1663,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -3453,6 +3483,27 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-testkit" +version = "0.1.0" +dependencies = [ + "flate2", + "futures-util", + "reqwest 0.12.28", + "rstest", + "semver", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "target-lexicon", + "tempfile", + "thiserror 2.0.19", + "tokio", + "toml", + "zip", +] + [[package]] name = "litellm-token-counter" version = "0.1.0" @@ -5206,6 +5257,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5537,6 +5597,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -5806,6 +5877,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -5822,9 +5917,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap 2.14.0", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.4", ] [[package]] @@ -5833,9 +5928,15 @@ version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tonic" version = "0.14.6" @@ -6597,6 +6698,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + [[package]] name = "winnow" version = "1.0.4" @@ -6659,6 +6766,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -6784,6 +6901,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.14.0", + "memchr", + "thiserror 2.0.19", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.7" @@ -6795,3 +6929,15 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 0c7236e807e..022e8f13311 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -81,6 +81,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" +flate2 = "1" +semver = "1" +tar = "0.4" +target-lexicon = "0.13.5" +tempfile = "3" +zip = { version = "2", default-features = false, features = ["deflate"] } moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" diff --git a/litellm-rust/crates/testkit/Cargo.toml b/litellm-rust/crates/testkit/Cargo.toml new file mode 100644 index 00000000000..98a36a1e87f --- /dev/null +++ b/litellm-rust/crates/testkit/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "litellm-testkit" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +flate2.workspace = true +reqwest.workspace = true +serde.workspace = true +semver.workspace = true +serde_json.workspace = true +sha2.workspace = true +tar.workspace = true +target-lexicon.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "process"] } +zip.workspace = true + +[dev-dependencies] +flate2.workspace = true +rstest.workspace = true +sha2.workspace = true +tar.workspace = true +target-lexicon.workspace = true +futures-util.workspace = true +tempfile.workspace = true +tokio.workspace = true +toml = "0.9" +zip.workspace = true diff --git a/litellm-rust/crates/testkit/src/agent/claude.rs b/litellm-rust/crates/testkit/src/agent/claude.rs new file mode 100644 index 00000000000..6870fff6bf8 --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/claude.rs @@ -0,0 +1,181 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use semver::Version; +use serde::Deserialize; + +use super::{ + Configure, Drive, Install, LaunchSpec, Outcome, Prompt, Settings, Usage, Wire, env, json_lines, + path_string, +}; +use crate::install::release::parse; +use crate::install::{Packaging, Release}; +use crate::{Error, Fetch, Target}; + +const RELEASES: &str = "https://downloads.claude.ai/claude-code-releases"; + +pub struct ClaudeCode; + +#[derive(Deserialize)] +struct Manifest { + platforms: BTreeMap, +} + +#[derive(Deserialize)] +struct Platform { + checksum: String, +} + +impl Install for ClaudeCode { + fn binary(&self) -> &'static str { + "claude" + } + + async fn release( + &self, + fetch: &impl Fetch, + version: &Version, + target: Target, + ) -> Result { + let manifest_url = format!("{RELEASES}/{version}/manifest.json"); + let manifest: Manifest = parse(&manifest_url, &fetch.get(&manifest_url).await?)?; + let key = format!( + "{}-{}{}", + target.os_name(), + target.arch_name(), + target.musl_suffix() + ); + let platform = manifest + .platforms + .get(&key) + .ok_or_else(|| Error::AssetNotFound(key.clone()))?; + Ok(Release { + url: format!("{RELEASES}/{version}/{key}/claude"), + asset: key, + sha256: platform.checksum.clone(), + packaging: Packaging::Bare, + }) + } +} + +impl Configure for ClaudeCode { + fn configure( + &self, + _version: &Version, + settings: &Settings, + home: &Path, + ) -> Result { + if settings.wire != Wire::Messages { + return Err(Error::UnsupportedWire { + agent: "claude", + wire: settings.wire, + }); + } + Ok(LaunchSpec { + env: env([ + ("HOME", path_string(home)), + ("CLAUDE_CONFIG_DIR", path_string(&home.join(".claude"))), + ("ANTHROPIC_BASE_URL", settings.base_url.clone()), + ("ANTHROPIC_AUTH_TOKEN", settings.api_key.clone()), + ("ANTHROPIC_MODEL", settings.model.clone()), + ("DISABLE_AUTOUPDATER", "1".to_owned()), + ]), + files: BTreeMap::new(), + }) + } +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Event { + Assistant { + message: AssistantMessage, + }, + Result(Finished), + #[serde(other)] + Other, +} + +#[derive(Deserialize)] +struct AssistantMessage { + content: Vec, +} + +#[derive(Deserialize)] +struct Block { + #[serde(rename = "type")] + kind: String, + name: Option, +} + +#[derive(Deserialize)] +struct Finished { + is_error: bool, + result: Option, + usage: Option, +} + +#[derive(Deserialize)] +struct TokenUsage { + input_tokens: u64, + output_tokens: u64, +} + +impl Drive for ClaudeCode { + fn args(&self, _version: &Version, settings: &Settings, prompt: &Prompt) -> Vec { + let base = [ + "-p", + &prompt.text, + "--output-format", + "stream-json", + "--verbose", + "--model", + &settings.model, + ]; + let tools = ["--allowedTools", "Bash,Read,Write,Edit"]; + base.into_iter() + .chain(tools.into_iter().filter(|_| prompt.allow_tools)) + .map(str::to_owned) + .collect() + } + + fn parse(&self, _version: &Version, stdout: &str) -> Outcome { + let events: Vec = json_lines(stdout).collect(); + let tool_calls = events + .iter() + .filter_map(|event| match event { + Event::Assistant { message } => Some(&message.content), + _ => None, + }) + .flatten() + .filter(|block| block.kind == "tool_use") + .filter_map(|block| block.name.clone()) + .collect(); + let finished = events.into_iter().find_map(|event| match event { + Event::Result(finished) => Some(finished), + _ => None, + }); + let Some(finished) = finished else { + return Outcome { + tool_calls, + ..Outcome::default() + }; + }; + let result = finished.result.unwrap_or_default(); + let (text, errors) = if finished.is_error { + (String::new(), vec![result]) + } else { + (result, Vec::new()) + }; + Outcome { + text, + tool_calls, + usage: finished.usage.map_or_else(Usage::default, |usage| Usage { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + }), + errors, + exit_code: None, + } + } +} diff --git a/litellm-rust/crates/testkit/src/agent/codex.rs b/litellm-rust/crates/testkit/src/agent/codex.rs new file mode 100644 index 00000000000..6749e81a471 --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/codex.rs @@ -0,0 +1,174 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use semver::Version; +use serde::Deserialize; + +use super::{ + Configure, Drive, Install, LaunchSpec, Outcome, Prompt, Settings, Usage, Wire, env, json_lines, + path_string, quoted, v1, +}; +use crate::install::release::github_release; +use crate::install::{Packaging, Release}; +use crate::target::{Arch, Os}; +use crate::{Error, Fetch, Target}; + +const RELEASES: &str = "https://api.github.com/repos/openai/codex/releases/tags"; + +pub struct Codex; + +fn triple(target: Target) -> String { + let arch = match target.arch { + Arch::Aarch64 => "aarch64", + Arch::X86_64 => "x86_64", + }; + match target.os { + Os::Macos => format!("{arch}-apple-darwin"), + Os::Linux => format!("{arch}-unknown-linux-musl"), + } +} + +impl Install for Codex { + fn binary(&self) -> &'static str { + "codex" + } + + async fn release( + &self, + fetch: &impl Fetch, + version: &Version, + target: Target, + ) -> Result { + let triple = triple(target); + github_release( + fetch, + RELEASES, + &format!("rust-v{version}"), + &format!("codex-{triple}.tar.gz"), + Packaging::TarGz { + member: format!("codex-{triple}"), + }, + ) + .await + } +} + +impl Configure for Codex { + fn configure( + &self, + _version: &Version, + settings: &Settings, + home: &Path, + ) -> Result { + if settings.wire != Wire::Responses { + return Err(Error::UnsupportedWire { + agent: "codex", + wire: settings.wire, + }); + } + let config = format!( + "model = {model}\nmodel_provider = \"litellm\"\n\n[model_providers.litellm]\nname = \"LiteLLM\"\nbase_url = {base_url}\nenv_key = \"LITELLM_API_KEY\"\nwire_api = \"responses\"\n", + model = quoted(&settings.model), + base_url = quoted(&v1(settings)), + ); + Ok(LaunchSpec { + env: env([ + ("HOME", path_string(home)), + ("CODEX_HOME", path_string(&home.join(".codex"))), + ("LITELLM_API_KEY", settings.api_key.clone()), + ]), + files: BTreeMap::from([(PathBuf::from(".codex/config.toml"), config)]), + }) + } +} + +#[derive(Deserialize)] +enum EventKind { + #[serde(rename = "item.completed")] + ItemCompleted, + #[serde(rename = "turn.completed")] + TurnCompleted, + #[serde(rename = "turn.failed")] + TurnFailed, + #[serde(other)] + Other, +} + +#[derive(Deserialize)] +struct Event { + #[serde(rename = "type")] + kind: EventKind, + item: Option, + usage: Option, + error: Option, +} + +#[derive(Deserialize)] +struct Item { + #[serde(rename = "type")] + kind: String, + text: Option, +} + +#[derive(Deserialize)] +struct TokenUsage { + input_tokens: u64, + output_tokens: u64, +} + +#[derive(Deserialize)] +struct Failure { + message: String, +} + +const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "error"]; + +impl Drive for Codex { + fn args(&self, _version: &Version, _settings: &Settings, prompt: &Prompt) -> Vec { + let sandbox = ["--sandbox", "workspace-write"]; + ["exec", "--json", "--skip-git-repo-check"] + .into_iter() + .chain(sandbox.into_iter().filter(|_| prompt.allow_tools)) + .chain([prompt.text.as_str()]) + .map(str::to_owned) + .collect() + } + + fn parse(&self, _version: &Version, stdout: &str) -> Outcome { + let events: Vec = json_lines(stdout).collect(); + let items: Vec<&Item> = events + .iter() + .filter(|event| matches!(event.kind, EventKind::ItemCompleted)) + .filter_map(|event| event.item.as_ref()) + .collect(); + Outcome { + text: items + .iter() + .rev() + .find(|item| item.kind == "agent_message") + .and_then(|item| item.text.clone()) + .unwrap_or_default(), + tool_calls: items + .iter() + .filter(|item| !NON_TOOL_ITEMS.contains(&item.kind.as_str())) + .map(|item| item.kind.clone()) + .collect(), + usage: events + .iter() + .filter(|event| matches!(event.kind, EventKind::TurnCompleted)) + .filter_map(|event| event.usage.as_ref()) + .map(|usage| Usage { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + }) + .fold(Usage::default(), |total, turn| total + turn), + errors: events + .iter() + .filter(|event| matches!(event.kind, EventKind::TurnFailed)) + .filter_map(|event| event.error.as_ref()) + .map(|failure| failure.message.clone()) + .collect(), + exit_code: None, + } + } +} diff --git a/litellm-rust/crates/testkit/src/agent/configure.rs b/litellm-rust/crates/testkit/src/agent/configure.rs new file mode 100644 index 00000000000..a095aacc92f --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/configure.rs @@ -0,0 +1,69 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use semver::Version; + +use crate::Error; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Wire { + ChatCompletions, + Messages, + Responses, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Settings { + pub base_url: String, + pub api_key: String, + pub model: String, + pub wire: Wire, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LaunchSpec { + pub env: BTreeMap, + pub files: BTreeMap, +} + +impl LaunchSpec { + pub fn write_files(&self, home: &Path) -> std::io::Result<()> { + self.files.iter().try_for_each(|(relative, contents)| { + let path = home.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, contents) + }) + } +} + +pub trait Configure { + fn configure( + &self, + version: &Version, + settings: &Settings, + home: &Path, + ) -> Result; +} + +pub(crate) fn env( + pairs: impl IntoIterator, +) -> BTreeMap { + pairs + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect() +} + +pub(crate) fn path_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +pub(crate) fn quoted(value: &str) -> String { + serde_json::Value::from(value).to_string() +} + +pub(crate) fn v1(settings: &Settings) -> String { + format!("{}/v1", settings.base_url.trim_end_matches('/')) +} diff --git a/litellm-rust/crates/testkit/src/agent/drive.rs b/litellm-rust/crates/testkit/src/agent/drive.rs new file mode 100644 index 00000000000..2c238843ed7 --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/drive.rs @@ -0,0 +1,57 @@ +use std::ops::Add; + +use semver::Version; + +use crate::Settings; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Prompt { + pub text: String, + pub allow_tools: bool, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Usage { + pub input_tokens: u64, + pub output_tokens: u64, +} + +impl Add for Usage { + type Output = Self; + + fn add(self, other: Self) -> Self { + Self { + input_tokens: self.input_tokens + other.input_tokens, + output_tokens: self.output_tokens + other.output_tokens, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Outcome { + pub text: String, + pub tool_calls: Vec, + pub usage: Usage, + pub errors: Vec, + pub exit_code: Option, +} + +impl Outcome { + pub fn succeeded(&self) -> bool { + self.exit_code == Some(0) && self.errors.is_empty() + } +} + +pub trait Drive { + fn args(&self, version: &Version, settings: &Settings, prompt: &Prompt) -> Vec; + + fn parse(&self, version: &Version, stdout: &str) -> Outcome; +} + +pub(crate) fn json_lines<'a, T: serde::de::DeserializeOwned + 'a>( + stdout: &'a str, +) -> impl Iterator + 'a { + stdout + .lines() + .filter_map(|line| serde_json::from_str(line).ok()) +} diff --git a/litellm-rust/crates/testkit/src/agent/install.rs b/litellm-rust/crates/testkit/src/agent/install.rs new file mode 100644 index 00000000000..3f1a0f8950d --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/install.rs @@ -0,0 +1,17 @@ +use std::future::Future; + +use semver::Version; + +use crate::install::Release; +use crate::{Error, Fetch, Target}; + +pub trait Install: Sync { + fn binary(&self) -> &'static str; + + fn release( + &self, + fetch: &impl Fetch, + version: &Version, + target: Target, + ) -> impl Future> + Send; +} diff --git a/litellm-rust/crates/testkit/src/agent/mod.rs b/litellm-rust/crates/testkit/src/agent/mod.rs new file mode 100644 index 00000000000..03a70583f66 --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/mod.rs @@ -0,0 +1,20 @@ +mod claude; +mod codex; +mod configure; +mod drive; +mod install; +mod opencode; + +pub use claude::ClaudeCode; +pub use codex::Codex; +pub use configure::{Configure, LaunchSpec, Settings, Wire}; +pub use drive::{Drive, Outcome, Prompt, Usage}; +pub use install::Install; +pub use opencode::Opencode; + +pub(crate) use configure::{env, path_string, quoted, v1}; +pub(crate) use drive::json_lines; + +pub trait Agent: Install + Configure + Drive {} + +impl Agent for T {} diff --git a/litellm-rust/crates/testkit/src/agent/opencode.rs b/litellm-rust/crates/testkit/src/agent/opencode.rs new file mode 100644 index 00000000000..a1b2fa01eef --- /dev/null +++ b/litellm-rust/crates/testkit/src/agent/opencode.rs @@ -0,0 +1,187 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use semver::Version; +use serde::Deserialize; + +use super::{ + Configure, Drive, Install, LaunchSpec, Outcome, Prompt, Settings, Usage, Wire, env, json_lines, + path_string, v1, +}; +use crate::install::release::github_release; +use crate::install::{Packaging, Release}; +use crate::target::Os; +use crate::{Error, Fetch, Target}; + +const RELEASES: &str = "https://api.github.com/repos/sst/opencode/releases/tags"; + +pub struct Opencode; + +impl Install for Opencode { + fn binary(&self) -> &'static str { + "opencode" + } + + async fn release( + &self, + fetch: &impl Fetch, + version: &Version, + target: Target, + ) -> Result { + let stem = format!( + "opencode-{}-{}{}", + target.os_name(), + target.arch_name(), + target.musl_suffix() + ); + let member = "opencode".to_owned(); + let (asset, packaging) = match target.os { + Os::Macos => (format!("{stem}.zip"), Packaging::Zip { member }), + Os::Linux => (format!("{stem}.tar.gz"), Packaging::TarGz { member }), + }; + github_release(fetch, RELEASES, &format!("v{version}"), &asset, packaging).await + } +} + +impl Configure for Opencode { + fn configure( + &self, + _version: &Version, + settings: &Settings, + home: &Path, + ) -> Result { + let npm = match settings.wire { + Wire::ChatCompletions => "@ai-sdk/openai-compatible", + Wire::Responses => "@ai-sdk/openai", + Wire::Messages => "@ai-sdk/anthropic", + }; + let config = serde_json::json!({ + "$schema": "https://opencode.ai/config.json", + "model": format!("litellm/{}", settings.model), + "provider": { + "litellm": { + "npm": npm, + "name": "LiteLLM", + "options": { "baseURL": v1(settings), "apiKey": settings.api_key }, + "models": { settings.model.clone(): { "name": settings.model } }, + } + }, + }); + Ok(LaunchSpec { + env: env([ + ("HOME", path_string(home)), + ("XDG_CONFIG_HOME", path_string(&home.join(".config"))), + ("XDG_DATA_HOME", path_string(&home.join(".local/share"))), + ("OPENCODE_DISABLE_AUTOUPDATE", "true".to_owned()), + ]), + files: BTreeMap::from([( + PathBuf::from(".config/opencode/opencode.json"), + config.to_string(), + )]), + }) + } +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Event { + Text { + part: TextPart, + }, + ToolUse { + part: ToolPart, + }, + StepFinish { + part: StepFinish, + }, + Error { + error: Failure, + }, + #[serde(other)] + Other, +} + +#[derive(Deserialize)] +struct TextPart { + text: String, +} + +#[derive(Deserialize)] +struct ToolPart { + tool: String, +} + +#[derive(Deserialize)] +struct StepFinish { + tokens: Tokens, +} + +#[derive(Deserialize)] +struct Tokens { + input: u64, + output: u64, +} + +#[derive(Deserialize)] +struct Failure { + name: String, + data: Option, +} + +#[derive(Deserialize)] +struct FailureData { + message: Option, +} + +impl Drive for Opencode { + fn args(&self, _version: &Version, _settings: &Settings, prompt: &Prompt) -> Vec { + ["run", "--format", "json", &prompt.text] + .map(str::to_owned) + .to_vec() + } + + fn parse(&self, _version: &Version, stdout: &str) -> Outcome { + let events: Vec = json_lines(stdout).collect(); + Outcome { + text: events + .iter() + .rev() + .find_map(|event| match event { + Event::Text { part } => Some(part.text.clone()), + _ => None, + }) + .unwrap_or_default(), + tool_calls: events + .iter() + .filter_map(|event| match event { + Event::ToolUse { part } => Some(part.tool.clone()), + _ => None, + }) + .collect(), + usage: events + .iter() + .filter_map(|event| match event { + Event::StepFinish { part } => Some(Usage { + input_tokens: part.tokens.input, + output_tokens: part.tokens.output, + }), + _ => None, + }) + .fold(Usage::default(), |total, step| total + step), + errors: events + .iter() + .filter_map(|event| match event { + Event::Error { error } => Some( + error + .data + .as_ref() + .and_then(|data| data.message.clone()) + .unwrap_or_else(|| error.name.clone()), + ), + _ => None, + }) + .collect(), + exit_code: None, + } + } +} diff --git a/litellm-rust/crates/testkit/src/error.rs b/litellm-rust/crates/testkit/src/error.rs new file mode 100644 index 00000000000..03520d827f1 --- /dev/null +++ b/litellm-rust/crates/testkit/src/error.rs @@ -0,0 +1,56 @@ +use std::io; +use std::path::PathBuf; + +use thiserror::Error; + +use crate::Wire; + +#[derive(Debug, Error)] +pub enum Error { + #[error("unsupported target {0}")] + UnsupportedTarget(String), + #[error("{0} is not a plain x.y.z release version")] + InvalidVersion(String), + #[error("request to {url} failed")] + Request { + url: String, + #[source] + source: reqwest::Error, + }, + #[error("{url} answered with status {status}")] + Status { url: String, status: u16 }, + #[error("release metadata at {url} is malformed")] + Metadata { + url: String, + #[source] + source: serde_json::Error, + }, + #[error("release has no asset named {0}")] + AssetNotFound(String), + #[error("release publishes no sha256 for {0}")] + MissingChecksum(String), + #[error("sha256 mismatch for {asset}: expected {expected}, got {actual}")] + ChecksumMismatch { + asset: String, + expected: String, + actual: String, + }, + #[error("archive does not contain {0}")] + ArchiveMemberNotFound(String), + #[error("archive is unreadable")] + Archive(#[source] io::Error), + #[error("zip archive is unreadable")] + Zip(#[from] zip::result::ZipError), + #[error("{binary} reports version '{reported}', expected {expected}")] + VersionMismatch { + binary: PathBuf, + expected: String, + reported: String, + }, + #[error("{agent} cannot talk to the gateway over {wire:?}")] + UnsupportedWire { agent: &'static str, wire: Wire }, + #[error("agent did not finish within {0:?}")] + Timeout(std::time::Duration), + #[error("io failure")] + Io(#[from] io::Error), +} diff --git a/litellm-rust/crates/testkit/src/install/archive.rs b/litellm-rust/crates/testkit/src/install/archive.rs new file mode 100644 index 00000000000..c8d08f66f47 --- /dev/null +++ b/litellm-rust/crates/testkit/src/install/archive.rs @@ -0,0 +1,52 @@ +use std::io::{Cursor, Read}; + +use flate2::read::GzDecoder; +use sha2::{Digest, Sha256}; + +use super::release::Packaging; +use crate::Error; + +pub(crate) fn verify_sha256(asset: &str, expected: &str, bytes: &[u8]) -> Result<(), Error> { + let actual = format!("{:x}", Sha256::digest(bytes)); + if actual.eq_ignore_ascii_case(expected) { + return Ok(()); + } + Err(Error::ChecksumMismatch { + asset: asset.to_owned(), + expected: expected.to_owned(), + actual, + }) +} + +pub(crate) fn extract_binary(packaging: &Packaging, bytes: &[u8]) -> Result, Error> { + match packaging { + Packaging::Bare => Ok(bytes.to_vec()), + Packaging::TarGz { member } => extract_tar_gz(member, bytes), + Packaging::Zip { member } => extract_zip(member, bytes), + } +} + +fn extract_tar_gz(member: &str, bytes: &[u8]) -> Result, Error> { + let mut archive = tar::Archive::new(GzDecoder::new(bytes)); + for entry in archive.entries().map_err(Error::Archive)? { + let mut entry = entry.map_err(Error::Archive)?; + let path = entry.path().map_err(Error::Archive)?; + if path.file_name().is_some_and(|name| name == member) { + let mut binary = Vec::new(); + entry.read_to_end(&mut binary).map_err(Error::Archive)?; + return Ok(binary); + } + } + Err(Error::ArchiveMemberNotFound(member.to_owned())) +} + +fn extract_zip(member: &str, bytes: &[u8]) -> Result, Error> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes))?; + let mut file = archive.by_name(member).map_err(|error| match error { + zip::result::ZipError::FileNotFound => Error::ArchiveMemberNotFound(member.to_owned()), + other => Error::Zip(other), + })?; + let mut binary = Vec::new(); + file.read_to_end(&mut binary).map_err(Error::Archive)?; + Ok(binary) +} diff --git a/litellm-rust/crates/testkit/src/install/fetch.rs b/litellm-rust/crates/testkit/src/install/fetch.rs new file mode 100644 index 00000000000..73008f7a0da --- /dev/null +++ b/litellm-rust/crates/testkit/src/install/fetch.rs @@ -0,0 +1,55 @@ +use std::future::Future; + +use crate::Error; + +pub trait Fetch: Sync { + fn get(&self, url: &str) -> impl Future, Error>> + Send; +} + +pub struct HttpFetch { + client: reqwest::Client, + github_token: Option, +} + +impl HttpFetch { + pub fn new(github_token: Option) -> Self { + Self { + client: reqwest::Client::new(), + github_token, + } + } + + pub fn from_env() -> Self { + Self::new(std::env::var("GITHUB_TOKEN").ok()) + } +} + +impl Fetch for HttpFetch { + async fn get(&self, url: &str) -> Result, Error> { + let request = self + .client + .get(url) + .header("user-agent", "litellm-testkit") + .header("accept", "application/json, application/octet-stream"); + let request = match ( + &self.github_token, + url.starts_with("https://api.github.com/"), + ) { + (Some(token), true) => request.bearer_auth(token), + _ => request, + }; + let request_error = |source| Error::Request { + url: url.to_owned(), + source, + }; + let response = request.send().await.map_err(request_error)?; + let status = response.status(); + if !status.is_success() { + return Err(Error::Status { + url: url.to_owned(), + status: status.as_u16(), + }); + } + Ok(response.bytes().await.map_err(request_error)?.to_vec()) + } +} diff --git a/litellm-rust/crates/testkit/src/install/mod.rs b/litellm-rust/crates/testkit/src/install/mod.rs new file mode 100644 index 00000000000..1104bcec102 --- /dev/null +++ b/litellm-rust/crates/testkit/src/install/mod.rs @@ -0,0 +1,118 @@ +mod archive; +mod fetch; +pub(crate) mod release; + +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::atomic::{AtomicU64, Ordering}; + +use semver::Version; +use tokio::fs; +use tokio::process::Command; + +use crate::{Error, Install, Target}; +use archive::{extract_binary, verify_sha256}; + +static STAGING_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Installed { + pub version: Version, + pub binary: PathBuf, +} + +pub struct Installer { + fetch: F, + cache_root: PathBuf, + target: Target, +} + +impl Installer { + pub fn new(fetch: F, cache_root: impl Into, target: Target) -> Self { + Self { + fetch, + cache_root: cache_root.into(), + target, + } + } + + pub async fn install( + &self, + agent: &impl Install, + version: &Version, + ) -> Result { + validate_release(version)?; + let dir = self + .cache_root + .join(agent.binary()) + .join(version.to_string()); + let binary = dir.join(agent.binary()); + let installed = Installed { + version: version.clone(), + binary: binary.clone(), + }; + if fs::try_exists(&binary).await? && probe_version(&binary, version).await.is_ok() { + return Ok(installed); + } + + let release = agent.release(&self.fetch, version, self.target).await?; + let archive = self.fetch.get(&release.url).await?; + verify_sha256(&release.asset, &release.sha256, &archive)?; + let contents = extract_binary(&release.packaging, &archive)?; + + fs::create_dir_all(&dir).await?; + let staging = dir.join(format!( + ".{}.{}.{}.partial", + agent.binary(), + std::process::id(), + STAGING_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&staging, contents).await?; + fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755)).await?; + fs::rename(&staging, &binary).await?; + + match probe_version(&binary, version).await { + Ok(()) => Ok(installed), + Err(error) => { + fs::remove_file(&binary).await?; + Err(error) + } + } + } +} + +fn validate_release(version: &Version) -> Result<(), Error> { + if version.pre.is_empty() && version.build.is_empty() { + return Ok(()); + } + Err(Error::InvalidVersion(version.to_string())) +} + +async fn probe_version(binary: &Path, expected: &Version) -> Result<(), Error> { + let home = std::env::temp_dir(); + let output = Command::new(binary) + .arg("--version") + .env_clear() + .env("HOME", home) + .env("DISABLE_AUTOUPDATER", "1") + .stdin(Stdio::null()) + .output() + .await?; + let stdout = String::from_utf8_lossy(&output.stdout); + if stdout + .split_whitespace() + .filter_map(|token| Version::parse(token).ok()) + .any(|reported| &reported == expected) + { + return Ok(()); + } + Err(Error::VersionMismatch { + binary: binary.to_owned(), + expected: expected.to_string(), + reported: stdout.trim().to_owned(), + }) +} + +pub use fetch::{Fetch, HttpFetch}; +pub use release::{Packaging, Release}; diff --git a/litellm-rust/crates/testkit/src/install/release.rs b/litellm-rust/crates/testkit/src/install/release.rs new file mode 100644 index 00000000000..a21b9a14f1b --- /dev/null +++ b/litellm-rust/crates/testkit/src/install/release.rs @@ -0,0 +1,65 @@ +use serde::Deserialize; + +use crate::{Error, Fetch}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Packaging { + Bare, + TarGz { member: String }, + Zip { member: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Release { + pub asset: String, + pub url: String, + pub sha256: String, + pub packaging: Packaging, +} + +#[derive(Deserialize)] +struct GithubRelease { + assets: Vec, +} + +#[derive(Deserialize)] +struct GithubAsset { + name: String, + digest: Option, + browser_download_url: String, +} + +pub(crate) async fn github_release( + fetch: &impl Fetch, + releases_url: &str, + tag: &str, + asset_name: &str, + packaging: Packaging, +) -> Result { + let url = format!("{releases_url}/{tag}"); + let release: GithubRelease = parse(&url, &fetch.get(&url).await?)?; + let asset = release + .assets + .into_iter() + .find(|asset| asset.name == asset_name) + .ok_or_else(|| Error::AssetNotFound(asset_name.to_owned()))?; + let sha256 = asset + .digest + .as_deref() + .and_then(|digest| digest.strip_prefix("sha256:")) + .ok_or_else(|| Error::MissingChecksum(asset_name.to_owned()))? + .to_owned(); + Ok(Release { + asset: asset.name, + url: asset.browser_download_url, + sha256, + packaging, + }) +} + +pub(crate) fn parse Deserialize<'de>>(url: &str, body: &[u8]) -> Result { + serde_json::from_slice(body).map_err(|source| Error::Metadata { + url: url.to_owned(), + source, + }) +} diff --git a/litellm-rust/crates/testkit/src/lib.rs b/litellm-rust/crates/testkit/src/lib.rs new file mode 100644 index 00000000000..9ea6123a176 --- /dev/null +++ b/litellm-rust/crates/testkit/src/lib.rs @@ -0,0 +1,15 @@ +mod agent; +mod error; +mod install; +mod session; +mod target; + +pub use agent::{ + Agent, ClaudeCode, Codex, Configure, Drive, Install, LaunchSpec, Opencode, Outcome, Prompt, + Settings, Usage, Wire, +}; +pub use error::Error; +pub use install::{Fetch, HttpFetch, Installed, Installer, Packaging, Release}; +pub use semver::Version; +pub use session::Session; +pub use target::{Arch, Os, Target}; diff --git a/litellm-rust/crates/testkit/src/session.rs b/litellm-rust/crates/testkit/src/session.rs new file mode 100644 index 00000000000..6b06e756cca --- /dev/null +++ b/litellm-rust/crates/testkit/src/session.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use semver::Version; +use tokio::process::Command; +use tokio::time::timeout; + +use crate::{Configure, Drive, Error, Installed, Outcome, Prompt, Settings}; + +const STDERR_LIMIT_CHARS: usize = 2000; + +pub struct Session { + binary: PathBuf, + home: PathBuf, + version: Version, + settings: Settings, + env: BTreeMap, +} + +impl Session { + pub fn prepare( + agent: &impl Configure, + installed: &Installed, + settings: Settings, + home: impl Into, + ) -> Result { + let home = home.into(); + let spec = agent.configure(&installed.version, &settings, &home)?; + spec.write_files(&home)?; + Ok(Self { + binary: installed.binary.clone(), + home, + version: installed.version.clone(), + settings, + env: spec.env, + }) + } + + pub async fn run( + &self, + agent: &impl Drive, + prompt: &Prompt, + limit: Duration, + ) -> Result { + let child = Command::new(&self.binary) + .args(agent.args(&self.version, &self.settings, prompt)) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .envs(&self.env) + .current_dir(&self.home) + .stdin(Stdio::null()) + .kill_on_drop(true) + .output(); + let output = timeout(limit, child) + .await + .map_err(|_| Error::Timeout(limit))??; + let parsed = agent.parse(&self.version, &String::from_utf8_lossy(&output.stdout)); + let failed_silently = !output.status.success() && parsed.errors.is_empty(); + Ok(Outcome { + errors: if failed_silently { + vec![ + String::from_utf8_lossy(&output.stderr) + .chars() + .take(STDERR_LIMIT_CHARS) + .collect(), + ] + } else { + parsed.errors + }, + exit_code: output.status.code(), + ..parsed + }) + } +} diff --git a/litellm-rust/crates/testkit/src/target.rs b/litellm-rust/crates/testkit/src/target.rs new file mode 100644 index 00000000000..a9d4b012d52 --- /dev/null +++ b/litellm-rust/crates/testkit/src/target.rs @@ -0,0 +1,69 @@ +use target_lexicon::{Architecture, Environment, OperatingSystem, Triple}; + +use crate::Error; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Os { + Macos, + Linux, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Arch { + Aarch64, + X86_64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Target { + pub os: Os, + pub arch: Arch, + pub musl: bool, +} + +impl Target { + pub fn host() -> Result { + Self::try_from(&Triple::host()) + } + + pub(crate) const fn os_name(self) -> &'static str { + match self.os { + Os::Macos => "darwin", + Os::Linux => "linux", + } + } + + pub(crate) const fn arch_name(self) -> &'static str { + match self.arch { + Arch::Aarch64 => "arm64", + Arch::X86_64 => "x64", + } + } + + pub(crate) const fn musl_suffix(self) -> &'static str { + if self.musl { "-musl" } else { "" } + } +} + +impl TryFrom<&Triple> for Target { + type Error = Error; + + fn try_from(triple: &Triple) -> Result { + let unsupported = || Error::UnsupportedTarget(triple.to_string()); + let os = match triple.operating_system { + OperatingSystem::Darwin(_) | OperatingSystem::MacOSX(_) => Os::Macos, + OperatingSystem::Linux => Os::Linux, + _ => return Err(unsupported()), + }; + let arch = match triple.architecture { + Architecture::Aarch64(_) => Arch::Aarch64, + Architecture::X86_64 => Arch::X86_64, + _ => return Err(unsupported()), + }; + Ok(Self { + os, + arch, + musl: triple.environment == Environment::Musl, + }) + } +} diff --git a/litellm-rust/crates/testkit/tests/configure.rs b/litellm-rust/crates/testkit/tests/configure.rs new file mode 100644 index 00000000000..ca3587c3474 --- /dev/null +++ b/litellm-rust/crates/testkit/tests/configure.rs @@ -0,0 +1,133 @@ +use std::path::Path; + +use litellm_testkit::{ClaudeCode, Codex, Configure, Error, Opencode, Settings, Version, Wire}; +use rstest::rstest; + +fn settings(wire: Wire) -> Settings { + Settings { + base_url: "http://localhost:4000/".to_owned(), + api_key: "sk-test \"quoted\"".to_owned(), + model: "some-model".to_owned(), + wire, + } +} + +fn version() -> Version { + Version::new(1, 2, 3) +} + +#[rstest] +#[case(&ClaudeCode, Wire::Messages)] +#[case(&Codex, Wire::Responses)] +#[case(&Opencode, Wire::ChatCompletions)] +fn every_agent_runs_inside_the_given_home(#[case] agent: &impl Configure, #[case] wire: Wire) { + let home = Path::new("/scratch/home"); + + let spec = agent.configure(&version(), &settings(wire), home).unwrap(); + + assert_eq!(spec.env["HOME"], "/scratch/home"); + assert!( + spec.env + .iter() + .filter(|(key, _)| key.ends_with("_HOME") || key.as_str() == "CLAUDE_CONFIG_DIR") + .all(|(_, value)| value.starts_with("/scratch/home")) + ); + assert!(spec.files.keys().all(|path| path.is_relative())); +} + +#[rstest] +#[case::claude_code(&ClaudeCode, &[Wire::ChatCompletions, Wire::Responses])] +#[case::codex(&Codex, &[Wire::ChatCompletions, Wire::Messages])] +fn wires_an_agent_cannot_speak_are_refused( + #[case] agent: &impl Configure, + #[case] refused: &[Wire], +) { + refused.iter().for_each(|wire| { + let result = agent.configure(&version(), &settings(*wire), Path::new("/h")); + + assert!(matches!(result, Err(Error::UnsupportedWire { wire: got, .. }) if got == *wire)); + }); +} + +#[test] +fn claude_code_points_at_the_gateway_root_with_the_key_and_model() { + let spec = ClaudeCode + .configure(&version(), &settings(Wire::Messages), Path::new("/h")) + .unwrap(); + + assert_eq!(spec.env["ANTHROPIC_BASE_URL"], "http://localhost:4000/"); + assert_eq!(spec.env["ANTHROPIC_AUTH_TOKEN"], "sk-test \"quoted\""); + assert_eq!(spec.env["ANTHROPIC_MODEL"], "some-model"); +} + +#[test] +fn codex_config_is_valid_toml_routing_the_responses_api_to_the_gateway() { + let dir = tempfile::tempdir().unwrap(); + let spec = Codex + .configure(&version(), &settings(Wire::Responses), dir.path()) + .unwrap(); + spec.write_files(dir.path()).unwrap(); + + let config: toml::Table = + toml::from_str(&std::fs::read_to_string(dir.path().join(".codex/config.toml")).unwrap()) + .unwrap(); + let provider = &config["model_providers"]["litellm"]; + + assert_eq!(config["model"].as_str(), Some("some-model")); + assert_eq!(config["model_provider"].as_str(), Some("litellm")); + assert_eq!( + provider["base_url"].as_str(), + Some("http://localhost:4000/v1") + ); + assert_eq!(provider["wire_api"].as_str(), Some("responses")); + let key_var = provider["env_key"].as_str().unwrap(); + assert_eq!(spec.env[key_var], "sk-test \"quoted\""); +} + +#[rstest] +#[case(Wire::ChatCompletions)] +#[case(Wire::Responses)] +#[case(Wire::Messages)] +fn opencode_config_is_valid_json_registering_the_gateway_model(#[case] wire: Wire) { + let dir = tempfile::tempdir().unwrap(); + let spec = Opencode + .configure(&version(), &settings(wire), dir.path()) + .unwrap(); + spec.write_files(dir.path()).unwrap(); + + let config: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(dir.path().join(".config/opencode/opencode.json")).unwrap(), + ) + .unwrap(); + let provider = &config["provider"]["litellm"]; + + assert_eq!(config["model"], "litellm/some-model"); + assert_eq!(provider["options"]["baseURL"], "http://localhost:4000/v1"); + assert_eq!(provider["options"]["apiKey"], "sk-test \"quoted\""); + assert!(provider["models"]["some-model"].is_object()); +} + +#[test] +fn opencode_uses_a_different_provider_package_for_every_wire() { + let package = |wire| { + let dir = tempfile::tempdir().unwrap(); + let spec = Opencode + .configure(&version(), &settings(wire), dir.path()) + .unwrap(); + let config: serde_json::Value = + serde_json::from_str(spec.files.values().next().unwrap()).unwrap(); + config["provider"]["litellm"]["npm"] + .as_str() + .unwrap() + .to_owned() + }; + let packages = [Wire::ChatCompletions, Wire::Responses, Wire::Messages].map(package); + + assert_eq!( + packages + .iter() + .collect::>() + .len(), + packages.len() + ); +} diff --git a/litellm-rust/crates/testkit/tests/install.rs b/litellm-rust/crates/testkit/tests/install.rs new file mode 100644 index 00000000000..7edaf27de6c --- /dev/null +++ b/litellm-rust/crates/testkit/tests/install.rs @@ -0,0 +1,262 @@ +mod support; + +use std::str::FromStr; + +use litellm_testkit::{ClaudeCode, Codex, Error, Installer, Opencode, Target, Version}; +use rstest::rstest; +use serde_json::json; +use support::{FakeFetch, script_printing, sha256, tar_gz, zip_archive}; +use target_lexicon::Triple; + +fn target(triple: &str) -> Target { + Target::try_from(&Triple::from_str(triple).unwrap()).unwrap() +} + +fn linux() -> Target { + target("x86_64-unknown-linux-gnu") +} +fn version() -> Version { + Version::new(9, 8, 7) +} + +fn github_release(asset: &str, download_url: &str, digest: Option) -> Vec { + json!({ + "assets": [ + { "name": "unrelated.txt", "digest": "sha256:00", "browser_download_url": "https://example.test/unrelated" }, + { "name": asset, "digest": digest, "browser_download_url": download_url }, + ] + }) + .to_string() + .into_bytes() +} + +fn claude_routes(binary: &[u8], checksum: &str) -> Vec<(String, Vec)> { + let base = "https://downloads.claude.ai/claude-code-releases/9.8.7"; + let manifest = json!({ "platforms": { "linux-x64": { "checksum": checksum } } }); + vec![ + ( + format!("{base}/manifest.json"), + manifest.to_string().into_bytes(), + ), + (format!("{base}/linux-x64/claude"), binary.to_vec()), + ] +} + +fn codex_routes(archive: Vec, digest: Option) -> Vec<(String, Vec)> { + let release = github_release( + "codex-x86_64-unknown-linux-musl.tar.gz", + "https://example.test/codex.tar.gz", + digest, + ); + vec![ + ( + "https://api.github.com/repos/openai/codex/releases/tags/rust-v9.8.7".to_owned(), + release, + ), + ("https://example.test/codex.tar.gz".to_owned(), archive), + ] +} + +#[tokio::test] +async fn claude_bare_binary_is_installed_and_runnable() { + let binary = script_printing("9.8.7 (Claude Code)"); + let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary))); + let cache = tempfile::tempdir().unwrap(); + + let installed = Installer::new(&fetch, cache.path(), linux()) + .install(&ClaudeCode, &version()) + .await + .unwrap(); + + assert_eq!(installed.binary, cache.path().join("claude/9.8.7/claude")); + assert_eq!(std::fs::read(&installed.binary).unwrap(), binary); +} + +#[tokio::test] +async fn codex_binary_is_extracted_from_the_tarball_under_its_own_name() { + let binary = script_printing("codex-cli 9.8.7"); + let archive = tar_gz("codex-x86_64-unknown-linux-musl", &binary); + let fetch = FakeFetch::new(codex_routes( + archive.clone(), + Some(format!("sha256:{}", sha256(&archive))), + )); + let cache = tempfile::tempdir().unwrap(); + + let installed = Installer::new(&fetch, cache.path(), linux()) + .install(&Codex, &version()) + .await + .unwrap(); + + assert_eq!(std::fs::read(&installed.binary).unwrap(), binary); + assert_eq!(installed.binary, cache.path().join("codex/9.8.7/codex")); +} + +#[tokio::test] +async fn opencode_binary_is_extracted_from_the_darwin_zip() { + let binary = script_printing("9.8.7"); + let archive = zip_archive("opencode", &binary); + let release = github_release( + "opencode-darwin-arm64.zip", + "https://example.test/opencode.zip", + Some(format!("sha256:{}", sha256(&archive))), + ); + let fetch = FakeFetch::new([ + ( + "https://api.github.com/repos/sst/opencode/releases/tags/v9.8.7".to_owned(), + release, + ), + ("https://example.test/opencode.zip".to_owned(), archive), + ]); + let cache = tempfile::tempdir().unwrap(); + + let installed = Installer::new(&fetch, cache.path(), target("aarch64-apple-darwin")) + .install(&Opencode, &version()) + .await + .unwrap(); + + assert_eq!(std::fs::read(&installed.binary).unwrap(), binary); +} + +#[tokio::test] +async fn tampered_download_is_rejected_and_nothing_is_left_behind() { + let binary = script_printing("9.8.7 (Claude Code)"); + let fetch = FakeFetch::new(claude_routes(&binary, &sha256(b"what the vendor signed"))); + let cache = tempfile::tempdir().unwrap(); + + let result = Installer::new(&fetch, cache.path(), linux()) + .install(&ClaudeCode, &version()) + .await; + + assert!(matches!(result, Err(Error::ChecksumMismatch { .. }))); + assert!(!cache.path().join("claude/9.8.7").exists()); +} + +#[tokio::test] +async fn github_asset_without_a_digest_is_refused() { + let archive = tar_gz( + "codex-x86_64-unknown-linux-musl", + &script_printing("codex-cli 9.8.7"), + ); + let fetch = FakeFetch::new(codex_routes(archive, None)); + let cache = tempfile::tempdir().unwrap(); + + let result = Installer::new(&fetch, cache.path(), linux()) + .install(&Codex, &version()) + .await; + + assert!(matches!(result, Err(Error::MissingChecksum(_)))); +} + +#[tokio::test] +async fn binary_reporting_a_different_version_is_removed() { + let binary = script_printing("1.0.0 (Claude Code)"); + let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary))); + let cache = tempfile::tempdir().unwrap(); + + let result = Installer::new(&fetch, cache.path(), linux()) + .install(&ClaudeCode, &version()) + .await; + + assert!(matches!(result, Err(Error::VersionMismatch { .. }))); + assert!(!cache.path().join("claude/9.8.7/claude").exists()); +} + +#[tokio::test] +async fn second_install_reuses_the_cached_binary_without_downloading() { + let binary = script_printing("9.8.7 (Claude Code)"); + let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary))); + let cache = tempfile::tempdir().unwrap(); + let installer = Installer::new(&fetch, cache.path(), linux()); + + let first = installer.install(&ClaudeCode, &version()).await.unwrap(); + let calls_after_first = fetch.calls(); + let second = installer.install(&ClaudeCode, &version()).await.unwrap(); + + assert_eq!(first, second); + assert_eq!(fetch.calls(), calls_after_first); +} + +#[tokio::test] +async fn corrupted_cache_entry_is_replaced_by_a_fresh_download() { + let binary = script_printing("9.8.7 (Claude Code)"); + let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary))); + let cache = tempfile::tempdir().unwrap(); + let installer = Installer::new(&fetch, cache.path(), linux()); + let installed = installer.install(&ClaudeCode, &version()).await.unwrap(); + std::fs::write(&installed.binary, script_printing("0.0.1")).unwrap(); + + installer.install(&ClaudeCode, &version()).await.unwrap(); + + assert_eq!(std::fs::read(&installed.binary).unwrap(), binary); +} + +#[rstest] +#[case("9.8.7-beta.1")] +#[case("9.8.7+build.5")] +#[tokio::test] +async fn pre_releases_never_reach_the_network_or_the_filesystem(#[case] version: &str) { + let fetch = FakeFetch::new([]); + let cache = tempfile::tempdir().unwrap(); + + let result = Installer::new(&fetch, cache.path(), linux()) + .install(&ClaudeCode, &Version::parse(version).unwrap()) + .await; + + assert!(matches!(result, Err(Error::InvalidVersion(_)))); + assert_eq!(fetch.calls(), 0); + assert_eq!(std::fs::read_dir(cache.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn musl_linux_picks_the_musl_claude_build() { + let binary = script_printing("9.8.7 (Claude Code)"); + let base = "https://downloads.claude.ai/claude-code-releases/9.8.7"; + let manifest = json!({ "platforms": { + "linux-x64": { "checksum": sha256(b"glibc build") }, + "linux-x64-musl": { "checksum": sha256(&binary) }, + } }); + let fetch = FakeFetch::new([ + ( + format!("{base}/manifest.json"), + manifest.to_string().into_bytes(), + ), + (format!("{base}/linux-x64-musl/claude"), binary.clone()), + ]); + let cache = tempfile::tempdir().unwrap(); + + let installed = Installer::new(&fetch, cache.path(), target("x86_64-unknown-linux-musl")) + .install(&ClaudeCode, &version()) + .await + .unwrap(); + + assert_eq!(std::fs::read(&installed.binary).unwrap(), binary); +} + +#[rstest] +#[case("x86_64-pc-windows-msvc")] +#[case("riscv64gc-unknown-linux-gnu")] +#[case("wasm32-unknown-unknown")] +fn targets_no_agent_ships_for_are_rejected(#[case] triple: &str) { + let result = Target::try_from(&Triple::from_str(triple).unwrap()); + + assert!(matches!(result, Err(Error::UnsupportedTarget(_)))); +} + +#[tokio::test] +async fn concurrent_installs_of_the_same_version_both_succeed() { + let binary = script_printing("9.8.7 (Claude Code)"); + let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary))); + let cache = tempfile::tempdir().unwrap(); + let installer = Installer::new(&fetch, cache.path(), linux()); + + let wanted = version(); + let installs = + futures_util::future::join_all((0..8).map(|_| installer.install(&ClaudeCode, &wanted))) + .await; + + assert!(installs.iter().all(Result::is_ok)); + assert_eq!( + std::fs::read(&installs[0].as_ref().unwrap().binary).unwrap(), + binary + ); +} diff --git a/litellm-rust/crates/testkit/tests/live.rs b/litellm-rust/crates/testkit/tests/live.rs new file mode 100644 index 00000000000..b805596879a --- /dev/null +++ b/litellm-rust/crates/testkit/tests/live.rs @@ -0,0 +1,133 @@ +//! Drives the real agents through a real gateway. Run with `cargo test -p litellm-testkit --test live -- --ignored` +//! after exporting `TESTKIT_GATEWAY_URL`, `TESTKIT_GATEWAY_KEY`, one `TESTKIT_MODEL_` per wire +//! (`MESSAGES`, `RESPONSES`, `CHAT_COMPLETIONS`) and one `TESTKIT__VERSION` per agent +//! (`CLAUDE`, `CODEX`, `OPENCODE`). `TESTKIT_CACHE_DIR` and `GITHUB_TOKEN` are optional. + +use std::path::PathBuf; +use std::time::Duration; + +use litellm_testkit::{ + Agent, ClaudeCode, Codex, HttpFetch, Installer, Opencode, Outcome, Prompt, Session, Settings, + Target, Version, Wire, +}; +use rstest::rstest; + +const LIMIT: Duration = Duration::from_secs(180); + +fn required(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set to run the live tests")) +} + +fn model_var(wire: Wire) -> &'static str { + match wire { + Wire::Messages => "TESTKIT_MODEL_MESSAGES", + Wire::Responses => "TESTKIT_MODEL_RESPONSES", + Wire::ChatCompletions => "TESTKIT_MODEL_CHAT_COMPLETIONS", + } +} + +async fn drive( + agent: &impl Agent, + version_var: &str, + wire: Wire, + model: Option<&str>, + prompt: Prompt, +) -> Outcome { + let cache = std::env::var("TESTKIT_CACHE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| std::env::temp_dir().join("litellm-testkit-cache")); + let installer = Installer::new(HttpFetch::from_env(), cache, Target::host().unwrap()); + let installed = installer + .install(agent, &Version::parse(&required(version_var)).unwrap()) + .await + .unwrap(); + let settings = Settings { + base_url: required("TESTKIT_GATEWAY_URL"), + api_key: required("TESTKIT_GATEWAY_KEY"), + model: model.map_or_else(|| required(model_var(wire)), str::to_owned), + wire, + }; + let home = tempfile::tempdir().unwrap(); + let session = Session::prepare(agent, &installed, settings, home.path()).unwrap(); + session.run(agent, &prompt, LIMIT).await.unwrap() +} + +fn text_prompt() -> Prompt { + Prompt { + text: "Reply with the single word: pong".to_owned(), + allow_tools: false, + } +} + +fn tool_prompt() -> Prompt { + Prompt { + text: "Run the shell command 'echo tool-ok' and reply with exactly its output.".to_owned(), + allow_tools: true, + } +} + +#[rstest] +#[case::claude_messages(&ClaudeCode, "TESTKIT_CLAUDE_VERSION", Wire::Messages)] +#[case::codex_responses(&Codex, "TESTKIT_CODEX_VERSION", Wire::Responses)] +#[case::opencode_chat(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::ChatCompletions)] +#[case::opencode_responses(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Responses)] +#[case::opencode_messages(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Messages)] +#[ignore = "needs a live gateway, see the module docs"] +#[tokio::test] +async fn plain_prompt_gets_an_answer_and_token_usage( + #[case] agent: &impl Agent, + #[case] version_var: &str, + #[case] wire: Wire, +) { + let outcome = drive(agent, version_var, wire, None, text_prompt()).await; + + assert!(outcome.succeeded(), "{outcome:?}"); + assert!(outcome.text.to_lowercase().contains("pong"), "{outcome:?}"); + assert!(outcome.usage.output_tokens > 0, "{outcome:?}"); +} + +#[rstest] +#[case::claude_messages(&ClaudeCode, "TESTKIT_CLAUDE_VERSION", Wire::Messages)] +#[case::codex_responses(&Codex, "TESTKIT_CODEX_VERSION", Wire::Responses)] +#[case::opencode_chat(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::ChatCompletions)] +#[case::opencode_responses(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Responses)] +#[case::opencode_messages(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Messages)] +#[ignore = "needs a live gateway, see the module docs"] +#[tokio::test] +async fn tool_use_is_reported_and_its_result_reaches_the_answer( + #[case] agent: &impl Agent, + #[case] version_var: &str, + #[case] wire: Wire, +) { + let outcome = drive(agent, version_var, wire, None, tool_prompt()).await; + + assert!(outcome.succeeded(), "{outcome:?}"); + assert!(!outcome.tool_calls.is_empty(), "{outcome:?}"); + assert!(outcome.text.contains("tool-ok"), "{outcome:?}"); +} + +#[rstest] +#[case::claude_messages(&ClaudeCode, "TESTKIT_CLAUDE_VERSION", Wire::Messages)] +#[case::codex_responses(&Codex, "TESTKIT_CODEX_VERSION", Wire::Responses)] +#[case::opencode_chat(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::ChatCompletions)] +#[case::opencode_responses(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Responses)] +#[case::opencode_messages(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Messages)] +#[ignore = "needs a live gateway, see the module docs"] +#[tokio::test] +async fn model_the_gateway_rejects_is_reported_as_an_error( + #[case] agent: &impl Agent, + #[case] version_var: &str, + #[case] wire: Wire, +) { + let outcome = drive( + agent, + version_var, + wire, + Some("testkit-no-such-model"), + text_prompt(), + ) + .await; + + assert!(!outcome.succeeded(), "{outcome:?}"); + assert!(!outcome.errors.is_empty(), "{outcome:?}"); +} diff --git a/litellm-rust/crates/testkit/tests/session.rs b/litellm-rust/crates/testkit/tests/session.rs new file mode 100644 index 00000000000..cd5e0dcbc71 --- /dev/null +++ b/litellm-rust/crates/testkit/tests/session.rs @@ -0,0 +1,155 @@ +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use litellm_testkit::{ + Configure, Drive, Error, Installed, LaunchSpec, Outcome, Prompt, Session, Settings, Version, + Wire, +}; + +struct Scripted; + +impl Configure for Scripted { + fn configure( + &self, + version: &Version, + _settings: &Settings, + home: &Path, + ) -> Result { + Ok(LaunchSpec { + env: [ + ("AGENT_HOME".to_owned(), home.to_string_lossy().into_owned()), + ("AGENT_SAW_VERSION".to_owned(), version.to_string()), + ] + .into(), + files: [( + PathBuf::from("conf/agent.toml"), + "configured = true\n".to_owned(), + )] + .into(), + }) + } +} + +impl Drive for Scripted { + fn args(&self, _version: &Version, _settings: &Settings, prompt: &Prompt) -> Vec { + vec!["--prompt".to_owned(), prompt.text.clone()] + } + + fn parse(&self, _version: &Version, stdout: &str) -> Outcome { + Outcome { + text: stdout.to_owned(), + ..Outcome::default() + } + } +} + +fn settings() -> Settings { + Settings { + base_url: "http://gateway.test".to_owned(), + api_key: "sk-test".to_owned(), + model: "some-model".to_owned(), + wire: Wire::Messages, + } +} + +fn prompt(text: &str) -> Prompt { + Prompt { + text: text.to_owned(), + allow_tools: false, + } +} + +fn session(script: &str) -> (Session, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let binary = dir.path().join("agent"); + std::fs::write(&binary, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)).unwrap(); + let home = dir.path().join("home"); + std::fs::create_dir(&home).unwrap(); + let installed = Installed { + version: Version::new(4, 5, 6), + binary, + }; + ( + Session::prepare(&Scripted, &installed, settings(), home).unwrap(), + dir, + ) +} + +const LIMIT: Duration = Duration::from_secs(20); + +#[tokio::test] +async fn prepare_writes_the_config_files_under_home() { + let (_session, dir) = session("true"); + + let written = std::fs::read_to_string(dir.path().join("home/conf/agent.toml")).unwrap(); + + assert_eq!(written, "configured = true\n"); +} + +#[tokio::test] +async fn configure_and_drive_are_given_the_installed_version() { + let (session, _dir) = session("echo \"$AGENT_SAW_VERSION\""); + + let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap(); + + assert_eq!(outcome.text.trim(), "4.5.6"); +} + +#[tokio::test] +async fn agent_runs_in_home_with_only_its_own_environment() { + let (session, dir) = session("pwd -P; env"); + + let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap(); + + let home = dir.path().join("home").canonicalize().unwrap(); + assert_eq!(outcome.text.lines().next().unwrap(), home.to_string_lossy()); + assert!(outcome.text.contains("AGENT_HOME=")); + assert!( + !outcome.text.contains("CARGO_"), + "test runner environment leaked into the agent" + ); +} + +#[tokio::test] +async fn prompt_reaches_the_agent_as_one_untouched_argument() { + let (session, _dir) = session("printf '%s|' \"$@\""); + let text = "two spaces; $(echo injected) 'quoted'"; + + let outcome = session.run(&Scripted, &prompt(text), LIMIT).await.unwrap(); + + assert_eq!(outcome.text, format!("--prompt|{text}|")); +} + +#[tokio::test] +async fn clean_exit_is_a_success() { + let (session, _dir) = session("echo done"); + + let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap(); + + assert_eq!(outcome.exit_code, Some(0)); + assert!(outcome.succeeded()); +} + +#[tokio::test] +async fn failing_exit_without_a_parsed_error_reports_stderr() { + let (session, _dir) = session("echo boom >&2; exit 3"); + + let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap(); + + assert_eq!(outcome.exit_code, Some(3)); + assert!(!outcome.succeeded()); + assert_eq!(outcome.errors, ["boom\n"]); +} + +#[tokio::test] +async fn agent_that_outlives_the_limit_is_stopped() { + let (session, _dir) = session("sleep 30"); + + let result = session + .run(&Scripted, &prompt("hi"), Duration::from_millis(200)) + .await; + + assert!(matches!(result, Err(Error::Timeout(_)))); +} diff --git a/litellm-rust/crates/testkit/tests/support/mod.rs b/litellm-rust/crates/testkit/tests/support/mod.rs new file mode 100644 index 00000000000..f4a13759941 --- /dev/null +++ b/litellm-rust/crates/testkit/tests/support/mod.rs @@ -0,0 +1,70 @@ +use std::collections::HashMap; +use std::io::Write; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use litellm_testkit::{Error, Fetch}; +use sha2::{Digest, Sha256}; + +pub struct FakeFetch { + routes: HashMap>, + calls: AtomicUsize, +} + +impl FakeFetch { + pub fn new(routes: impl IntoIterator)>) -> Self { + Self { + routes: routes.into_iter().collect(), + calls: AtomicUsize::new(0), + } + } + + pub fn calls(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } +} + +impl Fetch for FakeFetch { + async fn get(&self, url: &str) -> Result, Error> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.routes.get(url).cloned().ok_or_else(|| Error::Status { + url: url.to_owned(), + status: 404, + }) + } +} + +impl Fetch for &FakeFetch { + async fn get(&self, url: &str) -> Result, Error> { + (*self).get(url).await + } +} + +pub fn sha256(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +pub fn script_printing(output: &str) -> Vec { + format!("#!/bin/sh\necho '{output}'\n").into_bytes() +} + +pub fn tar_gz(member: &str, contents: &[u8]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(contents.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + builder.append_data(&mut header, member, contents).unwrap(); + let tarball = builder.into_inner().unwrap(); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(&tarball).unwrap(); + encoder.finish().unwrap() +} + +pub fn zip_archive(member: &str, contents: &[u8]) -> Vec { + let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + writer + .start_file(member, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(contents).unwrap(); + writer.finish().unwrap().into_inner() +}