mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
feat(testkit): agent clients for Claude Code, Codex and opencode (#43181)
* feat(testkit): install and configure Claude Code, Codex and opencode against a gateway Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * refactor(testkit): derive targets from target-lexicon and split agents behind a trait Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * refactor(testkit): group sources into agent and install folders Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * feat(testkit): split agents into install, configure and drive with semver-aware launch Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> * style(testkit): drop a needless borrow Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
081f73f021
commit
9c10e0f985
23 changed files with 2151 additions and 3 deletions
152
litellm-rust/Cargo.lock
generated
152
litellm-rust/Cargo.lock
generated
|
|
@ -73,6 +73,15 @@ version = "1.0.104"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
|
||||
dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "1.9.2"
|
||||
|
|
@ -1470,6 +1479,17 @@ dependencies = [
|
|||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.20.2"
|
||||
|
|
@ -1643,6 +1663,16 @@ version = "2.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
|
|
@ -3453,6 +3483,27 @@ dependencies = [
|
|||
"veil",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-testkit"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"futures-util",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tar",
|
||||
"target-lexicon",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"toml",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-token-counter"
|
||||
version = "0.1.0"
|
||||
|
|
@ -5206,6 +5257,15 @@ dependencies = [
|
|||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.7.1"
|
||||
|
|
@ -5537,6 +5597,17 @@ version = "0.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.13.5"
|
||||
|
|
@ -5806,6 +5877,30 @@ dependencies = [
|
|||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.9.12+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"serde_core",
|
||||
"serde_spanned",
|
||||
"toml_datetime 0.7.5+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"toml_writer",
|
||||
"winnow 0.7.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.7.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.1.1+spec-1.1.0"
|
||||
|
|
@ -5822,9 +5917,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"toml_datetime",
|
||||
"toml_datetime 1.1.1+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"winnow",
|
||||
"winnow 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -5833,9 +5928,15 @@ version = "1.1.3+spec-1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
|
||||
dependencies = [
|
||||
"winnow",
|
||||
"winnow 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_writer"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.6"
|
||||
|
|
@ -6597,6 +6698,12 @@ version = "0.52.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "1.0.4"
|
||||
|
|
@ -6659,6 +6766,16 @@ dependencies = [
|
|||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xattr"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xmlparser"
|
||||
version = "0.13.6"
|
||||
|
|
@ -6784,6 +6901,23 @@ dependencies = [
|
|||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"displaydoc",
|
||||
"flate2",
|
||||
"indexmap 2.14.0",
|
||||
"memchr",
|
||||
"thiserror 2.0.19",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.7"
|
||||
|
|
@ -6795,3 +6929,15 @@ name = "zmij"
|
|||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -81,6 +81,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"]
|
|||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
||||
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
||||
base64 = "0.22"
|
||||
flate2 = "1"
|
||||
semver = "1"
|
||||
tar = "0.4"
|
||||
target-lexicon = "0.13.5"
|
||||
tempfile = "3"
|
||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||
moka = { version = "0.12.16", features = ["future"] }
|
||||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
url = "2.5.8"
|
||||
|
|
|
|||
32
litellm-rust/crates/testkit/Cargo.toml
Normal file
32
litellm-rust/crates/testkit/Cargo.toml
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
[package]
|
||||
name = "litellm-testkit"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
flate2.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
semver.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
tar.workspace = true
|
||||
target-lexicon.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "process"] }
|
||||
zip.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
flate2.workspace = true
|
||||
rstest.workspace = true
|
||||
sha2.workspace = true
|
||||
tar.workspace = true
|
||||
target-lexicon.workspace = true
|
||||
futures-util.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio.workspace = true
|
||||
toml = "0.9"
|
||||
zip.workspace = true
|
||||
181
litellm-rust/crates/testkit/src/agent/claude.rs
Normal file
181
litellm-rust/crates/testkit/src/agent/claude.rs
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use semver::Version;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{
|
||||
Configure, Drive, Install, LaunchSpec, Outcome, Prompt, Settings, Usage, Wire, env, json_lines,
|
||||
path_string,
|
||||
};
|
||||
use crate::install::release::parse;
|
||||
use crate::install::{Packaging, Release};
|
||||
use crate::{Error, Fetch, Target};
|
||||
|
||||
const RELEASES: &str = "https://downloads.claude.ai/claude-code-releases";
|
||||
|
||||
pub struct ClaudeCode;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Manifest {
|
||||
platforms: BTreeMap<String, Platform>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Platform {
|
||||
checksum: String,
|
||||
}
|
||||
|
||||
impl Install for ClaudeCode {
|
||||
fn binary(&self) -> &'static str {
|
||||
"claude"
|
||||
}
|
||||
|
||||
async fn release(
|
||||
&self,
|
||||
fetch: &impl Fetch,
|
||||
version: &Version,
|
||||
target: Target,
|
||||
) -> Result<Release, Error> {
|
||||
let manifest_url = format!("{RELEASES}/{version}/manifest.json");
|
||||
let manifest: Manifest = parse(&manifest_url, &fetch.get(&manifest_url).await?)?;
|
||||
let key = format!(
|
||||
"{}-{}{}",
|
||||
target.os_name(),
|
||||
target.arch_name(),
|
||||
target.musl_suffix()
|
||||
);
|
||||
let platform = manifest
|
||||
.platforms
|
||||
.get(&key)
|
||||
.ok_or_else(|| Error::AssetNotFound(key.clone()))?;
|
||||
Ok(Release {
|
||||
url: format!("{RELEASES}/{version}/{key}/claude"),
|
||||
asset: key,
|
||||
sha256: platform.checksum.clone(),
|
||||
packaging: Packaging::Bare,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Configure for ClaudeCode {
|
||||
fn configure(
|
||||
&self,
|
||||
_version: &Version,
|
||||
settings: &Settings,
|
||||
home: &Path,
|
||||
) -> Result<LaunchSpec, Error> {
|
||||
if settings.wire != Wire::Messages {
|
||||
return Err(Error::UnsupportedWire {
|
||||
agent: "claude",
|
||||
wire: settings.wire,
|
||||
});
|
||||
}
|
||||
Ok(LaunchSpec {
|
||||
env: env([
|
||||
("HOME", path_string(home)),
|
||||
("CLAUDE_CONFIG_DIR", path_string(&home.join(".claude"))),
|
||||
("ANTHROPIC_BASE_URL", settings.base_url.clone()),
|
||||
("ANTHROPIC_AUTH_TOKEN", settings.api_key.clone()),
|
||||
("ANTHROPIC_MODEL", settings.model.clone()),
|
||||
("DISABLE_AUTOUPDATER", "1".to_owned()),
|
||||
]),
|
||||
files: BTreeMap::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum Event {
|
||||
Assistant {
|
||||
message: AssistantMessage,
|
||||
},
|
||||
Result(Finished),
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AssistantMessage {
|
||||
content: Vec<Block>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Block {
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Finished {
|
||||
is_error: bool,
|
||||
result: Option<String>,
|
||||
usage: Option<TokenUsage>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct TokenUsage {
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
}
|
||||
|
||||
impl Drive for ClaudeCode {
|
||||
fn args(&self, _version: &Version, settings: &Settings, prompt: &Prompt) -> Vec<String> {
|
||||
let base = [
|
||||
"-p",
|
||||
&prompt.text,
|
||||
"--output-format",
|
||||
"stream-json",
|
||||
"--verbose",
|
||||
"--model",
|
||||
&settings.model,
|
||||
];
|
||||
let tools = ["--allowedTools", "Bash,Read,Write,Edit"];
|
||||
base.into_iter()
|
||||
.chain(tools.into_iter().filter(|_| prompt.allow_tools))
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse(&self, _version: &Version, stdout: &str) -> Outcome {
|
||||
let events: Vec<Event> = json_lines(stdout).collect();
|
||||
let tool_calls = events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::Assistant { message } => Some(&message.content),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.filter(|block| block.kind == "tool_use")
|
||||
.filter_map(|block| block.name.clone())
|
||||
.collect();
|
||||
let finished = events.into_iter().find_map(|event| match event {
|
||||
Event::Result(finished) => Some(finished),
|
||||
_ => None,
|
||||
});
|
||||
let Some(finished) = finished else {
|
||||
return Outcome {
|
||||
tool_calls,
|
||||
..Outcome::default()
|
||||
};
|
||||
};
|
||||
let result = finished.result.unwrap_or_default();
|
||||
let (text, errors) = if finished.is_error {
|
||||
(String::new(), vec![result])
|
||||
} else {
|
||||
(result, Vec::new())
|
||||
};
|
||||
Outcome {
|
||||
text,
|
||||
tool_calls,
|
||||
usage: finished.usage.map_or_else(Usage::default, |usage| Usage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
}),
|
||||
errors,
|
||||
exit_code: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
174
litellm-rust/crates/testkit/src/agent/codex.rs
Normal file
174
litellm-rust/crates/testkit/src/agent/codex.rs
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use semver::Version;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{
|
||||
Configure, Drive, Install, LaunchSpec, Outcome, Prompt, Settings, Usage, Wire, env, json_lines,
|
||||
path_string, quoted, v1,
|
||||
};
|
||||
use crate::install::release::github_release;
|
||||
use crate::install::{Packaging, Release};
|
||||
use crate::target::{Arch, Os};
|
||||
use crate::{Error, Fetch, Target};
|
||||
|
||||
const RELEASES: &str = "https://api.github.com/repos/openai/codex/releases/tags";
|
||||
|
||||
pub struct Codex;
|
||||
|
||||
fn triple(target: Target) -> String {
|
||||
let arch = match target.arch {
|
||||
Arch::Aarch64 => "aarch64",
|
||||
Arch::X86_64 => "x86_64",
|
||||
};
|
||||
match target.os {
|
||||
Os::Macos => format!("{arch}-apple-darwin"),
|
||||
Os::Linux => format!("{arch}-unknown-linux-musl"),
|
||||
}
|
||||
}
|
||||
|
||||
impl Install for Codex {
|
||||
fn binary(&self) -> &'static str {
|
||||
"codex"
|
||||
}
|
||||
|
||||
async fn release(
|
||||
&self,
|
||||
fetch: &impl Fetch,
|
||||
version: &Version,
|
||||
target: Target,
|
||||
) -> Result<Release, Error> {
|
||||
let triple = triple(target);
|
||||
github_release(
|
||||
fetch,
|
||||
RELEASES,
|
||||
&format!("rust-v{version}"),
|
||||
&format!("codex-{triple}.tar.gz"),
|
||||
Packaging::TarGz {
|
||||
member: format!("codex-{triple}"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl Configure for Codex {
|
||||
fn configure(
|
||||
&self,
|
||||
_version: &Version,
|
||||
settings: &Settings,
|
||||
home: &Path,
|
||||
) -> Result<LaunchSpec, Error> {
|
||||
if settings.wire != Wire::Responses {
|
||||
return Err(Error::UnsupportedWire {
|
||||
agent: "codex",
|
||||
wire: settings.wire,
|
||||
});
|
||||
}
|
||||
let config = format!(
|
||||
"model = {model}\nmodel_provider = \"litellm\"\n\n[model_providers.litellm]\nname = \"LiteLLM\"\nbase_url = {base_url}\nenv_key = \"LITELLM_API_KEY\"\nwire_api = \"responses\"\n",
|
||||
model = quoted(&settings.model),
|
||||
base_url = quoted(&v1(settings)),
|
||||
);
|
||||
Ok(LaunchSpec {
|
||||
env: env([
|
||||
("HOME", path_string(home)),
|
||||
("CODEX_HOME", path_string(&home.join(".codex"))),
|
||||
("LITELLM_API_KEY", settings.api_key.clone()),
|
||||
]),
|
||||
files: BTreeMap::from([(PathBuf::from(".codex/config.toml"), config)]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
enum EventKind {
|
||||
#[serde(rename = "item.completed")]
|
||||
ItemCompleted,
|
||||
#[serde(rename = "turn.completed")]
|
||||
TurnCompleted,
|
||||
#[serde(rename = "turn.failed")]
|
||||
TurnFailed,
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Event {
|
||||
#[serde(rename = "type")]
|
||||
kind: EventKind,
|
||||
item: Option<Item>,
|
||||
usage: Option<TokenUsage>,
|
||||
error: Option<Failure>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Item {
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
text: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct TokenUsage {
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Failure {
|
||||
message: String,
|
||||
}
|
||||
|
||||
const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "error"];
|
||||
|
||||
impl Drive for Codex {
|
||||
fn args(&self, _version: &Version, _settings: &Settings, prompt: &Prompt) -> Vec<String> {
|
||||
let sandbox = ["--sandbox", "workspace-write"];
|
||||
["exec", "--json", "--skip-git-repo-check"]
|
||||
.into_iter()
|
||||
.chain(sandbox.into_iter().filter(|_| prompt.allow_tools))
|
||||
.chain([prompt.text.as_str()])
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse(&self, _version: &Version, stdout: &str) -> Outcome {
|
||||
let events: Vec<Event> = json_lines(stdout).collect();
|
||||
let items: Vec<&Item> = events
|
||||
.iter()
|
||||
.filter(|event| matches!(event.kind, EventKind::ItemCompleted))
|
||||
.filter_map(|event| event.item.as_ref())
|
||||
.collect();
|
||||
Outcome {
|
||||
text: items
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|item| item.kind == "agent_message")
|
||||
.and_then(|item| item.text.clone())
|
||||
.unwrap_or_default(),
|
||||
tool_calls: items
|
||||
.iter()
|
||||
.filter(|item| !NON_TOOL_ITEMS.contains(&item.kind.as_str()))
|
||||
.map(|item| item.kind.clone())
|
||||
.collect(),
|
||||
usage: events
|
||||
.iter()
|
||||
.filter(|event| matches!(event.kind, EventKind::TurnCompleted))
|
||||
.filter_map(|event| event.usage.as_ref())
|
||||
.map(|usage| Usage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
})
|
||||
.fold(Usage::default(), |total, turn| total + turn),
|
||||
errors: events
|
||||
.iter()
|
||||
.filter(|event| matches!(event.kind, EventKind::TurnFailed))
|
||||
.filter_map(|event| event.error.as_ref())
|
||||
.map(|failure| failure.message.clone())
|
||||
.collect(),
|
||||
exit_code: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
69
litellm-rust/crates/testkit/src/agent/configure.rs
Normal file
69
litellm-rust/crates/testkit/src/agent/configure.rs
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use semver::Version;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Wire {
|
||||
ChatCompletions,
|
||||
Messages,
|
||||
Responses,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Settings {
|
||||
pub base_url: String,
|
||||
pub api_key: String,
|
||||
pub model: String,
|
||||
pub wire: Wire,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct LaunchSpec {
|
||||
pub env: BTreeMap<String, String>,
|
||||
pub files: BTreeMap<PathBuf, String>,
|
||||
}
|
||||
|
||||
impl LaunchSpec {
|
||||
pub fn write_files(&self, home: &Path) -> std::io::Result<()> {
|
||||
self.files.iter().try_for_each(|(relative, contents)| {
|
||||
let path = home.join(relative);
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(path, contents)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Configure {
|
||||
fn configure(
|
||||
&self,
|
||||
version: &Version,
|
||||
settings: &Settings,
|
||||
home: &Path,
|
||||
) -> Result<LaunchSpec, Error>;
|
||||
}
|
||||
|
||||
pub(crate) fn env(
|
||||
pairs: impl IntoIterator<Item = (&'static str, String)>,
|
||||
) -> BTreeMap<String, String> {
|
||||
pairs
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key.to_owned(), value))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn path_string(path: &Path) -> String {
|
||||
path.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
pub(crate) fn quoted(value: &str) -> String {
|
||||
serde_json::Value::from(value).to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn v1(settings: &Settings) -> String {
|
||||
format!("{}/v1", settings.base_url.trim_end_matches('/'))
|
||||
}
|
||||
57
litellm-rust/crates/testkit/src/agent/drive.rs
Normal file
57
litellm-rust/crates/testkit/src/agent/drive.rs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
use std::ops::Add;
|
||||
|
||||
use semver::Version;
|
||||
|
||||
use crate::Settings;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Prompt {
|
||||
pub text: String,
|
||||
pub allow_tools: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Usage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
}
|
||||
|
||||
impl Add for Usage {
|
||||
type Output = Self;
|
||||
|
||||
fn add(self, other: Self) -> Self {
|
||||
Self {
|
||||
input_tokens: self.input_tokens + other.input_tokens,
|
||||
output_tokens: self.output_tokens + other.output_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Outcome {
|
||||
pub text: String,
|
||||
pub tool_calls: Vec<String>,
|
||||
pub usage: Usage,
|
||||
pub errors: Vec<String>,
|
||||
pub exit_code: Option<i32>,
|
||||
}
|
||||
|
||||
impl Outcome {
|
||||
pub fn succeeded(&self) -> bool {
|
||||
self.exit_code == Some(0) && self.errors.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Drive {
|
||||
fn args(&self, version: &Version, settings: &Settings, prompt: &Prompt) -> Vec<String>;
|
||||
|
||||
fn parse(&self, version: &Version, stdout: &str) -> Outcome;
|
||||
}
|
||||
|
||||
pub(crate) fn json_lines<'a, T: serde::de::DeserializeOwned + 'a>(
|
||||
stdout: &'a str,
|
||||
) -> impl Iterator<Item = T> + 'a {
|
||||
stdout
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str(line).ok())
|
||||
}
|
||||
17
litellm-rust/crates/testkit/src/agent/install.rs
Normal file
17
litellm-rust/crates/testkit/src/agent/install.rs
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
use std::future::Future;
|
||||
|
||||
use semver::Version;
|
||||
|
||||
use crate::install::Release;
|
||||
use crate::{Error, Fetch, Target};
|
||||
|
||||
pub trait Install: Sync {
|
||||
fn binary(&self) -> &'static str;
|
||||
|
||||
fn release(
|
||||
&self,
|
||||
fetch: &impl Fetch,
|
||||
version: &Version,
|
||||
target: Target,
|
||||
) -> impl Future<Output = Result<Release, Error>> + Send;
|
||||
}
|
||||
20
litellm-rust/crates/testkit/src/agent/mod.rs
Normal file
20
litellm-rust/crates/testkit/src/agent/mod.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
mod claude;
|
||||
mod codex;
|
||||
mod configure;
|
||||
mod drive;
|
||||
mod install;
|
||||
mod opencode;
|
||||
|
||||
pub use claude::ClaudeCode;
|
||||
pub use codex::Codex;
|
||||
pub use configure::{Configure, LaunchSpec, Settings, Wire};
|
||||
pub use drive::{Drive, Outcome, Prompt, Usage};
|
||||
pub use install::Install;
|
||||
pub use opencode::Opencode;
|
||||
|
||||
pub(crate) use configure::{env, path_string, quoted, v1};
|
||||
pub(crate) use drive::json_lines;
|
||||
|
||||
pub trait Agent: Install + Configure + Drive {}
|
||||
|
||||
impl<T: Install + Configure + Drive> Agent for T {}
|
||||
187
litellm-rust/crates/testkit/src/agent/opencode.rs
Normal file
187
litellm-rust/crates/testkit/src/agent/opencode.rs
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use semver::Version;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{
|
||||
Configure, Drive, Install, LaunchSpec, Outcome, Prompt, Settings, Usage, Wire, env, json_lines,
|
||||
path_string, v1,
|
||||
};
|
||||
use crate::install::release::github_release;
|
||||
use crate::install::{Packaging, Release};
|
||||
use crate::target::Os;
|
||||
use crate::{Error, Fetch, Target};
|
||||
|
||||
const RELEASES: &str = "https://api.github.com/repos/sst/opencode/releases/tags";
|
||||
|
||||
pub struct Opencode;
|
||||
|
||||
impl Install for Opencode {
|
||||
fn binary(&self) -> &'static str {
|
||||
"opencode"
|
||||
}
|
||||
|
||||
async fn release(
|
||||
&self,
|
||||
fetch: &impl Fetch,
|
||||
version: &Version,
|
||||
target: Target,
|
||||
) -> Result<Release, Error> {
|
||||
let stem = format!(
|
||||
"opencode-{}-{}{}",
|
||||
target.os_name(),
|
||||
target.arch_name(),
|
||||
target.musl_suffix()
|
||||
);
|
||||
let member = "opencode".to_owned();
|
||||
let (asset, packaging) = match target.os {
|
||||
Os::Macos => (format!("{stem}.zip"), Packaging::Zip { member }),
|
||||
Os::Linux => (format!("{stem}.tar.gz"), Packaging::TarGz { member }),
|
||||
};
|
||||
github_release(fetch, RELEASES, &format!("v{version}"), &asset, packaging).await
|
||||
}
|
||||
}
|
||||
|
||||
impl Configure for Opencode {
|
||||
fn configure(
|
||||
&self,
|
||||
_version: &Version,
|
||||
settings: &Settings,
|
||||
home: &Path,
|
||||
) -> Result<LaunchSpec, Error> {
|
||||
let npm = match settings.wire {
|
||||
Wire::ChatCompletions => "@ai-sdk/openai-compatible",
|
||||
Wire::Responses => "@ai-sdk/openai",
|
||||
Wire::Messages => "@ai-sdk/anthropic",
|
||||
};
|
||||
let config = serde_json::json!({
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"model": format!("litellm/{}", settings.model),
|
||||
"provider": {
|
||||
"litellm": {
|
||||
"npm": npm,
|
||||
"name": "LiteLLM",
|
||||
"options": { "baseURL": v1(settings), "apiKey": settings.api_key },
|
||||
"models": { settings.model.clone(): { "name": settings.model } },
|
||||
}
|
||||
},
|
||||
});
|
||||
Ok(LaunchSpec {
|
||||
env: env([
|
||||
("HOME", path_string(home)),
|
||||
("XDG_CONFIG_HOME", path_string(&home.join(".config"))),
|
||||
("XDG_DATA_HOME", path_string(&home.join(".local/share"))),
|
||||
("OPENCODE_DISABLE_AUTOUPDATE", "true".to_owned()),
|
||||
]),
|
||||
files: BTreeMap::from([(
|
||||
PathBuf::from(".config/opencode/opencode.json"),
|
||||
config.to_string(),
|
||||
)]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum Event {
|
||||
Text {
|
||||
part: TextPart,
|
||||
},
|
||||
ToolUse {
|
||||
part: ToolPart,
|
||||
},
|
||||
StepFinish {
|
||||
part: StepFinish,
|
||||
},
|
||||
Error {
|
||||
error: Failure,
|
||||
},
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct TextPart {
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ToolPart {
|
||||
tool: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StepFinish {
|
||||
tokens: Tokens,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Tokens {
|
||||
input: u64,
|
||||
output: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Failure {
|
||||
name: String,
|
||||
data: Option<FailureData>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct FailureData {
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
impl Drive for Opencode {
|
||||
fn args(&self, _version: &Version, _settings: &Settings, prompt: &Prompt) -> Vec<String> {
|
||||
["run", "--format", "json", &prompt.text]
|
||||
.map(str::to_owned)
|
||||
.to_vec()
|
||||
}
|
||||
|
||||
fn parse(&self, _version: &Version, stdout: &str) -> Outcome {
|
||||
let events: Vec<Event> = json_lines(stdout).collect();
|
||||
Outcome {
|
||||
text: events
|
||||
.iter()
|
||||
.rev()
|
||||
.find_map(|event| match event {
|
||||
Event::Text { part } => Some(part.text.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
tool_calls: events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::ToolUse { part } => Some(part.tool.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
usage: events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::StepFinish { part } => Some(Usage {
|
||||
input_tokens: part.tokens.input,
|
||||
output_tokens: part.tokens.output,
|
||||
}),
|
||||
_ => None,
|
||||
})
|
||||
.fold(Usage::default(), |total, step| total + step),
|
||||
errors: events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::Error { error } => Some(
|
||||
error
|
||||
.data
|
||||
.as_ref()
|
||||
.and_then(|data| data.message.clone())
|
||||
.unwrap_or_else(|| error.name.clone()),
|
||||
),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
exit_code: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
56
litellm-rust/crates/testkit/src/error.rs
Normal file
56
litellm-rust/crates/testkit/src/error.rs
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::Wire;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error("unsupported target {0}")]
|
||||
UnsupportedTarget(String),
|
||||
#[error("{0} is not a plain x.y.z release version")]
|
||||
InvalidVersion(String),
|
||||
#[error("request to {url} failed")]
|
||||
Request {
|
||||
url: String,
|
||||
#[source]
|
||||
source: reqwest::Error,
|
||||
},
|
||||
#[error("{url} answered with status {status}")]
|
||||
Status { url: String, status: u16 },
|
||||
#[error("release metadata at {url} is malformed")]
|
||||
Metadata {
|
||||
url: String,
|
||||
#[source]
|
||||
source: serde_json::Error,
|
||||
},
|
||||
#[error("release has no asset named {0}")]
|
||||
AssetNotFound(String),
|
||||
#[error("release publishes no sha256 for {0}")]
|
||||
MissingChecksum(String),
|
||||
#[error("sha256 mismatch for {asset}: expected {expected}, got {actual}")]
|
||||
ChecksumMismatch {
|
||||
asset: String,
|
||||
expected: String,
|
||||
actual: String,
|
||||
},
|
||||
#[error("archive does not contain {0}")]
|
||||
ArchiveMemberNotFound(String),
|
||||
#[error("archive is unreadable")]
|
||||
Archive(#[source] io::Error),
|
||||
#[error("zip archive is unreadable")]
|
||||
Zip(#[from] zip::result::ZipError),
|
||||
#[error("{binary} reports version '{reported}', expected {expected}")]
|
||||
VersionMismatch {
|
||||
binary: PathBuf,
|
||||
expected: String,
|
||||
reported: String,
|
||||
},
|
||||
#[error("{agent} cannot talk to the gateway over {wire:?}")]
|
||||
UnsupportedWire { agent: &'static str, wire: Wire },
|
||||
#[error("agent did not finish within {0:?}")]
|
||||
Timeout(std::time::Duration),
|
||||
#[error("io failure")]
|
||||
Io(#[from] io::Error),
|
||||
}
|
||||
52
litellm-rust/crates/testkit/src/install/archive.rs
Normal file
52
litellm-rust/crates/testkit/src/install/archive.rs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
use std::io::{Cursor, Read};
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::release::Packaging;
|
||||
use crate::Error;
|
||||
|
||||
pub(crate) fn verify_sha256(asset: &str, expected: &str, bytes: &[u8]) -> Result<(), Error> {
|
||||
let actual = format!("{:x}", Sha256::digest(bytes));
|
||||
if actual.eq_ignore_ascii_case(expected) {
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::ChecksumMismatch {
|
||||
asset: asset.to_owned(),
|
||||
expected: expected.to_owned(),
|
||||
actual,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn extract_binary(packaging: &Packaging, bytes: &[u8]) -> Result<Vec<u8>, Error> {
|
||||
match packaging {
|
||||
Packaging::Bare => Ok(bytes.to_vec()),
|
||||
Packaging::TarGz { member } => extract_tar_gz(member, bytes),
|
||||
Packaging::Zip { member } => extract_zip(member, bytes),
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tar_gz(member: &str, bytes: &[u8]) -> Result<Vec<u8>, Error> {
|
||||
let mut archive = tar::Archive::new(GzDecoder::new(bytes));
|
||||
for entry in archive.entries().map_err(Error::Archive)? {
|
||||
let mut entry = entry.map_err(Error::Archive)?;
|
||||
let path = entry.path().map_err(Error::Archive)?;
|
||||
if path.file_name().is_some_and(|name| name == member) {
|
||||
let mut binary = Vec::new();
|
||||
entry.read_to_end(&mut binary).map_err(Error::Archive)?;
|
||||
return Ok(binary);
|
||||
}
|
||||
}
|
||||
Err(Error::ArchiveMemberNotFound(member.to_owned()))
|
||||
}
|
||||
|
||||
fn extract_zip(member: &str, bytes: &[u8]) -> Result<Vec<u8>, Error> {
|
||||
let mut archive = zip::ZipArchive::new(Cursor::new(bytes))?;
|
||||
let mut file = archive.by_name(member).map_err(|error| match error {
|
||||
zip::result::ZipError::FileNotFound => Error::ArchiveMemberNotFound(member.to_owned()),
|
||||
other => Error::Zip(other),
|
||||
})?;
|
||||
let mut binary = Vec::new();
|
||||
file.read_to_end(&mut binary).map_err(Error::Archive)?;
|
||||
Ok(binary)
|
||||
}
|
||||
55
litellm-rust/crates/testkit/src/install/fetch.rs
Normal file
55
litellm-rust/crates/testkit/src/install/fetch.rs
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
use std::future::Future;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
pub trait Fetch: Sync {
|
||||
fn get(&self, url: &str) -> impl Future<Output = Result<Vec<u8>, Error>> + Send;
|
||||
}
|
||||
|
||||
pub struct HttpFetch {
|
||||
client: reqwest::Client,
|
||||
github_token: Option<String>,
|
||||
}
|
||||
|
||||
impl HttpFetch {
|
||||
pub fn new(github_token: Option<String>) -> Self {
|
||||
Self {
|
||||
client: reqwest::Client::new(),
|
||||
github_token,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_env() -> Self {
|
||||
Self::new(std::env::var("GITHUB_TOKEN").ok())
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetch for HttpFetch {
|
||||
async fn get(&self, url: &str) -> Result<Vec<u8>, Error> {
|
||||
let request = self
|
||||
.client
|
||||
.get(url)
|
||||
.header("user-agent", "litellm-testkit")
|
||||
.header("accept", "application/json, application/octet-stream");
|
||||
let request = match (
|
||||
&self.github_token,
|
||||
url.starts_with("https://api.github.com/"),
|
||||
) {
|
||||
(Some(token), true) => request.bearer_auth(token),
|
||||
_ => request,
|
||||
};
|
||||
let request_error = |source| Error::Request {
|
||||
url: url.to_owned(),
|
||||
source,
|
||||
};
|
||||
let response = request.send().await.map_err(request_error)?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(Error::Status {
|
||||
url: url.to_owned(),
|
||||
status: status.as_u16(),
|
||||
});
|
||||
}
|
||||
Ok(response.bytes().await.map_err(request_error)?.to_vec())
|
||||
}
|
||||
}
|
||||
118
litellm-rust/crates/testkit/src/install/mod.rs
Normal file
118
litellm-rust/crates/testkit/src/install/mod.rs
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
mod archive;
|
||||
mod fetch;
|
||||
pub(crate) mod release;
|
||||
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use semver::Version;
|
||||
use tokio::fs;
|
||||
use tokio::process::Command;
|
||||
|
||||
use crate::{Error, Install, Target};
|
||||
use archive::{extract_binary, verify_sha256};
|
||||
|
||||
static STAGING_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Installed {
|
||||
pub version: Version,
|
||||
pub binary: PathBuf,
|
||||
}
|
||||
|
||||
pub struct Installer<F> {
|
||||
fetch: F,
|
||||
cache_root: PathBuf,
|
||||
target: Target,
|
||||
}
|
||||
|
||||
impl<F: Fetch> Installer<F> {
|
||||
pub fn new(fetch: F, cache_root: impl Into<PathBuf>, target: Target) -> Self {
|
||||
Self {
|
||||
fetch,
|
||||
cache_root: cache_root.into(),
|
||||
target,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn install(
|
||||
&self,
|
||||
agent: &impl Install,
|
||||
version: &Version,
|
||||
) -> Result<Installed, Error> {
|
||||
validate_release(version)?;
|
||||
let dir = self
|
||||
.cache_root
|
||||
.join(agent.binary())
|
||||
.join(version.to_string());
|
||||
let binary = dir.join(agent.binary());
|
||||
let installed = Installed {
|
||||
version: version.clone(),
|
||||
binary: binary.clone(),
|
||||
};
|
||||
if fs::try_exists(&binary).await? && probe_version(&binary, version).await.is_ok() {
|
||||
return Ok(installed);
|
||||
}
|
||||
|
||||
let release = agent.release(&self.fetch, version, self.target).await?;
|
||||
let archive = self.fetch.get(&release.url).await?;
|
||||
verify_sha256(&release.asset, &release.sha256, &archive)?;
|
||||
let contents = extract_binary(&release.packaging, &archive)?;
|
||||
|
||||
fs::create_dir_all(&dir).await?;
|
||||
let staging = dir.join(format!(
|
||||
".{}.{}.{}.partial",
|
||||
agent.binary(),
|
||||
std::process::id(),
|
||||
STAGING_COUNTER.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
fs::write(&staging, contents).await?;
|
||||
fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755)).await?;
|
||||
fs::rename(&staging, &binary).await?;
|
||||
|
||||
match probe_version(&binary, version).await {
|
||||
Ok(()) => Ok(installed),
|
||||
Err(error) => {
|
||||
fs::remove_file(&binary).await?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_release(version: &Version) -> Result<(), Error> {
|
||||
if version.pre.is_empty() && version.build.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::InvalidVersion(version.to_string()))
|
||||
}
|
||||
|
||||
async fn probe_version(binary: &Path, expected: &Version) -> Result<(), Error> {
|
||||
let home = std::env::temp_dir();
|
||||
let output = Command::new(binary)
|
||||
.arg("--version")
|
||||
.env_clear()
|
||||
.env("HOME", home)
|
||||
.env("DISABLE_AUTOUPDATER", "1")
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.await?;
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
if stdout
|
||||
.split_whitespace()
|
||||
.filter_map(|token| Version::parse(token).ok())
|
||||
.any(|reported| &reported == expected)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::VersionMismatch {
|
||||
binary: binary.to_owned(),
|
||||
expected: expected.to_string(),
|
||||
reported: stdout.trim().to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub use fetch::{Fetch, HttpFetch};
|
||||
pub use release::{Packaging, Release};
|
||||
65
litellm-rust/crates/testkit/src/install/release.rs
Normal file
65
litellm-rust/crates/testkit/src/install/release.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
use serde::Deserialize;
|
||||
|
||||
use crate::{Error, Fetch};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Packaging {
|
||||
Bare,
|
||||
TarGz { member: String },
|
||||
Zip { member: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Release {
|
||||
pub asset: String,
|
||||
pub url: String,
|
||||
pub sha256: String,
|
||||
pub packaging: Packaging,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GithubRelease {
|
||||
assets: Vec<GithubAsset>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GithubAsset {
|
||||
name: String,
|
||||
digest: Option<String>,
|
||||
browser_download_url: String,
|
||||
}
|
||||
|
||||
pub(crate) async fn github_release(
|
||||
fetch: &impl Fetch,
|
||||
releases_url: &str,
|
||||
tag: &str,
|
||||
asset_name: &str,
|
||||
packaging: Packaging,
|
||||
) -> Result<Release, Error> {
|
||||
let url = format!("{releases_url}/{tag}");
|
||||
let release: GithubRelease = parse(&url, &fetch.get(&url).await?)?;
|
||||
let asset = release
|
||||
.assets
|
||||
.into_iter()
|
||||
.find(|asset| asset.name == asset_name)
|
||||
.ok_or_else(|| Error::AssetNotFound(asset_name.to_owned()))?;
|
||||
let sha256 = asset
|
||||
.digest
|
||||
.as_deref()
|
||||
.and_then(|digest| digest.strip_prefix("sha256:"))
|
||||
.ok_or_else(|| Error::MissingChecksum(asset_name.to_owned()))?
|
||||
.to_owned();
|
||||
Ok(Release {
|
||||
asset: asset.name,
|
||||
url: asset.browser_download_url,
|
||||
sha256,
|
||||
packaging,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn parse<T: for<'de> Deserialize<'de>>(url: &str, body: &[u8]) -> Result<T, Error> {
|
||||
serde_json::from_slice(body).map_err(|source| Error::Metadata {
|
||||
url: url.to_owned(),
|
||||
source,
|
||||
})
|
||||
}
|
||||
15
litellm-rust/crates/testkit/src/lib.rs
Normal file
15
litellm-rust/crates/testkit/src/lib.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
mod agent;
|
||||
mod error;
|
||||
mod install;
|
||||
mod session;
|
||||
mod target;
|
||||
|
||||
pub use agent::{
|
||||
Agent, ClaudeCode, Codex, Configure, Drive, Install, LaunchSpec, Opencode, Outcome, Prompt,
|
||||
Settings, Usage, Wire,
|
||||
};
|
||||
pub use error::Error;
|
||||
pub use install::{Fetch, HttpFetch, Installed, Installer, Packaging, Release};
|
||||
pub use semver::Version;
|
||||
pub use session::Session;
|
||||
pub use target::{Arch, Os, Target};
|
||||
76
litellm-rust/crates/testkit/src/session.rs
Normal file
76
litellm-rust/crates/testkit/src/session.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
|
||||
use semver::Version;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::timeout;
|
||||
|
||||
use crate::{Configure, Drive, Error, Installed, Outcome, Prompt, Settings};
|
||||
|
||||
const STDERR_LIMIT_CHARS: usize = 2000;
|
||||
|
||||
pub struct Session {
|
||||
binary: PathBuf,
|
||||
home: PathBuf,
|
||||
version: Version,
|
||||
settings: Settings,
|
||||
env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub fn prepare(
|
||||
agent: &impl Configure,
|
||||
installed: &Installed,
|
||||
settings: Settings,
|
||||
home: impl Into<PathBuf>,
|
||||
) -> Result<Self, Error> {
|
||||
let home = home.into();
|
||||
let spec = agent.configure(&installed.version, &settings, &home)?;
|
||||
spec.write_files(&home)?;
|
||||
Ok(Self {
|
||||
binary: installed.binary.clone(),
|
||||
home,
|
||||
version: installed.version.clone(),
|
||||
settings,
|
||||
env: spec.env,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
&self,
|
||||
agent: &impl Drive,
|
||||
prompt: &Prompt,
|
||||
limit: Duration,
|
||||
) -> Result<Outcome, Error> {
|
||||
let child = Command::new(&self.binary)
|
||||
.args(agent.args(&self.version, &self.settings, prompt))
|
||||
.env_clear()
|
||||
.env("PATH", "/usr/bin:/bin")
|
||||
.envs(&self.env)
|
||||
.current_dir(&self.home)
|
||||
.stdin(Stdio::null())
|
||||
.kill_on_drop(true)
|
||||
.output();
|
||||
let output = timeout(limit, child)
|
||||
.await
|
||||
.map_err(|_| Error::Timeout(limit))??;
|
||||
let parsed = agent.parse(&self.version, &String::from_utf8_lossy(&output.stdout));
|
||||
let failed_silently = !output.status.success() && parsed.errors.is_empty();
|
||||
Ok(Outcome {
|
||||
errors: if failed_silently {
|
||||
vec![
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
.chars()
|
||||
.take(STDERR_LIMIT_CHARS)
|
||||
.collect(),
|
||||
]
|
||||
} else {
|
||||
parsed.errors
|
||||
},
|
||||
exit_code: output.status.code(),
|
||||
..parsed
|
||||
})
|
||||
}
|
||||
}
|
||||
69
litellm-rust/crates/testkit/src/target.rs
Normal file
69
litellm-rust/crates/testkit/src/target.rs
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
use target_lexicon::{Architecture, Environment, OperatingSystem, Triple};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Os {
|
||||
Macos,
|
||||
Linux,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Arch {
|
||||
Aarch64,
|
||||
X86_64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Target {
|
||||
pub os: Os,
|
||||
pub arch: Arch,
|
||||
pub musl: bool,
|
||||
}
|
||||
|
||||
impl Target {
|
||||
pub fn host() -> Result<Self, Error> {
|
||||
Self::try_from(&Triple::host())
|
||||
}
|
||||
|
||||
pub(crate) const fn os_name(self) -> &'static str {
|
||||
match self.os {
|
||||
Os::Macos => "darwin",
|
||||
Os::Linux => "linux",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn arch_name(self) -> &'static str {
|
||||
match self.arch {
|
||||
Arch::Aarch64 => "arm64",
|
||||
Arch::X86_64 => "x64",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn musl_suffix(self) -> &'static str {
|
||||
if self.musl { "-musl" } else { "" }
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&Triple> for Target {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(triple: &Triple) -> Result<Self, Error> {
|
||||
let unsupported = || Error::UnsupportedTarget(triple.to_string());
|
||||
let os = match triple.operating_system {
|
||||
OperatingSystem::Darwin(_) | OperatingSystem::MacOSX(_) => Os::Macos,
|
||||
OperatingSystem::Linux => Os::Linux,
|
||||
_ => return Err(unsupported()),
|
||||
};
|
||||
let arch = match triple.architecture {
|
||||
Architecture::Aarch64(_) => Arch::Aarch64,
|
||||
Architecture::X86_64 => Arch::X86_64,
|
||||
_ => return Err(unsupported()),
|
||||
};
|
||||
Ok(Self {
|
||||
os,
|
||||
arch,
|
||||
musl: triple.environment == Environment::Musl,
|
||||
})
|
||||
}
|
||||
}
|
||||
133
litellm-rust/crates/testkit/tests/configure.rs
Normal file
133
litellm-rust/crates/testkit/tests/configure.rs
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
use std::path::Path;
|
||||
|
||||
use litellm_testkit::{ClaudeCode, Codex, Configure, Error, Opencode, Settings, Version, Wire};
|
||||
use rstest::rstest;
|
||||
|
||||
fn settings(wire: Wire) -> Settings {
|
||||
Settings {
|
||||
base_url: "http://localhost:4000/".to_owned(),
|
||||
api_key: "sk-test \"quoted\"".to_owned(),
|
||||
model: "some-model".to_owned(),
|
||||
wire,
|
||||
}
|
||||
}
|
||||
|
||||
fn version() -> Version {
|
||||
Version::new(1, 2, 3)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(&ClaudeCode, Wire::Messages)]
|
||||
#[case(&Codex, Wire::Responses)]
|
||||
#[case(&Opencode, Wire::ChatCompletions)]
|
||||
fn every_agent_runs_inside_the_given_home(#[case] agent: &impl Configure, #[case] wire: Wire) {
|
||||
let home = Path::new("/scratch/home");
|
||||
|
||||
let spec = agent.configure(&version(), &settings(wire), home).unwrap();
|
||||
|
||||
assert_eq!(spec.env["HOME"], "/scratch/home");
|
||||
assert!(
|
||||
spec.env
|
||||
.iter()
|
||||
.filter(|(key, _)| key.ends_with("_HOME") || key.as_str() == "CLAUDE_CONFIG_DIR")
|
||||
.all(|(_, value)| value.starts_with("/scratch/home"))
|
||||
);
|
||||
assert!(spec.files.keys().all(|path| path.is_relative()));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::claude_code(&ClaudeCode, &[Wire::ChatCompletions, Wire::Responses])]
|
||||
#[case::codex(&Codex, &[Wire::ChatCompletions, Wire::Messages])]
|
||||
fn wires_an_agent_cannot_speak_are_refused(
|
||||
#[case] agent: &impl Configure,
|
||||
#[case] refused: &[Wire],
|
||||
) {
|
||||
refused.iter().for_each(|wire| {
|
||||
let result = agent.configure(&version(), &settings(*wire), Path::new("/h"));
|
||||
|
||||
assert!(matches!(result, Err(Error::UnsupportedWire { wire: got, .. }) if got == *wire));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claude_code_points_at_the_gateway_root_with_the_key_and_model() {
|
||||
let spec = ClaudeCode
|
||||
.configure(&version(), &settings(Wire::Messages), Path::new("/h"))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(spec.env["ANTHROPIC_BASE_URL"], "http://localhost:4000/");
|
||||
assert_eq!(spec.env["ANTHROPIC_AUTH_TOKEN"], "sk-test \"quoted\"");
|
||||
assert_eq!(spec.env["ANTHROPIC_MODEL"], "some-model");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_config_is_valid_toml_routing_the_responses_api_to_the_gateway() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let spec = Codex
|
||||
.configure(&version(), &settings(Wire::Responses), dir.path())
|
||||
.unwrap();
|
||||
spec.write_files(dir.path()).unwrap();
|
||||
|
||||
let config: toml::Table =
|
||||
toml::from_str(&std::fs::read_to_string(dir.path().join(".codex/config.toml")).unwrap())
|
||||
.unwrap();
|
||||
let provider = &config["model_providers"]["litellm"];
|
||||
|
||||
assert_eq!(config["model"].as_str(), Some("some-model"));
|
||||
assert_eq!(config["model_provider"].as_str(), Some("litellm"));
|
||||
assert_eq!(
|
||||
provider["base_url"].as_str(),
|
||||
Some("http://localhost:4000/v1")
|
||||
);
|
||||
assert_eq!(provider["wire_api"].as_str(), Some("responses"));
|
||||
let key_var = provider["env_key"].as_str().unwrap();
|
||||
assert_eq!(spec.env[key_var], "sk-test \"quoted\"");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(Wire::ChatCompletions)]
|
||||
#[case(Wire::Responses)]
|
||||
#[case(Wire::Messages)]
|
||||
fn opencode_config_is_valid_json_registering_the_gateway_model(#[case] wire: Wire) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let spec = Opencode
|
||||
.configure(&version(), &settings(wire), dir.path())
|
||||
.unwrap();
|
||||
spec.write_files(dir.path()).unwrap();
|
||||
|
||||
let config: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(dir.path().join(".config/opencode/opencode.json")).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let provider = &config["provider"]["litellm"];
|
||||
|
||||
assert_eq!(config["model"], "litellm/some-model");
|
||||
assert_eq!(provider["options"]["baseURL"], "http://localhost:4000/v1");
|
||||
assert_eq!(provider["options"]["apiKey"], "sk-test \"quoted\"");
|
||||
assert!(provider["models"]["some-model"].is_object());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opencode_uses_a_different_provider_package_for_every_wire() {
|
||||
let package = |wire| {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let spec = Opencode
|
||||
.configure(&version(), &settings(wire), dir.path())
|
||||
.unwrap();
|
||||
let config: serde_json::Value =
|
||||
serde_json::from_str(spec.files.values().next().unwrap()).unwrap();
|
||||
config["provider"]["litellm"]["npm"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned()
|
||||
};
|
||||
let packages = [Wire::ChatCompletions, Wire::Responses, Wire::Messages].map(package);
|
||||
|
||||
assert_eq!(
|
||||
packages
|
||||
.iter()
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.len(),
|
||||
packages.len()
|
||||
);
|
||||
}
|
||||
262
litellm-rust/crates/testkit/tests/install.rs
Normal file
262
litellm-rust/crates/testkit/tests/install.rs
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
mod support;
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use litellm_testkit::{ClaudeCode, Codex, Error, Installer, Opencode, Target, Version};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
use support::{FakeFetch, script_printing, sha256, tar_gz, zip_archive};
|
||||
use target_lexicon::Triple;
|
||||
|
||||
fn target(triple: &str) -> Target {
|
||||
Target::try_from(&Triple::from_str(triple).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn linux() -> Target {
|
||||
target("x86_64-unknown-linux-gnu")
|
||||
}
|
||||
fn version() -> Version {
|
||||
Version::new(9, 8, 7)
|
||||
}
|
||||
|
||||
fn github_release(asset: &str, download_url: &str, digest: Option<String>) -> Vec<u8> {
|
||||
json!({
|
||||
"assets": [
|
||||
{ "name": "unrelated.txt", "digest": "sha256:00", "browser_download_url": "https://example.test/unrelated" },
|
||||
{ "name": asset, "digest": digest, "browser_download_url": download_url },
|
||||
]
|
||||
})
|
||||
.to_string()
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn claude_routes(binary: &[u8], checksum: &str) -> Vec<(String, Vec<u8>)> {
|
||||
let base = "https://downloads.claude.ai/claude-code-releases/9.8.7";
|
||||
let manifest = json!({ "platforms": { "linux-x64": { "checksum": checksum } } });
|
||||
vec![
|
||||
(
|
||||
format!("{base}/manifest.json"),
|
||||
manifest.to_string().into_bytes(),
|
||||
),
|
||||
(format!("{base}/linux-x64/claude"), binary.to_vec()),
|
||||
]
|
||||
}
|
||||
|
||||
fn codex_routes(archive: Vec<u8>, digest: Option<String>) -> Vec<(String, Vec<u8>)> {
|
||||
let release = github_release(
|
||||
"codex-x86_64-unknown-linux-musl.tar.gz",
|
||||
"https://example.test/codex.tar.gz",
|
||||
digest,
|
||||
);
|
||||
vec![
|
||||
(
|
||||
"https://api.github.com/repos/openai/codex/releases/tags/rust-v9.8.7".to_owned(),
|
||||
release,
|
||||
),
|
||||
("https://example.test/codex.tar.gz".to_owned(), archive),
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn claude_bare_binary_is_installed_and_runnable() {
|
||||
let binary = script_printing("9.8.7 (Claude Code)");
|
||||
let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary)));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let installed = Installer::new(&fetch, cache.path(), linux())
|
||||
.install(&ClaudeCode, &version())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(installed.binary, cache.path().join("claude/9.8.7/claude"));
|
||||
assert_eq!(std::fs::read(&installed.binary).unwrap(), binary);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_binary_is_extracted_from_the_tarball_under_its_own_name() {
|
||||
let binary = script_printing("codex-cli 9.8.7");
|
||||
let archive = tar_gz("codex-x86_64-unknown-linux-musl", &binary);
|
||||
let fetch = FakeFetch::new(codex_routes(
|
||||
archive.clone(),
|
||||
Some(format!("sha256:{}", sha256(&archive))),
|
||||
));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let installed = Installer::new(&fetch, cache.path(), linux())
|
||||
.install(&Codex, &version())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&installed.binary).unwrap(), binary);
|
||||
assert_eq!(installed.binary, cache.path().join("codex/9.8.7/codex"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opencode_binary_is_extracted_from_the_darwin_zip() {
|
||||
let binary = script_printing("9.8.7");
|
||||
let archive = zip_archive("opencode", &binary);
|
||||
let release = github_release(
|
||||
"opencode-darwin-arm64.zip",
|
||||
"https://example.test/opencode.zip",
|
||||
Some(format!("sha256:{}", sha256(&archive))),
|
||||
);
|
||||
let fetch = FakeFetch::new([
|
||||
(
|
||||
"https://api.github.com/repos/sst/opencode/releases/tags/v9.8.7".to_owned(),
|
||||
release,
|
||||
),
|
||||
("https://example.test/opencode.zip".to_owned(), archive),
|
||||
]);
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let installed = Installer::new(&fetch, cache.path(), target("aarch64-apple-darwin"))
|
||||
.install(&Opencode, &version())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&installed.binary).unwrap(), binary);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tampered_download_is_rejected_and_nothing_is_left_behind() {
|
||||
let binary = script_printing("9.8.7 (Claude Code)");
|
||||
let fetch = FakeFetch::new(claude_routes(&binary, &sha256(b"what the vendor signed")));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = Installer::new(&fetch, cache.path(), linux())
|
||||
.install(&ClaudeCode, &version())
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::ChecksumMismatch { .. })));
|
||||
assert!(!cache.path().join("claude/9.8.7").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn github_asset_without_a_digest_is_refused() {
|
||||
let archive = tar_gz(
|
||||
"codex-x86_64-unknown-linux-musl",
|
||||
&script_printing("codex-cli 9.8.7"),
|
||||
);
|
||||
let fetch = FakeFetch::new(codex_routes(archive, None));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = Installer::new(&fetch, cache.path(), linux())
|
||||
.install(&Codex, &version())
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::MissingChecksum(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn binary_reporting_a_different_version_is_removed() {
|
||||
let binary = script_printing("1.0.0 (Claude Code)");
|
||||
let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary)));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = Installer::new(&fetch, cache.path(), linux())
|
||||
.install(&ClaudeCode, &version())
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::VersionMismatch { .. })));
|
||||
assert!(!cache.path().join("claude/9.8.7/claude").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn second_install_reuses_the_cached_binary_without_downloading() {
|
||||
let binary = script_printing("9.8.7 (Claude Code)");
|
||||
let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary)));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let installer = Installer::new(&fetch, cache.path(), linux());
|
||||
|
||||
let first = installer.install(&ClaudeCode, &version()).await.unwrap();
|
||||
let calls_after_first = fetch.calls();
|
||||
let second = installer.install(&ClaudeCode, &version()).await.unwrap();
|
||||
|
||||
assert_eq!(first, second);
|
||||
assert_eq!(fetch.calls(), calls_after_first);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupted_cache_entry_is_replaced_by_a_fresh_download() {
|
||||
let binary = script_printing("9.8.7 (Claude Code)");
|
||||
let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary)));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let installer = Installer::new(&fetch, cache.path(), linux());
|
||||
let installed = installer.install(&ClaudeCode, &version()).await.unwrap();
|
||||
std::fs::write(&installed.binary, script_printing("0.0.1")).unwrap();
|
||||
|
||||
installer.install(&ClaudeCode, &version()).await.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&installed.binary).unwrap(), binary);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("9.8.7-beta.1")]
|
||||
#[case("9.8.7+build.5")]
|
||||
#[tokio::test]
|
||||
async fn pre_releases_never_reach_the_network_or_the_filesystem(#[case] version: &str) {
|
||||
let fetch = FakeFetch::new([]);
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = Installer::new(&fetch, cache.path(), linux())
|
||||
.install(&ClaudeCode, &Version::parse(version).unwrap())
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::InvalidVersion(_))));
|
||||
assert_eq!(fetch.calls(), 0);
|
||||
assert_eq!(std::fs::read_dir(cache.path()).unwrap().count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn musl_linux_picks_the_musl_claude_build() {
|
||||
let binary = script_printing("9.8.7 (Claude Code)");
|
||||
let base = "https://downloads.claude.ai/claude-code-releases/9.8.7";
|
||||
let manifest = json!({ "platforms": {
|
||||
"linux-x64": { "checksum": sha256(b"glibc build") },
|
||||
"linux-x64-musl": { "checksum": sha256(&binary) },
|
||||
} });
|
||||
let fetch = FakeFetch::new([
|
||||
(
|
||||
format!("{base}/manifest.json"),
|
||||
manifest.to_string().into_bytes(),
|
||||
),
|
||||
(format!("{base}/linux-x64-musl/claude"), binary.clone()),
|
||||
]);
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
|
||||
let installed = Installer::new(&fetch, cache.path(), target("x86_64-unknown-linux-musl"))
|
||||
.install(&ClaudeCode, &version())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&installed.binary).unwrap(), binary);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("x86_64-pc-windows-msvc")]
|
||||
#[case("riscv64gc-unknown-linux-gnu")]
|
||||
#[case("wasm32-unknown-unknown")]
|
||||
fn targets_no_agent_ships_for_are_rejected(#[case] triple: &str) {
|
||||
let result = Target::try_from(&Triple::from_str(triple).unwrap());
|
||||
|
||||
assert!(matches!(result, Err(Error::UnsupportedTarget(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_installs_of_the_same_version_both_succeed() {
|
||||
let binary = script_printing("9.8.7 (Claude Code)");
|
||||
let fetch = FakeFetch::new(claude_routes(&binary, &sha256(&binary)));
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let installer = Installer::new(&fetch, cache.path(), linux());
|
||||
|
||||
let wanted = version();
|
||||
let installs =
|
||||
futures_util::future::join_all((0..8).map(|_| installer.install(&ClaudeCode, &wanted)))
|
||||
.await;
|
||||
|
||||
assert!(installs.iter().all(Result::is_ok));
|
||||
assert_eq!(
|
||||
std::fs::read(&installs[0].as_ref().unwrap().binary).unwrap(),
|
||||
binary
|
||||
);
|
||||
}
|
||||
133
litellm-rust/crates/testkit/tests/live.rs
Normal file
133
litellm-rust/crates/testkit/tests/live.rs
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
//! Drives the real agents through a real gateway. Run with `cargo test -p litellm-testkit --test live -- --ignored`
|
||||
//! after exporting `TESTKIT_GATEWAY_URL`, `TESTKIT_GATEWAY_KEY`, one `TESTKIT_MODEL_<WIRE>` per wire
|
||||
//! (`MESSAGES`, `RESPONSES`, `CHAT_COMPLETIONS`) and one `TESTKIT_<AGENT>_VERSION` per agent
|
||||
//! (`CLAUDE`, `CODEX`, `OPENCODE`). `TESTKIT_CACHE_DIR` and `GITHUB_TOKEN` are optional.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_testkit::{
|
||||
Agent, ClaudeCode, Codex, HttpFetch, Installer, Opencode, Outcome, Prompt, Session, Settings,
|
||||
Target, Version, Wire,
|
||||
};
|
||||
use rstest::rstest;
|
||||
|
||||
const LIMIT: Duration = Duration::from_secs(180);
|
||||
|
||||
fn required(name: &str) -> String {
|
||||
std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set to run the live tests"))
|
||||
}
|
||||
|
||||
fn model_var(wire: Wire) -> &'static str {
|
||||
match wire {
|
||||
Wire::Messages => "TESTKIT_MODEL_MESSAGES",
|
||||
Wire::Responses => "TESTKIT_MODEL_RESPONSES",
|
||||
Wire::ChatCompletions => "TESTKIT_MODEL_CHAT_COMPLETIONS",
|
||||
}
|
||||
}
|
||||
|
||||
async fn drive(
|
||||
agent: &impl Agent,
|
||||
version_var: &str,
|
||||
wire: Wire,
|
||||
model: Option<&str>,
|
||||
prompt: Prompt,
|
||||
) -> Outcome {
|
||||
let cache = std::env::var("TESTKIT_CACHE_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| std::env::temp_dir().join("litellm-testkit-cache"));
|
||||
let installer = Installer::new(HttpFetch::from_env(), cache, Target::host().unwrap());
|
||||
let installed = installer
|
||||
.install(agent, &Version::parse(&required(version_var)).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let settings = Settings {
|
||||
base_url: required("TESTKIT_GATEWAY_URL"),
|
||||
api_key: required("TESTKIT_GATEWAY_KEY"),
|
||||
model: model.map_or_else(|| required(model_var(wire)), str::to_owned),
|
||||
wire,
|
||||
};
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
let session = Session::prepare(agent, &installed, settings, home.path()).unwrap();
|
||||
session.run(agent, &prompt, LIMIT).await.unwrap()
|
||||
}
|
||||
|
||||
fn text_prompt() -> Prompt {
|
||||
Prompt {
|
||||
text: "Reply with the single word: pong".to_owned(),
|
||||
allow_tools: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_prompt() -> Prompt {
|
||||
Prompt {
|
||||
text: "Run the shell command 'echo tool-ok' and reply with exactly its output.".to_owned(),
|
||||
allow_tools: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::claude_messages(&ClaudeCode, "TESTKIT_CLAUDE_VERSION", Wire::Messages)]
|
||||
#[case::codex_responses(&Codex, "TESTKIT_CODEX_VERSION", Wire::Responses)]
|
||||
#[case::opencode_chat(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::ChatCompletions)]
|
||||
#[case::opencode_responses(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Responses)]
|
||||
#[case::opencode_messages(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Messages)]
|
||||
#[ignore = "needs a live gateway, see the module docs"]
|
||||
#[tokio::test]
|
||||
async fn plain_prompt_gets_an_answer_and_token_usage(
|
||||
#[case] agent: &impl Agent,
|
||||
#[case] version_var: &str,
|
||||
#[case] wire: Wire,
|
||||
) {
|
||||
let outcome = drive(agent, version_var, wire, None, text_prompt()).await;
|
||||
|
||||
assert!(outcome.succeeded(), "{outcome:?}");
|
||||
assert!(outcome.text.to_lowercase().contains("pong"), "{outcome:?}");
|
||||
assert!(outcome.usage.output_tokens > 0, "{outcome:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::claude_messages(&ClaudeCode, "TESTKIT_CLAUDE_VERSION", Wire::Messages)]
|
||||
#[case::codex_responses(&Codex, "TESTKIT_CODEX_VERSION", Wire::Responses)]
|
||||
#[case::opencode_chat(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::ChatCompletions)]
|
||||
#[case::opencode_responses(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Responses)]
|
||||
#[case::opencode_messages(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Messages)]
|
||||
#[ignore = "needs a live gateway, see the module docs"]
|
||||
#[tokio::test]
|
||||
async fn tool_use_is_reported_and_its_result_reaches_the_answer(
|
||||
#[case] agent: &impl Agent,
|
||||
#[case] version_var: &str,
|
||||
#[case] wire: Wire,
|
||||
) {
|
||||
let outcome = drive(agent, version_var, wire, None, tool_prompt()).await;
|
||||
|
||||
assert!(outcome.succeeded(), "{outcome:?}");
|
||||
assert!(!outcome.tool_calls.is_empty(), "{outcome:?}");
|
||||
assert!(outcome.text.contains("tool-ok"), "{outcome:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::claude_messages(&ClaudeCode, "TESTKIT_CLAUDE_VERSION", Wire::Messages)]
|
||||
#[case::codex_responses(&Codex, "TESTKIT_CODEX_VERSION", Wire::Responses)]
|
||||
#[case::opencode_chat(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::ChatCompletions)]
|
||||
#[case::opencode_responses(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Responses)]
|
||||
#[case::opencode_messages(&Opencode, "TESTKIT_OPENCODE_VERSION", Wire::Messages)]
|
||||
#[ignore = "needs a live gateway, see the module docs"]
|
||||
#[tokio::test]
|
||||
async fn model_the_gateway_rejects_is_reported_as_an_error(
|
||||
#[case] agent: &impl Agent,
|
||||
#[case] version_var: &str,
|
||||
#[case] wire: Wire,
|
||||
) {
|
||||
let outcome = drive(
|
||||
agent,
|
||||
version_var,
|
||||
wire,
|
||||
Some("testkit-no-such-model"),
|
||||
text_prompt(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(!outcome.succeeded(), "{outcome:?}");
|
||||
assert!(!outcome.errors.is_empty(), "{outcome:?}");
|
||||
}
|
||||
155
litellm-rust/crates/testkit/tests/session.rs
Normal file
155
litellm-rust/crates/testkit/tests/session.rs
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_testkit::{
|
||||
Configure, Drive, Error, Installed, LaunchSpec, Outcome, Prompt, Session, Settings, Version,
|
||||
Wire,
|
||||
};
|
||||
|
||||
struct Scripted;
|
||||
|
||||
impl Configure for Scripted {
|
||||
fn configure(
|
||||
&self,
|
||||
version: &Version,
|
||||
_settings: &Settings,
|
||||
home: &Path,
|
||||
) -> Result<LaunchSpec, Error> {
|
||||
Ok(LaunchSpec {
|
||||
env: [
|
||||
("AGENT_HOME".to_owned(), home.to_string_lossy().into_owned()),
|
||||
("AGENT_SAW_VERSION".to_owned(), version.to_string()),
|
||||
]
|
||||
.into(),
|
||||
files: [(
|
||||
PathBuf::from("conf/agent.toml"),
|
||||
"configured = true\n".to_owned(),
|
||||
)]
|
||||
.into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drive for Scripted {
|
||||
fn args(&self, _version: &Version, _settings: &Settings, prompt: &Prompt) -> Vec<String> {
|
||||
vec!["--prompt".to_owned(), prompt.text.clone()]
|
||||
}
|
||||
|
||||
fn parse(&self, _version: &Version, stdout: &str) -> Outcome {
|
||||
Outcome {
|
||||
text: stdout.to_owned(),
|
||||
..Outcome::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn settings() -> Settings {
|
||||
Settings {
|
||||
base_url: "http://gateway.test".to_owned(),
|
||||
api_key: "sk-test".to_owned(),
|
||||
model: "some-model".to_owned(),
|
||||
wire: Wire::Messages,
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt(text: &str) -> Prompt {
|
||||
Prompt {
|
||||
text: text.to_owned(),
|
||||
allow_tools: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn session(script: &str) -> (Session, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let binary = dir.path().join("agent");
|
||||
std::fs::write(&binary, format!("#!/bin/sh\n{script}\n")).unwrap();
|
||||
std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
let home = dir.path().join("home");
|
||||
std::fs::create_dir(&home).unwrap();
|
||||
let installed = Installed {
|
||||
version: Version::new(4, 5, 6),
|
||||
binary,
|
||||
};
|
||||
(
|
||||
Session::prepare(&Scripted, &installed, settings(), home).unwrap(),
|
||||
dir,
|
||||
)
|
||||
}
|
||||
|
||||
const LIMIT: Duration = Duration::from_secs(20);
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_writes_the_config_files_under_home() {
|
||||
let (_session, dir) = session("true");
|
||||
|
||||
let written = std::fs::read_to_string(dir.path().join("home/conf/agent.toml")).unwrap();
|
||||
|
||||
assert_eq!(written, "configured = true\n");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_and_drive_are_given_the_installed_version() {
|
||||
let (session, _dir) = session("echo \"$AGENT_SAW_VERSION\"");
|
||||
|
||||
let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap();
|
||||
|
||||
assert_eq!(outcome.text.trim(), "4.5.6");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn agent_runs_in_home_with_only_its_own_environment() {
|
||||
let (session, dir) = session("pwd -P; env");
|
||||
|
||||
let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap();
|
||||
|
||||
let home = dir.path().join("home").canonicalize().unwrap();
|
||||
assert_eq!(outcome.text.lines().next().unwrap(), home.to_string_lossy());
|
||||
assert!(outcome.text.contains("AGENT_HOME="));
|
||||
assert!(
|
||||
!outcome.text.contains("CARGO_"),
|
||||
"test runner environment leaked into the agent"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prompt_reaches_the_agent_as_one_untouched_argument() {
|
||||
let (session, _dir) = session("printf '%s|' \"$@\"");
|
||||
let text = "two spaces; $(echo injected) 'quoted'";
|
||||
|
||||
let outcome = session.run(&Scripted, &prompt(text), LIMIT).await.unwrap();
|
||||
|
||||
assert_eq!(outcome.text, format!("--prompt|{text}|"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clean_exit_is_a_success() {
|
||||
let (session, _dir) = session("echo done");
|
||||
|
||||
let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap();
|
||||
|
||||
assert_eq!(outcome.exit_code, Some(0));
|
||||
assert!(outcome.succeeded());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failing_exit_without_a_parsed_error_reports_stderr() {
|
||||
let (session, _dir) = session("echo boom >&2; exit 3");
|
||||
|
||||
let outcome = session.run(&Scripted, &prompt("hi"), LIMIT).await.unwrap();
|
||||
|
||||
assert_eq!(outcome.exit_code, Some(3));
|
||||
assert!(!outcome.succeeded());
|
||||
assert_eq!(outcome.errors, ["boom\n"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn agent_that_outlives_the_limit_is_stopped() {
|
||||
let (session, _dir) = session("sleep 30");
|
||||
|
||||
let result = session
|
||||
.run(&Scripted, &prompt("hi"), Duration::from_millis(200))
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::Timeout(_))));
|
||||
}
|
||||
70
litellm-rust/crates/testkit/tests/support/mod.rs
Normal file
70
litellm-rust/crates/testkit/tests/support/mod.rs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
use std::collections::HashMap;
|
||||
use std::io::Write;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use litellm_testkit::{Error, Fetch};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub struct FakeFetch {
|
||||
routes: HashMap<String, Vec<u8>>,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FakeFetch {
|
||||
pub fn new(routes: impl IntoIterator<Item = (String, Vec<u8>)>) -> Self {
|
||||
Self {
|
||||
routes: routes.into_iter().collect(),
|
||||
calls: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calls(&self) -> usize {
|
||||
self.calls.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetch for FakeFetch {
|
||||
async fn get(&self, url: &str) -> Result<Vec<u8>, Error> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
self.routes.get(url).cloned().ok_or_else(|| Error::Status {
|
||||
url: url.to_owned(),
|
||||
status: 404,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetch for &FakeFetch {
|
||||
async fn get(&self, url: &str) -> Result<Vec<u8>, Error> {
|
||||
(*self).get(url).await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sha256(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
pub fn script_printing(output: &str) -> Vec<u8> {
|
||||
format!("#!/bin/sh\necho '{output}'\n").into_bytes()
|
||||
}
|
||||
|
||||
pub fn tar_gz(member: &str, contents: &[u8]) -> Vec<u8> {
|
||||
let mut builder = tar::Builder::new(Vec::new());
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(contents.len() as u64);
|
||||
header.set_mode(0o755);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, member, contents).unwrap();
|
||||
let tarball = builder.into_inner().unwrap();
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(&tarball).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
pub fn zip_archive(member: &str, contents: &[u8]) -> Vec<u8> {
|
||||
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
|
||||
writer
|
||||
.start_file(member, zip::write::SimpleFileOptions::default())
|
||||
.unwrap();
|
||||
writer.write_all(contents).unwrap();
|
||||
writer.finish().unwrap().into_inner()
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue