This commit is contained in:
Alex Fournier 2026-10-28 09:15:43 -04:00 • committed by GitHub
commit bf60250d97
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 2066 additions and 30 deletions

View file

@ -101,6 +101,8 @@ jobs:
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
- run: cargo clippy -p litellm-gateway-inference -p litellm-gateway --all-targets --features switchyard --locked -- -D warnings
rust-test:
runs-on: ubuntu-latest
timeout-minutes: 30
@ -147,6 +149,8 @@ jobs:
- run: cargo nextest run --workspace --locked
- run: cargo nextest run -p litellm-gateway-inference -p litellm-gateway --features switchyard --locked
- run: cargo test --workspace --doc --locked
- name: Test token counter feature combinations

238
litellm-rust/Cargo.lock generated
View file

@ -209,9 +209,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.91"
version = "0.1.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
dependencies = [
"proc-macro2",
"quote",
@ -2122,6 +2122,16 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fraction"
version = "0.15.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872"
dependencies = [
"lazy_static",
"num",
]
[[package]]
name = "fraction"
version = "0.17.0"
@ -3249,6 +3259,45 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "jsonptr"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72c22d99362c7aac7d77efab6b49c2181674b165d7e6e7c755b3432ded64bf51"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "jsonschema"
version = "0.49.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2"
dependencies = [
"ahash",
"bytecount",
"data-encoding",
"email_address",
"fancy-regex 0.19.2",
"fraction 0.15.4",
"getrandom 0.3.4",
"idna",
"itoa",
"jsonschema-regex 0.49.9",
"jsonschema-value 0.49.9",
"num-cmp",
"num-traits",
"percent-encoding",
"referencing 0.49.9",
"regex",
"serde",
"serde_json",
"strum",
"unicode-general-category",
"uuid-simd",
]
[[package]]
name = "jsonschema"
version = "0.55.1"
@ -3260,15 +3309,15 @@ dependencies = [
"data-encoding",
"email_address",
"fancy-regex 0.19.2",
"fraction",
"fraction 0.17.0",
"getrandom 0.3.4",
"itoa",
"jsonschema-regex",
"jsonschema-value",
"jsonschema-regex 0.55.1",
"jsonschema-value 0.55.1",
"num-cmp",
"num-traits",
"percent-encoding",
"referencing",
"referencing 0.55.1",
"regex",
"serde",
"serde_json",
@ -3277,6 +3326,15 @@ dependencies = [
"uuid-simd",
]
[[package]]
name = "jsonschema-regex"
version = "0.49.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85"
dependencies = [
"regex-syntax",
]
[[package]]
name = "jsonschema-regex"
version = "0.55.1"
@ -3286,6 +3344,20 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "jsonschema-value"
version = "0.49.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb"
dependencies = [
"ahash",
"bytecount",
"fraction 0.15.4",
"num-cmp",
"num-traits",
"serde_json",
]
[[package]]
name = "jsonschema-value"
version = "0.55.1"
@ -3294,7 +3366,7 @@ checksum = "0230ac05e09c6111e96c147b75c390579f5cbd45654b980c68ac60fe17b3f129"
dependencies = [
"ahash",
"bytecount",
"fraction",
"fraction 0.17.0",
"getrandom 0.3.4",
"num-cmp",
"num-traits",
@ -3817,6 +3889,7 @@ dependencies = [
name = "litellm-gateway-inference"
version = "0.1.0"
dependencies = [
"async-trait",
"axum",
"base64 0.22.1",
"bytes",
@ -3824,6 +3897,7 @@ dependencies = [
"litellm-auth",
"litellm-cache-memory",
"litellm-cache-response",
"litellm-config",
"litellm-gateway-auth",
"litellm-host",
"litellm-host-http",
@ -3841,6 +3915,12 @@ dependencies = [
"rstest",
"serde",
"serde_json",
"strum",
"switchyard-libsy",
"switchyard-llm-client",
"switchyard-protocol",
"switchyard-runner",
"switchyard-translation",
"thiserror 2.0.19",
"tokio",
"tower",
@ -4228,7 +4308,7 @@ name = "litellm-model-catalog"
version = "0.1.0"
dependencies = [
"indexmap 2.14.0",
"jsonschema",
"jsonschema 0.55.1",
"litellm-llms-types",
"rstest",
"schemars 1.2.2",
@ -4563,7 +4643,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.19",
"tokio",
"toml",
"toml 0.9.12+spec-1.1.0",
"zip",
]
@ -5878,6 +5958,23 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "referencing"
version = "0.49.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278"
dependencies = [
"ahash",
"fluent-uri",
"getrandom 0.3.4",
"hashbrown 0.17.1",
"itoa",
"micromap",
"parking_lot",
"percent-encoding",
"serde_json",
]
[[package]]
name = "referencing"
version = "0.55.1"
@ -7099,6 +7196,100 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "switchyard-libsy"
version = "0.3.0"
source = "git+https://github.com/NVIDIA-NeMo/Switchyard.git?rev=4bdf5719ce1c43902df92ef64407b858d1f99394#4bdf5719ce1c43902df92ef64407b858d1f99394"
dependencies = [
"async-trait",
"futures",
"http 1.4.2",
"jsonptr",
"jsonschema 0.49.9",
"opentelemetry",
"parking_lot",
"rand 0.10.2",
"regex",
"serde",
"serde_json",
"shlex",
"switchyard-protocol",
"thiserror 2.0.19",
"tokio",
"tokio-stream",
"tracing",
"tracing-opentelemetry",
"uuid",
]
[[package]]
name = "switchyard-llm-client"
version = "0.3.0"
source = "git+https://github.com/NVIDIA-NeMo/Switchyard.git?rev=4bdf5719ce1c43902df92ef64407b858d1f99394#4bdf5719ce1c43902df92ef64407b858d1f99394"
dependencies = [
"async-trait",
"futures",
"futures-util",
"http 1.4.2",
"httpdate",
"opentelemetry",
"parking_lot",
"reqwest 0.13.5",
"serde",
"serde_json",
"switchyard-libsy",
"switchyard-protocol",
"switchyard-translation",
"tokio",
"tracing",
"tracing-opentelemetry",
]
[[package]]
name = "switchyard-protocol"
version = "0.3.0"
source = "git+https://github.com/NVIDIA-NeMo/Switchyard.git?rev=4bdf5719ce1c43902df92ef64407b858d1f99394#4bdf5719ce1c43902df92ef64407b858d1f99394"
dependencies = [
"async-trait",
"futures",
"http 1.4.2",
"serde",
"serde_json",
"thiserror 2.0.19",
]
[[package]]
name = "switchyard-runner"
version = "0.3.0"
source = "git+https://github.com/NVIDIA-NeMo/Switchyard.git?rev=4bdf5719ce1c43902df92ef64407b858d1f99394#4bdf5719ce1c43902df92ef64407b858d1f99394"
dependencies = [
"reqwest 0.13.5",
"serde",
"serde_json",
"strum_macros",
"switchyard-libsy",
"switchyard-llm-client",
"switchyard-protocol",
"thiserror 2.0.19",
"toml 1.1.6+spec-1.1.0",
"tracing",
]
[[package]]
name = "switchyard-translation"
version = "0.3.0"
source = "git+https://github.com/NVIDIA-NeMo/Switchyard.git?rev=4bdf5719ce1c43902df92ef64407b858d1f99394#4bdf5719ce1c43902df92ef64407b858d1f99394"
dependencies = [
"async-stream",
"base64 0.22.1",
"futures",
"serde",
"serde_json",
"switchyard-protocol",
"thiserror 2.0.19",
"tracing",
]
[[package]]
name = "syn"
version = "1.0.109"
@ -7412,6 +7603,7 @@ dependencies = [
"bytes",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.6.5",
@ -7506,6 +7698,21 @@ dependencies = [
"winnow 0.7.15",
]
[[package]]
name = "toml"
version = "1.1.6+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a"
dependencies = [
"indexmap 2.14.0",
"serde_core",
"serde_spanned",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
"toml_writer",
"winnow 1.0.4",
]
[[package]]
name = "toml_datetime"
version = "0.7.5+spec-1.1.0"
@ -7775,6 +7982,17 @@ dependencies = [
"tracing",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-opentelemetry"
version = "0.33.0"
@ -7783,8 +8001,10 @@ checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
dependencies = [
"js-sys",
"opentelemetry",
"smallvec",
"tracing",
"tracing-core",
"tracing-log",
"tracing-subscriber",
"web-time",
]

View file

@ -5,11 +5,24 @@ edition.workspace = true
license.workspace = true
repository.workspace = true
[features]
switchyard = [
"dep:async-trait",
"dep:futures-util",
"dep:switchyard-libsy",
"dep:switchyard-llm-client",
"dep:switchyard-protocol",
"dep:switchyard-runner",
"dep:switchyard-translation",
]
[dependencies]
async-trait = { version = "0.1.92", optional = true }
litellm-cache-response.workspace = true
axum = { workspace = true, features = ["json", "multipart", "original-uri"] }
base64.workspace = true
bytes.workspace = true
futures-util = { workspace = true, optional = true }
litellm-auth.workspace = true
litellm-gateway-auth.workspace = true
litellm-inference.workspace = true
@ -18,6 +31,7 @@ litellm-inference-responses.workspace = true
litellm-inference-messages.workspace = true
litellm-inference-chat.workspace = true
litellm-inference-ocr.workspace = true
litellm-config.workspace = true
litellm-host-http.workspace = true
litellm-host.workspace = true
litellm-http.workspace = true
@ -27,6 +41,12 @@ litellm-secrets.workspace = true
litellm-llms-types.workspace = true
serde.workspace = true
serde_json.workspace = true
switchyard-libsy = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
switchyard-llm-client = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
switchyard-protocol = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
switchyard-runner = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
switchyard-translation = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]

View file

@ -0,0 +1,329 @@
use std::{
collections::{HashMap, HashSet},
future::Future,
pin::Pin,
sync::Arc,
};
use axum::{
Json,
extract::{Path, State},
http::HeaderMap,
response::{IntoResponse, Response},
};
use litellm_cache_response::CacheOptions;
use litellm_config::Model;
use litellm_gateway_auth::AuthenticatedRequest;
use serde_json::{Map, Value};
use crate::{AutoRoutingConfigError, Error, Gateway, JsonObject, ModelRouter, RequestId, request};
#[cfg(feature = "switchyard")]
mod switchyard;
#[derive(Clone, Copy, strum::VariantArray)]
pub enum ApiFormat {
Chat,
Responses,
Messages,
}
pub struct StrategyRequest {
pub format: ApiFormat,
pub body: Map<String, Value>,
pub headers: HeaderMap,
pub cache_options: CacheOptions,
}
pub trait AutoRoutingStrategy: Send + Sync {
fn execute<'a>(
&'a self,
gateway: &'a Gateway,
request: StrategyRequest,
) -> Pin<Box<dyn Future<Output = Result<Response, Error>> + Send + 'a>>;
}
#[derive(Default)]
pub(crate) struct Registry(HashMap<String, Arc<dyn AutoRoutingStrategy>>);
impl Registry {
pub(crate) fn insert(&mut self, model: String, strategy: Arc<dyn AutoRoutingStrategy>) {
self.0.insert(model, strategy);
}
pub(crate) fn from_models(
models: &[Model],
deployments: &ModelRouter,
) -> Result<Self, AutoRoutingConfigError> {
let mut seen = HashSet::new();
let mut strategies = HashMap::new();
for model in models.iter().rev() {
if !seen.insert(&model.model_name)
|| model.litellm_params.model != "switchyard/algorithm"
{
continue;
}
if deployments
.get(&model.model_name)
.is_none_or(|deployment| deployment.model != model.litellm_params.model)
{
return Err(AutoRoutingConfigError {
model: model.model_name.clone(),
reason: "model does not match its configured deployment".into(),
});
}
strategies.insert(
model.model_name.clone(),
build_strategy(model, models, deployments)?,
);
}
Ok(Self(strategies))
}
fn get(&self, body: &Map<String, Value>) -> Option<&dyn AutoRoutingStrategy> {
self.0.get(body.get("model")?.as_str()?).map(AsRef::as_ref)
}
}
#[cfg(feature = "switchyard")]
fn build_strategy(
model: &Model,
models: &[Model],
deployments: &ModelRouter,
) -> Result<Arc<dyn AutoRoutingStrategy>, AutoRoutingConfigError> {
Ok(Arc::new(switchyard::SwitchyardStrategy::from_model(
model,
models,
deployments,
)?))
}
#[cfg(not(feature = "switchyard"))]
fn build_strategy(
model: &Model,
_: &[Model],
_: &ModelRouter,
) -> Result<Arc<dyn AutoRoutingStrategy>, AutoRoutingConfigError> {
Err(AutoRoutingConfigError {
model: model.model_name.clone(),
reason: "this build does not enable the switchyard feature".into(),
})
}
async fn execute(
strategy: &dyn AutoRoutingStrategy,
gateway: &Gateway,
identity: &AuthenticatedRequest,
format: ApiFormat,
headers: HeaderMap,
body: Map<String, Value>,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(gateway, &body)?;
request::authorize_model(identity, deployment, &body).await?;
let (body, cache_options) = crate::caching::prepare(identity, body)?;
strategy
.execute(
gateway,
StrategyRequest {
format,
body,
headers,
cache_options,
},
)
.await
}
pub(crate) async fn chat(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<Response, Error> {
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Chat,
headers,
body,
)
.await;
}
crate::chat_completions::create(State(gateway), identity, JsonObject(body))
.await
.map(IntoResponse::into_response)
}
pub(crate) async fn chat_from_model_path(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
headers: HeaderMap,
Path(model): Path<String>,
JsonObject(mut body): JsonObject,
) -> Result<Response, Error> {
if body.get("model").is_none_or(Value::is_null) {
body.insert("model".into(), Value::String(model.clone()));
}
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Chat,
headers,
body,
)
.await;
}
crate::chat_completions::create_from_model_path(
State(gateway),
identity,
Path(model),
JsonObject(body),
)
.await
.map(IntoResponse::into_response)
}
pub(crate) async fn responses(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<Response, Error> {
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Responses,
headers,
body,
)
.await;
}
crate::responses::create(State(gateway), identity, JsonObject(body)).await
}
pub(crate) async fn messages(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
RequestId(request_id): RequestId,
headers: HeaderMap,
body: Result<JsonObject, Error>,
) -> Response {
match body {
Ok(JsonObject(body)) => {
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Messages,
headers,
body,
)
.await
.unwrap_or_else(|error| {
(error.status(), Json(error.body(request_id.as_deref()))).into_response()
});
}
crate::messages::create(
State(gateway),
identity,
RequestId(request_id),
headers,
Ok(JsonObject(body)),
)
.await
.into_response()
}
Err(error) => crate::messages::create(
State(gateway),
identity,
RequestId(request_id),
headers,
Err(error),
)
.await
.into_response(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use litellm_config::Config;
use rstest::rstest;
#[rstest]
fn overwritten_virtual_models_do_not_initialize_a_strategy() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: public
litellm_params: {model: switchyard/algorithm}
- model_name: public
litellm_params: {model: openai/model}
"#,
)
.unwrap();
let registry = Registry::from_models(
&config.model_list,
&ModelRouter::from_model_list(&config.model_list),
)
.unwrap();
assert!(
registry
.get(
&serde_json::json!({"model":"public"})
.as_object()
.unwrap()
.clone()
)
.is_none()
);
}
#[rstest]
fn strategy_configuration_must_match_the_deployment_registry() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: public
litellm_params: {model: switchyard/algorithm}
"#,
)
.unwrap();
let error = Registry::from_models(&config.model_list, &ModelRouter::default())
.err()
.unwrap();
assert!(error.to_string().contains("does not match"));
}
#[cfg(not(feature = "switchyard"))]
#[rstest]
fn disabled_strategies_fail_at_configuration_time() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: public
litellm_params: {model: switchyard/algorithm}
"#,
)
.unwrap();
let error = Registry::from_models(
&config.model_list,
&ModelRouter::from_model_list(&config.model_list),
)
.err()
.unwrap();
assert!(
error
.to_string()
.contains("does not enable the switchyard feature")
);
}
}

View file

@ -0,0 +1,635 @@
use super::{ApiFormat, AutoRoutingStrategy, StrategyRequest};
use litellm_inference_chat::types::ChatCompletionsRequest;
use litellm_inference_messages::{MessagesCall, messages_body};
use litellm_inference_responses::types::ResponsesCall;
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
use std::{
collections::{BTreeMap, HashMap},
convert::Infallible,
sync::Arc,
};
use async_trait::async_trait;
use axum::{
Json,
http::HeaderMap,
response::{
IntoResponse, Response as HttpResponse,
sse::{Event, Sse},
},
};
use futures_util::{StreamExt, stream};
use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache};
use litellm_config::Model;
use litellm_host::call::CallOutput;
use litellm_http::transport::Error as TransportError;
use litellm_inference::Error as InferenceError;
use litellm_inference_chat::ChatCompletionsRoute;
use litellm_inference_messages::MessagesRoute;
use litellm_inference_responses::ResponsesRoute;
use serde::Deserialize;
use serde_json::{Map, Value, json};
use switchyard_libsy::{Algorithm, LibsyError, RuntimeModels};
use switchyard_llm_client::ClientRouter;
use switchyard_protocol::{
BoxError, LlmClientError, LlmResponse, Metadata, ModelId, Request, Response, RoutedLlmClient,
WireFormat,
};
use switchyard_runner::AlgorithmSpec;
use switchyard_translation::{
LlmStreamError, RawEventStream, decode_aggregated_response, decode_request, decode_stream,
encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions,
};
use crate::{
AutoRoutingConfigError, Deployment, Error, Gateway, ModelRouter, caching::CacheHeaders,
};
const SWITCHYARD_MODEL: &str = "switchyard/algorithm";
pub(super) struct SwitchyardStrategy {
algorithm: Arc<dyn Algorithm>,
runtime_models: Arc<RuntimeModels>,
targets: Arc<HashMap<ModelId, Target>>,
}
impl SwitchyardStrategy {
pub(super) fn from_model(
model: &Model,
model_list: &[Model],
deployments: &ModelRouter,
) -> Result<Self, AutoRoutingConfigError> {
let config = model
.litellm_params
.extra
.get("switchyard_config")
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
let config = AlgorithmSpec::deserialize(config)
.map_err(|error| invalid(model, error.to_string()))?;
let targets = targets(model, &config, model_list, deployments)?;
let (algorithm, runtime_models) = config
.build_with_runtime_models(&model.model_name, &targets.models)
.map_err(|error| invalid(model, error.to_string()))?;
Ok(Self {
algorithm,
runtime_models: Arc::new(runtime_models),
targets: Arc::new(targets.by_model),
})
}
pub(crate) async fn execute(
&self,
gateway: &Gateway,
cache_options: CacheOptions,
wire_format: WireFormat,
body: Map<String, Value>,
headers: HeaderMap,
) -> Result<HttpResponse, Error> {
let llm_request = decode_request(wire_format, &Value::Object(body))
.map_err(|error| Error::InvalidBody(error.to_string()))?;
let extensions = llm_request.extensions.clone();
let mut metadata = Metadata::from_headers(&headers);
metadata.http_headers = Some(headers);
let request = Request {
llm_request,
raw_request: None,
metadata: Some(metadata),
};
let client = Arc::new(LiteLlmClient {
routes: ProviderRoutes::new(gateway, &cache_options),
targets: Arc::clone(&self.targets),
cache_policy: cache_options.policy,
});
let (selected_model, response) = switchyard_llm_client::run(
Arc::clone(&self.algorithm),
ClientRouter::single(client),
request,
Arc::clone(&self.runtime_models),
None,
)
.await
.map_err(map_libsy_error)?;
let served_model = response.served_model().cloned().unwrap_or(selected_model);
let upstream_headers = response.upstream_headers;
let mut response = match response.llm_response {
LlmResponse::Agg(response) => {
let body = encode_aggregated_response_with_extensions(
&response,
wire_format,
Some(served_model.as_str()),
&extensions,
)
.map_err(|error| Error::Internal(error.to_string()))?;
Json(body).into_response()
}
LlmResponse::Stream(response) => {
let events = encode_stream_with_extensions(
response,
wire_format,
Some(served_model.into_string()),
&extensions,
)
.map_err(|error| Error::Internal(error.to_string()))?;
stream_response(events, wire_format)
}
};
response.headers_mut().extend(upstream_headers);
Ok(response)
}
}
struct Target {
deployment: Deployment,
wire_format: WireFormat,
}
struct ProviderRoutes {
chat: ChatCompletionsRoute,
responses: ResponsesRoute,
messages: MessagesRoute,
}
impl ProviderRoutes {
fn new(gateway: &Gateway, options: &CacheOptions) -> Self {
let mut routes = Self {
chat: gateway.chat_completions.clone(),
responses: gateway.responses.clone(),
messages: gateway.messages.clone(),
};
if let Some(cache) = &gateway.cache {
let scoped = || ScopedCache::new(Arc::clone(cache), options.scope.clone());
routes.chat = routes.chat.with_cache(scoped());
routes.responses = routes.responses.with_cache(scoped());
routes.messages = routes.messages.with_cache(scoped());
}
routes
}
}
struct LiteLlmClient {
routes: ProviderRoutes,
targets: Arc<HashMap<ModelId, Target>>,
cache_policy: CachePolicy,
}
#[async_trait]
impl RoutedLlmClient for LiteLlmClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
let Some(model) = request.llm_request.model.as_deref() else {
return Err(LlmClientError::InvalidRequest {
message: "routed request has no model".to_string(),
});
};
let target = self
.targets
.get(model)
.ok_or_else(|| LlmClientError::Configuration {
message: format!("unknown LiteLLM deployment {model}"),
})?;
let Request {
llm_request,
metadata,
..
} = request;
let Value::Object(body) = encode_request(&llm_request, target.wire_format)
.map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?
else {
return Err(LlmClientError::RequestEncoding(format!(
"{} request is not an object",
target.wire_format
)));
};
let cache_headers = CacheHeaders::default();
let response = match target.wire_format {
WireFormat::OpenAiChat => {
let response = self
.routes
.chat
.execute(
chat_request(&target.deployment, body),
&cache_headers,
self.cache_policy,
)
.await
.map_err(map_route_error)?;
decode_complete(response, target.wire_format)?
}
WireFormat::OpenAiResponses => {
let response = self
.routes
.responses
.execute(
responses_request(&target.deployment, body),
&cache_headers,
self.cache_policy,
)
.await
.map_err(map_route_error)?;
decode_output(response, target.wire_format)?
}
WireFormat::AnthropicMessages => {
let empty_headers = HeaderMap::new();
let headers = metadata
.as_ref()
.and_then(|metadata| metadata.http_headers.as_ref())
.unwrap_or(&empty_headers);
let call =
messages_request(&target.deployment, body, headers).map_err(map_route_error)?;
let response = self
.routes
.messages
.execute(call, &cache_headers, self.cache_policy)
.await
.map_err(map_route_error)?;
decode_output(response, target.wire_format)?
}
};
let mut response = Response {
llm_response: response,
metadata,
upstream_headers: HeaderMap::new(),
};
cache_headers.apply_to(&mut response.upstream_headers);
Ok(response)
}
}
fn decode_complete(
response: impl serde::Serialize,
wire_format: WireFormat,
) -> Result<LlmResponse, LlmClientError> {
let body = serde_json::to_value(response)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
decode_aggregated_response(&body, wire_format)
.map(LlmResponse::Agg)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))
}
fn decode_output<T, H>(
response: CallOutput<T, H, bytes::Bytes, InferenceError>,
wire_format: WireFormat,
) -> Result<LlmResponse, LlmClientError>
where
T: serde::Serialize,
{
match response {
CallOutput::Complete(response) => decode_complete(response, wire_format),
CallOutput::Stream { chunks, .. } => {
let bytes =
chunks.map(|chunk| chunk.map(|bytes| bytes.to_vec()).map_err(map_route_error));
decode_stream(bytes, wire_format).map(LlmResponse::Stream)
}
}
}
// A stream error ends the response; Chat's success sentinel must not follow it.
fn stream_response(events: RawEventStream, wire_format: WireFormat) -> HttpResponse {
let frames = stream::unfold(Some(events), move |events| async move {
let mut events = events?;
let (event, next) = match events.next().await {
Some(Ok(value)) => (frame_event(&value, wire_format), Some(events)),
Some(Err(LlmStreamError::Upstream(value))) => (frame_event(&value, wire_format), None),
Some(Err(LlmStreamError::Client(error))) => {
let error = map_client_error(error);
let value = match wire_format {
WireFormat::OpenAiChat => error.openai_body(),
WireFormat::AnthropicMessages => error.body(None),
WireFormat::OpenAiResponses => json!({
"type": "error", "code": error.status().as_u16().to_string(),
"message": error.to_string(), "param": null
}),
};
(frame_event(&value, wire_format), None)
}
None if wire_format == WireFormat::OpenAiChat => {
(Event::default().data("[DONE]"), None)
}
None => return None,
};
Some((Ok::<_, Infallible>(event), next))
});
Sse::new(frames).into_response()
}
fn frame_event(value: &Value, wire_format: WireFormat) -> Event {
let event = Event::default().data(value.to_string());
match wire_format {
WireFormat::OpenAiChat => event,
WireFormat::OpenAiResponses | WireFormat::AnthropicMessages => {
let event_type = value
.get("type")
.and_then(Value::as_str)
.filter(|name| !name.contains(['\r', '\n']))
.unwrap_or("message");
event.event(event_type)
}
}
}
struct Targets {
models: BTreeMap<String, ModelId>,
by_model: HashMap<ModelId, Target>,
}
fn targets(
route: &Model,
config: &AlgorithmSpec,
model_list: &[Model],
deployments: &ModelRouter,
) -> Result<Targets, AutoRoutingConfigError> {
let mut models = BTreeMap::new();
let mut by_model = HashMap::new();
for target in config.callable_target_names() {
let configured = model_list
.iter()
.rev()
.find(|model| model.model_name == target)
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
let deployment = deployments
.get(target)
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
if deployment.model == SWITCHYARD_MODEL {
return Err(invalid(
route,
format!("target {target:?} is another Switchyard route"),
));
}
let model = ModelId::from(target);
models.insert(target.to_string(), model.clone());
by_model.insert(
model,
Target {
deployment: deployment.clone(),
wire_format: target_wire_format(route, configured)?,
},
);
}
Ok(Targets { models, by_model })
}
// Caller format and target execution format are independent, so each target declares its own.
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, AutoRoutingConfigError> {
let value = target
.litellm_params
.extra
.get("switchyard_wire_format")
.ok_or_else(|| {
invalid(
route,
format!(
"target {:?} requires switchyard_wire_format",
target.model_name
),
)
})?;
WireFormat::deserialize(value).map_err(|error| {
invalid(
route,
format!(
"target {:?} has invalid switchyard_wire_format: {error}",
target.model_name,
),
)
})
}
fn invalid(model: &Model, message: impl Into<String>) -> AutoRoutingConfigError {
AutoRoutingConfigError {
model: model.model_name.clone(),
reason: message.into(),
}
}
fn map_route_error(error: InferenceError) -> LlmClientError {
match error {
InferenceError::Transport(TransportError::Http { status, body }) => {
match axum::http::StatusCode::from_u16(status) {
Ok(status) => LlmClientError::UpstreamHttp { status, body },
Err(source) => LlmClientError::InvalidResponse {
source: Box::new(source),
},
}
}
// Only Connect is known to precede dispatch, so only it is safe to fall back from.
error @ InferenceError::Transport(TransportError::Connect(_)) => {
LlmClientError::Transport {
source: Box::new(error),
}
}
error @ InferenceError::InvalidResponse(_) => LlmClientError::InvalidResponse {
source: Box::new(error),
},
error => LlmClientError::Host {
source: Box::new(error),
},
}
}
fn map_libsy_error(error: LibsyError) -> Error {
match error {
LibsyError::ClientCall { source, .. } => map_client_error(source),
error => Error::Internal(error.to_string()),
}
}
fn map_client_error(error: LlmClientError) -> Error {
match error {
LlmClientError::InvalidRequest { message }
| LlmClientError::RequestTranslation(message) => {
Error::Route(InferenceError::InvalidRequest(message.into()))
}
LlmClientError::UpstreamHttp { status, body } => {
Error::Route(InferenceError::Transport(TransportError::Http {
status: status.as_u16(),
body,
}))
}
LlmClientError::Transport { source }
| LlmClientError::Timeout { source }
| LlmClientError::InvalidResponse { source }
| LlmClientError::Host { source }
| LlmClientError::Ffi { source } => map_source(source),
LlmClientError::ContextWindowExceeded { message, .. } => {
Error::Route(InferenceError::InvalidRequest(message.into()))
}
LlmClientError::ResponseTranslation(message) => {
Error::Route(InferenceError::InvalidResponse(message.into()))
}
error => Error::Internal(error.to_string()),
}
}
fn map_source(source: BoxError) -> Error {
match source.downcast::<InferenceError>() {
Ok(error) => Error::Route(*error),
Err(source) => Error::Internal(source.to_string()),
}
}
const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"];
const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-metadata",
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
fn chat_request(
deployment: &Deployment,
mut body: Map<String, Value>,
) -> ChatCompletionsRequest<'_> {
let messages = body.remove("messages").unwrap_or_default();
body.remove("model");
ChatCompletionsRequest {
model: &deployment.model,
messages,
optional_params: body,
api_key: deployment.api_key.as_deref(),
api_base: deployment.api_base.as_deref(),
custom_llm_provider: deployment.custom_llm_provider.as_deref(),
extra_headers: None,
timeout: deployment.timeout,
}
}
fn responses_request(deployment: &Deployment, mut body: Map<String, Value>) -> ResponsesCall {
let input = body.remove("input").unwrap_or_default();
body.remove("model");
ResponsesCall {
model: deployment.model.clone(),
input,
optional_params: body,
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
}
}
fn messages_request(
deployment: &Deployment,
mut body: Map<String, Value>,
headers: &HeaderMap,
) -> Result<MessagesCall, InferenceError> {
if let Some(model) = body.get_mut("model") {
*model = Value::from(deployment.model.as_str());
}
Ok(MessagesCall {
body: messages_body(body)?,
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
litellm_params: deployment.litellm_params.clone(),
extra_headers: None,
provider_specific_header: anthropic_api_headers(headers),
timeout: deployment.timeout,
shaping: deployment.shaping.clone(),
})
}
fn anthropic_api_headers(headers: &HeaderMap) -> Option<ProviderSpecificHeaders> {
let scopes = [
(
ANTHROPIC_API_HEADER_PROVIDERS,
ANTHROPIC_API_HEADERS.as_slice(),
),
("openrouter", OPENROUTER_API_HEADERS.as_slice()),
];
let forwarded: Vec<ProviderSpecificHeader> = scopes
.into_iter()
.filter_map(|(provider, names)| {
let extra_headers: Map<String, Value> = names
.iter()
.filter_map(|name| {
let value = headers.get(*name)?.to_str().ok()?;
Some(((*name).to_owned(), Value::from(value)))
})
.collect();
(!extra_headers.is_empty()).then(|| ProviderSpecificHeader {
custom_llm_provider: provider.into(),
extra_headers,
})
})
.collect();
(!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded))
}
impl AutoRoutingStrategy for SwitchyardStrategy {
fn execute<'a>(
&'a self,
gateway: &'a Gateway,
request: StrategyRequest,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<HttpResponse, Error>> + Send + 'a>>
{
let format = match request.format {
ApiFormat::Chat => WireFormat::OpenAiChat,
ApiFormat::Responses => WireFormat::OpenAiResponses,
ApiFormat::Messages => WireFormat::AnthropicMessages,
};
Box::pin(SwitchyardStrategy::execute(
self,
gateway,
request.cache_options,
format,
request.body,
request.headers,
))
}
}
#[cfg(test)]
mod tests {
use axum::body::to_bytes;
use rstest::rstest;
use super::*;
#[rstest]
fn host_errors_keep_their_gateway_classification() {
let error = InferenceError::Auth(litellm_auth::Error::MissingApiKey {
provider: "OpenAI",
environment_variable: "OPENAI_API_KEY",
});
assert!(matches!(
map_client_error(map_route_error(error)),
Error::Route(InferenceError::Auth(_))
));
}
#[test]
fn targets_require_an_explicit_wire_format() {
let config = litellm_config::Config::from_yaml(
"model_list:\n - model_name: target\n litellm_params: { model: upstream }\n - model_name: route\n litellm_params: { model: switchyard/algorithm }\n",
)
.expect("config");
let error = target_wire_format(&config.model_list[1], &config.model_list[0])
.expect_err("ambiguous target format");
assert!(error.reason.contains("requires switchyard_wire_format"));
}
#[rstest]
#[tokio::test]
async fn a_stream_error_is_terminal() {
let events: RawEventStream = Box::pin(stream::iter([
Ok(json!({"chunk": "before"})),
Err(LlmStreamError::Upstream(json!({"error": "failed"}))),
Ok(json!({"chunk": "after"})),
]));
let body = to_bytes(
stream_response(events, WireFormat::OpenAiChat).into_body(),
4096,
)
.await
.expect("read stream");
let body = std::str::from_utf8(&body).expect("stream is UTF-8");
assert!(body.contains("before"));
assert!(body.contains("failed"));
assert!(!body.contains("after"));
assert!(!body.contains("[DONE]"));
}
}

View file

@ -101,11 +101,15 @@ impl litellm_host::interceptors::Interceptors<litellm_inference::Error> for Cach
impl CacheHeaders {
pub(crate) fn apply(&self, mut response: axum::response::Response) -> axum::response::Response {
self.apply_to(response.headers_mut());
response
}
pub(crate) fn apply_to(&self, headers: &mut axum::http::HeaderMap) {
if let Some(key) = self.0.get()
&& let Ok(value) = axum::http::HeaderValue::from_str(key)
{
response.headers_mut().insert("x-litellm-cache-key", value);
headers.insert("x-litellm-cache-key", value);
}
response
}
}

View file

@ -8,6 +8,13 @@ use litellm_inference::Error as InferenceError;
use litellm_llms::base_llm::ocr::error::Error as OcrError;
use serde_json::{Map, Value, json};
#[derive(Debug, thiserror::Error)]
#[error("invalid auto-routing model {model}: {reason}")]
pub struct AutoRoutingConfigError {
pub(crate) model: String,
pub(crate) reason: String,
}
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error(transparent)]
@ -76,21 +83,21 @@ impl Error {
}
pub fn openai_response(self) -> Response {
(self.status(), Json(self.openai_body())).into_response()
}
pub(crate) fn openai_body(&self) -> Value {
let status = self.status();
let message = match &self {
let message = match self {
Self::UnknownModel(model) => format!("Invalid model name passed in model={model}"),
_ => self.to_string(),
};
(
status,
Json(json!({"error": {
"message": message,
"type": error_type(status),
"param": null,
"code": status.as_u16(),
}})),
)
.into_response()
json!({"error": {
"message": message,
"type": error_type(status),
"param": null,
"code": status.as_u16(),
}})
}
/// The Anthropic error envelope Python's `AnthropicExceptionMapping` builds: an upstream

View file

@ -4,6 +4,7 @@
//! maps a public model name to its deployment and runs the core route.
mod audio_transcription;
pub mod auto_routing;
mod caching;
mod chat_completions;
mod error;
@ -25,12 +26,14 @@ use litellm_inference_transcription::AudioTranscriptionRoute;
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
use litellm_secrets::source::SecretSource;
pub use error::AutoRoutingConfigError;
pub use error::Error;
pub use litellm_router::{Deployment, Router as ModelRouter};
pub use request::{JsonObject, RequestId};
pub struct Gateway {
cache: Option<Arc<dyn litellm_cache_response::ResponseCacheService>>,
auto_routing: auto_routing::Registry,
pub audio_transcription: AudioTranscriptionRoute,
pub chat_completions: ChatCompletionsRoute,
pub messages: MessagesRoute,
@ -60,6 +63,7 @@ impl Gateway {
let auth = resources.auth.clone();
Ok(Self {
cache: None,
auto_routing: auto_routing::Registry::default(),
audio_transcription: AudioTranscriptionRoute::new(
provider.clone(),
auth.clone(),
@ -86,15 +90,38 @@ impl Gateway {
http,
})
}
pub fn with_auto_routing_strategy(
mut self,
model: String,
strategy: Arc<dyn auto_routing::AutoRoutingStrategy>,
) -> Result<Self, AutoRoutingConfigError> {
if self.models.get(&model).is_none() {
return Err(AutoRoutingConfigError {
model,
reason: "model is not configured".into(),
});
}
self.auto_routing.insert(model, strategy);
Ok(self)
}
pub fn with_auto_routing_models(
mut self,
models: &[litellm_config::Model],
) -> Result<Self, AutoRoutingConfigError> {
self.auto_routing = auto_routing::Registry::from_models(models, &self.models)?;
Ok(self)
}
}
pub fn router(gateway: Arc<Gateway>) -> Router {
Router::new()
.route("/v1/messages", post(messages::create))
.route("/v1/messages", post(auto_routing::messages))
.route("/ocr", post(ocr::create))
.route("/v1/ocr", post(ocr::create))
.route("/chat/completions", post(chat_completions::create))
.route("/v1/chat/completions", post(chat_completions::create))
.route("/chat/completions", post(auto_routing::chat))
.route("/v1/chat/completions", post(auto_routing::chat))
.nest("/engines/{model}", model_routes())
.nest("/openai/deployments/{model}", model_routes())
.route("/audio/transcriptions", post(audio_transcription::create))
@ -102,8 +129,8 @@ pub fn router(gateway: Arc<Gateway>) -> Router {
"/v1/audio/transcriptions",
post(audio_transcription::create),
)
.route("/responses", post(responses::create))
.route("/v1/responses", post(responses::create))
.route("/responses", post(auto_routing::responses))
.route("/v1/responses", post(auto_routing::responses))
.route("/embeddings", post(request::unsupported))
.route("/v1/embeddings", post(request::unsupported))
.route("/completions", post(request::unsupported))
@ -118,7 +145,7 @@ fn model_routes() -> Router<Arc<Gateway>> {
Router::new()
.route(
"/chat/completions",
post(chat_completions::create_from_model_path),
post(auto_routing::chat_from_model_path),
)
.route("/embeddings", post(request::unsupported))
.route("/completions", post(request::unsupported))

View file

@ -0,0 +1,267 @@
mod support;
use axum::{
Json,
response::{IntoResponse, Response},
};
use litellm_gateway_auth::Permissions;
use litellm_gateway_inference::{
Deployment, Error, Gateway,
auto_routing::{ApiFormat, AutoRoutingStrategy, StrategyRequest},
};
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
use litellm_inference::resources::CoreResources;
use rstest::{fixture, rstest};
use serde_json::{Value, json};
use std::{
future::Future,
pin::Pin,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
struct EchoStrategy(Arc<AtomicUsize>);
impl AutoRoutingStrategy for EchoStrategy {
fn execute<'a>(
&'a self,
_: &'a Gateway,
request: StrategyRequest,
) -> Pin<Box<dyn Future<Output = Result<Response, Error>> + Send + 'a>> {
self.0.fetch_add(1, Ordering::SeqCst);
Box::pin(async move {
if request.body.get("fail").and_then(Value::as_bool) == Some(true) {
return Err(Error::InvalidBody("strategy rejected request".into()));
}
let format = match request.format {
ApiFormat::Chat => "chat",
ApiFormat::Responses => "responses",
ApiFormat::Messages => "messages",
};
Ok(Json(json!({"format": format, "body": request.body, "request_id": request.headers.get("x-request-id").and_then(|v| v.to_str().ok()), "no_store": request.cache_options.policy.no_store})).into_response())
})
}
}
#[fixture]
fn gateway() -> Gateway {
Gateway::new(
CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))),
Resolution::from(&HttpSettings::default()).config,
Arc::new(support::NoSecrets),
[(
"virtual".into(),
Deployment {
model: "virtual-target".into(),
..Default::default()
},
)]
.into_iter()
.collect(),
)
.unwrap()
}
#[rstest]
#[case::chat("/v1/chat/completions", "chat", json!({"model":"virtual"}))]
#[case::chat_alias("/chat/completions", "chat", json!({"model":"virtual"}))]
#[case::responses("/v1/responses", "responses", json!({"model":"virtual"}))]
#[case::responses_alias("/responses", "responses", json!({"model":"virtual"}))]
#[case::messages("/v1/messages", "messages", json!({"model":"virtual"}))]
#[case::engine("/engines/virtual/chat/completions", "chat", json!({}))]
#[case::deployment("/openai/deployments/virtual/chat/completions", "chat", json!({"model":null}))]
#[case::body_wins("/engines/unknown/chat/completions", "chat", json!({"model":"virtual"}))]
#[tokio::test]
async fn virtual_models_invoke_the_strategy_once(
gateway: Gateway,
#[case] path: &str,
#[case] format: &str,
#[case] body: Value,
) {
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
body,
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["format"], format);
assert_eq!(body["body"]["model"], "virtual");
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn denied_virtual_models_never_invoke_the_strategy(gateway: Gateway, #[case] path: &str) {
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::Only(Arc::from([])), None),
path,
json!({"model":"virtual"}),
)
.await;
assert_eq!(response.status(), 403);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[rstest]
#[case::chat("/v1/chat/completions", false)]
#[case::responses("/v1/responses", false)]
#[case::messages("/v1/messages", true)]
#[tokio::test]
async fn strategy_errors_keep_the_callers_error_format(
gateway: Gateway,
#[case] path: &str,
#[case] anthropic: bool,
) {
let gateway = gateway
.with_auto_routing_strategy(
"virtual".into(),
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
)
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
json!({"model":"virtual", "fail":true}),
)
.await;
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.unwrap()
.contains("strategy rejected request")
);
assert_eq!(body.get("type").is_some(), anthropic);
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn cache_controls_are_prepared_before_strategy_execution(
gateway: Gateway,
#[case] path: &str,
) {
let gateway = gateway
.with_auto_routing_strategy(
"virtual".into(),
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
)
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
json!({"model":"virtual", "cache":{"no-store":true}, "caching":false}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["no_store"], true);
assert!(body["body"].get("cache").is_none());
assert!(body["body"].get("caching").is_none());
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn incoming_headers_reach_the_strategy(gateway: Gateway, #[case] path: &str) {
use axum::{body::Body, http::Request};
use tower::ServiceExt;
let gateway = gateway
.with_auto_routing_strategy(
"virtual".into(),
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
)
.unwrap();
let response = support::authenticated_app(gateway, Permissions::All, None)
.oneshot(
Request::post(path)
.header("content-type", "application/json")
.header("x-request-id", "request-context")
.body(Body::from(r#"{"model":"virtual"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(
support::json(response).await["request_id"],
"request-context"
);
}
#[rstest]
#[case::chat("/v1/chat/completions", false)]
#[case::responses("/v1/responses", false)]
#[case::messages("/v1/messages", true)]
#[tokio::test]
async fn malformed_bodies_preserve_api_error_envelopes(
gateway: Gateway,
#[case] path: &str,
#[case] anthropic: bool,
) {
use axum::{body::Body, http::Request};
use tower::ServiceExt;
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::authenticated_app(gateway, Permissions::All, None)
.oneshot(
Request::post(path)
.header("content-type", "application/json")
.header("x-request-id", "malformed-request")
.body(Body::from("{"))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body.get("type").is_some(), anthropic);
if anthropic {
assert_eq!(body["request_id"], "malformed-request");
}
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn invalid_cache_settings_do_not_invoke_the_strategy(gateway: Gateway, #[case] path: &str) {
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
json!({"model":"virtual", "cache":{"ttl":-1}}),
)
.await;
assert_eq!(response.status(), 400);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}

View file

@ -102,6 +102,14 @@ fn configured_app(
Some(cache) => gateway.with_cache(cache),
None => gateway,
};
authenticated_app(gateway, permissions, principal)
}
pub fn authenticated_app(
gateway: Gateway,
permissions: litellm_gateway_auth::Permissions,
principal: Option<litellm_gateway_auth::Principal>,
) -> Router {
router(Arc::new(gateway)).layer(axum::middleware::from_fn_with_state(
(permissions, principal),
test_identity,

View file

@ -0,0 +1,482 @@
#![cfg(feature = "switchyard")]
mod support;
use std::{sync::Arc, time::Duration};
use futures_util::StreamExt;
use litellm_config::Config;
use litellm_gateway_auth::{AccessRequest, Permissions};
use litellm_gateway_inference::{Gateway, ModelRouter};
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
use litellm_inference::resources::CoreResources;
use rstest::rstest;
use serde_json::json;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{body_partial_json, method, path},
};
#[rstest]
#[tokio::test]
async fn configured_switchyard_routes_answer_and_fall_back() {
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(json!({"model": "judge-model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"model": "judge-model",
"choices": [{
"message": {
"content": r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#
}
}]
})))
.expect(1)
.mount(&upstream)
.await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(json!({
"model": "fast-model",
"messages": [{"role": "user", "content": "fix this"}],
"service_tier": "flex"
})))
.respond_with(ResponseTemplate::new(503).set_body_json(json!({
"error": {"message": "unavailable"}
})))
.expect(1)
.mount(&upstream)
.await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(json!({
"model": "strong-model",
"messages": [{"role": "user", "content": "fix this"}],
"service_tier": "flex"
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"model": "strong-model",
"choices": [{
"message": {"content": "fallback answer"}
}]
})))
.expect(1)
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: fast
litellm_params:
model: fast-model
api_base: {base}/v1
api_key: test-key
custom_llm_provider: openai_like
switchyard_wire_format: openai_chat
- model_name: strong
litellm_params:
model: strong-model
api_base: {base}/v1
api_key: test-key
custom_llm_provider: openai_like
switchyard_wire_format: openai_chat
- model_name: judge
litellm_params:
model: judge-model
api_base: {base}/v1
api_key: test-key
custom_llm_provider: openai_like
switchyard_wire_format: openai_chat
- model_name: coding-router
litellm_params:
model: switchyard/algorithm
switchyard_config:
type: llm_classifier
mode: capability
classifier_target: judge
weak_target: fast
strong_target: strong
base_threshold: 0.5
- model_name: noop-router
litellm_params:
model: switchyard/algorithm
switchyard_config:
type: noop
"#,
base = upstream.uri()
))
.expect("valid config");
let app = switchyard_app(
&config,
Permissions::Only(Arc::from([
AccessRequest::Model {
name: "coding-router".into(),
deployment: "switchyard/algorithm".into(),
},
AccessRequest::Model {
name: "noop-router".into(),
deployment: "switchyard/algorithm".into(),
},
])),
);
let response = support::post(
app.clone(),
"/v1/chat/completions",
json!({
"model": "coding-router",
"messages": [{"role": "user", "content": "fix this"}],
"service_tier": "flex"
}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "strong");
assert_eq!(body["choices"][0]["message"]["content"], "fallback answer");
let response = support::post(
app,
"/v1/chat/completions",
json!({
"model": "noop-router",
"messages": [{"role": "user", "content": "health check"}]
}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "OK");
}
// The caller API and selected target API are independent and use the same host bridge.
#[rstest]
#[tokio::test]
async fn configured_switchyard_routes_translate_responses_and_messages() {
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/responses"))
.and(body_partial_json(json!({"model": "responses-model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "resp_1",
"model": "responses-model",
"status": "completed",
"output": [{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "responses answer"}]
}],
"provider_extra": true,
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
})))
.expect(2)
.mount(&upstream)
.await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.and(body_partial_json(json!({"model": "messages-model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "messages-model",
"content": [{"type": "text", "text": "messages answer"}],
"provider_extra": true,
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1}
})))
.expect(1)
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: responses-target
litellm_params:
model: openai/responses-model
api_base: {base}
api_key: test-key
custom_llm_provider: openai
switchyard_wire_format: openai_responses
- model_name: messages-target
litellm_params:
model: anthropic/messages-model
api_base: {base}
api_key: test-key
custom_llm_provider: anthropic
switchyard_wire_format: anthropic_messages
- model_name: responses-router
litellm_params:
model: switchyard/algorithm
switchyard_config:
type: passthrough
target: responses-target
- model_name: messages-router
litellm_params:
model: switchyard/algorithm
switchyard_config:
type: passthrough
target: messages-target
"#,
base = upstream.uri()
))
.expect("valid config");
let app = switchyard_app(&config, Permissions::All);
let response = support::post(
app.clone(),
"/v1/responses",
json!({"model": "responses-router", "input": "hello"}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "responses-target");
assert_eq!(body["output"][0]["content"][0]["text"], "responses answer");
assert_eq!(body["provider_extra"], true);
let response = support::post(
app.clone(),
"/v1/messages",
json!({
"model": "messages-router",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 16
}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "messages-target");
assert_eq!(body["content"][0]["text"], "messages answer");
assert_eq!(body["provider_extra"], true);
let response = support::post(
app,
"/v1/chat/completions",
json!({
"model": "responses-router",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "responses-target");
assert_eq!(body["choices"][0]["message"]["content"], "responses answer");
}
#[rstest]
#[tokio::test]
async fn streaming_chat_target_keeps_litellm_core_rejection() {
let upstream = MockServer::start().await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: target
litellm_params: {{ model: test-model, api_base: {base}/v1,
custom_llm_provider: openai_like, switchyard_wire_format: openai_chat }}
- model_name: route
litellm_params: {{ model: switchyard/algorithm,
switchyard_config: {{ type: passthrough, target: target }} }}
"#,
base = upstream.uri(),
))
.expect("valid config");
let response = support::post(
switchyard_app(&config, Permissions::All),
"/v1/chat/completions",
json!({
"model": "route",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), 501);
let body = support::json(response).await;
assert!(
body["error"]["message"]
.as_str()
.unwrap()
.contains("streaming")
);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[tokio::test]
async fn switchyard_streams_across_formats_and_cancels_on_drop() {
let (base, upstream) = streaming_responses_upstream().await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: responses-target
litellm_params: {{ model: openai/responses-model, api_base: {base}, api_key: test-key,
custom_llm_provider: openai, switchyard_wire_format: openai_responses }}
- model_name: responses-router
litellm_params: {{ model: switchyard/algorithm,
switchyard_config: {{ type: passthrough, target: responses-target }} }}
"#
))
.expect("valid config");
let response = tokio::time::timeout(
Duration::from_secs(2),
support::post(
switchyard_app(&config, Permissions::All),
"/v1/messages",
json!({
"model": "responses-router",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 16,
"stream": true
}),
),
)
.await
.expect("response starts while upstream remains open");
assert_eq!(response.status(), 200);
assert_eq!(response.headers()["content-type"], "text/event-stream");
let mut body = response.into_body().into_data_stream();
let first = tokio::time::timeout(Duration::from_secs(2), body.next())
.await
.expect("first event arrives while upstream remains open")
.expect("stream has a first event")
.expect("first event is readable");
assert!(
std::str::from_utf8(&first)
.expect("event is UTF-8")
.contains("event: message_start")
);
drop(body);
tokio::time::timeout(Duration::from_secs(2), upstream)
.await
.expect("dropping the caller closes the upstream stream")
.expect("upstream task succeeds");
}
async fn streaming_responses_upstream() -> (String, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind upstream");
let base = format!(
"http://{}",
listener.local_addr().expect("upstream address")
);
let upstream = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accept request");
let response = concat!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n",
"content-length: 4096\r\n\r\n",
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":",
"{\"id\":\"resp_1\",\"model\":\"responses-model\"}}\n\n"
);
socket
.write_all(response.as_bytes())
.await
.expect("write first event");
let mut pending = [0; 1];
while matches!(socket.read(&mut pending).await, Ok(read) if read > 0) {}
});
(base, upstream)
}
fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router {
support::authenticated_app(switchyard_gateway(config), permissions, None)
}
fn switchyard_gateway(config: &Config) -> Gateway {
let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))));
let http = Resolution::from(&HttpSettings::default()).config;
Gateway::new(
resources,
http,
Arc::new(support::NoSecrets),
ModelRouter::from_model_list(&config.model_list),
)
.expect("gateway")
.with_auto_routing_models(&config.model_list)
.expect("Switchyard routes")
}
#[rstest]
#[tokio::test]
async fn routed_answers_preserve_cache_hits_and_caller_isolation() {
use litellm_cache_memory::InMemoryCache;
use litellm_cache_response::ResponseCache;
use litellm_gateway_auth::{Principal, PrincipalKind};
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"model":"answer", "choices":[{"message":{"role":"assistant","content":"cached answer"}}]
})))
.expect(2)
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: answer
litellm_params:
model: answer
custom_llm_provider: openai_like
api_base: {}/v1
api_key: test-key
switchyard_wire_format: openai_chat
- model_name: virtual
litellm_params:
model: switchyard/algorithm
switchyard_config: {{type: passthrough, target: answer}}
"#,
upstream.uri()
))
.unwrap();
let cache = Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new(
Some(100),
Some(Duration::from_secs(60)),
))));
let first_app = support::authenticated_app(
switchyard_gateway(&config).with_cache(cache.clone()),
Permissions::All,
Some(Principal::new(
"issuer".into(),
"first".into(),
PrincipalKind::Service,
)),
);
let second_app = support::authenticated_app(
switchyard_gateway(&config).with_cache(cache),
Permissions::All,
Some(Principal::new(
"issuer".into(),
"second".into(),
PrincipalKind::Service,
)),
);
let request = json!({"model":"virtual", "messages":[{"role":"user", "content":"hello"}]});
let first = support::post(first_app.clone(), "/v1/chat/completions", request.clone()).await;
assert_eq!(first.status(), 200);
assert!(!first.headers().contains_key("x-litellm-cache-key"));
let first_body = support::json(first).await;
let hit = support::post(first_app, "/v1/chat/completions", request.clone()).await;
assert_eq!(hit.status(), 200);
let first_key = hit.headers().get("x-litellm-cache-key").unwrap().clone();
assert_eq!(support::json(hit).await, first_body);
let second = support::post(second_app.clone(), "/v1/chat/completions", request.clone()).await;
assert_eq!(second.status(), 200);
assert!(!second.headers().contains_key("x-litellm-cache-key"));
let hit = support::post(second_app, "/v1/chat/completions", request).await;
assert_eq!(hit.status(), 200);
assert_ne!(hit.headers().get("x-litellm-cache-key").unwrap(), first_key);
upstream.verify().await;
}

View file

@ -5,6 +5,9 @@ edition.workspace = true
license.workspace = true
repository.workspace = true
[features]
switchyard = ["litellm-gateway-inference/switchyard"]
[dependencies]
axum.workspace = true
envy = "0.4.2"

View file

@ -19,4 +19,6 @@ pub enum Error {
RedisConnect(#[from] litellm_redis_client::ConnectError),
#[error("MCP requires a configured master key")]
Auth(#[from] litellm_gateway_auth::Error),
#[error(transparent)]
AutoRouting(#[from] litellm_gateway_inference::AutoRoutingConfigError),
}

View file

@ -64,12 +64,14 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
Arc::new(EnvironmentSecrets::python_compatible(client)),
));
let resources = CoreResources::new(pool);
Ok(Arc::new(Gateway::new(
let gateway = Gateway::new(
resources,
http,
secrets,
ModelRouter::from_model_list(&config.model_list),
)?))
)?;
let gateway = gateway.with_auto_routing_models(&config.model_list)?;
Ok(Arc::new(gateway))
}
pub async fn build_redis(config: &Config) -> Result<RedisClients, Error> {

View file

@ -33,6 +33,32 @@ fn inference() -> Arc<Gateway> {
litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap()
}
#[cfg(feature = "switchyard")]
#[rstest]
fn validates_only_winning_switchyard_models() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: coding-router
litellm_params:
model: switchyard/algorithm
switchyard_config: { type: passthrough, target: missing }
"#,
)
.unwrap();
assert!(litellm_gateway::build_inference(&config).is_err());
let config = Config::from_yaml(
r#"
model_list:
- { model_name: coding-router, litellm_params: { model: switchyard/algorithm, switchyard_config: { type: passthrough, target: missing } } }
- { model_name: coding-router, litellm_params: { model: openai/model } }
"#,
)
.unwrap();
assert!(litellm_gateway::build_inference(&config).is_ok());
}
#[rstest]
#[case::authorized("/v1/messages", Some("Bearer gateway-key"), Some("gateway-key"), 400)]
#[case::missing_token("/v1/messages", None, Some("gateway-key"), 401)]