refactor(gateway): isolate auto-routing strategy orchestration

This commit is contained in:
Joshua Valluru 2026-10-25 19:19:54 -07:00
parent dd5fe2e550
commit c3827fef75
14 changed files with 932 additions and 244 deletions

View file

@ -101,6 +101,8 @@ jobs:
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
- run: cargo clippy -p litellm-gateway-inference -p litellm-gateway --all-targets --features switchyard --locked -- -D warnings
rust-test:
runs-on: ubuntu-latest
timeout-minutes: 30
@ -147,6 +149,8 @@ jobs:
- run: cargo nextest run --workspace --locked
- run: cargo nextest run -p litellm-gateway-inference -p litellm-gateway --features switchyard --locked
- run: cargo test --workspace --doc --locked
- name: Test token counter feature combinations

View file

@ -3882,6 +3882,7 @@ dependencies = [
"rstest",
"serde",
"serde_json",
"strum",
"switchyard-libsy",
"switchyard-llm-client",
"switchyard-protocol",

View file

@ -9,7 +9,6 @@ repository.workspace = true
switchyard = [
"dep:async-trait",
"dep:futures-util",
"dep:litellm-config",
"dep:switchyard-libsy",
"dep:switchyard-llm-client",
"dep:switchyard-protocol",
@ -32,7 +31,7 @@ litellm-inference-responses.workspace = true
litellm-inference-messages.workspace = true
litellm-inference-chat.workspace = true
litellm-inference-ocr.workspace = true
litellm-config = { workspace = true, optional = true }
litellm-config.workspace = true
litellm-host-http.workspace = true
litellm-host.workspace = true
litellm-http.workspace = true
@ -47,6 +46,7 @@ switchyard-llm-client = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git",
switchyard-protocol = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
switchyard-runner = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
switchyard-translation = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]

View file

@ -0,0 +1,329 @@
use std::{
collections::{HashMap, HashSet},
future::Future,
pin::Pin,
sync::Arc,
};
use axum::{
Json,
extract::{Path, State},
http::HeaderMap,
response::{IntoResponse, Response},
};
use litellm_cache_response::CacheOptions;
use litellm_config::Model;
use litellm_gateway_auth::AuthenticatedRequest;
use serde_json::{Map, Value};
use crate::{AutoRoutingConfigError, Error, Gateway, JsonObject, ModelRouter, RequestId, request};
#[cfg(feature = "switchyard")]
mod switchyard;
#[derive(Clone, Copy, strum::VariantArray)]
pub enum ApiFormat {
Chat,
Responses,
Messages,
}
pub struct StrategyRequest {
pub format: ApiFormat,
pub body: Map<String, Value>,
pub headers: HeaderMap,
pub cache_options: CacheOptions,
}
pub trait AutoRoutingStrategy: Send + Sync {
fn execute<'a>(
&'a self,
gateway: &'a Gateway,
request: StrategyRequest,
) -> Pin<Box<dyn Future<Output = Result<Response, Error>> + Send + 'a>>;
}
#[derive(Default)]
pub(crate) struct Registry(HashMap<String, Arc<dyn AutoRoutingStrategy>>);
impl Registry {
pub(crate) fn insert(&mut self, model: String, strategy: Arc<dyn AutoRoutingStrategy>) {
self.0.insert(model, strategy);
}
pub(crate) fn from_models(
models: &[Model],
deployments: &ModelRouter,
) -> Result<Self, AutoRoutingConfigError> {
let mut seen = HashSet::new();
let mut strategies = HashMap::new();
for model in models.iter().rev() {
if !seen.insert(&model.model_name)
|| model.litellm_params.model != "switchyard/algorithm"
{
continue;
}
if deployments
.get(&model.model_name)
.is_none_or(|deployment| deployment.model != model.litellm_params.model)
{
return Err(AutoRoutingConfigError {
model: model.model_name.clone(),
reason: "model does not match its configured deployment".into(),
});
}
strategies.insert(
model.model_name.clone(),
build_strategy(model, models, deployments)?,
);
}
Ok(Self(strategies))
}
fn get(&self, body: &Map<String, Value>) -> Option<&dyn AutoRoutingStrategy> {
self.0.get(body.get("model")?.as_str()?).map(AsRef::as_ref)
}
}
#[cfg(feature = "switchyard")]
fn build_strategy(
model: &Model,
models: &[Model],
deployments: &ModelRouter,
) -> Result<Arc<dyn AutoRoutingStrategy>, AutoRoutingConfigError> {
Ok(Arc::new(switchyard::SwitchyardStrategy::from_model(
model,
models,
deployments,
)?))
}
#[cfg(not(feature = "switchyard"))]
fn build_strategy(
model: &Model,
_: &[Model],
_: &ModelRouter,
) -> Result<Arc<dyn AutoRoutingStrategy>, AutoRoutingConfigError> {
Err(AutoRoutingConfigError {
model: model.model_name.clone(),
reason: "this build does not enable the switchyard feature".into(),
})
}
async fn execute(
strategy: &dyn AutoRoutingStrategy,
gateway: &Gateway,
identity: &AuthenticatedRequest,
format: ApiFormat,
headers: HeaderMap,
body: Map<String, Value>,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(gateway, &body)?;
request::authorize_model(identity, deployment, &body).await?;
let (body, cache_options) = crate::caching::prepare(identity, body)?;
strategy
.execute(
gateway,
StrategyRequest {
format,
body,
headers,
cache_options,
},
)
.await
}
pub(crate) async fn chat(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<Response, Error> {
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Chat,
headers,
body,
)
.await;
}
crate::chat_completions::create(State(gateway), identity, JsonObject(body))
.await
.map(IntoResponse::into_response)
}
pub(crate) async fn chat_from_model_path(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
headers: HeaderMap,
Path(model): Path<String>,
JsonObject(mut body): JsonObject,
) -> Result<Response, Error> {
if body.get("model").is_none_or(Value::is_null) {
body.insert("model".into(), Value::String(model.clone()));
}
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Chat,
headers,
body,
)
.await;
}
crate::chat_completions::create_from_model_path(
State(gateway),
identity,
Path(model),
JsonObject(body),
)
.await
.map(IntoResponse::into_response)
}
pub(crate) async fn responses(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<Response, Error> {
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Responses,
headers,
body,
)
.await;
}
crate::responses::create(State(gateway), identity, JsonObject(body)).await
}
pub(crate) async fn messages(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
RequestId(request_id): RequestId,
headers: HeaderMap,
body: Result<JsonObject, Error>,
) -> Response {
match body {
Ok(JsonObject(body)) => {
if let Some(strategy) = gateway.auto_routing.get(&body) {
return execute(
strategy,
&gateway,
&identity,
ApiFormat::Messages,
headers,
body,
)
.await
.unwrap_or_else(|error| {
(error.status(), Json(error.body(request_id.as_deref()))).into_response()
});
}
crate::messages::create(
State(gateway),
identity,
RequestId(request_id),
headers,
Ok(JsonObject(body)),
)
.await
.into_response()
}
Err(error) => crate::messages::create(
State(gateway),
identity,
RequestId(request_id),
headers,
Err(error),
)
.await
.into_response(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use litellm_config::Config;
use rstest::rstest;
#[rstest]
fn overwritten_virtual_models_do_not_initialize_a_strategy() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: public
litellm_params: {model: switchyard/algorithm}
- model_name: public
litellm_params: {model: openai/model}
"#,
)
.unwrap();
let registry = Registry::from_models(
&config.model_list,
&ModelRouter::from_model_list(&config.model_list),
)
.unwrap();
assert!(
registry
.get(
&serde_json::json!({"model":"public"})
.as_object()
.unwrap()
.clone()
)
.is_none()
);
}
#[rstest]
fn strategy_configuration_must_match_the_deployment_registry() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: public
litellm_params: {model: switchyard/algorithm}
"#,
)
.unwrap();
let error = Registry::from_models(&config.model_list, &ModelRouter::default())
.err()
.unwrap();
assert!(error.to_string().contains("does not match"));
}
#[cfg(not(feature = "switchyard"))]
#[rstest]
fn disabled_strategies_fail_at_configuration_time() {
let config = Config::from_yaml(
r#"
model_list:
- model_name: public
litellm_params: {model: switchyard/algorithm}
"#,
)
.unwrap();
let error = Registry::from_models(
&config.model_list,
&ModelRouter::from_model_list(&config.model_list),
)
.err()
.unwrap();
assert!(
error
.to_string()
.contains("does not enable the switchyard feature")
);
}
}

View file

@ -1,5 +1,10 @@
use super::{ApiFormat, AutoRoutingStrategy, StrategyRequest};
use litellm_inference_chat::types::ChatCompletionsRequest;
use litellm_inference_messages::{MessagesCall, messages_body};
use litellm_inference_responses::types::ResponsesCall;
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
use std::{
collections::{BTreeMap, HashMap, HashSet},
collections::{BTreeMap, HashMap},
convert::Infallible,
sync::Arc,
};
@ -18,7 +23,7 @@ use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache};
use litellm_config::Model;
use litellm_host::call::CallOutput;
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error as InferenceError;
use litellm_inference_chat::ChatCompletionsRoute;
use litellm_inference_messages::MessagesRoute;
use litellm_inference_responses::ResponsesRoute;
@ -37,70 +42,41 @@ use switchyard_translation::{
};
use crate::{
Deployment, Error, Gateway, ModelRouter, SwitchyardConfigError, caching::CacheHeaders,
chat_completions, messages, responses,
AutoRoutingConfigError, Deployment, Error, Gateway, ModelRouter, caching::CacheHeaders,
};
const SWITCHYARD_MODEL: &str = "switchyard/algorithm";
#[derive(Default)]
pub(crate) struct Routes(HashMap<String, Route>);
impl Routes {
pub(crate) fn from_models(
model_list: &[Model],
deployments: &ModelRouter,
) -> Result<Self, SwitchyardConfigError> {
let mut routes = HashMap::new();
let mut seen = HashSet::new();
// ModelRouter uses the final entry when public model names repeat.
for model in model_list.iter().rev() {
if !seen.insert(model.model_name.as_str())
|| model.litellm_params.model != SWITCHYARD_MODEL
{
continue;
}
if deployments
.get(&model.model_name)
.is_none_or(|deployment| deployment.model != SWITCHYARD_MODEL)
{
return Err(invalid(model, "model is not a Switchyard route"));
}
let config = model
.litellm_params
.extra
.get("switchyard_config")
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
let config = AlgorithmSpec::deserialize(config)
.map_err(|error| invalid(model, error.to_string()))?;
let targets = targets(model, &config, model_list, deployments)?;
let (algorithm, runtime_models) = config
.build_with_runtime_models(&model.model_name, &targets.models)
.map_err(|error| invalid(model, error.to_string()))?;
routes.insert(
model.model_name.clone(),
Route {
algorithm,
runtime_models: Arc::new(runtime_models),
targets: Arc::new(targets.by_model),
},
);
}
Ok(Self(routes))
}
pub(crate) fn get(&self, model: &str) -> Option<&Route> {
self.0.get(model)
}
}
pub(crate) struct Route {
pub(super) struct SwitchyardStrategy {
algorithm: Arc<dyn Algorithm>,
runtime_models: Arc<RuntimeModels>,
targets: Arc<HashMap<ModelId, Target>>,
}
impl Route {
impl SwitchyardStrategy {
pub(super) fn from_model(
model: &Model,
model_list: &[Model],
deployments: &ModelRouter,
) -> Result<Self, AutoRoutingConfigError> {
let config = model
.litellm_params
.extra
.get("switchyard_config")
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
let config = AlgorithmSpec::deserialize(config)
.map_err(|error| invalid(model, error.to_string()))?;
let targets = targets(model, &config, model_list, deployments)?;
let (algorithm, runtime_models) = config
.build_with_runtime_models(&model.model_name, &targets.models)
.map_err(|error| invalid(model, error.to_string()))?;
Ok(Self {
algorithm,
runtime_models: Arc::new(runtime_models),
targets: Arc::new(targets.by_model),
})
}
pub(crate) async fn execute(
&self,
gateway: &Gateway,
@ -230,7 +206,7 @@ impl RoutedLlmClient for LiteLlmClient {
.routes
.chat
.execute(
chat_completions::provider_request(&target.deployment, body),
chat_request(&target.deployment, body),
&cache_headers,
self.cache_policy,
)
@ -243,7 +219,7 @@ impl RoutedLlmClient for LiteLlmClient {
.routes
.responses
.execute(
responses::provider_request(&target.deployment, body),
responses_request(&target.deployment, body),
&cache_headers,
self.cache_policy,
)
@ -257,8 +233,8 @@ impl RoutedLlmClient for LiteLlmClient {
.as_ref()
.and_then(|metadata| metadata.http_headers.as_ref())
.unwrap_or(&empty_headers);
let call = messages::provider_request(&target.deployment, body, headers)
.map_err(map_route_error)?;
let call =
messages_request(&target.deployment, body, headers).map_err(map_route_error)?;
let response = self
.routes
.messages
@ -290,7 +266,7 @@ fn decode_complete(
}
fn decode_output<T, H>(
response: CallOutput<T, H, bytes::Bytes, RouteError>,
response: CallOutput<T, H, bytes::Bytes, InferenceError>,
wire_format: WireFormat,
) -> Result<LlmResponse, LlmClientError>
where
@ -360,7 +336,7 @@ fn targets(
config: &AlgorithmSpec,
model_list: &[Model],
deployments: &ModelRouter,
) -> Result<Targets, SwitchyardConfigError> {
) -> Result<Targets, AutoRoutingConfigError> {
let mut models = BTreeMap::new();
let mut by_model = HashMap::new();
for target in config.callable_target_names() {
@ -392,7 +368,7 @@ fn targets(
}
// Caller format and target execution format are independent, so each target declares its own.
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, SwitchyardConfigError> {
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, AutoRoutingConfigError> {
let value = target
.litellm_params
.extra
@ -417,16 +393,16 @@ fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, Switc
})
}
fn invalid(model: &Model, message: impl Into<String>) -> SwitchyardConfigError {
SwitchyardConfigError {
fn invalid(model: &Model, message: impl Into<String>) -> AutoRoutingConfigError {
AutoRoutingConfigError {
model: model.model_name.clone(),
reason: message.into(),
}
}
fn map_route_error(error: RouteError) -> LlmClientError {
fn map_route_error(error: InferenceError) -> LlmClientError {
match error {
RouteError::Transport(TransportError::Http { status, body }) => {
InferenceError::Transport(TransportError::Http { status, body }) => {
match axum::http::StatusCode::from_u16(status) {
Ok(status) => LlmClientError::UpstreamHttp { status, body },
Err(source) => LlmClientError::InvalidResponse {
@ -435,10 +411,12 @@ fn map_route_error(error: RouteError) -> LlmClientError {
}
}
// Only Connect is known to precede dispatch, so only it is safe to fall back from.
error @ RouteError::Transport(TransportError::Connect(_)) => LlmClientError::Transport {
source: Box::new(error),
},
error @ RouteError::InvalidResponse(_) => LlmClientError::InvalidResponse {
error @ InferenceError::Transport(TransportError::Connect(_)) => {
LlmClientError::Transport {
source: Box::new(error),
}
}
error @ InferenceError::InvalidResponse(_) => LlmClientError::InvalidResponse {
source: Box::new(error),
},
error => LlmClientError::Host {
@ -458,10 +436,10 @@ fn map_client_error(error: LlmClientError) -> Error {
match error {
LlmClientError::InvalidRequest { message }
| LlmClientError::RequestTranslation(message) => {
Error::Route(RouteError::InvalidRequest(message.into()))
Error::Route(InferenceError::InvalidRequest(message.into()))
}
LlmClientError::UpstreamHttp { status, body } => {
Error::Route(RouteError::Transport(TransportError::Http {
Error::Route(InferenceError::Transport(TransportError::Http {
status: status.as_u16(),
body,
}))
@ -472,22 +450,134 @@ fn map_client_error(error: LlmClientError) -> Error {
| LlmClientError::Host { source }
| LlmClientError::Ffi { source } => map_source(source),
LlmClientError::ContextWindowExceeded { message, .. } => {
Error::Route(RouteError::InvalidRequest(message.into()))
Error::Route(InferenceError::InvalidRequest(message.into()))
}
LlmClientError::ResponseTranslation(message) => {
Error::Route(RouteError::InvalidResponse(message.into()))
Error::Route(InferenceError::InvalidResponse(message.into()))
}
error => Error::Internal(error.to_string()),
}
}
fn map_source(source: BoxError) -> Error {
match source.downcast::<RouteError>() {
match source.downcast::<InferenceError>() {
Ok(error) => Error::Route(*error),
Err(source) => Error::Internal(source.to_string()),
}
}
const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"];
const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-metadata",
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
fn chat_request(
deployment: &Deployment,
mut body: Map<String, Value>,
) -> ChatCompletionsRequest<'_> {
let messages = body.remove("messages").unwrap_or_default();
body.remove("model");
ChatCompletionsRequest {
model: &deployment.model,
messages,
optional_params: body,
api_key: deployment.api_key.as_deref(),
api_base: deployment.api_base.as_deref(),
custom_llm_provider: deployment.custom_llm_provider.as_deref(),
extra_headers: None,
timeout: deployment.timeout,
}
}
fn responses_request(deployment: &Deployment, mut body: Map<String, Value>) -> ResponsesCall {
let input = body.remove("input").unwrap_or_default();
body.remove("model");
ResponsesCall {
model: deployment.model.clone(),
input,
optional_params: body,
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
}
}
fn messages_request(
deployment: &Deployment,
mut body: Map<String, Value>,
headers: &HeaderMap,
) -> Result<MessagesCall, InferenceError> {
if let Some(model) = body.get_mut("model") {
*model = Value::from(deployment.model.as_str());
}
Ok(MessagesCall {
body: messages_body(body)?,
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
litellm_params: deployment.litellm_params.clone(),
extra_headers: None,
provider_specific_header: anthropic_api_headers(headers),
timeout: deployment.timeout,
shaping: deployment.shaping.clone(),
})
}
fn anthropic_api_headers(headers: &HeaderMap) -> Option<ProviderSpecificHeaders> {
let scopes = [
(
ANTHROPIC_API_HEADER_PROVIDERS,
ANTHROPIC_API_HEADERS.as_slice(),
),
("openrouter", OPENROUTER_API_HEADERS.as_slice()),
];
let forwarded: Vec<ProviderSpecificHeader> = scopes
.into_iter()
.filter_map(|(provider, names)| {
let extra_headers: Map<String, Value> = names
.iter()
.filter_map(|name| {
let value = headers.get(*name)?.to_str().ok()?;
Some(((*name).to_owned(), Value::from(value)))
})
.collect();
(!extra_headers.is_empty()).then(|| ProviderSpecificHeader {
custom_llm_provider: provider.into(),
extra_headers,
})
})
.collect();
(!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded))
}
impl AutoRoutingStrategy for SwitchyardStrategy {
fn execute<'a>(
&'a self,
gateway: &'a Gateway,
request: StrategyRequest,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<HttpResponse, Error>> + Send + 'a>>
{
let format = match request.format {
ApiFormat::Chat => WireFormat::OpenAiChat,
ApiFormat::Responses => WireFormat::OpenAiResponses,
ApiFormat::Messages => WireFormat::AnthropicMessages,
};
Box::pin(SwitchyardStrategy::execute(
self,
gateway,
request.cache_options,
format,
request.body,
request.headers,
))
}
}
#[cfg(test)]
mod tests {
use axum::body::to_bytes;
@ -497,14 +587,14 @@ mod tests {
#[rstest]
fn host_errors_keep_their_gateway_classification() {
let error = RouteError::Auth(litellm_auth::Error::MissingApiKey {
let error = InferenceError::Auth(litellm_auth::Error::MissingApiKey {
provider: "OpenAI",
environment_variable: "OPENAI_API_KEY",
});
assert!(matches!(
map_client_error(map_route_error(error)),
Error::Route(RouteError::Auth(_))
Error::Route(InferenceError::Auth(_))
));
}

View file

@ -1,40 +1,27 @@
use litellm_gateway_auth::AuthenticatedRequest;
use std::sync::Arc;
#[cfg(feature = "switchyard")]
use axum::http::HeaderMap;
use axum::{
Json,
extract::{Path, State},
response::{IntoResponse, Response},
};
use litellm_inference_chat::types::{ChatCompletionsCall, ChatCompletionsRequest};
use litellm_inference_chat::types::ChatCompletionsCall;
use serde_json::{Map, Value};
#[cfg(feature = "switchyard")]
use switchyard_protocol::WireFormat;
use crate::{Deployment, Error, Gateway, JsonObject, request};
use crate::{Error, Gateway, JsonObject, request};
pub(crate) async fn create(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
#[cfg(feature = "switchyard")] headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<impl IntoResponse, Error> {
handle(
&gateway,
&identity,
#[cfg(feature = "switchyard")]
headers,
body,
)
.await
handle(&gateway, &identity, body).await
}
pub(crate) async fn create_from_model_path(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
#[cfg(feature = "switchyard")] headers: HeaderMap,
Path(model): Path<String>,
JsonObject(body): JsonObject,
) -> Result<impl IntoResponse, Error> {
@ -45,42 +32,17 @@ pub(crate) async fn create_from_model_path(
.collect(),
Some(_) => body,
};
handle(
&gateway,
&identity,
#[cfg(feature = "switchyard")]
headers,
body,
)
.await
handle(&gateway, &identity, body).await
}
async fn handle(
gateway: &Gateway,
identity: &AuthenticatedRequest,
#[cfg(feature = "switchyard")] inbound_headers: HeaderMap,
body: Map<String, Value>,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(gateway, &body)?;
request::authorize_model(identity, deployment, &body).await?;
#[cfg(feature = "switchyard")]
let switchyard_route = body
.get("model")
.and_then(Value::as_str)
.and_then(|model| gateway.switchyard_routes.get(model));
let (body, cache_options) = crate::caching::prepare(identity, body)?;
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
return switchyard_route
.execute(
gateway,
cache_options,
WireFormat::OpenAiChat,
body,
inbound_headers,
)
.await;
}
let route = gateway.chat_completions.clone();
let route = match &gateway.cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
@ -89,10 +51,24 @@ async fn handle(
)),
None => route,
};
let messages = body.get("messages").cloned().unwrap_or_default();
let headers = crate::caching::CacheHeaders::default();
let response = litellm_host_http::serve_unary(
route.machine(
ChatCompletionsCall::from(provider_request(deployment, body)),
ChatCompletionsCall {
model: deployment.model.clone(),
messages,
optional_params: body
.into_iter()
.filter(|(name, _)| !matches!(name.as_str(), "model" | "messages"))
.collect(),
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
},
cache_options.policy,
),
(),
@ -103,21 +79,3 @@ async fn handle(
.await?;
Ok(headers.apply(response))
}
pub(crate) fn provider_request(
deployment: &Deployment,
mut body: Map<String, Value>,
) -> ChatCompletionsRequest<'_> {
let messages = body.remove("messages").unwrap_or_default();
body.remove("model");
ChatCompletionsRequest {
model: &deployment.model,
messages,
optional_params: body,
api_key: deployment.api_key.as_deref(),
api_base: deployment.api_base.as_deref(),
custom_llm_provider: deployment.custom_llm_provider.as_deref(),
extra_headers: None,
timeout: deployment.timeout,
}
}

View file

@ -8,11 +8,9 @@ use litellm_inference::Error as InferenceError;
use litellm_llms::base_llm::ocr::error::Error as OcrError;
use serde_json::{Map, Value, json};
/// Invalid configuration for a Switchyard virtual model.
#[cfg(feature = "switchyard")]
#[derive(Debug, thiserror::Error)]
#[error("invalid Switchyard model {model}: {reason}")]
pub struct SwitchyardConfigError {
#[error("invalid auto-routing model {model}: {reason}")]
pub struct AutoRoutingConfigError {
pub(crate) model: String,
pub(crate) reason: String,
}

View file

@ -4,6 +4,7 @@
//! maps a public model name to its deployment and runs the core route.
mod audio_transcription;
pub mod auto_routing;
mod caching;
mod chat_completions;
mod error;
@ -11,8 +12,6 @@ pub mod messages;
mod ocr;
mod request;
mod responses;
#[cfg(feature = "switchyard")]
mod switchyard;
use std::sync::Arc;
@ -27,16 +26,14 @@ use litellm_inference_transcription::AudioTranscriptionRoute;
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
use litellm_secrets::source::SecretSource;
pub use error::AutoRoutingConfigError;
pub use error::Error;
#[cfg(feature = "switchyard")]
pub use error::SwitchyardConfigError;
pub use litellm_router::{Deployment, Router as ModelRouter};
pub use request::{JsonObject, RequestId};
pub struct Gateway {
cache: Option<Arc<dyn litellm_cache_response::ResponseCacheService>>,
#[cfg(feature = "switchyard")]
switchyard_routes: switchyard::Routes,
auto_routing: auto_routing::Registry,
pub audio_transcription: AudioTranscriptionRoute,
pub chat_completions: ChatCompletionsRoute,
pub messages: MessagesRoute,
@ -66,8 +63,7 @@ impl Gateway {
let auth = resources.auth.clone();
Ok(Self {
cache: None,
#[cfg(feature = "switchyard")]
switchyard_routes: switchyard::Routes::default(),
auto_routing: auto_routing::Registry::default(),
audio_transcription: AudioTranscriptionRoute::new(
provider.clone(),
auth.clone(),
@ -95,24 +91,37 @@ impl Gateway {
})
}
/// Configure Switchyard virtual models from LiteLLM's model list.
#[cfg(feature = "switchyard")]
pub fn with_switchyard_models(
pub fn with_auto_routing_strategy(
mut self,
model: String,
strategy: Arc<dyn auto_routing::AutoRoutingStrategy>,
) -> Result<Self, AutoRoutingConfigError> {
if self.models.get(&model).is_none() {
return Err(AutoRoutingConfigError {
model,
reason: "model is not configured".into(),
});
}
self.auto_routing.insert(model, strategy);
Ok(self)
}
pub fn with_auto_routing_models(
mut self,
models: &[litellm_config::Model],
) -> Result<Self, SwitchyardConfigError> {
self.switchyard_routes = switchyard::Routes::from_models(models, &self.models)?;
) -> Result<Self, AutoRoutingConfigError> {
self.auto_routing = auto_routing::Registry::from_models(models, &self.models)?;
Ok(self)
}
}
pub fn router(gateway: Arc<Gateway>) -> Router {
Router::new()
.route("/v1/messages", post(messages::create))
.route("/v1/messages", post(auto_routing::messages))
.route("/ocr", post(ocr::create))
.route("/v1/ocr", post(ocr::create))
.route("/chat/completions", post(chat_completions::create))
.route("/v1/chat/completions", post(chat_completions::create))
.route("/chat/completions", post(auto_routing::chat))
.route("/v1/chat/completions", post(auto_routing::chat))
.nest("/engines/{model}", model_routes())
.nest("/openai/deployments/{model}", model_routes())
.route("/audio/transcriptions", post(audio_transcription::create))
@ -120,8 +129,8 @@ pub fn router(gateway: Arc<Gateway>) -> Router {
"/v1/audio/transcriptions",
post(audio_transcription::create),
)
.route("/responses", post(responses::create))
.route("/v1/responses", post(responses::create))
.route("/responses", post(auto_routing::responses))
.route("/v1/responses", post(auto_routing::responses))
.route("/embeddings", post(request::unsupported))
.route("/v1/embeddings", post(request::unsupported))
.route("/completions", post(request::unsupported))
@ -136,7 +145,7 @@ fn model_routes() -> Router<Arc<Gateway>> {
Router::new()
.route(
"/chat/completions",
post(chat_completions::create_from_model_path),
post(auto_routing::chat_from_model_path),
)
.route("/embeddings", post(request::unsupported))
.route("/completions", post(request::unsupported))

View file

@ -14,8 +14,6 @@ use litellm_host_http::Sse;
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
use serde_json::{Map, Value};
#[cfg(feature = "switchyard")]
use switchyard_protocol::WireFormat;
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
@ -36,7 +34,7 @@ pub async fn create(
body: Result<JsonObject, Error>,
) -> impl IntoResponse {
let result = match body {
Ok(JsonObject(body)) => handle(&gateway, &identity, headers, body).await,
Ok(JsonObject(body)) => handle(&gateway, &identity, &headers, body).await,
Err(error) => Err(error),
};
result.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
@ -45,29 +43,12 @@ pub async fn create(
async fn handle(
gateway: &Gateway,
identity: &AuthenticatedRequest,
headers: HeaderMap,
headers: &HeaderMap,
body: Map<String, Value>,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(gateway, &body)?;
request::authorize_model(identity, deployment, &body).await?;
#[cfg(feature = "switchyard")]
let switchyard_route = body
.get("model")
.and_then(Value::as_str)
.and_then(|model| gateway.switchyard_routes.get(model));
let (body, cache_options) = crate::caching::prepare(identity, body)?;
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
return switchyard_route
.execute(
gateway,
cache_options,
WireFormat::AnthropicMessages,
body,
headers,
)
.await;
}
let route = gateway.messages.clone();
let route = match &gateway.cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
@ -77,7 +58,7 @@ async fn handle(
None => route,
};
let call = provider_request(deployment, body, &headers)?;
let call = project(deployment, body, headers)?;
let machine = route.machine(call, cache_options.policy);
let stream =
Sse::<Messages, _, _>::new(Json, |error| Bytes::from(Error::from(error).sse_frame()));
@ -86,14 +67,18 @@ async fn handle(
Ok(headers.apply(response))
}
pub(crate) fn provider_request(
fn project(
deployment: &Deployment,
mut body: Map<String, Value>,
body: Map<String, Value>,
headers: &HeaderMap,
) -> Result<MessagesCall, litellm_inference::RouteError> {
if let Some(model) = body.get_mut("model") {
*model = Value::from(deployment.model.as_str());
}
) -> Result<MessagesCall, Error> {
let body = body
.into_iter()
.map(|(name, value)| match name.as_str() {
"model" => (name, Value::from(deployment.model.as_str())),
_ => (name, value),
})
.collect();
Ok(MessagesCall {
body: messages_body(body)?,
api_key: deployment.api_key.clone(),

View file

@ -1,43 +1,21 @@
use std::sync::Arc;
#[cfg(feature = "switchyard")]
use axum::http::HeaderMap;
use axum::{Json, body::Bytes, extract::State, response::Response};
use litellm_gateway_auth::AuthenticatedRequest;
use litellm_host_http::Sse;
use litellm_inference_responses::{route::Responses, types::ResponsesCall};
use serde_json::{Map, Value, json};
#[cfg(feature = "switchyard")]
use switchyard_protocol::WireFormat;
use serde_json::json;
use crate::{Deployment, Error, Gateway, JsonObject, request};
use crate::{Error, Gateway, JsonObject, request};
pub(crate) async fn create(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
#[cfg(feature = "switchyard")] headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(&gateway, &body)?;
request::authorize_model(&identity, deployment, &body).await?;
#[cfg(feature = "switchyard")]
let switchyard_route = body
.get("model")
.and_then(Value::as_str)
.and_then(|model| gateway.switchyard_routes.get(model));
let (body, cache_options) = crate::caching::prepare(&identity, body)?;
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
return switchyard_route
.execute(
&gateway,
cache_options,
WireFormat::OpenAiResponses,
body,
headers,
)
.await;
}
let route = gateway.responses.clone();
let route = match &gateway.cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
@ -47,7 +25,19 @@ pub(crate) async fn create(
None => route,
};
let call = provider_request(deployment, body);
let call = ResponsesCall {
model: deployment.model.clone(),
input: body.get("input").cloned().unwrap_or_default(),
optional_params: body
.into_iter()
.filter(|(name, _)| !matches!(name.as_str(), "model" | "input"))
.collect(),
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
};
let machine = route.machine(call, cache_options.policy);
let stream = Sse::<Responses, _, _>::new(Json, |error| {
let error = Error::from(error);
@ -60,21 +50,3 @@ pub(crate) async fn create(
let response = litellm_host_http::serve(machine, (), headers.clone(), stream, None).await?;
Ok(headers.apply(response))
}
pub(crate) fn provider_request(
deployment: &Deployment,
mut body: Map<String, Value>,
) -> ResponsesCall {
let input = body.remove("input").unwrap_or_default();
body.remove("model");
ResponsesCall {
model: deployment.model.clone(),
input,
optional_params: body,
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
}
}

View file

@ -0,0 +1,267 @@
mod support;
use axum::{
Json,
response::{IntoResponse, Response},
};
use litellm_gateway_auth::Permissions;
use litellm_gateway_inference::{
Deployment, Error, Gateway,
auto_routing::{ApiFormat, AutoRoutingStrategy, StrategyRequest},
};
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
use litellm_inference::resources::CoreResources;
use rstest::{fixture, rstest};
use serde_json::{Value, json};
use std::{
future::Future,
pin::Pin,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
struct EchoStrategy(Arc<AtomicUsize>);
impl AutoRoutingStrategy for EchoStrategy {
fn execute<'a>(
&'a self,
_: &'a Gateway,
request: StrategyRequest,
) -> Pin<Box<dyn Future<Output = Result<Response, Error>> + Send + 'a>> {
self.0.fetch_add(1, Ordering::SeqCst);
Box::pin(async move {
if request.body.get("fail").and_then(Value::as_bool) == Some(true) {
return Err(Error::InvalidBody("strategy rejected request".into()));
}
let format = match request.format {
ApiFormat::Chat => "chat",
ApiFormat::Responses => "responses",
ApiFormat::Messages => "messages",
};
Ok(Json(json!({"format": format, "body": request.body, "request_id": request.headers.get("x-request-id").and_then(|v| v.to_str().ok()), "no_store": request.cache_options.policy.no_store})).into_response())
})
}
}
#[fixture]
fn gateway() -> Gateway {
Gateway::new(
CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))),
Resolution::from(&HttpSettings::default()).config,
Arc::new(support::NoSecrets),
[(
"virtual".into(),
Deployment {
model: "virtual-target".into(),
..Default::default()
},
)]
.into_iter()
.collect(),
)
.unwrap()
}
#[rstest]
#[case::chat("/v1/chat/completions", "chat", json!({"model":"virtual"}))]
#[case::chat_alias("/chat/completions", "chat", json!({"model":"virtual"}))]
#[case::responses("/v1/responses", "responses", json!({"model":"virtual"}))]
#[case::responses_alias("/responses", "responses", json!({"model":"virtual"}))]
#[case::messages("/v1/messages", "messages", json!({"model":"virtual"}))]
#[case::engine("/engines/virtual/chat/completions", "chat", json!({}))]
#[case::deployment("/openai/deployments/virtual/chat/completions", "chat", json!({"model":null}))]
#[case::body_wins("/engines/unknown/chat/completions", "chat", json!({"model":"virtual"}))]
#[tokio::test]
async fn virtual_models_invoke_the_strategy_once(
gateway: Gateway,
#[case] path: &str,
#[case] format: &str,
#[case] body: Value,
) {
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
body,
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["format"], format);
assert_eq!(body["body"]["model"], "virtual");
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn denied_virtual_models_never_invoke_the_strategy(gateway: Gateway, #[case] path: &str) {
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::Only(Arc::from([])), None),
path,
json!({"model":"virtual"}),
)
.await;
assert_eq!(response.status(), 403);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[rstest]
#[case::chat("/v1/chat/completions", false)]
#[case::responses("/v1/responses", false)]
#[case::messages("/v1/messages", true)]
#[tokio::test]
async fn strategy_errors_keep_the_callers_error_format(
gateway: Gateway,
#[case] path: &str,
#[case] anthropic: bool,
) {
let gateway = gateway
.with_auto_routing_strategy(
"virtual".into(),
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
)
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
json!({"model":"virtual", "fail":true}),
)
.await;
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.unwrap()
.contains("strategy rejected request")
);
assert_eq!(body.get("type").is_some(), anthropic);
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn cache_controls_are_prepared_before_strategy_execution(
gateway: Gateway,
#[case] path: &str,
) {
let gateway = gateway
.with_auto_routing_strategy(
"virtual".into(),
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
)
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
json!({"model":"virtual", "cache":{"no-store":true}, "caching":false}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["no_store"], true);
assert!(body["body"].get("cache").is_none());
assert!(body["body"].get("caching").is_none());
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn incoming_headers_reach_the_strategy(gateway: Gateway, #[case] path: &str) {
use axum::{body::Body, http::Request};
use tower::ServiceExt;
let gateway = gateway
.with_auto_routing_strategy(
"virtual".into(),
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
)
.unwrap();
let response = support::authenticated_app(gateway, Permissions::All, None)
.oneshot(
Request::post(path)
.header("content-type", "application/json")
.header("x-request-id", "request-context")
.body(Body::from(r#"{"model":"virtual"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(
support::json(response).await["request_id"],
"request-context"
);
}
#[rstest]
#[case::chat("/v1/chat/completions", false)]
#[case::responses("/v1/responses", false)]
#[case::messages("/v1/messages", true)]
#[tokio::test]
async fn malformed_bodies_preserve_api_error_envelopes(
gateway: Gateway,
#[case] path: &str,
#[case] anthropic: bool,
) {
use axum::{body::Body, http::Request};
use tower::ServiceExt;
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::authenticated_app(gateway, Permissions::All, None)
.oneshot(
Request::post(path)
.header("content-type", "application/json")
.header("x-request-id", "malformed-request")
.body(Body::from("{"))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body.get("type").is_some(), anthropic);
if anthropic {
assert_eq!(body["request_id"], "malformed-request");
}
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[rstest]
#[case::chat("/v1/chat/completions")]
#[case::responses("/v1/responses")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn invalid_cache_settings_do_not_invoke_the_strategy(gateway: Gateway, #[case] path: &str) {
let calls = Arc::new(AtomicUsize::new(0));
let gateway = gateway
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
.unwrap();
let response = support::post(
support::authenticated_app(gateway, Permissions::All, None),
path,
json!({"model":"virtual", "cache":{"ttl":-1}}),
)
.await;
assert_eq!(response.status(), 400);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}

View file

@ -390,16 +390,93 @@ async fn streaming_responses_upstream() -> (String, tokio::task::JoinHandle<()>)
}
fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router {
support::authenticated_app(switchyard_gateway(config), permissions, None)
}
fn switchyard_gateway(config: &Config) -> Gateway {
let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))));
let http = Resolution::from(&HttpSettings::default()).config;
let gateway = Gateway::new(
Gateway::new(
resources,
http,
Arc::new(support::NoSecrets),
ModelRouter::from_model_list(&config.model_list),
)
.expect("gateway")
.with_switchyard_models(&config.model_list)
.expect("Switchyard routes");
support::authenticated_app(gateway, permissions, None)
.with_auto_routing_models(&config.model_list)
.expect("Switchyard routes")
}
#[rstest]
#[tokio::test]
async fn routed_answers_preserve_cache_hits_and_caller_isolation() {
use litellm_cache_memory::InMemoryCache;
use litellm_cache_response::ResponseCache;
use litellm_gateway_auth::{Principal, PrincipalKind};
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"model":"answer", "choices":[{"message":{"role":"assistant","content":"cached answer"}}]
})))
.expect(2)
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: answer
litellm_params:
model: answer
custom_llm_provider: openai_like
api_base: {}/v1
api_key: test-key
switchyard_wire_format: openai_chat
- model_name: virtual
litellm_params:
model: switchyard/algorithm
switchyard_config: {{type: passthrough, target: answer}}
"#,
upstream.uri()
))
.unwrap();
let cache = Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new(
Some(100),
Some(Duration::from_secs(60)),
))));
let first_app = support::authenticated_app(
switchyard_gateway(&config).with_cache(cache.clone()),
Permissions::All,
Some(Principal::new(
"issuer".into(),
"first".into(),
PrincipalKind::Service,
)),
);
let second_app = support::authenticated_app(
switchyard_gateway(&config).with_cache(cache),
Permissions::All,
Some(Principal::new(
"issuer".into(),
"second".into(),
PrincipalKind::Service,
)),
);
let request = json!({"model":"virtual", "messages":[{"role":"user", "content":"hello"}]});
let first = support::post(first_app.clone(), "/v1/chat/completions", request.clone()).await;
assert_eq!(first.status(), 200);
assert!(!first.headers().contains_key("x-litellm-cache-key"));
let first_body = support::json(first).await;
let hit = support::post(first_app, "/v1/chat/completions", request.clone()).await;
assert_eq!(hit.status(), 200);
let first_key = hit.headers().get("x-litellm-cache-key").unwrap().clone();
assert_eq!(support::json(hit).await, first_body);
let second = support::post(second_app.clone(), "/v1/chat/completions", request.clone()).await;
assert_eq!(second.status(), 200);
assert!(!second.headers().contains_key("x-litellm-cache-key"));
let hit = support::post(second_app, "/v1/chat/completions", request).await;
assert_eq!(hit.status(), 200);
assert_ne!(hit.headers().get("x-litellm-cache-key").unwrap(), first_key);
upstream.verify().await;
}

View file

@ -10,7 +10,6 @@ pub enum Error {
Mcp(#[from] litellm_gateway_mcp::ConnectError),
#[error("MCP requires a configured master key")]
Auth(#[from] litellm_gateway_auth::Error),
#[cfg(feature = "switchyard")]
#[error(transparent)]
Switchyard(#[from] litellm_gateway_inference::SwitchyardConfigError),
AutoRouting(#[from] litellm_gateway_inference::AutoRoutingConfigError),
}

View file

@ -68,8 +68,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
secrets,
ModelRouter::from_model_list(&config.model_list),
)?;
#[cfg(feature = "switchyard")]
let gateway = gateway.with_switchyard_models(&config.model_list)?;
let gateway = gateway.with_auto_routing_models(&config.model_list)?;
Ok(Arc::new(gateway))
}