From c3827fef75a3f381580fd6145ae76f2a27d27795 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:19:54 -0700 Subject: [PATCH] refactor(gateway): isolate auto-routing strategy orchestration --- .github/workflows/test-rust.yml | 4 + litellm-rust/Cargo.lock | 1 + .../crates/gateway-inference/Cargo.toml | 4 +- .../gateway-inference/src/auto_routing/mod.rs | 329 ++++++++++++++++++ .../src/{ => auto_routing}/switchyard.rs | 248 ++++++++----- .../gateway-inference/src/chat_completions.rs | 80 +---- .../crates/gateway-inference/src/error.rs | 6 +- .../crates/gateway-inference/src/lib.rs | 47 ++- .../crates/gateway-inference/src/messages.rs | 41 +-- .../crates/gateway-inference/src/responses.rs | 58 +-- .../gateway-inference/tests/auto_routing.rs | 267 ++++++++++++++ .../gateway-inference/tests/switchyard.rs | 85 ++++- litellm-rust/crates/gateway/src/error.rs | 3 +- litellm-rust/crates/gateway/src/lib.rs | 3 +- 14 files changed, 932 insertions(+), 244 deletions(-) create mode 100644 litellm-rust/crates/gateway-inference/src/auto_routing/mod.rs rename litellm-rust/crates/gateway-inference/src/{ => auto_routing}/switchyard.rs (70%) create mode 100644 litellm-rust/crates/gateway-inference/tests/auto_routing.rs diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 636a5031e5f..c60e3ff292d 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -101,6 +101,8 @@ jobs: - run: cargo clippy --workspace --all-targets --locked -- -D warnings + - run: cargo clippy -p litellm-gateway-inference -p litellm-gateway --all-targets --features switchyard --locked -- -D warnings + rust-test: runs-on: ubuntu-latest timeout-minutes: 30 @@ -147,6 +149,8 @@ jobs: - run: cargo nextest run --workspace --locked + - run: cargo nextest run -p litellm-gateway-inference -p litellm-gateway --features switchyard --locked + - run: cargo test --workspace --doc --locked - name: Test token counter feature combinations diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f5a87a0efe8..45e0729d988 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3882,6 +3882,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "strum", "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 8c6cb9efccf..3131dd960a8 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true switchyard = [ "dep:async-trait", "dep:futures-util", - "dep:litellm-config", "dep:switchyard-libsy", "dep:switchyard-llm-client", "dep:switchyard-protocol", @@ -32,7 +31,7 @@ litellm-inference-responses.workspace = true litellm-inference-messages.workspace = true litellm-inference-chat.workspace = true litellm-inference-ocr.workspace = true -litellm-config = { workspace = true, optional = true } +litellm-config.workspace = true litellm-host-http.workspace = true litellm-host.workspace = true litellm-http.workspace = true @@ -47,6 +46,7 @@ switchyard-llm-client = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", switchyard-protocol = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true } switchyard-runner = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true } switchyard-translation = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true } +strum.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/gateway-inference/src/auto_routing/mod.rs b/litellm-rust/crates/gateway-inference/src/auto_routing/mod.rs new file mode 100644 index 00000000000..d403e04e0e9 --- /dev/null +++ b/litellm-rust/crates/gateway-inference/src/auto_routing/mod.rs @@ -0,0 +1,329 @@ +use std::{ + collections::{HashMap, HashSet}, + future::Future, + pin::Pin, + sync::Arc, +}; + +use axum::{ + Json, + extract::{Path, State}, + http::HeaderMap, + response::{IntoResponse, Response}, +}; +use litellm_cache_response::CacheOptions; +use litellm_config::Model; +use litellm_gateway_auth::AuthenticatedRequest; +use serde_json::{Map, Value}; + +use crate::{AutoRoutingConfigError, Error, Gateway, JsonObject, ModelRouter, RequestId, request}; + +#[cfg(feature = "switchyard")] +mod switchyard; + +#[derive(Clone, Copy, strum::VariantArray)] +pub enum ApiFormat { + Chat, + Responses, + Messages, +} + +pub struct StrategyRequest { + pub format: ApiFormat, + pub body: Map, + pub headers: HeaderMap, + pub cache_options: CacheOptions, +} + +pub trait AutoRoutingStrategy: Send + Sync { + fn execute<'a>( + &'a self, + gateway: &'a Gateway, + request: StrategyRequest, + ) -> Pin> + Send + 'a>>; +} + +#[derive(Default)] +pub(crate) struct Registry(HashMap>); + +impl Registry { + pub(crate) fn insert(&mut self, model: String, strategy: Arc) { + self.0.insert(model, strategy); + } + + pub(crate) fn from_models( + models: &[Model], + deployments: &ModelRouter, + ) -> Result { + let mut seen = HashSet::new(); + let mut strategies = HashMap::new(); + for model in models.iter().rev() { + if !seen.insert(&model.model_name) + || model.litellm_params.model != "switchyard/algorithm" + { + continue; + } + if deployments + .get(&model.model_name) + .is_none_or(|deployment| deployment.model != model.litellm_params.model) + { + return Err(AutoRoutingConfigError { + model: model.model_name.clone(), + reason: "model does not match its configured deployment".into(), + }); + } + strategies.insert( + model.model_name.clone(), + build_strategy(model, models, deployments)?, + ); + } + Ok(Self(strategies)) + } + + fn get(&self, body: &Map) -> Option<&dyn AutoRoutingStrategy> { + self.0.get(body.get("model")?.as_str()?).map(AsRef::as_ref) + } +} + +#[cfg(feature = "switchyard")] +fn build_strategy( + model: &Model, + models: &[Model], + deployments: &ModelRouter, +) -> Result, AutoRoutingConfigError> { + Ok(Arc::new(switchyard::SwitchyardStrategy::from_model( + model, + models, + deployments, + )?)) +} + +#[cfg(not(feature = "switchyard"))] +fn build_strategy( + model: &Model, + _: &[Model], + _: &ModelRouter, +) -> Result, AutoRoutingConfigError> { + Err(AutoRoutingConfigError { + model: model.model_name.clone(), + reason: "this build does not enable the switchyard feature".into(), + }) +} + +async fn execute( + strategy: &dyn AutoRoutingStrategy, + gateway: &Gateway, + identity: &AuthenticatedRequest, + format: ApiFormat, + headers: HeaderMap, + body: Map, +) -> Result { + let deployment = request::resolve_deployment(gateway, &body)?; + request::authorize_model(identity, deployment, &body).await?; + let (body, cache_options) = crate::caching::prepare(identity, body)?; + strategy + .execute( + gateway, + StrategyRequest { + format, + body, + headers, + cache_options, + }, + ) + .await +} + +pub(crate) async fn chat( + State(gateway): State>, + identity: AuthenticatedRequest, + headers: HeaderMap, + JsonObject(body): JsonObject, +) -> Result { + if let Some(strategy) = gateway.auto_routing.get(&body) { + return execute( + strategy, + &gateway, + &identity, + ApiFormat::Chat, + headers, + body, + ) + .await; + } + crate::chat_completions::create(State(gateway), identity, JsonObject(body)) + .await + .map(IntoResponse::into_response) +} + +pub(crate) async fn chat_from_model_path( + State(gateway): State>, + identity: AuthenticatedRequest, + headers: HeaderMap, + Path(model): Path, + JsonObject(mut body): JsonObject, +) -> Result { + if body.get("model").is_none_or(Value::is_null) { + body.insert("model".into(), Value::String(model.clone())); + } + if let Some(strategy) = gateway.auto_routing.get(&body) { + return execute( + strategy, + &gateway, + &identity, + ApiFormat::Chat, + headers, + body, + ) + .await; + } + crate::chat_completions::create_from_model_path( + State(gateway), + identity, + Path(model), + JsonObject(body), + ) + .await + .map(IntoResponse::into_response) +} + +pub(crate) async fn responses( + State(gateway): State>, + identity: AuthenticatedRequest, + headers: HeaderMap, + JsonObject(body): JsonObject, +) -> Result { + if let Some(strategy) = gateway.auto_routing.get(&body) { + return execute( + strategy, + &gateway, + &identity, + ApiFormat::Responses, + headers, + body, + ) + .await; + } + crate::responses::create(State(gateway), identity, JsonObject(body)).await +} + +pub(crate) async fn messages( + State(gateway): State>, + identity: AuthenticatedRequest, + RequestId(request_id): RequestId, + headers: HeaderMap, + body: Result, +) -> Response { + match body { + Ok(JsonObject(body)) => { + if let Some(strategy) = gateway.auto_routing.get(&body) { + return execute( + strategy, + &gateway, + &identity, + ApiFormat::Messages, + headers, + body, + ) + .await + .unwrap_or_else(|error| { + (error.status(), Json(error.body(request_id.as_deref()))).into_response() + }); + } + crate::messages::create( + State(gateway), + identity, + RequestId(request_id), + headers, + Ok(JsonObject(body)), + ) + .await + .into_response() + } + Err(error) => crate::messages::create( + State(gateway), + identity, + RequestId(request_id), + headers, + Err(error), + ) + .await + .into_response(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use litellm_config::Config; + use rstest::rstest; + + #[rstest] + fn overwritten_virtual_models_do_not_initialize_a_strategy() { + let config = Config::from_yaml( + r#" +model_list: + - model_name: public + litellm_params: {model: switchyard/algorithm} + - model_name: public + litellm_params: {model: openai/model} +"#, + ) + .unwrap(); + let registry = Registry::from_models( + &config.model_list, + &ModelRouter::from_model_list(&config.model_list), + ) + .unwrap(); + assert!( + registry + .get( + &serde_json::json!({"model":"public"}) + .as_object() + .unwrap() + .clone() + ) + .is_none() + ); + } + + #[rstest] + fn strategy_configuration_must_match_the_deployment_registry() { + let config = Config::from_yaml( + r#" +model_list: + - model_name: public + litellm_params: {model: switchyard/algorithm} +"#, + ) + .unwrap(); + let error = Registry::from_models(&config.model_list, &ModelRouter::default()) + .err() + .unwrap(); + assert!(error.to_string().contains("does not match")); + } + + #[cfg(not(feature = "switchyard"))] + #[rstest] + fn disabled_strategies_fail_at_configuration_time() { + let config = Config::from_yaml( + r#" +model_list: + - model_name: public + litellm_params: {model: switchyard/algorithm} +"#, + ) + .unwrap(); + let error = Registry::from_models( + &config.model_list, + &ModelRouter::from_model_list(&config.model_list), + ) + .err() + .unwrap(); + assert!( + error + .to_string() + .contains("does not enable the switchyard feature") + ); + } +} diff --git a/litellm-rust/crates/gateway-inference/src/switchyard.rs b/litellm-rust/crates/gateway-inference/src/auto_routing/switchyard.rs similarity index 70% rename from litellm-rust/crates/gateway-inference/src/switchyard.rs rename to litellm-rust/crates/gateway-inference/src/auto_routing/switchyard.rs index 11fd8729e02..265723beca3 100644 --- a/litellm-rust/crates/gateway-inference/src/switchyard.rs +++ b/litellm-rust/crates/gateway-inference/src/auto_routing/switchyard.rs @@ -1,5 +1,10 @@ +use super::{ApiFormat, AutoRoutingStrategy, StrategyRequest}; +use litellm_inference_chat::types::ChatCompletionsRequest; +use litellm_inference_messages::{MessagesCall, messages_body}; +use litellm_inference_responses::types::ResponsesCall; +use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders}; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap}, convert::Infallible, sync::Arc, }; @@ -18,7 +23,7 @@ use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache}; use litellm_config::Model; use litellm_host::call::CallOutput; use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error as InferenceError; use litellm_inference_chat::ChatCompletionsRoute; use litellm_inference_messages::MessagesRoute; use litellm_inference_responses::ResponsesRoute; @@ -37,70 +42,41 @@ use switchyard_translation::{ }; use crate::{ - Deployment, Error, Gateway, ModelRouter, SwitchyardConfigError, caching::CacheHeaders, - chat_completions, messages, responses, + AutoRoutingConfigError, Deployment, Error, Gateway, ModelRouter, caching::CacheHeaders, }; const SWITCHYARD_MODEL: &str = "switchyard/algorithm"; -#[derive(Default)] -pub(crate) struct Routes(HashMap); - -impl Routes { - pub(crate) fn from_models( - model_list: &[Model], - deployments: &ModelRouter, - ) -> Result { - let mut routes = HashMap::new(); - let mut seen = HashSet::new(); - // ModelRouter uses the final entry when public model names repeat. - for model in model_list.iter().rev() { - if !seen.insert(model.model_name.as_str()) - || model.litellm_params.model != SWITCHYARD_MODEL - { - continue; - } - if deployments - .get(&model.model_name) - .is_none_or(|deployment| deployment.model != SWITCHYARD_MODEL) - { - return Err(invalid(model, "model is not a Switchyard route")); - } - let config = model - .litellm_params - .extra - .get("switchyard_config") - .ok_or_else(|| invalid(model, "switchyard_config is required"))?; - let config = AlgorithmSpec::deserialize(config) - .map_err(|error| invalid(model, error.to_string()))?; - let targets = targets(model, &config, model_list, deployments)?; - let (algorithm, runtime_models) = config - .build_with_runtime_models(&model.model_name, &targets.models) - .map_err(|error| invalid(model, error.to_string()))?; - routes.insert( - model.model_name.clone(), - Route { - algorithm, - runtime_models: Arc::new(runtime_models), - targets: Arc::new(targets.by_model), - }, - ); - } - Ok(Self(routes)) - } - - pub(crate) fn get(&self, model: &str) -> Option<&Route> { - self.0.get(model) - } -} - -pub(crate) struct Route { +pub(super) struct SwitchyardStrategy { algorithm: Arc, runtime_models: Arc, targets: Arc>, } -impl Route { +impl SwitchyardStrategy { + pub(super) fn from_model( + model: &Model, + model_list: &[Model], + deployments: &ModelRouter, + ) -> Result { + let config = model + .litellm_params + .extra + .get("switchyard_config") + .ok_or_else(|| invalid(model, "switchyard_config is required"))?; + let config = AlgorithmSpec::deserialize(config) + .map_err(|error| invalid(model, error.to_string()))?; + let targets = targets(model, &config, model_list, deployments)?; + let (algorithm, runtime_models) = config + .build_with_runtime_models(&model.model_name, &targets.models) + .map_err(|error| invalid(model, error.to_string()))?; + Ok(Self { + algorithm, + runtime_models: Arc::new(runtime_models), + targets: Arc::new(targets.by_model), + }) + } + pub(crate) async fn execute( &self, gateway: &Gateway, @@ -230,7 +206,7 @@ impl RoutedLlmClient for LiteLlmClient { .routes .chat .execute( - chat_completions::provider_request(&target.deployment, body), + chat_request(&target.deployment, body), &cache_headers, self.cache_policy, ) @@ -243,7 +219,7 @@ impl RoutedLlmClient for LiteLlmClient { .routes .responses .execute( - responses::provider_request(&target.deployment, body), + responses_request(&target.deployment, body), &cache_headers, self.cache_policy, ) @@ -257,8 +233,8 @@ impl RoutedLlmClient for LiteLlmClient { .as_ref() .and_then(|metadata| metadata.http_headers.as_ref()) .unwrap_or(&empty_headers); - let call = messages::provider_request(&target.deployment, body, headers) - .map_err(map_route_error)?; + let call = + messages_request(&target.deployment, body, headers).map_err(map_route_error)?; let response = self .routes .messages @@ -290,7 +266,7 @@ fn decode_complete( } fn decode_output( - response: CallOutput, + response: CallOutput, wire_format: WireFormat, ) -> Result where @@ -360,7 +336,7 @@ fn targets( config: &AlgorithmSpec, model_list: &[Model], deployments: &ModelRouter, -) -> Result { +) -> Result { let mut models = BTreeMap::new(); let mut by_model = HashMap::new(); for target in config.callable_target_names() { @@ -392,7 +368,7 @@ fn targets( } // Caller format and target execution format are independent, so each target declares its own. -fn target_wire_format(route: &Model, target: &Model) -> Result { +fn target_wire_format(route: &Model, target: &Model) -> Result { let value = target .litellm_params .extra @@ -417,16 +393,16 @@ fn target_wire_format(route: &Model, target: &Model) -> Result) -> SwitchyardConfigError { - SwitchyardConfigError { +fn invalid(model: &Model, message: impl Into) -> AutoRoutingConfigError { + AutoRoutingConfigError { model: model.model_name.clone(), reason: message.into(), } } -fn map_route_error(error: RouteError) -> LlmClientError { +fn map_route_error(error: InferenceError) -> LlmClientError { match error { - RouteError::Transport(TransportError::Http { status, body }) => { + InferenceError::Transport(TransportError::Http { status, body }) => { match axum::http::StatusCode::from_u16(status) { Ok(status) => LlmClientError::UpstreamHttp { status, body }, Err(source) => LlmClientError::InvalidResponse { @@ -435,10 +411,12 @@ fn map_route_error(error: RouteError) -> LlmClientError { } } // Only Connect is known to precede dispatch, so only it is safe to fall back from. - error @ RouteError::Transport(TransportError::Connect(_)) => LlmClientError::Transport { - source: Box::new(error), - }, - error @ RouteError::InvalidResponse(_) => LlmClientError::InvalidResponse { + error @ InferenceError::Transport(TransportError::Connect(_)) => { + LlmClientError::Transport { + source: Box::new(error), + } + } + error @ InferenceError::InvalidResponse(_) => LlmClientError::InvalidResponse { source: Box::new(error), }, error => LlmClientError::Host { @@ -458,10 +436,10 @@ fn map_client_error(error: LlmClientError) -> Error { match error { LlmClientError::InvalidRequest { message } | LlmClientError::RequestTranslation(message) => { - Error::Route(RouteError::InvalidRequest(message.into())) + Error::Route(InferenceError::InvalidRequest(message.into())) } LlmClientError::UpstreamHttp { status, body } => { - Error::Route(RouteError::Transport(TransportError::Http { + Error::Route(InferenceError::Transport(TransportError::Http { status: status.as_u16(), body, })) @@ -472,22 +450,134 @@ fn map_client_error(error: LlmClientError) -> Error { | LlmClientError::Host { source } | LlmClientError::Ffi { source } => map_source(source), LlmClientError::ContextWindowExceeded { message, .. } => { - Error::Route(RouteError::InvalidRequest(message.into())) + Error::Route(InferenceError::InvalidRequest(message.into())) } LlmClientError::ResponseTranslation(message) => { - Error::Route(RouteError::InvalidResponse(message.into())) + Error::Route(InferenceError::InvalidResponse(message.into())) } error => Error::Internal(error.to_string()), } } fn map_source(source: BoxError) -> Error { - match source.downcast::() { + match source.downcast::() { Ok(error) => Error::Route(*error), Err(source) => Error::Internal(source.to_string()), } } +const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"]; +const OPENROUTER_API_HEADERS: [&str; 2] = [ + "x-openrouter-metadata", + "x-openrouter-experimental-metadata", +]; +const ANTHROPIC_API_HEADER_PROVIDERS: &str = + "anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek"; + +fn chat_request( + deployment: &Deployment, + mut body: Map, +) -> ChatCompletionsRequest<'_> { + let messages = body.remove("messages").unwrap_or_default(); + body.remove("model"); + ChatCompletionsRequest { + model: &deployment.model, + messages, + optional_params: body, + api_key: deployment.api_key.as_deref(), + api_base: deployment.api_base.as_deref(), + custom_llm_provider: deployment.custom_llm_provider.as_deref(), + extra_headers: None, + timeout: deployment.timeout, + } +} + +fn responses_request(deployment: &Deployment, mut body: Map) -> ResponsesCall { + let input = body.remove("input").unwrap_or_default(); + body.remove("model"); + ResponsesCall { + model: deployment.model.clone(), + input, + optional_params: body, + api_key: deployment.api_key.clone(), + api_base: deployment.api_base.clone(), + custom_llm_provider: deployment.custom_llm_provider.clone(), + extra_headers: None, + timeout: deployment.timeout, + } +} + +fn messages_request( + deployment: &Deployment, + mut body: Map, + headers: &HeaderMap, +) -> Result { + if let Some(model) = body.get_mut("model") { + *model = Value::from(deployment.model.as_str()); + } + Ok(MessagesCall { + body: messages_body(body)?, + api_key: deployment.api_key.clone(), + api_base: deployment.api_base.clone(), + custom_llm_provider: deployment.custom_llm_provider.clone(), + litellm_params: deployment.litellm_params.clone(), + extra_headers: None, + provider_specific_header: anthropic_api_headers(headers), + timeout: deployment.timeout, + shaping: deployment.shaping.clone(), + }) +} + +fn anthropic_api_headers(headers: &HeaderMap) -> Option { + let scopes = [ + ( + ANTHROPIC_API_HEADER_PROVIDERS, + ANTHROPIC_API_HEADERS.as_slice(), + ), + ("openrouter", OPENROUTER_API_HEADERS.as_slice()), + ]; + let forwarded: Vec = scopes + .into_iter() + .filter_map(|(provider, names)| { + let extra_headers: Map = names + .iter() + .filter_map(|name| { + let value = headers.get(*name)?.to_str().ok()?; + Some(((*name).to_owned(), Value::from(value))) + }) + .collect(); + (!extra_headers.is_empty()).then(|| ProviderSpecificHeader { + custom_llm_provider: provider.into(), + extra_headers, + }) + }) + .collect(); + (!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded)) +} + +impl AutoRoutingStrategy for SwitchyardStrategy { + fn execute<'a>( + &'a self, + gateway: &'a Gateway, + request: StrategyRequest, + ) -> std::pin::Pin> + Send + 'a>> + { + let format = match request.format { + ApiFormat::Chat => WireFormat::OpenAiChat, + ApiFormat::Responses => WireFormat::OpenAiResponses, + ApiFormat::Messages => WireFormat::AnthropicMessages, + }; + Box::pin(SwitchyardStrategy::execute( + self, + gateway, + request.cache_options, + format, + request.body, + request.headers, + )) + } +} + #[cfg(test)] mod tests { use axum::body::to_bytes; @@ -497,14 +587,14 @@ mod tests { #[rstest] fn host_errors_keep_their_gateway_classification() { - let error = RouteError::Auth(litellm_auth::Error::MissingApiKey { + let error = InferenceError::Auth(litellm_auth::Error::MissingApiKey { provider: "OpenAI", environment_variable: "OPENAI_API_KEY", }); assert!(matches!( map_client_error(map_route_error(error)), - Error::Route(RouteError::Auth(_)) + Error::Route(InferenceError::Auth(_)) )); } diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index c920b874786..742a373b286 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -1,40 +1,27 @@ use litellm_gateway_auth::AuthenticatedRequest; use std::sync::Arc; -#[cfg(feature = "switchyard")] -use axum::http::HeaderMap; use axum::{ Json, extract::{Path, State}, response::{IntoResponse, Response}, }; -use litellm_inference_chat::types::{ChatCompletionsCall, ChatCompletionsRequest}; +use litellm_inference_chat::types::ChatCompletionsCall; use serde_json::{Map, Value}; -#[cfg(feature = "switchyard")] -use switchyard_protocol::WireFormat; -use crate::{Deployment, Error, Gateway, JsonObject, request}; +use crate::{Error, Gateway, JsonObject, request}; pub(crate) async fn create( State(gateway): State>, identity: AuthenticatedRequest, - #[cfg(feature = "switchyard")] headers: HeaderMap, JsonObject(body): JsonObject, ) -> Result { - handle( - &gateway, - &identity, - #[cfg(feature = "switchyard")] - headers, - body, - ) - .await + handle(&gateway, &identity, body).await } pub(crate) async fn create_from_model_path( State(gateway): State>, identity: AuthenticatedRequest, - #[cfg(feature = "switchyard")] headers: HeaderMap, Path(model): Path, JsonObject(body): JsonObject, ) -> Result { @@ -45,42 +32,17 @@ pub(crate) async fn create_from_model_path( .collect(), Some(_) => body, }; - handle( - &gateway, - &identity, - #[cfg(feature = "switchyard")] - headers, - body, - ) - .await + handle(&gateway, &identity, body).await } async fn handle( gateway: &Gateway, identity: &AuthenticatedRequest, - #[cfg(feature = "switchyard")] inbound_headers: HeaderMap, body: Map, ) -> Result { let deployment = request::resolve_deployment(gateway, &body)?; request::authorize_model(identity, deployment, &body).await?; - #[cfg(feature = "switchyard")] - let switchyard_route = body - .get("model") - .and_then(Value::as_str) - .and_then(|model| gateway.switchyard_routes.get(model)); let (body, cache_options) = crate::caching::prepare(identity, body)?; - #[cfg(feature = "switchyard")] - if let Some(switchyard_route) = switchyard_route { - return switchyard_route - .execute( - gateway, - cache_options, - WireFormat::OpenAiChat, - body, - inbound_headers, - ) - .await; - } let route = gateway.chat_completions.clone(); let route = match &gateway.cache { Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( @@ -89,10 +51,24 @@ async fn handle( )), None => route, }; + + let messages = body.get("messages").cloned().unwrap_or_default(); let headers = crate::caching::CacheHeaders::default(); let response = litellm_host_http::serve_unary( route.machine( - ChatCompletionsCall::from(provider_request(deployment, body)), + ChatCompletionsCall { + model: deployment.model.clone(), + messages, + optional_params: body + .into_iter() + .filter(|(name, _)| !matches!(name.as_str(), "model" | "messages")) + .collect(), + api_key: deployment.api_key.clone(), + api_base: deployment.api_base.clone(), + custom_llm_provider: deployment.custom_llm_provider.clone(), + extra_headers: None, + timeout: deployment.timeout, + }, cache_options.policy, ), (), @@ -103,21 +79,3 @@ async fn handle( .await?; Ok(headers.apply(response)) } - -pub(crate) fn provider_request( - deployment: &Deployment, - mut body: Map, -) -> ChatCompletionsRequest<'_> { - let messages = body.remove("messages").unwrap_or_default(); - body.remove("model"); - ChatCompletionsRequest { - model: &deployment.model, - messages, - optional_params: body, - api_key: deployment.api_key.as_deref(), - api_base: deployment.api_base.as_deref(), - custom_llm_provider: deployment.custom_llm_provider.as_deref(), - extra_headers: None, - timeout: deployment.timeout, - } -} diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index 8869bd58b31..a623353e478 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -8,11 +8,9 @@ use litellm_inference::Error as InferenceError; use litellm_llms::base_llm::ocr::error::Error as OcrError; use serde_json::{Map, Value, json}; -/// Invalid configuration for a Switchyard virtual model. -#[cfg(feature = "switchyard")] #[derive(Debug, thiserror::Error)] -#[error("invalid Switchyard model {model}: {reason}")] -pub struct SwitchyardConfigError { +#[error("invalid auto-routing model {model}: {reason}")] +pub struct AutoRoutingConfigError { pub(crate) model: String, pub(crate) reason: String, } diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 06d8aa5949d..65ceb8166b3 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -4,6 +4,7 @@ //! maps a public model name to its deployment and runs the core route. mod audio_transcription; +pub mod auto_routing; mod caching; mod chat_completions; mod error; @@ -11,8 +12,6 @@ pub mod messages; mod ocr; mod request; mod responses; -#[cfg(feature = "switchyard")] -mod switchyard; use std::sync::Arc; @@ -27,16 +26,14 @@ use litellm_inference_transcription::AudioTranscriptionRoute; use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings}; use litellm_secrets::source::SecretSource; +pub use error::AutoRoutingConfigError; pub use error::Error; -#[cfg(feature = "switchyard")] -pub use error::SwitchyardConfigError; pub use litellm_router::{Deployment, Router as ModelRouter}; pub use request::{JsonObject, RequestId}; pub struct Gateway { cache: Option>, - #[cfg(feature = "switchyard")] - switchyard_routes: switchyard::Routes, + auto_routing: auto_routing::Registry, pub audio_transcription: AudioTranscriptionRoute, pub chat_completions: ChatCompletionsRoute, pub messages: MessagesRoute, @@ -66,8 +63,7 @@ impl Gateway { let auth = resources.auth.clone(); Ok(Self { cache: None, - #[cfg(feature = "switchyard")] - switchyard_routes: switchyard::Routes::default(), + auto_routing: auto_routing::Registry::default(), audio_transcription: AudioTranscriptionRoute::new( provider.clone(), auth.clone(), @@ -95,24 +91,37 @@ impl Gateway { }) } - /// Configure Switchyard virtual models from LiteLLM's model list. - #[cfg(feature = "switchyard")] - pub fn with_switchyard_models( + pub fn with_auto_routing_strategy( + mut self, + model: String, + strategy: Arc, + ) -> Result { + if self.models.get(&model).is_none() { + return Err(AutoRoutingConfigError { + model, + reason: "model is not configured".into(), + }); + } + self.auto_routing.insert(model, strategy); + Ok(self) + } + + pub fn with_auto_routing_models( mut self, models: &[litellm_config::Model], - ) -> Result { - self.switchyard_routes = switchyard::Routes::from_models(models, &self.models)?; + ) -> Result { + self.auto_routing = auto_routing::Registry::from_models(models, &self.models)?; Ok(self) } } pub fn router(gateway: Arc) -> Router { Router::new() - .route("/v1/messages", post(messages::create)) + .route("/v1/messages", post(auto_routing::messages)) .route("/ocr", post(ocr::create)) .route("/v1/ocr", post(ocr::create)) - .route("/chat/completions", post(chat_completions::create)) - .route("/v1/chat/completions", post(chat_completions::create)) + .route("/chat/completions", post(auto_routing::chat)) + .route("/v1/chat/completions", post(auto_routing::chat)) .nest("/engines/{model}", model_routes()) .nest("/openai/deployments/{model}", model_routes()) .route("/audio/transcriptions", post(audio_transcription::create)) @@ -120,8 +129,8 @@ pub fn router(gateway: Arc) -> Router { "/v1/audio/transcriptions", post(audio_transcription::create), ) - .route("/responses", post(responses::create)) - .route("/v1/responses", post(responses::create)) + .route("/responses", post(auto_routing::responses)) + .route("/v1/responses", post(auto_routing::responses)) .route("/embeddings", post(request::unsupported)) .route("/v1/embeddings", post(request::unsupported)) .route("/completions", post(request::unsupported)) @@ -136,7 +145,7 @@ fn model_routes() -> Router> { Router::new() .route( "/chat/completions", - post(chat_completions::create_from_model_path), + post(auto_routing::chat_from_model_path), ) .route("/embeddings", post(request::unsupported)) .route("/completions", post(request::unsupported)) diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index 8e9850d769a..d612a042e03 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -14,8 +14,6 @@ use litellm_host_http::Sse; use litellm_inference_messages::{MessagesCall, messages_body, route::Messages}; use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders}; use serde_json::{Map, Value}; -#[cfg(feature = "switchyard")] -use switchyard_protocol::WireFormat; use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request}; @@ -36,7 +34,7 @@ pub async fn create( body: Result, ) -> impl IntoResponse { let result = match body { - Ok(JsonObject(body)) => handle(&gateway, &identity, headers, body).await, + Ok(JsonObject(body)) => handle(&gateway, &identity, &headers, body).await, Err(error) => Err(error), }; result.map_err(|error| (error.status(), Json(error.body(request_id.as_deref())))) @@ -45,29 +43,12 @@ pub async fn create( async fn handle( gateway: &Gateway, identity: &AuthenticatedRequest, - headers: HeaderMap, + headers: &HeaderMap, body: Map, ) -> Result { let deployment = request::resolve_deployment(gateway, &body)?; request::authorize_model(identity, deployment, &body).await?; - #[cfg(feature = "switchyard")] - let switchyard_route = body - .get("model") - .and_then(Value::as_str) - .and_then(|model| gateway.switchyard_routes.get(model)); let (body, cache_options) = crate::caching::prepare(identity, body)?; - #[cfg(feature = "switchyard")] - if let Some(switchyard_route) = switchyard_route { - return switchyard_route - .execute( - gateway, - cache_options, - WireFormat::AnthropicMessages, - body, - headers, - ) - .await; - } let route = gateway.messages.clone(); let route = match &gateway.cache { Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( @@ -77,7 +58,7 @@ async fn handle( None => route, }; - let call = provider_request(deployment, body, &headers)?; + let call = project(deployment, body, headers)?; let machine = route.machine(call, cache_options.policy); let stream = Sse::::new(Json, |error| Bytes::from(Error::from(error).sse_frame())); @@ -86,14 +67,18 @@ async fn handle( Ok(headers.apply(response)) } -pub(crate) fn provider_request( +fn project( deployment: &Deployment, - mut body: Map, + body: Map, headers: &HeaderMap, -) -> Result { - if let Some(model) = body.get_mut("model") { - *model = Value::from(deployment.model.as_str()); - } +) -> Result { + let body = body + .into_iter() + .map(|(name, value)| match name.as_str() { + "model" => (name, Value::from(deployment.model.as_str())), + _ => (name, value), + }) + .collect(); Ok(MessagesCall { body: messages_body(body)?, api_key: deployment.api_key.clone(), diff --git a/litellm-rust/crates/gateway-inference/src/responses.rs b/litellm-rust/crates/gateway-inference/src/responses.rs index 0379d4fe584..77bebf610e1 100644 --- a/litellm-rust/crates/gateway-inference/src/responses.rs +++ b/litellm-rust/crates/gateway-inference/src/responses.rs @@ -1,43 +1,21 @@ use std::sync::Arc; -#[cfg(feature = "switchyard")] -use axum::http::HeaderMap; use axum::{Json, body::Bytes, extract::State, response::Response}; use litellm_gateway_auth::AuthenticatedRequest; use litellm_host_http::Sse; use litellm_inference_responses::{route::Responses, types::ResponsesCall}; -use serde_json::{Map, Value, json}; -#[cfg(feature = "switchyard")] -use switchyard_protocol::WireFormat; +use serde_json::json; -use crate::{Deployment, Error, Gateway, JsonObject, request}; +use crate::{Error, Gateway, JsonObject, request}; pub(crate) async fn create( State(gateway): State>, identity: AuthenticatedRequest, - #[cfg(feature = "switchyard")] headers: HeaderMap, JsonObject(body): JsonObject, ) -> Result { let deployment = request::resolve_deployment(&gateway, &body)?; request::authorize_model(&identity, deployment, &body).await?; - #[cfg(feature = "switchyard")] - let switchyard_route = body - .get("model") - .and_then(Value::as_str) - .and_then(|model| gateway.switchyard_routes.get(model)); let (body, cache_options) = crate::caching::prepare(&identity, body)?; - #[cfg(feature = "switchyard")] - if let Some(switchyard_route) = switchyard_route { - return switchyard_route - .execute( - &gateway, - cache_options, - WireFormat::OpenAiResponses, - body, - headers, - ) - .await; - } let route = gateway.responses.clone(); let route = match &gateway.cache { Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( @@ -47,7 +25,19 @@ pub(crate) async fn create( None => route, }; - let call = provider_request(deployment, body); + let call = ResponsesCall { + model: deployment.model.clone(), + input: body.get("input").cloned().unwrap_or_default(), + optional_params: body + .into_iter() + .filter(|(name, _)| !matches!(name.as_str(), "model" | "input")) + .collect(), + api_key: deployment.api_key.clone(), + api_base: deployment.api_base.clone(), + custom_llm_provider: deployment.custom_llm_provider.clone(), + extra_headers: None, + timeout: deployment.timeout, + }; let machine = route.machine(call, cache_options.policy); let stream = Sse::::new(Json, |error| { let error = Error::from(error); @@ -60,21 +50,3 @@ pub(crate) async fn create( let response = litellm_host_http::serve(machine, (), headers.clone(), stream, None).await?; Ok(headers.apply(response)) } - -pub(crate) fn provider_request( - deployment: &Deployment, - mut body: Map, -) -> ResponsesCall { - let input = body.remove("input").unwrap_or_default(); - body.remove("model"); - ResponsesCall { - model: deployment.model.clone(), - input, - optional_params: body, - api_key: deployment.api_key.clone(), - api_base: deployment.api_base.clone(), - custom_llm_provider: deployment.custom_llm_provider.clone(), - extra_headers: None, - timeout: deployment.timeout, - } -} diff --git a/litellm-rust/crates/gateway-inference/tests/auto_routing.rs b/litellm-rust/crates/gateway-inference/tests/auto_routing.rs new file mode 100644 index 00000000000..9068899ea57 --- /dev/null +++ b/litellm-rust/crates/gateway-inference/tests/auto_routing.rs @@ -0,0 +1,267 @@ +mod support; + +use axum::{ + Json, + response::{IntoResponse, Response}, +}; +use litellm_gateway_auth::Permissions; +use litellm_gateway_inference::{ + Deployment, Error, Gateway, + auto_routing::{ApiFormat, AutoRoutingStrategy, StrategyRequest}, +}; +use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; +use litellm_inference::resources::CoreResources; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use std::{ + future::Future, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +struct EchoStrategy(Arc); + +impl AutoRoutingStrategy for EchoStrategy { + fn execute<'a>( + &'a self, + _: &'a Gateway, + request: StrategyRequest, + ) -> Pin> + Send + 'a>> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + if request.body.get("fail").and_then(Value::as_bool) == Some(true) { + return Err(Error::InvalidBody("strategy rejected request".into())); + } + let format = match request.format { + ApiFormat::Chat => "chat", + ApiFormat::Responses => "responses", + ApiFormat::Messages => "messages", + }; + Ok(Json(json!({"format": format, "body": request.body, "request_id": request.headers.get("x-request-id").and_then(|v| v.to_str().ok()), "no_store": request.cache_options.policy.no_store})).into_response()) + }) + } +} + +#[fixture] +fn gateway() -> Gateway { + Gateway::new( + CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))), + Resolution::from(&HttpSettings::default()).config, + Arc::new(support::NoSecrets), + [( + "virtual".into(), + Deployment { + model: "virtual-target".into(), + ..Default::default() + }, + )] + .into_iter() + .collect(), + ) + .unwrap() +} + +#[rstest] +#[case::chat("/v1/chat/completions", "chat", json!({"model":"virtual"}))] +#[case::chat_alias("/chat/completions", "chat", json!({"model":"virtual"}))] +#[case::responses("/v1/responses", "responses", json!({"model":"virtual"}))] +#[case::responses_alias("/responses", "responses", json!({"model":"virtual"}))] +#[case::messages("/v1/messages", "messages", json!({"model":"virtual"}))] +#[case::engine("/engines/virtual/chat/completions", "chat", json!({}))] +#[case::deployment("/openai/deployments/virtual/chat/completions", "chat", json!({"model":null}))] +#[case::body_wins("/engines/unknown/chat/completions", "chat", json!({"model":"virtual"}))] +#[tokio::test] +async fn virtual_models_invoke_the_strategy_once( + gateway: Gateway, + #[case] path: &str, + #[case] format: &str, + #[case] body: Value, +) { + let calls = Arc::new(AtomicUsize::new(0)); + let gateway = gateway + .with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone()))) + .unwrap(); + let response = support::post( + support::authenticated_app(gateway, Permissions::All, None), + path, + body, + ) + .await; + assert_eq!(response.status(), 200); + let body = support::json(response).await; + assert_eq!(body["format"], format); + assert_eq!(body["body"]["model"], "virtual"); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[rstest] +#[case::chat("/v1/chat/completions")] +#[case::responses("/v1/responses")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn denied_virtual_models_never_invoke_the_strategy(gateway: Gateway, #[case] path: &str) { + let calls = Arc::new(AtomicUsize::new(0)); + let gateway = gateway + .with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone()))) + .unwrap(); + let response = support::post( + support::authenticated_app(gateway, Permissions::Only(Arc::from([])), None), + path, + json!({"model":"virtual"}), + ) + .await; + assert_eq!(response.status(), 403); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[rstest] +#[case::chat("/v1/chat/completions", false)] +#[case::responses("/v1/responses", false)] +#[case::messages("/v1/messages", true)] +#[tokio::test] +async fn strategy_errors_keep_the_callers_error_format( + gateway: Gateway, + #[case] path: &str, + #[case] anthropic: bool, +) { + let gateway = gateway + .with_auto_routing_strategy( + "virtual".into(), + Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))), + ) + .unwrap(); + let response = support::post( + support::authenticated_app(gateway, Permissions::All, None), + path, + json!({"model":"virtual", "fail":true}), + ) + .await; + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert!( + body["error"]["message"] + .as_str() + .unwrap() + .contains("strategy rejected request") + ); + assert_eq!(body.get("type").is_some(), anthropic); +} + +#[rstest] +#[case::chat("/v1/chat/completions")] +#[case::responses("/v1/responses")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn cache_controls_are_prepared_before_strategy_execution( + gateway: Gateway, + #[case] path: &str, +) { + let gateway = gateway + .with_auto_routing_strategy( + "virtual".into(), + Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))), + ) + .unwrap(); + let response = support::post( + support::authenticated_app(gateway, Permissions::All, None), + path, + json!({"model":"virtual", "cache":{"no-store":true}, "caching":false}), + ) + .await; + assert_eq!(response.status(), 200); + let body = support::json(response).await; + assert_eq!(body["no_store"], true); + assert!(body["body"].get("cache").is_none()); + assert!(body["body"].get("caching").is_none()); +} + +#[rstest] +#[case::chat("/v1/chat/completions")] +#[case::responses("/v1/responses")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn incoming_headers_reach_the_strategy(gateway: Gateway, #[case] path: &str) { + use axum::{body::Body, http::Request}; + use tower::ServiceExt; + let gateway = gateway + .with_auto_routing_strategy( + "virtual".into(), + Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))), + ) + .unwrap(); + let response = support::authenticated_app(gateway, Permissions::All, None) + .oneshot( + Request::post(path) + .header("content-type", "application/json") + .header("x-request-id", "request-context") + .body(Body::from(r#"{"model":"virtual"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!( + support::json(response).await["request_id"], + "request-context" + ); +} + +#[rstest] +#[case::chat("/v1/chat/completions", false)] +#[case::responses("/v1/responses", false)] +#[case::messages("/v1/messages", true)] +#[tokio::test] +async fn malformed_bodies_preserve_api_error_envelopes( + gateway: Gateway, + #[case] path: &str, + #[case] anthropic: bool, +) { + use axum::{body::Body, http::Request}; + use tower::ServiceExt; + let calls = Arc::new(AtomicUsize::new(0)); + let gateway = gateway + .with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone()))) + .unwrap(); + let response = support::authenticated_app(gateway, Permissions::All, None) + .oneshot( + Request::post(path) + .header("content-type", "application/json") + .header("x-request-id", "malformed-request") + .body(Body::from("{")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert_eq!(body.get("type").is_some(), anthropic); + if anthropic { + assert_eq!(body["request_id"], "malformed-request"); + } + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[rstest] +#[case::chat("/v1/chat/completions")] +#[case::responses("/v1/responses")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn invalid_cache_settings_do_not_invoke_the_strategy(gateway: Gateway, #[case] path: &str) { + let calls = Arc::new(AtomicUsize::new(0)); + let gateway = gateway + .with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone()))) + .unwrap(); + let response = support::post( + support::authenticated_app(gateway, Permissions::All, None), + path, + json!({"model":"virtual", "cache":{"ttl":-1}}), + ) + .await; + assert_eq!(response.status(), 400); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} diff --git a/litellm-rust/crates/gateway-inference/tests/switchyard.rs b/litellm-rust/crates/gateway-inference/tests/switchyard.rs index 902473f7773..479b5906bac 100644 --- a/litellm-rust/crates/gateway-inference/tests/switchyard.rs +++ b/litellm-rust/crates/gateway-inference/tests/switchyard.rs @@ -390,16 +390,93 @@ async fn streaming_responses_upstream() -> (String, tokio::task::JoinHandle<()>) } fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router { + support::authenticated_app(switchyard_gateway(config), permissions, None) +} + +fn switchyard_gateway(config: &Config) -> Gateway { let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))); let http = Resolution::from(&HttpSettings::default()).config; - let gateway = Gateway::new( + Gateway::new( resources, http, Arc::new(support::NoSecrets), ModelRouter::from_model_list(&config.model_list), ) .expect("gateway") - .with_switchyard_models(&config.model_list) - .expect("Switchyard routes"); - support::authenticated_app(gateway, permissions, None) + .with_auto_routing_models(&config.model_list) + .expect("Switchyard routes") +} + +#[rstest] +#[tokio::test] +async fn routed_answers_preserve_cache_hits_and_caller_isolation() { + use litellm_cache_memory::InMemoryCache; + use litellm_cache_response::ResponseCache; + use litellm_gateway_auth::{Principal, PrincipalKind}; + + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "model":"answer", "choices":[{"message":{"role":"assistant","content":"cached answer"}}] + }))) + .expect(2) + .mount(&upstream) + .await; + let config = Config::from_yaml(&format!( + r#" +model_list: + - model_name: answer + litellm_params: + model: answer + custom_llm_provider: openai_like + api_base: {}/v1 + api_key: test-key + switchyard_wire_format: openai_chat + - model_name: virtual + litellm_params: + model: switchyard/algorithm + switchyard_config: {{type: passthrough, target: answer}} +"#, + upstream.uri() + )) + .unwrap(); + let cache = Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new( + Some(100), + Some(Duration::from_secs(60)), + )))); + let first_app = support::authenticated_app( + switchyard_gateway(&config).with_cache(cache.clone()), + Permissions::All, + Some(Principal::new( + "issuer".into(), + "first".into(), + PrincipalKind::Service, + )), + ); + let second_app = support::authenticated_app( + switchyard_gateway(&config).with_cache(cache), + Permissions::All, + Some(Principal::new( + "issuer".into(), + "second".into(), + PrincipalKind::Service, + )), + ); + let request = json!({"model":"virtual", "messages":[{"role":"user", "content":"hello"}]}); + let first = support::post(first_app.clone(), "/v1/chat/completions", request.clone()).await; + assert_eq!(first.status(), 200); + assert!(!first.headers().contains_key("x-litellm-cache-key")); + let first_body = support::json(first).await; + let hit = support::post(first_app, "/v1/chat/completions", request.clone()).await; + assert_eq!(hit.status(), 200); + let first_key = hit.headers().get("x-litellm-cache-key").unwrap().clone(); + assert_eq!(support::json(hit).await, first_body); + let second = support::post(second_app.clone(), "/v1/chat/completions", request.clone()).await; + assert_eq!(second.status(), 200); + assert!(!second.headers().contains_key("x-litellm-cache-key")); + let hit = support::post(second_app, "/v1/chat/completions", request).await; + assert_eq!(hit.status(), 200); + assert_ne!(hit.headers().get("x-litellm-cache-key").unwrap(), first_key); + upstream.verify().await; } diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 579a55e3c1c..0471ff884d6 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -10,7 +10,6 @@ pub enum Error { Mcp(#[from] litellm_gateway_mcp::ConnectError), #[error("MCP requires a configured master key")] Auth(#[from] litellm_gateway_auth::Error), - #[cfg(feature = "switchyard")] #[error(transparent)] - Switchyard(#[from] litellm_gateway_inference::SwitchyardConfigError), + AutoRouting(#[from] litellm_gateway_inference::AutoRoutingConfigError), } diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 1ff1b9de966..125eeb36b3e 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -68,8 +68,7 @@ pub fn build_inference(config: &Config) -> Result, Error> { secrets, ModelRouter::from_model_list(&config.model_list), )?; - #[cfg(feature = "switchyard")] - let gateway = gateway.with_switchyard_models(&config.model_list)?; + let gateway = gateway.with_auto_routing_models(&config.model_list)?; Ok(Arc::new(gateway)) }