mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(gateway): isolate auto-routing strategy orchestration
This commit is contained in:
parent
dd5fe2e550
commit
c3827fef75
14 changed files with 932 additions and 244 deletions
4
.github/workflows/test-rust.yml
vendored
4
.github/workflows/test-rust.yml
vendored
|
|
@ -101,6 +101,8 @@ jobs:
|
|||
|
||||
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
|
||||
- run: cargo clippy -p litellm-gateway-inference -p litellm-gateway --all-targets --features switchyard --locked -- -D warnings
|
||||
|
||||
rust-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
|
|
@ -147,6 +149,8 @@ jobs:
|
|||
|
||||
- run: cargo nextest run --workspace --locked
|
||||
|
||||
- run: cargo nextest run -p litellm-gateway-inference -p litellm-gateway --features switchyard --locked
|
||||
|
||||
- run: cargo test --workspace --doc --locked
|
||||
|
||||
- name: Test token counter feature combinations
|
||||
|
|
|
|||
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -3882,6 +3882,7 @@ dependencies = [
|
|||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"switchyard-libsy",
|
||||
"switchyard-llm-client",
|
||||
"switchyard-protocol",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ repository.workspace = true
|
|||
switchyard = [
|
||||
"dep:async-trait",
|
||||
"dep:futures-util",
|
||||
"dep:litellm-config",
|
||||
"dep:switchyard-libsy",
|
||||
"dep:switchyard-llm-client",
|
||||
"dep:switchyard-protocol",
|
||||
|
|
@ -32,7 +31,7 @@ litellm-inference-responses.workspace = true
|
|||
litellm-inference-messages.workspace = true
|
||||
litellm-inference-chat.workspace = true
|
||||
litellm-inference-ocr.workspace = true
|
||||
litellm-config = { workspace = true, optional = true }
|
||||
litellm-config.workspace = true
|
||||
litellm-host-http.workspace = true
|
||||
litellm-host.workspace = true
|
||||
litellm-http.workspace = true
|
||||
|
|
@ -47,6 +46,7 @@ switchyard-llm-client = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git",
|
|||
switchyard-protocol = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
|
||||
switchyard-runner = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
|
||||
switchyard-translation = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", rev = "4bdf5719ce1c43902df92ef64407b858d1f99394", optional = true }
|
||||
strum.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
329
litellm-rust/crates/gateway-inference/src/auto_routing/mod.rs
Normal file
329
litellm-rust/crates/gateway-inference/src/auto_routing/mod.rs
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
http::HeaderMap,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use litellm_cache_response::CacheOptions;
|
||||
use litellm_config::Model;
|
||||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{AutoRoutingConfigError, Error, Gateway, JsonObject, ModelRouter, RequestId, request};
|
||||
|
||||
#[cfg(feature = "switchyard")]
|
||||
mod switchyard;
|
||||
|
||||
#[derive(Clone, Copy, strum::VariantArray)]
|
||||
pub enum ApiFormat {
|
||||
Chat,
|
||||
Responses,
|
||||
Messages,
|
||||
}
|
||||
|
||||
pub struct StrategyRequest {
|
||||
pub format: ApiFormat,
|
||||
pub body: Map<String, Value>,
|
||||
pub headers: HeaderMap,
|
||||
pub cache_options: CacheOptions,
|
||||
}
|
||||
|
||||
pub trait AutoRoutingStrategy: Send + Sync {
|
||||
fn execute<'a>(
|
||||
&'a self,
|
||||
gateway: &'a Gateway,
|
||||
request: StrategyRequest,
|
||||
) -> Pin<Box<dyn Future<Output = Result<Response, Error>> + Send + 'a>>;
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Registry(HashMap<String, Arc<dyn AutoRoutingStrategy>>);
|
||||
|
||||
impl Registry {
|
||||
pub(crate) fn insert(&mut self, model: String, strategy: Arc<dyn AutoRoutingStrategy>) {
|
||||
self.0.insert(model, strategy);
|
||||
}
|
||||
|
||||
pub(crate) fn from_models(
|
||||
models: &[Model],
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<Self, AutoRoutingConfigError> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut strategies = HashMap::new();
|
||||
for model in models.iter().rev() {
|
||||
if !seen.insert(&model.model_name)
|
||||
|| model.litellm_params.model != "switchyard/algorithm"
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if deployments
|
||||
.get(&model.model_name)
|
||||
.is_none_or(|deployment| deployment.model != model.litellm_params.model)
|
||||
{
|
||||
return Err(AutoRoutingConfigError {
|
||||
model: model.model_name.clone(),
|
||||
reason: "model does not match its configured deployment".into(),
|
||||
});
|
||||
}
|
||||
strategies.insert(
|
||||
model.model_name.clone(),
|
||||
build_strategy(model, models, deployments)?,
|
||||
);
|
||||
}
|
||||
Ok(Self(strategies))
|
||||
}
|
||||
|
||||
fn get(&self, body: &Map<String, Value>) -> Option<&dyn AutoRoutingStrategy> {
|
||||
self.0.get(body.get("model")?.as_str()?).map(AsRef::as_ref)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "switchyard")]
|
||||
fn build_strategy(
|
||||
model: &Model,
|
||||
models: &[Model],
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<Arc<dyn AutoRoutingStrategy>, AutoRoutingConfigError> {
|
||||
Ok(Arc::new(switchyard::SwitchyardStrategy::from_model(
|
||||
model,
|
||||
models,
|
||||
deployments,
|
||||
)?))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "switchyard"))]
|
||||
fn build_strategy(
|
||||
model: &Model,
|
||||
_: &[Model],
|
||||
_: &ModelRouter,
|
||||
) -> Result<Arc<dyn AutoRoutingStrategy>, AutoRoutingConfigError> {
|
||||
Err(AutoRoutingConfigError {
|
||||
model: model.model_name.clone(),
|
||||
reason: "this build does not enable the switchyard feature".into(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
strategy: &dyn AutoRoutingStrategy,
|
||||
gateway: &Gateway,
|
||||
identity: &AuthenticatedRequest,
|
||||
format: ApiFormat,
|
||||
headers: HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<Response, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
let (body, cache_options) = crate::caching::prepare(identity, body)?;
|
||||
strategy
|
||||
.execute(
|
||||
gateway,
|
||||
StrategyRequest {
|
||||
format,
|
||||
body,
|
||||
headers,
|
||||
cache_options,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn chat(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
headers: HeaderMap,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<Response, Error> {
|
||||
if let Some(strategy) = gateway.auto_routing.get(&body) {
|
||||
return execute(
|
||||
strategy,
|
||||
&gateway,
|
||||
&identity,
|
||||
ApiFormat::Chat,
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
crate::chat_completions::create(State(gateway), identity, JsonObject(body))
|
||||
.await
|
||||
.map(IntoResponse::into_response)
|
||||
}
|
||||
|
||||
pub(crate) async fn chat_from_model_path(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
headers: HeaderMap,
|
||||
Path(model): Path<String>,
|
||||
JsonObject(mut body): JsonObject,
|
||||
) -> Result<Response, Error> {
|
||||
if body.get("model").is_none_or(Value::is_null) {
|
||||
body.insert("model".into(), Value::String(model.clone()));
|
||||
}
|
||||
if let Some(strategy) = gateway.auto_routing.get(&body) {
|
||||
return execute(
|
||||
strategy,
|
||||
&gateway,
|
||||
&identity,
|
||||
ApiFormat::Chat,
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
crate::chat_completions::create_from_model_path(
|
||||
State(gateway),
|
||||
identity,
|
||||
Path(model),
|
||||
JsonObject(body),
|
||||
)
|
||||
.await
|
||||
.map(IntoResponse::into_response)
|
||||
}
|
||||
|
||||
pub(crate) async fn responses(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
headers: HeaderMap,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<Response, Error> {
|
||||
if let Some(strategy) = gateway.auto_routing.get(&body) {
|
||||
return execute(
|
||||
strategy,
|
||||
&gateway,
|
||||
&identity,
|
||||
ApiFormat::Responses,
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
crate::responses::create(State(gateway), identity, JsonObject(body)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn messages(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
RequestId(request_id): RequestId,
|
||||
headers: HeaderMap,
|
||||
body: Result<JsonObject, Error>,
|
||||
) -> Response {
|
||||
match body {
|
||||
Ok(JsonObject(body)) => {
|
||||
if let Some(strategy) = gateway.auto_routing.get(&body) {
|
||||
return execute(
|
||||
strategy,
|
||||
&gateway,
|
||||
&identity,
|
||||
ApiFormat::Messages,
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| {
|
||||
(error.status(), Json(error.body(request_id.as_deref()))).into_response()
|
||||
});
|
||||
}
|
||||
crate::messages::create(
|
||||
State(gateway),
|
||||
identity,
|
||||
RequestId(request_id),
|
||||
headers,
|
||||
Ok(JsonObject(body)),
|
||||
)
|
||||
.await
|
||||
.into_response()
|
||||
}
|
||||
Err(error) => crate::messages::create(
|
||||
State(gateway),
|
||||
identity,
|
||||
RequestId(request_id),
|
||||
headers,
|
||||
Err(error),
|
||||
)
|
||||
.await
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use litellm_config::Config;
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
fn overwritten_virtual_models_do_not_initialize_a_strategy() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: public
|
||||
litellm_params: {model: switchyard/algorithm}
|
||||
- model_name: public
|
||||
litellm_params: {model: openai/model}
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let registry = Registry::from_models(
|
||||
&config.model_list,
|
||||
&ModelRouter::from_model_list(&config.model_list),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.get(
|
||||
&serde_json::json!({"model":"public"})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone()
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn strategy_configuration_must_match_the_deployment_registry() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: public
|
||||
litellm_params: {model: switchyard/algorithm}
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let error = Registry::from_models(&config.model_list, &ModelRouter::default())
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(error.to_string().contains("does not match"));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "switchyard"))]
|
||||
#[rstest]
|
||||
fn disabled_strategies_fail_at_configuration_time() {
|
||||
let config = Config::from_yaml(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: public
|
||||
litellm_params: {model: switchyard/algorithm}
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let error = Registry::from_models(
|
||||
&config.model_list,
|
||||
&ModelRouter::from_model_list(&config.model_list),
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("does not enable the switchyard feature")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,10 @@
|
|||
use super::{ApiFormat, AutoRoutingStrategy, StrategyRequest};
|
||||
use litellm_inference_chat::types::ChatCompletionsRequest;
|
||||
use litellm_inference_messages::{MessagesCall, messages_body};
|
||||
use litellm_inference_responses::types::ResponsesCall;
|
||||
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap, HashSet},
|
||||
collections::{BTreeMap, HashMap},
|
||||
convert::Infallible,
|
||||
sync::Arc,
|
||||
};
|
||||
|
|
@ -18,7 +23,7 @@ use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache};
|
|||
use litellm_config::Model;
|
||||
use litellm_host::call::CallOutput;
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error as InferenceError;
|
||||
use litellm_inference_chat::ChatCompletionsRoute;
|
||||
use litellm_inference_messages::MessagesRoute;
|
||||
use litellm_inference_responses::ResponsesRoute;
|
||||
|
|
@ -37,70 +42,41 @@ use switchyard_translation::{
|
|||
};
|
||||
|
||||
use crate::{
|
||||
Deployment, Error, Gateway, ModelRouter, SwitchyardConfigError, caching::CacheHeaders,
|
||||
chat_completions, messages, responses,
|
||||
AutoRoutingConfigError, Deployment, Error, Gateway, ModelRouter, caching::CacheHeaders,
|
||||
};
|
||||
|
||||
const SWITCHYARD_MODEL: &str = "switchyard/algorithm";
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Routes(HashMap<String, Route>);
|
||||
|
||||
impl Routes {
|
||||
pub(crate) fn from_models(
|
||||
model_list: &[Model],
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<Self, SwitchyardConfigError> {
|
||||
let mut routes = HashMap::new();
|
||||
let mut seen = HashSet::new();
|
||||
// ModelRouter uses the final entry when public model names repeat.
|
||||
for model in model_list.iter().rev() {
|
||||
if !seen.insert(model.model_name.as_str())
|
||||
|| model.litellm_params.model != SWITCHYARD_MODEL
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if deployments
|
||||
.get(&model.model_name)
|
||||
.is_none_or(|deployment| deployment.model != SWITCHYARD_MODEL)
|
||||
{
|
||||
return Err(invalid(model, "model is not a Switchyard route"));
|
||||
}
|
||||
let config = model
|
||||
.litellm_params
|
||||
.extra
|
||||
.get("switchyard_config")
|
||||
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
|
||||
let config = AlgorithmSpec::deserialize(config)
|
||||
.map_err(|error| invalid(model, error.to_string()))?;
|
||||
let targets = targets(model, &config, model_list, deployments)?;
|
||||
let (algorithm, runtime_models) = config
|
||||
.build_with_runtime_models(&model.model_name, &targets.models)
|
||||
.map_err(|error| invalid(model, error.to_string()))?;
|
||||
routes.insert(
|
||||
model.model_name.clone(),
|
||||
Route {
|
||||
algorithm,
|
||||
runtime_models: Arc::new(runtime_models),
|
||||
targets: Arc::new(targets.by_model),
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Self(routes))
|
||||
}
|
||||
|
||||
pub(crate) fn get(&self, model: &str) -> Option<&Route> {
|
||||
self.0.get(model)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Route {
|
||||
pub(super) struct SwitchyardStrategy {
|
||||
algorithm: Arc<dyn Algorithm>,
|
||||
runtime_models: Arc<RuntimeModels>,
|
||||
targets: Arc<HashMap<ModelId, Target>>,
|
||||
}
|
||||
|
||||
impl Route {
|
||||
impl SwitchyardStrategy {
|
||||
pub(super) fn from_model(
|
||||
model: &Model,
|
||||
model_list: &[Model],
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<Self, AutoRoutingConfigError> {
|
||||
let config = model
|
||||
.litellm_params
|
||||
.extra
|
||||
.get("switchyard_config")
|
||||
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
|
||||
let config = AlgorithmSpec::deserialize(config)
|
||||
.map_err(|error| invalid(model, error.to_string()))?;
|
||||
let targets = targets(model, &config, model_list, deployments)?;
|
||||
let (algorithm, runtime_models) = config
|
||||
.build_with_runtime_models(&model.model_name, &targets.models)
|
||||
.map_err(|error| invalid(model, error.to_string()))?;
|
||||
Ok(Self {
|
||||
algorithm,
|
||||
runtime_models: Arc::new(runtime_models),
|
||||
targets: Arc::new(targets.by_model),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn execute(
|
||||
&self,
|
||||
gateway: &Gateway,
|
||||
|
|
@ -230,7 +206,7 @@ impl RoutedLlmClient for LiteLlmClient {
|
|||
.routes
|
||||
.chat
|
||||
.execute(
|
||||
chat_completions::provider_request(&target.deployment, body),
|
||||
chat_request(&target.deployment, body),
|
||||
&cache_headers,
|
||||
self.cache_policy,
|
||||
)
|
||||
|
|
@ -243,7 +219,7 @@ impl RoutedLlmClient for LiteLlmClient {
|
|||
.routes
|
||||
.responses
|
||||
.execute(
|
||||
responses::provider_request(&target.deployment, body),
|
||||
responses_request(&target.deployment, body),
|
||||
&cache_headers,
|
||||
self.cache_policy,
|
||||
)
|
||||
|
|
@ -257,8 +233,8 @@ impl RoutedLlmClient for LiteLlmClient {
|
|||
.as_ref()
|
||||
.and_then(|metadata| metadata.http_headers.as_ref())
|
||||
.unwrap_or(&empty_headers);
|
||||
let call = messages::provider_request(&target.deployment, body, headers)
|
||||
.map_err(map_route_error)?;
|
||||
let call =
|
||||
messages_request(&target.deployment, body, headers).map_err(map_route_error)?;
|
||||
let response = self
|
||||
.routes
|
||||
.messages
|
||||
|
|
@ -290,7 +266,7 @@ fn decode_complete(
|
|||
}
|
||||
|
||||
fn decode_output<T, H>(
|
||||
response: CallOutput<T, H, bytes::Bytes, RouteError>,
|
||||
response: CallOutput<T, H, bytes::Bytes, InferenceError>,
|
||||
wire_format: WireFormat,
|
||||
) -> Result<LlmResponse, LlmClientError>
|
||||
where
|
||||
|
|
@ -360,7 +336,7 @@ fn targets(
|
|||
config: &AlgorithmSpec,
|
||||
model_list: &[Model],
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<Targets, SwitchyardConfigError> {
|
||||
) -> Result<Targets, AutoRoutingConfigError> {
|
||||
let mut models = BTreeMap::new();
|
||||
let mut by_model = HashMap::new();
|
||||
for target in config.callable_target_names() {
|
||||
|
|
@ -392,7 +368,7 @@ fn targets(
|
|||
}
|
||||
|
||||
// Caller format and target execution format are independent, so each target declares its own.
|
||||
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, SwitchyardConfigError> {
|
||||
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, AutoRoutingConfigError> {
|
||||
let value = target
|
||||
.litellm_params
|
||||
.extra
|
||||
|
|
@ -417,16 +393,16 @@ fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, Switc
|
|||
})
|
||||
}
|
||||
|
||||
fn invalid(model: &Model, message: impl Into<String>) -> SwitchyardConfigError {
|
||||
SwitchyardConfigError {
|
||||
fn invalid(model: &Model, message: impl Into<String>) -> AutoRoutingConfigError {
|
||||
AutoRoutingConfigError {
|
||||
model: model.model_name.clone(),
|
||||
reason: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn map_route_error(error: RouteError) -> LlmClientError {
|
||||
fn map_route_error(error: InferenceError) -> LlmClientError {
|
||||
match error {
|
||||
RouteError::Transport(TransportError::Http { status, body }) => {
|
||||
InferenceError::Transport(TransportError::Http { status, body }) => {
|
||||
match axum::http::StatusCode::from_u16(status) {
|
||||
Ok(status) => LlmClientError::UpstreamHttp { status, body },
|
||||
Err(source) => LlmClientError::InvalidResponse {
|
||||
|
|
@ -435,10 +411,12 @@ fn map_route_error(error: RouteError) -> LlmClientError {
|
|||
}
|
||||
}
|
||||
// Only Connect is known to precede dispatch, so only it is safe to fall back from.
|
||||
error @ RouteError::Transport(TransportError::Connect(_)) => LlmClientError::Transport {
|
||||
source: Box::new(error),
|
||||
},
|
||||
error @ RouteError::InvalidResponse(_) => LlmClientError::InvalidResponse {
|
||||
error @ InferenceError::Transport(TransportError::Connect(_)) => {
|
||||
LlmClientError::Transport {
|
||||
source: Box::new(error),
|
||||
}
|
||||
}
|
||||
error @ InferenceError::InvalidResponse(_) => LlmClientError::InvalidResponse {
|
||||
source: Box::new(error),
|
||||
},
|
||||
error => LlmClientError::Host {
|
||||
|
|
@ -458,10 +436,10 @@ fn map_client_error(error: LlmClientError) -> Error {
|
|||
match error {
|
||||
LlmClientError::InvalidRequest { message }
|
||||
| LlmClientError::RequestTranslation(message) => {
|
||||
Error::Route(RouteError::InvalidRequest(message.into()))
|
||||
Error::Route(InferenceError::InvalidRequest(message.into()))
|
||||
}
|
||||
LlmClientError::UpstreamHttp { status, body } => {
|
||||
Error::Route(RouteError::Transport(TransportError::Http {
|
||||
Error::Route(InferenceError::Transport(TransportError::Http {
|
||||
status: status.as_u16(),
|
||||
body,
|
||||
}))
|
||||
|
|
@ -472,22 +450,134 @@ fn map_client_error(error: LlmClientError) -> Error {
|
|||
| LlmClientError::Host { source }
|
||||
| LlmClientError::Ffi { source } => map_source(source),
|
||||
LlmClientError::ContextWindowExceeded { message, .. } => {
|
||||
Error::Route(RouteError::InvalidRequest(message.into()))
|
||||
Error::Route(InferenceError::InvalidRequest(message.into()))
|
||||
}
|
||||
LlmClientError::ResponseTranslation(message) => {
|
||||
Error::Route(RouteError::InvalidResponse(message.into()))
|
||||
Error::Route(InferenceError::InvalidResponse(message.into()))
|
||||
}
|
||||
error => Error::Internal(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn map_source(source: BoxError) -> Error {
|
||||
match source.downcast::<RouteError>() {
|
||||
match source.downcast::<InferenceError>() {
|
||||
Ok(error) => Error::Route(*error),
|
||||
Err(source) => Error::Internal(source.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"];
|
||||
const OPENROUTER_API_HEADERS: [&str; 2] = [
|
||||
"x-openrouter-metadata",
|
||||
"x-openrouter-experimental-metadata",
|
||||
];
|
||||
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
|
||||
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
|
||||
|
||||
fn chat_request(
|
||||
deployment: &Deployment,
|
||||
mut body: Map<String, Value>,
|
||||
) -> ChatCompletionsRequest<'_> {
|
||||
let messages = body.remove("messages").unwrap_or_default();
|
||||
body.remove("model");
|
||||
ChatCompletionsRequest {
|
||||
model: &deployment.model,
|
||||
messages,
|
||||
optional_params: body,
|
||||
api_key: deployment.api_key.as_deref(),
|
||||
api_base: deployment.api_base.as_deref(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.as_deref(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
}
|
||||
}
|
||||
|
||||
fn responses_request(deployment: &Deployment, mut body: Map<String, Value>) -> ResponsesCall {
|
||||
let input = body.remove("input").unwrap_or_default();
|
||||
body.remove("model");
|
||||
ResponsesCall {
|
||||
model: deployment.model.clone(),
|
||||
input,
|
||||
optional_params: body,
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
}
|
||||
}
|
||||
|
||||
fn messages_request(
|
||||
deployment: &Deployment,
|
||||
mut body: Map<String, Value>,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<MessagesCall, InferenceError> {
|
||||
if let Some(model) = body.get_mut("model") {
|
||||
*model = Value::from(deployment.model.as_str());
|
||||
}
|
||||
Ok(MessagesCall {
|
||||
body: messages_body(body)?,
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
litellm_params: deployment.litellm_params.clone(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: anthropic_api_headers(headers),
|
||||
timeout: deployment.timeout,
|
||||
shaping: deployment.shaping.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn anthropic_api_headers(headers: &HeaderMap) -> Option<ProviderSpecificHeaders> {
|
||||
let scopes = [
|
||||
(
|
||||
ANTHROPIC_API_HEADER_PROVIDERS,
|
||||
ANTHROPIC_API_HEADERS.as_slice(),
|
||||
),
|
||||
("openrouter", OPENROUTER_API_HEADERS.as_slice()),
|
||||
];
|
||||
let forwarded: Vec<ProviderSpecificHeader> = scopes
|
||||
.into_iter()
|
||||
.filter_map(|(provider, names)| {
|
||||
let extra_headers: Map<String, Value> = names
|
||||
.iter()
|
||||
.filter_map(|name| {
|
||||
let value = headers.get(*name)?.to_str().ok()?;
|
||||
Some(((*name).to_owned(), Value::from(value)))
|
||||
})
|
||||
.collect();
|
||||
(!extra_headers.is_empty()).then(|| ProviderSpecificHeader {
|
||||
custom_llm_provider: provider.into(),
|
||||
extra_headers,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
(!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded))
|
||||
}
|
||||
|
||||
impl AutoRoutingStrategy for SwitchyardStrategy {
|
||||
fn execute<'a>(
|
||||
&'a self,
|
||||
gateway: &'a Gateway,
|
||||
request: StrategyRequest,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<HttpResponse, Error>> + Send + 'a>>
|
||||
{
|
||||
let format = match request.format {
|
||||
ApiFormat::Chat => WireFormat::OpenAiChat,
|
||||
ApiFormat::Responses => WireFormat::OpenAiResponses,
|
||||
ApiFormat::Messages => WireFormat::AnthropicMessages,
|
||||
};
|
||||
Box::pin(SwitchyardStrategy::execute(
|
||||
self,
|
||||
gateway,
|
||||
request.cache_options,
|
||||
format,
|
||||
request.body,
|
||||
request.headers,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::body::to_bytes;
|
||||
|
|
@ -497,14 +587,14 @@ mod tests {
|
|||
|
||||
#[rstest]
|
||||
fn host_errors_keep_their_gateway_classification() {
|
||||
let error = RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
let error = InferenceError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "OpenAI",
|
||||
environment_variable: "OPENAI_API_KEY",
|
||||
});
|
||||
|
||||
assert!(matches!(
|
||||
map_client_error(map_route_error(error)),
|
||||
Error::Route(RouteError::Auth(_))
|
||||
Error::Route(InferenceError::Auth(_))
|
||||
));
|
||||
}
|
||||
|
||||
|
|
@ -1,40 +1,27 @@
|
|||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "switchyard")]
|
||||
use axum::http::HeaderMap;
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use litellm_inference_chat::types::{ChatCompletionsCall, ChatCompletionsRequest};
|
||||
use litellm_inference_chat::types::ChatCompletionsCall;
|
||||
use serde_json::{Map, Value};
|
||||
#[cfg(feature = "switchyard")]
|
||||
use switchyard_protocol::WireFormat;
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, request};
|
||||
use crate::{Error, Gateway, JsonObject, request};
|
||||
|
||||
pub(crate) async fn create(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
#[cfg(feature = "switchyard")] headers: HeaderMap,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<impl IntoResponse, Error> {
|
||||
handle(
|
||||
&gateway,
|
||||
&identity,
|
||||
#[cfg(feature = "switchyard")]
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await
|
||||
handle(&gateway, &identity, body).await
|
||||
}
|
||||
|
||||
pub(crate) async fn create_from_model_path(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
#[cfg(feature = "switchyard")] headers: HeaderMap,
|
||||
Path(model): Path<String>,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<impl IntoResponse, Error> {
|
||||
|
|
@ -45,42 +32,17 @@ pub(crate) async fn create_from_model_path(
|
|||
.collect(),
|
||||
Some(_) => body,
|
||||
};
|
||||
handle(
|
||||
&gateway,
|
||||
&identity,
|
||||
#[cfg(feature = "switchyard")]
|
||||
headers,
|
||||
body,
|
||||
)
|
||||
.await
|
||||
handle(&gateway, &identity, body).await
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
gateway: &Gateway,
|
||||
identity: &AuthenticatedRequest,
|
||||
#[cfg(feature = "switchyard")] inbound_headers: HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<Response, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
let switchyard_route = body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|model| gateway.switchyard_routes.get(model));
|
||||
let (body, cache_options) = crate::caching::prepare(identity, body)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
return switchyard_route
|
||||
.execute(
|
||||
gateway,
|
||||
cache_options,
|
||||
WireFormat::OpenAiChat,
|
||||
body,
|
||||
inbound_headers,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let route = gateway.chat_completions.clone();
|
||||
let route = match &gateway.cache {
|
||||
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
|
||||
|
|
@ -89,10 +51,24 @@ async fn handle(
|
|||
)),
|
||||
None => route,
|
||||
};
|
||||
|
||||
let messages = body.get("messages").cloned().unwrap_or_default();
|
||||
let headers = crate::caching::CacheHeaders::default();
|
||||
let response = litellm_host_http::serve_unary(
|
||||
route.machine(
|
||||
ChatCompletionsCall::from(provider_request(deployment, body)),
|
||||
ChatCompletionsCall {
|
||||
model: deployment.model.clone(),
|
||||
messages,
|
||||
optional_params: body
|
||||
.into_iter()
|
||||
.filter(|(name, _)| !matches!(name.as_str(), "model" | "messages"))
|
||||
.collect(),
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
},
|
||||
cache_options.policy,
|
||||
),
|
||||
(),
|
||||
|
|
@ -103,21 +79,3 @@ async fn handle(
|
|||
.await?;
|
||||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
pub(crate) fn provider_request(
|
||||
deployment: &Deployment,
|
||||
mut body: Map<String, Value>,
|
||||
) -> ChatCompletionsRequest<'_> {
|
||||
let messages = body.remove("messages").unwrap_or_default();
|
||||
body.remove("model");
|
||||
ChatCompletionsRequest {
|
||||
model: &deployment.model,
|
||||
messages,
|
||||
optional_params: body,
|
||||
api_key: deployment.api_key.as_deref(),
|
||||
api_base: deployment.api_base.as_deref(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.as_deref(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,11 +8,9 @@ use litellm_inference::Error as InferenceError;
|
|||
use litellm_llms::base_llm::ocr::error::Error as OcrError;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
/// Invalid configuration for a Switchyard virtual model.
|
||||
#[cfg(feature = "switchyard")]
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("invalid Switchyard model {model}: {reason}")]
|
||||
pub struct SwitchyardConfigError {
|
||||
#[error("invalid auto-routing model {model}: {reason}")]
|
||||
pub struct AutoRoutingConfigError {
|
||||
pub(crate) model: String,
|
||||
pub(crate) reason: String,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
//! maps a public model name to its deployment and runs the core route.
|
||||
|
||||
mod audio_transcription;
|
||||
pub mod auto_routing;
|
||||
mod caching;
|
||||
mod chat_completions;
|
||||
mod error;
|
||||
|
|
@ -11,8 +12,6 @@ pub mod messages;
|
|||
mod ocr;
|
||||
mod request;
|
||||
mod responses;
|
||||
#[cfg(feature = "switchyard")]
|
||||
mod switchyard;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -27,16 +26,14 @@ use litellm_inference_transcription::AudioTranscriptionRoute;
|
|||
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
||||
pub use error::AutoRoutingConfigError;
|
||||
pub use error::Error;
|
||||
#[cfg(feature = "switchyard")]
|
||||
pub use error::SwitchyardConfigError;
|
||||
pub use litellm_router::{Deployment, Router as ModelRouter};
|
||||
pub use request::{JsonObject, RequestId};
|
||||
|
||||
pub struct Gateway {
|
||||
cache: Option<Arc<dyn litellm_cache_response::ResponseCacheService>>,
|
||||
#[cfg(feature = "switchyard")]
|
||||
switchyard_routes: switchyard::Routes,
|
||||
auto_routing: auto_routing::Registry,
|
||||
pub audio_transcription: AudioTranscriptionRoute,
|
||||
pub chat_completions: ChatCompletionsRoute,
|
||||
pub messages: MessagesRoute,
|
||||
|
|
@ -66,8 +63,7 @@ impl Gateway {
|
|||
let auth = resources.auth.clone();
|
||||
Ok(Self {
|
||||
cache: None,
|
||||
#[cfg(feature = "switchyard")]
|
||||
switchyard_routes: switchyard::Routes::default(),
|
||||
auto_routing: auto_routing::Registry::default(),
|
||||
audio_transcription: AudioTranscriptionRoute::new(
|
||||
provider.clone(),
|
||||
auth.clone(),
|
||||
|
|
@ -95,24 +91,37 @@ impl Gateway {
|
|||
})
|
||||
}
|
||||
|
||||
/// Configure Switchyard virtual models from LiteLLM's model list.
|
||||
#[cfg(feature = "switchyard")]
|
||||
pub fn with_switchyard_models(
|
||||
pub fn with_auto_routing_strategy(
|
||||
mut self,
|
||||
model: String,
|
||||
strategy: Arc<dyn auto_routing::AutoRoutingStrategy>,
|
||||
) -> Result<Self, AutoRoutingConfigError> {
|
||||
if self.models.get(&model).is_none() {
|
||||
return Err(AutoRoutingConfigError {
|
||||
model,
|
||||
reason: "model is not configured".into(),
|
||||
});
|
||||
}
|
||||
self.auto_routing.insert(model, strategy);
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub fn with_auto_routing_models(
|
||||
mut self,
|
||||
models: &[litellm_config::Model],
|
||||
) -> Result<Self, SwitchyardConfigError> {
|
||||
self.switchyard_routes = switchyard::Routes::from_models(models, &self.models)?;
|
||||
) -> Result<Self, AutoRoutingConfigError> {
|
||||
self.auto_routing = auto_routing::Registry::from_models(models, &self.models)?;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn router(gateway: Arc<Gateway>) -> Router {
|
||||
Router::new()
|
||||
.route("/v1/messages", post(messages::create))
|
||||
.route("/v1/messages", post(auto_routing::messages))
|
||||
.route("/ocr", post(ocr::create))
|
||||
.route("/v1/ocr", post(ocr::create))
|
||||
.route("/chat/completions", post(chat_completions::create))
|
||||
.route("/v1/chat/completions", post(chat_completions::create))
|
||||
.route("/chat/completions", post(auto_routing::chat))
|
||||
.route("/v1/chat/completions", post(auto_routing::chat))
|
||||
.nest("/engines/{model}", model_routes())
|
||||
.nest("/openai/deployments/{model}", model_routes())
|
||||
.route("/audio/transcriptions", post(audio_transcription::create))
|
||||
|
|
@ -120,8 +129,8 @@ pub fn router(gateway: Arc<Gateway>) -> Router {
|
|||
"/v1/audio/transcriptions",
|
||||
post(audio_transcription::create),
|
||||
)
|
||||
.route("/responses", post(responses::create))
|
||||
.route("/v1/responses", post(responses::create))
|
||||
.route("/responses", post(auto_routing::responses))
|
||||
.route("/v1/responses", post(auto_routing::responses))
|
||||
.route("/embeddings", post(request::unsupported))
|
||||
.route("/v1/embeddings", post(request::unsupported))
|
||||
.route("/completions", post(request::unsupported))
|
||||
|
|
@ -136,7 +145,7 @@ fn model_routes() -> Router<Arc<Gateway>> {
|
|||
Router::new()
|
||||
.route(
|
||||
"/chat/completions",
|
||||
post(chat_completions::create_from_model_path),
|
||||
post(auto_routing::chat_from_model_path),
|
||||
)
|
||||
.route("/embeddings", post(request::unsupported))
|
||||
.route("/completions", post(request::unsupported))
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ use litellm_host_http::Sse;
|
|||
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
|
||||
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
|
||||
use serde_json::{Map, Value};
|
||||
#[cfg(feature = "switchyard")]
|
||||
use switchyard_protocol::WireFormat;
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
|
||||
|
||||
|
|
@ -36,7 +34,7 @@ pub async fn create(
|
|||
body: Result<JsonObject, Error>,
|
||||
) -> impl IntoResponse {
|
||||
let result = match body {
|
||||
Ok(JsonObject(body)) => handle(&gateway, &identity, headers, body).await,
|
||||
Ok(JsonObject(body)) => handle(&gateway, &identity, &headers, body).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
result.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
|
||||
|
|
@ -45,29 +43,12 @@ pub async fn create(
|
|||
async fn handle(
|
||||
gateway: &Gateway,
|
||||
identity: &AuthenticatedRequest,
|
||||
headers: HeaderMap,
|
||||
headers: &HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<Response, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
let switchyard_route = body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|model| gateway.switchyard_routes.get(model));
|
||||
let (body, cache_options) = crate::caching::prepare(identity, body)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
return switchyard_route
|
||||
.execute(
|
||||
gateway,
|
||||
cache_options,
|
||||
WireFormat::AnthropicMessages,
|
||||
body,
|
||||
headers,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let route = gateway.messages.clone();
|
||||
let route = match &gateway.cache {
|
||||
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
|
||||
|
|
@ -77,7 +58,7 @@ async fn handle(
|
|||
None => route,
|
||||
};
|
||||
|
||||
let call = provider_request(deployment, body, &headers)?;
|
||||
let call = project(deployment, body, headers)?;
|
||||
let machine = route.machine(call, cache_options.policy);
|
||||
let stream =
|
||||
Sse::<Messages, _, _>::new(Json, |error| Bytes::from(Error::from(error).sse_frame()));
|
||||
|
|
@ -86,14 +67,18 @@ async fn handle(
|
|||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
pub(crate) fn provider_request(
|
||||
fn project(
|
||||
deployment: &Deployment,
|
||||
mut body: Map<String, Value>,
|
||||
body: Map<String, Value>,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<MessagesCall, litellm_inference::RouteError> {
|
||||
if let Some(model) = body.get_mut("model") {
|
||||
*model = Value::from(deployment.model.as_str());
|
||||
}
|
||||
) -> Result<MessagesCall, Error> {
|
||||
let body = body
|
||||
.into_iter()
|
||||
.map(|(name, value)| match name.as_str() {
|
||||
"model" => (name, Value::from(deployment.model.as_str())),
|
||||
_ => (name, value),
|
||||
})
|
||||
.collect();
|
||||
Ok(MessagesCall {
|
||||
body: messages_body(body)?,
|
||||
api_key: deployment.api_key.clone(),
|
||||
|
|
|
|||
|
|
@ -1,43 +1,21 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "switchyard")]
|
||||
use axum::http::HeaderMap;
|
||||
use axum::{Json, body::Bytes, extract::State, response::Response};
|
||||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use litellm_host_http::Sse;
|
||||
use litellm_inference_responses::{route::Responses, types::ResponsesCall};
|
||||
use serde_json::{Map, Value, json};
|
||||
#[cfg(feature = "switchyard")]
|
||||
use switchyard_protocol::WireFormat;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, request};
|
||||
use crate::{Error, Gateway, JsonObject, request};
|
||||
|
||||
pub(crate) async fn create(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
#[cfg(feature = "switchyard")] headers: HeaderMap,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<Response, Error> {
|
||||
let deployment = request::resolve_deployment(&gateway, &body)?;
|
||||
request::authorize_model(&identity, deployment, &body).await?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
let switchyard_route = body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|model| gateway.switchyard_routes.get(model));
|
||||
let (body, cache_options) = crate::caching::prepare(&identity, body)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
return switchyard_route
|
||||
.execute(
|
||||
&gateway,
|
||||
cache_options,
|
||||
WireFormat::OpenAiResponses,
|
||||
body,
|
||||
headers,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let route = gateway.responses.clone();
|
||||
let route = match &gateway.cache {
|
||||
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
|
||||
|
|
@ -47,7 +25,19 @@ pub(crate) async fn create(
|
|||
None => route,
|
||||
};
|
||||
|
||||
let call = provider_request(deployment, body);
|
||||
let call = ResponsesCall {
|
||||
model: deployment.model.clone(),
|
||||
input: body.get("input").cloned().unwrap_or_default(),
|
||||
optional_params: body
|
||||
.into_iter()
|
||||
.filter(|(name, _)| !matches!(name.as_str(), "model" | "input"))
|
||||
.collect(),
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
};
|
||||
let machine = route.machine(call, cache_options.policy);
|
||||
let stream = Sse::<Responses, _, _>::new(Json, |error| {
|
||||
let error = Error::from(error);
|
||||
|
|
@ -60,21 +50,3 @@ pub(crate) async fn create(
|
|||
let response = litellm_host_http::serve(machine, (), headers.clone(), stream, None).await?;
|
||||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
pub(crate) fn provider_request(
|
||||
deployment: &Deployment,
|
||||
mut body: Map<String, Value>,
|
||||
) -> ResponsesCall {
|
||||
let input = body.remove("input").unwrap_or_default();
|
||||
body.remove("model");
|
||||
ResponsesCall {
|
||||
model: deployment.model.clone(),
|
||||
input,
|
||||
optional_params: body,
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
267
litellm-rust/crates/gateway-inference/tests/auto_routing.rs
Normal file
267
litellm-rust/crates/gateway-inference/tests/auto_routing.rs
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
mod support;
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use litellm_gateway_auth::Permissions;
|
||||
use litellm_gateway_inference::{
|
||||
Deployment, Error, Gateway,
|
||||
auto_routing::{ApiFormat, AutoRoutingStrategy, StrategyRequest},
|
||||
};
|
||||
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
|
||||
use litellm_inference::resources::CoreResources;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
struct EchoStrategy(Arc<AtomicUsize>);
|
||||
|
||||
impl AutoRoutingStrategy for EchoStrategy {
|
||||
fn execute<'a>(
|
||||
&'a self,
|
||||
_: &'a Gateway,
|
||||
request: StrategyRequest,
|
||||
) -> Pin<Box<dyn Future<Output = Result<Response, Error>> + Send + 'a>> {
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
Box::pin(async move {
|
||||
if request.body.get("fail").and_then(Value::as_bool) == Some(true) {
|
||||
return Err(Error::InvalidBody("strategy rejected request".into()));
|
||||
}
|
||||
let format = match request.format {
|
||||
ApiFormat::Chat => "chat",
|
||||
ApiFormat::Responses => "responses",
|
||||
ApiFormat::Messages => "messages",
|
||||
};
|
||||
Ok(Json(json!({"format": format, "body": request.body, "request_id": request.headers.get("x-request-id").and_then(|v| v.to_str().ok()), "no_store": request.cache_options.policy.no_store})).into_response())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn gateway() -> Gateway {
|
||||
Gateway::new(
|
||||
CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))),
|
||||
Resolution::from(&HttpSettings::default()).config,
|
||||
Arc::new(support::NoSecrets),
|
||||
[(
|
||||
"virtual".into(),
|
||||
Deployment {
|
||||
model: "virtual-target".into(),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions", "chat", json!({"model":"virtual"}))]
|
||||
#[case::chat_alias("/chat/completions", "chat", json!({"model":"virtual"}))]
|
||||
#[case::responses("/v1/responses", "responses", json!({"model":"virtual"}))]
|
||||
#[case::responses_alias("/responses", "responses", json!({"model":"virtual"}))]
|
||||
#[case::messages("/v1/messages", "messages", json!({"model":"virtual"}))]
|
||||
#[case::engine("/engines/virtual/chat/completions", "chat", json!({}))]
|
||||
#[case::deployment("/openai/deployments/virtual/chat/completions", "chat", json!({"model":null}))]
|
||||
#[case::body_wins("/engines/unknown/chat/completions", "chat", json!({"model":"virtual"}))]
|
||||
#[tokio::test]
|
||||
async fn virtual_models_invoke_the_strategy_once(
|
||||
gateway: Gateway,
|
||||
#[case] path: &str,
|
||||
#[case] format: &str,
|
||||
#[case] body: Value,
|
||||
) {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
|
||||
.unwrap();
|
||||
let response = support::post(
|
||||
support::authenticated_app(gateway, Permissions::All, None),
|
||||
path,
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["format"], format);
|
||||
assert_eq!(body["body"]["model"], "virtual");
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions")]
|
||||
#[case::responses("/v1/responses")]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[tokio::test]
|
||||
async fn denied_virtual_models_never_invoke_the_strategy(gateway: Gateway, #[case] path: &str) {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
|
||||
.unwrap();
|
||||
let response = support::post(
|
||||
support::authenticated_app(gateway, Permissions::Only(Arc::from([])), None),
|
||||
path,
|
||||
json!({"model":"virtual"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 403);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions", false)]
|
||||
#[case::responses("/v1/responses", false)]
|
||||
#[case::messages("/v1/messages", true)]
|
||||
#[tokio::test]
|
||||
async fn strategy_errors_keep_the_callers_error_format(
|
||||
gateway: Gateway,
|
||||
#[case] path: &str,
|
||||
#[case] anthropic: bool,
|
||||
) {
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy(
|
||||
"virtual".into(),
|
||||
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
|
||||
)
|
||||
.unwrap();
|
||||
let response = support::post(
|
||||
support::authenticated_app(gateway, Permissions::All, None),
|
||||
path,
|
||||
json!({"model":"virtual", "fail":true}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("strategy rejected request")
|
||||
);
|
||||
assert_eq!(body.get("type").is_some(), anthropic);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions")]
|
||||
#[case::responses("/v1/responses")]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[tokio::test]
|
||||
async fn cache_controls_are_prepared_before_strategy_execution(
|
||||
gateway: Gateway,
|
||||
#[case] path: &str,
|
||||
) {
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy(
|
||||
"virtual".into(),
|
||||
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
|
||||
)
|
||||
.unwrap();
|
||||
let response = support::post(
|
||||
support::authenticated_app(gateway, Permissions::All, None),
|
||||
path,
|
||||
json!({"model":"virtual", "cache":{"no-store":true}, "caching":false}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["no_store"], true);
|
||||
assert!(body["body"].get("cache").is_none());
|
||||
assert!(body["body"].get("caching").is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions")]
|
||||
#[case::responses("/v1/responses")]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[tokio::test]
|
||||
async fn incoming_headers_reach_the_strategy(gateway: Gateway, #[case] path: &str) {
|
||||
use axum::{body::Body, http::Request};
|
||||
use tower::ServiceExt;
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy(
|
||||
"virtual".into(),
|
||||
Arc::new(EchoStrategy(Arc::new(AtomicUsize::new(0)))),
|
||||
)
|
||||
.unwrap();
|
||||
let response = support::authenticated_app(gateway, Permissions::All, None)
|
||||
.oneshot(
|
||||
Request::post(path)
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", "request-context")
|
||||
.body(Body::from(r#"{"model":"virtual"}"#))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(
|
||||
support::json(response).await["request_id"],
|
||||
"request-context"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions", false)]
|
||||
#[case::responses("/v1/responses", false)]
|
||||
#[case::messages("/v1/messages", true)]
|
||||
#[tokio::test]
|
||||
async fn malformed_bodies_preserve_api_error_envelopes(
|
||||
gateway: Gateway,
|
||||
#[case] path: &str,
|
||||
#[case] anthropic: bool,
|
||||
) {
|
||||
use axum::{body::Body, http::Request};
|
||||
use tower::ServiceExt;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
|
||||
.unwrap();
|
||||
let response = support::authenticated_app(gateway, Permissions::All, None)
|
||||
.oneshot(
|
||||
Request::post(path)
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", "malformed-request")
|
||||
.body(Body::from("{"))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 400);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert_eq!(body.get("type").is_some(), anthropic);
|
||||
if anthropic {
|
||||
assert_eq!(body["request_id"], "malformed-request");
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat("/v1/chat/completions")]
|
||||
#[case::responses("/v1/responses")]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[tokio::test]
|
||||
async fn invalid_cache_settings_do_not_invoke_the_strategy(gateway: Gateway, #[case] path: &str) {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let gateway = gateway
|
||||
.with_auto_routing_strategy("virtual".into(), Arc::new(EchoStrategy(calls.clone())))
|
||||
.unwrap();
|
||||
let response = support::post(
|
||||
support::authenticated_app(gateway, Permissions::All, None),
|
||||
path,
|
||||
json!({"model":"virtual", "cache":{"ttl":-1}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
|
@ -390,16 +390,93 @@ async fn streaming_responses_upstream() -> (String, tokio::task::JoinHandle<()>)
|
|||
}
|
||||
|
||||
fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router {
|
||||
support::authenticated_app(switchyard_gateway(config), permissions, None)
|
||||
}
|
||||
|
||||
fn switchyard_gateway(config: &Config) -> Gateway {
|
||||
let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))));
|
||||
let http = Resolution::from(&HttpSettings::default()).config;
|
||||
let gateway = Gateway::new(
|
||||
Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
Arc::new(support::NoSecrets),
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)
|
||||
.expect("gateway")
|
||||
.with_switchyard_models(&config.model_list)
|
||||
.expect("Switchyard routes");
|
||||
support::authenticated_app(gateway, permissions, None)
|
||||
.with_auto_routing_models(&config.model_list)
|
||||
.expect("Switchyard routes")
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn routed_answers_preserve_cache_hits_and_caller_isolation() {
|
||||
use litellm_cache_memory::InMemoryCache;
|
||||
use litellm_cache_response::ResponseCache;
|
||||
use litellm_gateway_auth::{Principal, PrincipalKind};
|
||||
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"model":"answer", "choices":[{"message":{"role":"assistant","content":"cached answer"}}]
|
||||
})))
|
||||
.expect(2)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let config = Config::from_yaml(&format!(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: answer
|
||||
litellm_params:
|
||||
model: answer
|
||||
custom_llm_provider: openai_like
|
||||
api_base: {}/v1
|
||||
api_key: test-key
|
||||
switchyard_wire_format: openai_chat
|
||||
- model_name: virtual
|
||||
litellm_params:
|
||||
model: switchyard/algorithm
|
||||
switchyard_config: {{type: passthrough, target: answer}}
|
||||
"#,
|
||||
upstream.uri()
|
||||
))
|
||||
.unwrap();
|
||||
let cache = Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new(
|
||||
Some(100),
|
||||
Some(Duration::from_secs(60)),
|
||||
))));
|
||||
let first_app = support::authenticated_app(
|
||||
switchyard_gateway(&config).with_cache(cache.clone()),
|
||||
Permissions::All,
|
||||
Some(Principal::new(
|
||||
"issuer".into(),
|
||||
"first".into(),
|
||||
PrincipalKind::Service,
|
||||
)),
|
||||
);
|
||||
let second_app = support::authenticated_app(
|
||||
switchyard_gateway(&config).with_cache(cache),
|
||||
Permissions::All,
|
||||
Some(Principal::new(
|
||||
"issuer".into(),
|
||||
"second".into(),
|
||||
PrincipalKind::Service,
|
||||
)),
|
||||
);
|
||||
let request = json!({"model":"virtual", "messages":[{"role":"user", "content":"hello"}]});
|
||||
let first = support::post(first_app.clone(), "/v1/chat/completions", request.clone()).await;
|
||||
assert_eq!(first.status(), 200);
|
||||
assert!(!first.headers().contains_key("x-litellm-cache-key"));
|
||||
let first_body = support::json(first).await;
|
||||
let hit = support::post(first_app, "/v1/chat/completions", request.clone()).await;
|
||||
assert_eq!(hit.status(), 200);
|
||||
let first_key = hit.headers().get("x-litellm-cache-key").unwrap().clone();
|
||||
assert_eq!(support::json(hit).await, first_body);
|
||||
let second = support::post(second_app.clone(), "/v1/chat/completions", request.clone()).await;
|
||||
assert_eq!(second.status(), 200);
|
||||
assert!(!second.headers().contains_key("x-litellm-cache-key"));
|
||||
let hit = support::post(second_app, "/v1/chat/completions", request).await;
|
||||
assert_eq!(hit.status(), 200);
|
||||
assert_ne!(hit.headers().get("x-litellm-cache-key").unwrap(), first_key);
|
||||
upstream.verify().await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ pub enum Error {
|
|||
Mcp(#[from] litellm_gateway_mcp::ConnectError),
|
||||
#[error("MCP requires a configured master key")]
|
||||
Auth(#[from] litellm_gateway_auth::Error),
|
||||
#[cfg(feature = "switchyard")]
|
||||
#[error(transparent)]
|
||||
Switchyard(#[from] litellm_gateway_inference::SwitchyardConfigError),
|
||||
AutoRouting(#[from] litellm_gateway_inference::AutoRoutingConfigError),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,8 +68,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
|||
secrets,
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
let gateway = gateway.with_switchyard_models(&config.model_list)?;
|
||||
let gateway = gateway.with_auto_routing_models(&config.model_list)?;
|
||||
Ok(Arc::new(gateway))
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue