mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
feat(gateway): route buffered APIs through Switchyard
Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
parent
1bc923cce9
commit
bcf0bdafd3
6 changed files with 443 additions and 121 deletions
|
|
@ -10,6 +10,8 @@ use axum::{
|
|||
};
|
||||
use litellm_inference_chat::types::{ChatCompletionsCall, ChatCompletionsRequest};
|
||||
use serde_json::{Map, Value};
|
||||
#[cfg(feature = "switchyard")]
|
||||
use switchyard_protocol::WireFormat;
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, request};
|
||||
|
||||
|
|
@ -67,6 +69,18 @@ async fn handle(
|
|||
.and_then(Value::as_str)
|
||||
.and_then(|model| gateway.switchyard_routes.get(model));
|
||||
let (body, cache_options) = crate::caching::prepare(identity, body)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
return switchyard_route
|
||||
.execute(
|
||||
gateway,
|
||||
cache_options,
|
||||
WireFormat::OpenAiChat,
|
||||
body,
|
||||
inbound_headers,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let route = gateway.chat_completions.clone();
|
||||
let route = match &gateway.cache {
|
||||
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
|
||||
|
|
@ -75,16 +89,6 @@ async fn handle(
|
|||
)),
|
||||
None => route,
|
||||
};
|
||||
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
let (body, upstream_headers) = switchyard_route
|
||||
.execute(route, cache_options.policy, body, inbound_headers)
|
||||
.await?;
|
||||
let mut response = Json(body).into_response();
|
||||
response.headers_mut().extend(upstream_headers);
|
||||
return Ok(response);
|
||||
}
|
||||
let headers = crate::caching::CacheHeaders::default();
|
||||
let response = litellm_host_http::serve_unary(
|
||||
route.machine(
|
||||
|
|
|
|||
|
|
@ -101,8 +101,7 @@ impl Gateway {
|
|||
mut self,
|
||||
models: &[litellm_config::Model],
|
||||
) -> Result<Self, SwitchyardConfigError> {
|
||||
self.switchyard_routes =
|
||||
switchyard::Routes::from_models(models, Arc::new(self.models.clone()))?;
|
||||
self.switchyard_routes = switchyard::Routes::from_models(models, &self.models)?;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ use litellm_host_http::Sse;
|
|||
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
|
||||
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
|
||||
use serde_json::{Map, Value};
|
||||
#[cfg(feature = "switchyard")]
|
||||
use switchyard_protocol::WireFormat;
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
|
||||
|
||||
|
|
@ -34,7 +36,7 @@ pub async fn create(
|
|||
body: Result<JsonObject, Error>,
|
||||
) -> impl IntoResponse {
|
||||
let result = match body {
|
||||
Ok(JsonObject(body)) => handle(&gateway, &identity, &headers, body).await,
|
||||
Ok(JsonObject(body)) => handle(&gateway, &identity, headers, body).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
result.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
|
||||
|
|
@ -43,12 +45,29 @@ pub async fn create(
|
|||
async fn handle(
|
||||
gateway: &Gateway,
|
||||
identity: &AuthenticatedRequest,
|
||||
headers: &HeaderMap,
|
||||
headers: HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<Response, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
let switchyard_route = body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|model| gateway.switchyard_routes.get(model));
|
||||
let (body, cache_options) = crate::caching::prepare(identity, body)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
return switchyard_route
|
||||
.execute(
|
||||
gateway,
|
||||
cache_options,
|
||||
WireFormat::AnthropicMessages,
|
||||
body,
|
||||
headers,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let route = gateway.messages.clone();
|
||||
let route = match &gateway.cache {
|
||||
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
|
||||
|
|
@ -58,7 +77,7 @@ async fn handle(
|
|||
None => route,
|
||||
};
|
||||
|
||||
let call = project(deployment, body, headers)?;
|
||||
let call = provider_request(deployment, body, &headers)?;
|
||||
let machine = route.machine(call, cache_options.policy);
|
||||
let stream =
|
||||
Sse::<Messages, _, _>::new(Json, |error| Bytes::from(Error::from(error).sse_frame()));
|
||||
|
|
@ -67,18 +86,14 @@ async fn handle(
|
|||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
fn project(
|
||||
pub(crate) fn provider_request(
|
||||
deployment: &Deployment,
|
||||
body: Map<String, Value>,
|
||||
mut body: Map<String, Value>,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<MessagesCall, Error> {
|
||||
let body = body
|
||||
.into_iter()
|
||||
.map(|(name, value)| match name.as_str() {
|
||||
"model" => (name, Value::from(deployment.model.as_str())),
|
||||
_ => (name, value),
|
||||
})
|
||||
.collect();
|
||||
) -> Result<MessagesCall, litellm_inference::RouteError> {
|
||||
if let Some(model) = body.get_mut("model") {
|
||||
*model = Value::from(deployment.model.as_str());
|
||||
}
|
||||
Ok(MessagesCall {
|
||||
body: messages_body(body)?,
|
||||
api_key: deployment.api_key.clone(),
|
||||
|
|
|
|||
|
|
@ -1,21 +1,43 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "switchyard")]
|
||||
use axum::http::HeaderMap;
|
||||
use axum::{Json, body::Bytes, extract::State, response::Response};
|
||||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use litellm_host_http::Sse;
|
||||
use litellm_inference_responses::{route::Responses, types::ResponsesCall};
|
||||
use serde_json::json;
|
||||
use serde_json::{Map, Value, json};
|
||||
#[cfg(feature = "switchyard")]
|
||||
use switchyard_protocol::WireFormat;
|
||||
|
||||
use crate::{Error, Gateway, JsonObject, request};
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, request};
|
||||
|
||||
pub(crate) async fn create(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
#[cfg(feature = "switchyard")] headers: HeaderMap,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<Response, Error> {
|
||||
let deployment = request::resolve_deployment(&gateway, &body)?;
|
||||
request::authorize_model(&identity, deployment, &body).await?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
let switchyard_route = body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|model| gateway.switchyard_routes.get(model));
|
||||
let (body, cache_options) = crate::caching::prepare(&identity, body)?;
|
||||
#[cfg(feature = "switchyard")]
|
||||
if let Some(switchyard_route) = switchyard_route {
|
||||
return switchyard_route
|
||||
.execute(
|
||||
&gateway,
|
||||
cache_options,
|
||||
WireFormat::OpenAiResponses,
|
||||
body,
|
||||
headers,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let route = gateway.responses.clone();
|
||||
let route = match &gateway.cache {
|
||||
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
|
||||
|
|
@ -25,19 +47,7 @@ pub(crate) async fn create(
|
|||
None => route,
|
||||
};
|
||||
|
||||
let call = ResponsesCall {
|
||||
model: deployment.model.clone(),
|
||||
input: body.get("input").cloned().unwrap_or_default(),
|
||||
optional_params: body
|
||||
.into_iter()
|
||||
.filter(|(name, _)| !matches!(name.as_str(), "model" | "input"))
|
||||
.collect(),
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
};
|
||||
let call = provider_request(deployment, body);
|
||||
let machine = route.machine(call, cache_options.policy);
|
||||
let stream = Sse::<Responses, _, _>::new(Json, |error| {
|
||||
let error = Error::from(error);
|
||||
|
|
@ -50,3 +60,21 @@ pub(crate) async fn create(
|
|||
let response = litellm_host_http::serve(machine, (), headers.clone(), stream, None).await?;
|
||||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
pub(crate) fn provider_request(
|
||||
deployment: &Deployment,
|
||||
mut body: Map<String, Value>,
|
||||
) -> ResponsesCall {
|
||||
let input = body.remove("input").unwrap_or_default();
|
||||
body.remove("model");
|
||||
ResponsesCall {
|
||||
model: deployment.model.clone(),
|
||||
input,
|
||||
optional_params: body,
|
||||
api_key: deployment.api_key.clone(),
|
||||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,18 @@ use std::{
|
|||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use axum::http::HeaderMap;
|
||||
use litellm_cache_response::CachePolicy;
|
||||
use axum::{
|
||||
Json,
|
||||
http::HeaderMap,
|
||||
response::{IntoResponse, Response as HttpResponse},
|
||||
};
|
||||
use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache};
|
||||
use litellm_config::Model;
|
||||
use litellm_host::call::CallOutput;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference_chat::ChatCompletionsRoute;
|
||||
use litellm_inference_messages::MessagesRoute;
|
||||
use litellm_inference_responses::ResponsesRoute;
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -25,8 +32,8 @@ use switchyard_translation::{
|
|||
};
|
||||
|
||||
use crate::{
|
||||
Error, ModelRouter, SwitchyardConfigError, caching::CacheHeaders,
|
||||
chat_completions::provider_request,
|
||||
Deployment, Error, Gateway, ModelRouter, SwitchyardConfigError, caching::CacheHeaders,
|
||||
chat_completions, messages, responses,
|
||||
};
|
||||
|
||||
const SWITCHYARD_MODEL: &str = "switchyard/algorithm";
|
||||
|
|
@ -37,7 +44,7 @@ pub(crate) struct Routes(HashMap<String, Route>);
|
|||
impl Routes {
|
||||
pub(crate) fn from_models(
|
||||
model_list: &[Model],
|
||||
deployments: Arc<ModelRouter>,
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<Self, SwitchyardConfigError> {
|
||||
let mut routes = HashMap::new();
|
||||
let mut seen = HashSet::new();
|
||||
|
|
@ -61,16 +68,16 @@ impl Routes {
|
|||
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
|
||||
let config = AlgorithmSpec::deserialize(config)
|
||||
.map_err(|error| invalid(model, error.to_string()))?;
|
||||
let targets = targets(model, &config, &deployments)?;
|
||||
let targets = targets(model, &config, model_list, deployments)?;
|
||||
let (algorithm, runtime_models) = config
|
||||
.build_with_runtime_models(&model.model_name, &targets)
|
||||
.build_with_runtime_models(&model.model_name, &targets.models)
|
||||
.map_err(|error| invalid(model, error.to_string()))?;
|
||||
routes.insert(
|
||||
model.model_name.clone(),
|
||||
Route {
|
||||
algorithm,
|
||||
runtime_models: Arc::new(runtime_models),
|
||||
deployments: Arc::clone(&deployments),
|
||||
targets: Arc::new(targets.by_model),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -85,38 +92,39 @@ impl Routes {
|
|||
pub(crate) struct Route {
|
||||
algorithm: Arc<dyn Algorithm>,
|
||||
runtime_models: Arc<RuntimeModels>,
|
||||
deployments: Arc<ModelRouter>,
|
||||
targets: Arc<HashMap<ModelId, Target>>,
|
||||
}
|
||||
|
||||
impl Route {
|
||||
pub(crate) async fn execute(
|
||||
&self,
|
||||
route: ChatCompletionsRoute,
|
||||
cache_policy: CachePolicy,
|
||||
gateway: &Gateway,
|
||||
cache_options: CacheOptions,
|
||||
wire_format: WireFormat,
|
||||
body: Map<String, Value>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<(Value, HeaderMap), Error> {
|
||||
let llm_request = decode_request(WireFormat::OpenAiChat, &Value::Object(body))
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let llm_request = decode_request(wire_format, &Value::Object(body))
|
||||
.map_err(|error| Error::InvalidBody(error.to_string()))?;
|
||||
// Reject unsupported streaming before a classifier or judge can make a model call.
|
||||
// Switchyard routes currently support buffered calls only.
|
||||
if llm_request.stream {
|
||||
return Err(Error::Route(RouteError::Unsupported("streaming")));
|
||||
}
|
||||
let extensions = llm_request.extensions.clone();
|
||||
let mut metadata = Metadata::from_headers(&headers);
|
||||
metadata.http_headers = Some(headers);
|
||||
metadata.wire_format = Some(WireFormat::OpenAiChat);
|
||||
metadata.wire_format = None;
|
||||
let request = Request {
|
||||
llm_request,
|
||||
raw_request: None,
|
||||
metadata: Some(metadata),
|
||||
};
|
||||
let client = Arc::new(LiteLlmClient {
|
||||
route,
|
||||
deployments: Arc::clone(&self.deployments),
|
||||
cache_policy,
|
||||
routes: ProviderRoutes::new(gateway, &cache_options),
|
||||
targets: Arc::clone(&self.targets),
|
||||
cache_policy: cache_options.policy,
|
||||
});
|
||||
let (_, response) = switchyard_llm_client::run(
|
||||
let (selected_model, response) = switchyard_llm_client::run(
|
||||
Arc::clone(&self.algorithm),
|
||||
ClientRouter::single(client),
|
||||
request,
|
||||
|
|
@ -125,6 +133,7 @@ impl Route {
|
|||
)
|
||||
.await
|
||||
.map_err(map_libsy_error)?;
|
||||
let served_model = response.served_model().cloned().unwrap_or(selected_model);
|
||||
let upstream_headers = response.upstream_headers;
|
||||
let response = response
|
||||
.llm_response
|
||||
|
|
@ -133,18 +142,48 @@ impl Route {
|
|||
.map_err(map_client_error)?;
|
||||
let body = encode_aggregated_response_with_extensions(
|
||||
&response,
|
||||
WireFormat::OpenAiChat,
|
||||
None,
|
||||
wire_format,
|
||||
Some(served_model.as_str()),
|
||||
&extensions,
|
||||
)
|
||||
.map_err(|error| Error::Internal(error.to_string()))?;
|
||||
Ok((body, upstream_headers))
|
||||
let mut response = Json(body).into_response();
|
||||
response.headers_mut().extend(upstream_headers);
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
struct Target {
|
||||
deployment: Deployment,
|
||||
wire_format: WireFormat,
|
||||
}
|
||||
|
||||
struct ProviderRoutes {
|
||||
chat: ChatCompletionsRoute,
|
||||
responses: ResponsesRoute,
|
||||
messages: MessagesRoute,
|
||||
}
|
||||
|
||||
impl ProviderRoutes {
|
||||
fn new(gateway: &Gateway, options: &CacheOptions) -> Self {
|
||||
let mut routes = Self {
|
||||
chat: gateway.chat_completions.clone(),
|
||||
responses: gateway.responses.clone(),
|
||||
messages: gateway.messages.clone(),
|
||||
};
|
||||
if let Some(cache) = &gateway.cache {
|
||||
let scoped = || ScopedCache::new(Arc::clone(cache), options.scope.clone());
|
||||
routes.chat = routes.chat.with_cache(scoped());
|
||||
routes.responses = routes.responses.with_cache(scoped());
|
||||
routes.messages = routes.messages.with_cache(scoped());
|
||||
}
|
||||
routes
|
||||
}
|
||||
}
|
||||
|
||||
struct LiteLlmClient {
|
||||
route: ChatCompletionsRoute,
|
||||
deployments: Arc<ModelRouter>,
|
||||
routes: ProviderRoutes,
|
||||
targets: Arc<HashMap<ModelId, Target>>,
|
||||
cache_policy: CachePolicy,
|
||||
}
|
||||
|
||||
|
|
@ -156,67 +195,169 @@ impl RoutedLlmClient for LiteLlmClient {
|
|||
message: "routed request has no model".to_string(),
|
||||
});
|
||||
};
|
||||
let deployment =
|
||||
self.deployments
|
||||
.get(model)
|
||||
.ok_or_else(|| LlmClientError::Configuration {
|
||||
message: format!("unknown LiteLLM deployment {model}"),
|
||||
})?;
|
||||
let target = self
|
||||
.targets
|
||||
.get(model)
|
||||
.ok_or_else(|| LlmClientError::Configuration {
|
||||
message: format!("unknown LiteLLM deployment {model}"),
|
||||
})?;
|
||||
let Request {
|
||||
llm_request,
|
||||
metadata,
|
||||
..
|
||||
} = request;
|
||||
let Value::Object(body) = encode_request(&llm_request, WireFormat::OpenAiChat)
|
||||
let Value::Object(body) = encode_request(&llm_request, target.wire_format)
|
||||
.map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?
|
||||
else {
|
||||
return Err(LlmClientError::RequestEncoding(
|
||||
"OpenAI Chat request is not an object".to_string(),
|
||||
));
|
||||
return Err(LlmClientError::RequestEncoding(format!(
|
||||
"{} request is not an object",
|
||||
target.wire_format
|
||||
)));
|
||||
};
|
||||
let cache_headers = CacheHeaders::default();
|
||||
let response = self
|
||||
.route
|
||||
.execute(
|
||||
provider_request(deployment, body),
|
||||
&cache_headers,
|
||||
self.cache_policy,
|
||||
)
|
||||
.await
|
||||
.map_err(map_route_error)?;
|
||||
let body = serde_json::to_value(response)
|
||||
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
|
||||
let response = decode_aggregated_response(&body, WireFormat::OpenAiChat)
|
||||
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
|
||||
let response = match target.wire_format {
|
||||
WireFormat::OpenAiChat => {
|
||||
let response = self
|
||||
.routes
|
||||
.chat
|
||||
.execute(
|
||||
chat_completions::provider_request(&target.deployment, body),
|
||||
&cache_headers,
|
||||
self.cache_policy,
|
||||
)
|
||||
.await
|
||||
.map_err(map_route_error)?;
|
||||
decode_complete(response, target.wire_format)?
|
||||
}
|
||||
WireFormat::OpenAiResponses => {
|
||||
let response = self
|
||||
.routes
|
||||
.responses
|
||||
.execute(
|
||||
responses::provider_request(&target.deployment, body),
|
||||
&cache_headers,
|
||||
self.cache_policy,
|
||||
)
|
||||
.await
|
||||
.map_err(map_route_error)?;
|
||||
decode_output(response, target.wire_format)?
|
||||
}
|
||||
WireFormat::AnthropicMessages => {
|
||||
let empty_headers = HeaderMap::new();
|
||||
let headers = metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.http_headers.as_ref())
|
||||
.unwrap_or(&empty_headers);
|
||||
let call = messages::provider_request(&target.deployment, body, headers)
|
||||
.map_err(map_route_error)?;
|
||||
let response = self
|
||||
.routes
|
||||
.messages
|
||||
.execute(call, &cache_headers, self.cache_policy)
|
||||
.await
|
||||
.map_err(map_route_error)?;
|
||||
decode_output(response, target.wire_format)?
|
||||
}
|
||||
};
|
||||
Ok(Response {
|
||||
llm_response: LlmResponse::Agg(response),
|
||||
llm_response: response,
|
||||
metadata,
|
||||
upstream_headers: cache_headers.headers(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_complete(
|
||||
response: impl serde::Serialize,
|
||||
wire_format: WireFormat,
|
||||
) -> Result<LlmResponse, LlmClientError> {
|
||||
let body = serde_json::to_value(response)
|
||||
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
|
||||
decode_aggregated_response(&body, wire_format)
|
||||
.map(LlmResponse::Agg)
|
||||
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))
|
||||
}
|
||||
|
||||
fn decode_output<T, H>(
|
||||
response: CallOutput<T, H, bytes::Bytes, RouteError>,
|
||||
wire_format: WireFormat,
|
||||
) -> Result<LlmResponse, LlmClientError>
|
||||
where
|
||||
T: serde::Serialize,
|
||||
{
|
||||
match response {
|
||||
CallOutput::Complete(response) => decode_complete(response, wire_format),
|
||||
CallOutput::Stream { .. } => Err(LlmClientError::ResponseTranslation(
|
||||
"LiteLLM streamed a response for a buffered Switchyard call".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
struct Targets {
|
||||
models: BTreeMap<String, ModelId>,
|
||||
by_model: HashMap<ModelId, Target>,
|
||||
}
|
||||
|
||||
fn targets(
|
||||
route: &Model,
|
||||
config: &AlgorithmSpec,
|
||||
model_list: &[Model],
|
||||
deployments: &ModelRouter,
|
||||
) -> Result<BTreeMap<String, ModelId>, SwitchyardConfigError> {
|
||||
config
|
||||
.callable_target_names()
|
||||
.into_iter()
|
||||
.map(|target| {
|
||||
let deployment = deployments
|
||||
.get(target)
|
||||
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
|
||||
if deployment.model == SWITCHYARD_MODEL {
|
||||
return Err(invalid(
|
||||
route,
|
||||
format!("target {target:?} is another Switchyard route"),
|
||||
));
|
||||
}
|
||||
Ok((target.to_string(), ModelId::from(target)))
|
||||
})
|
||||
.collect()
|
||||
) -> Result<Targets, SwitchyardConfigError> {
|
||||
let mut models = BTreeMap::new();
|
||||
let mut by_model = HashMap::new();
|
||||
for target in config.callable_target_names() {
|
||||
let configured = model_list
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|model| model.model_name == target)
|
||||
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
|
||||
let deployment = deployments
|
||||
.get(target)
|
||||
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
|
||||
if deployment.model == SWITCHYARD_MODEL {
|
||||
return Err(invalid(
|
||||
route,
|
||||
format!("target {target:?} is another Switchyard route"),
|
||||
));
|
||||
}
|
||||
let model = ModelId::from(target);
|
||||
models.insert(target.to_string(), model.clone());
|
||||
by_model.insert(
|
||||
model,
|
||||
Target {
|
||||
deployment: deployment.clone(),
|
||||
wire_format: target_wire_format(route, configured)?,
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Targets { models, by_model })
|
||||
}
|
||||
|
||||
// Caller format and target execution format are independent, so each target declares its own.
|
||||
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, SwitchyardConfigError> {
|
||||
let value = target
|
||||
.litellm_params
|
||||
.extra
|
||||
.get("switchyard_wire_format")
|
||||
.ok_or_else(|| {
|
||||
invalid(
|
||||
route,
|
||||
format!(
|
||||
"target {:?} requires switchyard_wire_format",
|
||||
target.model_name
|
||||
),
|
||||
)
|
||||
})?;
|
||||
WireFormat::deserialize(value).map_err(|error| {
|
||||
invalid(
|
||||
route,
|
||||
format!(
|
||||
"target {:?} has invalid switchyard_wire_format: {error}",
|
||||
target.model_name,
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn invalid(model: &Model, message: impl Into<String>) -> SwitchyardConfigError {
|
||||
|
|
@ -308,4 +449,16 @@ mod tests {
|
|||
Error::Route(RouteError::Auth(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn targets_require_an_explicit_wire_format() {
|
||||
let config = litellm_config::Config::from_yaml(
|
||||
"model_list:\n - model_name: target\n litellm_params: { model: upstream }\n - model_name: route\n litellm_params: { model: switchyard/algorithm }\n",
|
||||
)
|
||||
.expect("config");
|
||||
|
||||
let error = target_wire_format(&config.model_list[1], &config.model_list[0])
|
||||
.expect_err("ambiguous target format");
|
||||
assert!(error.reason.contains("requires switchyard_wire_format"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,18 +73,21 @@ model_list:
|
|||
api_base: {base}/v1
|
||||
api_key: test-key
|
||||
custom_llm_provider: openai_like
|
||||
switchyard_wire_format: openai_chat
|
||||
- model_name: strong
|
||||
litellm_params:
|
||||
model: strong-model
|
||||
api_base: {base}/v1
|
||||
api_key: test-key
|
||||
custom_llm_provider: openai_like
|
||||
switchyard_wire_format: openai_chat
|
||||
- model_name: judge
|
||||
litellm_params:
|
||||
model: judge-model
|
||||
api_base: {base}/v1
|
||||
api_key: test-key
|
||||
custom_llm_provider: openai_like
|
||||
switchyard_wire_format: openai_chat
|
||||
- model_name: coding-router
|
||||
litellm_params:
|
||||
model: switchyard/algorithm
|
||||
|
|
@ -104,20 +107,8 @@ model_list:
|
|||
base = upstream.uri()
|
||||
))
|
||||
.expect("valid config");
|
||||
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
|
||||
let resources = CoreResources::new(pool);
|
||||
let http = Resolution::from(&HttpSettings::default()).config;
|
||||
let gateway = Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
Arc::new(support::NoSecrets),
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)
|
||||
.expect("gateway")
|
||||
.with_switchyard_models(&config.model_list)
|
||||
.expect("Switchyard routes");
|
||||
let app = support::authenticated_app(
|
||||
gateway,
|
||||
let app = switchyard_app(
|
||||
&config,
|
||||
Permissions::Only(Arc::from([
|
||||
AccessRequest::Model {
|
||||
name: "coding-router".into(),
|
||||
|
|
@ -128,7 +119,6 @@ model_list:
|
|||
deployment: "switchyard/algorithm".into(),
|
||||
},
|
||||
])),
|
||||
None,
|
||||
);
|
||||
|
||||
let response = support::post(
|
||||
|
|
@ -155,7 +145,7 @@ model_list:
|
|||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["model"], "strong-model");
|
||||
assert_eq!(body["model"], "strong");
|
||||
assert_eq!(body["choices"][0]["message"]["content"], "fallback answer");
|
||||
|
||||
let response = support::post(
|
||||
|
|
@ -171,3 +161,136 @@ model_list:
|
|||
let body = support::json(response).await;
|
||||
assert_eq!(body["choices"][0]["message"]["content"], "OK");
|
||||
}
|
||||
|
||||
// The caller API and selected target API are independent and use the same host bridge.
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn configured_switchyard_routes_translate_responses_and_messages() {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/responses"))
|
||||
.and(body_partial_json(json!({"model": "responses-model"})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"id": "resp_1",
|
||||
"model": "responses-model",
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "responses answer"}]
|
||||
}],
|
||||
"provider_extra": true,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
|
||||
})))
|
||||
.expect(2)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages"))
|
||||
.and(body_partial_json(json!({"model": "messages-model"})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "messages-model",
|
||||
"content": [{"type": "text", "text": "messages answer"}],
|
||||
"provider_extra": true,
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
|
||||
let config = Config::from_yaml(&format!(
|
||||
r#"
|
||||
model_list:
|
||||
- model_name: responses-target
|
||||
litellm_params:
|
||||
model: openai/responses-model
|
||||
api_base: {base}
|
||||
api_key: test-key
|
||||
custom_llm_provider: openai
|
||||
switchyard_wire_format: openai_responses
|
||||
- model_name: messages-target
|
||||
litellm_params:
|
||||
model: anthropic/messages-model
|
||||
api_base: {base}
|
||||
api_key: test-key
|
||||
custom_llm_provider: anthropic
|
||||
switchyard_wire_format: anthropic_messages
|
||||
- model_name: responses-router
|
||||
litellm_params:
|
||||
model: switchyard/algorithm
|
||||
switchyard_config:
|
||||
type: passthrough
|
||||
target: responses-target
|
||||
- model_name: messages-router
|
||||
litellm_params:
|
||||
model: switchyard/algorithm
|
||||
switchyard_config:
|
||||
type: passthrough
|
||||
target: messages-target
|
||||
"#,
|
||||
base = upstream.uri()
|
||||
))
|
||||
.expect("valid config");
|
||||
let app = switchyard_app(&config, Permissions::All);
|
||||
|
||||
let response = support::post(
|
||||
app.clone(),
|
||||
"/v1/responses",
|
||||
json!({"model": "responses-router", "input": "hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["model"], "responses-target");
|
||||
assert_eq!(body["output"][0]["content"][0]["text"], "responses answer");
|
||||
assert_eq!(body["provider_extra"], true);
|
||||
|
||||
let response = support::post(
|
||||
app.clone(),
|
||||
"/v1/messages",
|
||||
json!({
|
||||
"model": "messages-router",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 16
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["model"], "messages-target");
|
||||
assert_eq!(body["content"][0]["text"], "messages answer");
|
||||
assert_eq!(body["provider_extra"], true);
|
||||
|
||||
let response = support::post(
|
||||
app,
|
||||
"/v1/chat/completions",
|
||||
json!({
|
||||
"model": "responses-router",
|
||||
"messages": [{"role": "user", "content": "hello"}]
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["model"], "responses-target");
|
||||
assert_eq!(body["choices"][0]["message"]["content"], "responses answer");
|
||||
}
|
||||
|
||||
fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router {
|
||||
let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))));
|
||||
let http = Resolution::from(&HttpSettings::default()).config;
|
||||
let gateway = Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
Arc::new(support::NoSecrets),
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)
|
||||
.expect("gateway")
|
||||
.with_switchyard_models(&config.model_list)
|
||||
.expect("Switchyard routes");
|
||||
support::authenticated_app(gateway, permissions, None)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue