feat(gateway): route buffered APIs through Switchyard

Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
Alex Fournier 2026-10-07 12:11:16 -07:00 • committed by Joshua Valluru
parent 1bc923cce9
commit bcf0bdafd3
6 changed files with 443 additions and 121 deletions

View file

@ -10,6 +10,8 @@ use axum::{
};
use litellm_inference_chat::types::{ChatCompletionsCall, ChatCompletionsRequest};
use serde_json::{Map, Value};
#[cfg(feature = "switchyard")]
use switchyard_protocol::WireFormat;
use crate::{Deployment, Error, Gateway, JsonObject, request};
@ -67,6 +69,18 @@ async fn handle(
.and_then(Value::as_str)
.and_then(|model| gateway.switchyard_routes.get(model));
let (body, cache_options) = crate::caching::prepare(identity, body)?;
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
return switchyard_route
.execute(
gateway,
cache_options,
WireFormat::OpenAiChat,
body,
inbound_headers,
)
.await;
}
let route = gateway.chat_completions.clone();
let route = match &gateway.cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
@ -75,16 +89,6 @@ async fn handle(
)),
None => route,
};
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
let (body, upstream_headers) = switchyard_route
.execute(route, cache_options.policy, body, inbound_headers)
.await?;
let mut response = Json(body).into_response();
response.headers_mut().extend(upstream_headers);
return Ok(response);
}
let headers = crate::caching::CacheHeaders::default();
let response = litellm_host_http::serve_unary(
route.machine(

View file

@ -101,8 +101,7 @@ impl Gateway {
mut self,
models: &[litellm_config::Model],
) -> Result<Self, SwitchyardConfigError> {
self.switchyard_routes =
switchyard::Routes::from_models(models, Arc::new(self.models.clone()))?;
self.switchyard_routes = switchyard::Routes::from_models(models, &self.models)?;
Ok(self)
}
}

View file

@ -14,6 +14,8 @@ use litellm_host_http::Sse;
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
use serde_json::{Map, Value};
#[cfg(feature = "switchyard")]
use switchyard_protocol::WireFormat;
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
@ -34,7 +36,7 @@ pub async fn create(
body: Result<JsonObject, Error>,
) -> impl IntoResponse {
let result = match body {
Ok(JsonObject(body)) => handle(&gateway, &identity, &headers, body).await,
Ok(JsonObject(body)) => handle(&gateway, &identity, headers, body).await,
Err(error) => Err(error),
};
result.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
@ -43,12 +45,29 @@ pub async fn create(
async fn handle(
gateway: &Gateway,
identity: &AuthenticatedRequest,
headers: &HeaderMap,
headers: HeaderMap,
body: Map<String, Value>,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(gateway, &body)?;
request::authorize_model(identity, deployment, &body).await?;
#[cfg(feature = "switchyard")]
let switchyard_route = body
.get("model")
.and_then(Value::as_str)
.and_then(|model| gateway.switchyard_routes.get(model));
let (body, cache_options) = crate::caching::prepare(identity, body)?;
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
return switchyard_route
.execute(
gateway,
cache_options,
WireFormat::AnthropicMessages,
body,
headers,
)
.await;
}
let route = gateway.messages.clone();
let route = match &gateway.cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
@ -58,7 +77,7 @@ async fn handle(
None => route,
};
let call = project(deployment, body, headers)?;
let call = provider_request(deployment, body, &headers)?;
let machine = route.machine(call, cache_options.policy);
let stream =
Sse::<Messages, _, _>::new(Json, |error| Bytes::from(Error::from(error).sse_frame()));
@ -67,18 +86,14 @@ async fn handle(
Ok(headers.apply(response))
}
fn project(
pub(crate) fn provider_request(
deployment: &Deployment,
body: Map<String, Value>,
mut body: Map<String, Value>,
headers: &HeaderMap,
) -> Result<MessagesCall, Error> {
let body = body
.into_iter()
.map(|(name, value)| match name.as_str() {
"model" => (name, Value::from(deployment.model.as_str())),
_ => (name, value),
})
.collect();
) -> Result<MessagesCall, litellm_inference::RouteError> {
if let Some(model) = body.get_mut("model") {
*model = Value::from(deployment.model.as_str());
}
Ok(MessagesCall {
body: messages_body(body)?,
api_key: deployment.api_key.clone(),

View file

@ -1,21 +1,43 @@
use std::sync::Arc;
#[cfg(feature = "switchyard")]
use axum::http::HeaderMap;
use axum::{Json, body::Bytes, extract::State, response::Response};
use litellm_gateway_auth::AuthenticatedRequest;
use litellm_host_http::Sse;
use litellm_inference_responses::{route::Responses, types::ResponsesCall};
use serde_json::json;
use serde_json::{Map, Value, json};
#[cfg(feature = "switchyard")]
use switchyard_protocol::WireFormat;
use crate::{Error, Gateway, JsonObject, request};
use crate::{Deployment, Error, Gateway, JsonObject, request};
pub(crate) async fn create(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
#[cfg(feature = "switchyard")] headers: HeaderMap,
JsonObject(body): JsonObject,
) -> Result<Response, Error> {
let deployment = request::resolve_deployment(&gateway, &body)?;
request::authorize_model(&identity, deployment, &body).await?;
#[cfg(feature = "switchyard")]
let switchyard_route = body
.get("model")
.and_then(Value::as_str)
.and_then(|model| gateway.switchyard_routes.get(model));
let (body, cache_options) = crate::caching::prepare(&identity, body)?;
#[cfg(feature = "switchyard")]
if let Some(switchyard_route) = switchyard_route {
return switchyard_route
.execute(
&gateway,
cache_options,
WireFormat::OpenAiResponses,
body,
headers,
)
.await;
}
let route = gateway.responses.clone();
let route = match &gateway.cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
@ -25,19 +47,7 @@ pub(crate) async fn create(
None => route,
};
let call = ResponsesCall {
model: deployment.model.clone(),
input: body.get("input").cloned().unwrap_or_default(),
optional_params: body
.into_iter()
.filter(|(name, _)| !matches!(name.as_str(), "model" | "input"))
.collect(),
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
};
let call = provider_request(deployment, body);
let machine = route.machine(call, cache_options.policy);
let stream = Sse::<Responses, _, _>::new(Json, |error| {
let error = Error::from(error);
@ -50,3 +60,21 @@ pub(crate) async fn create(
let response = litellm_host_http::serve(machine, (), headers.clone(), stream, None).await?;
Ok(headers.apply(response))
}
pub(crate) fn provider_request(
deployment: &Deployment,
mut body: Map<String, Value>,
) -> ResponsesCall {
let input = body.remove("input").unwrap_or_default();
body.remove("model");
ResponsesCall {
model: deployment.model.clone(),
input,
optional_params: body,
api_key: deployment.api_key.clone(),
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
extra_headers: None,
timeout: deployment.timeout,
}
}

View file

@ -4,11 +4,18 @@ use std::{
};
use async_trait::async_trait;
use axum::http::HeaderMap;
use litellm_cache_response::CachePolicy;
use axum::{
Json,
http::HeaderMap,
response::{IntoResponse, Response as HttpResponse},
};
use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache};
use litellm_config::Model;
use litellm_host::call::CallOutput;
use litellm_inference::RouteError;
use litellm_inference_chat::ChatCompletionsRoute;
use litellm_inference_messages::MessagesRoute;
use litellm_inference_responses::ResponsesRoute;
use litellm_http::transport::Error as TransportError;
use serde::Deserialize;
use serde_json::{Map, Value};
@ -25,8 +32,8 @@ use switchyard_translation::{
};
use crate::{
Error, ModelRouter, SwitchyardConfigError, caching::CacheHeaders,
chat_completions::provider_request,
Deployment, Error, Gateway, ModelRouter, SwitchyardConfigError, caching::CacheHeaders,
chat_completions, messages, responses,
};
const SWITCHYARD_MODEL: &str = "switchyard/algorithm";
@ -37,7 +44,7 @@ pub(crate) struct Routes(HashMap<String, Route>);
impl Routes {
pub(crate) fn from_models(
model_list: &[Model],
deployments: Arc<ModelRouter>,
deployments: &ModelRouter,
) -> Result<Self, SwitchyardConfigError> {
let mut routes = HashMap::new();
let mut seen = HashSet::new();
@ -61,16 +68,16 @@ impl Routes {
.ok_or_else(|| invalid(model, "switchyard_config is required"))?;
let config = AlgorithmSpec::deserialize(config)
.map_err(|error| invalid(model, error.to_string()))?;
let targets = targets(model, &config, &deployments)?;
let targets = targets(model, &config, model_list, deployments)?;
let (algorithm, runtime_models) = config
.build_with_runtime_models(&model.model_name, &targets)
.build_with_runtime_models(&model.model_name, &targets.models)
.map_err(|error| invalid(model, error.to_string()))?;
routes.insert(
model.model_name.clone(),
Route {
algorithm,
runtime_models: Arc::new(runtime_models),
deployments: Arc::clone(&deployments),
targets: Arc::new(targets.by_model),
},
);
}
@ -85,38 +92,39 @@ impl Routes {
pub(crate) struct Route {
algorithm: Arc<dyn Algorithm>,
runtime_models: Arc<RuntimeModels>,
deployments: Arc<ModelRouter>,
targets: Arc<HashMap<ModelId, Target>>,
}
impl Route {
pub(crate) async fn execute(
&self,
route: ChatCompletionsRoute,
cache_policy: CachePolicy,
gateway: &Gateway,
cache_options: CacheOptions,
wire_format: WireFormat,
body: Map<String, Value>,
headers: HeaderMap,
) -> Result<(Value, HeaderMap), Error> {
let llm_request = decode_request(WireFormat::OpenAiChat, &Value::Object(body))
) -> Result<HttpResponse, Error> {
let llm_request = decode_request(wire_format, &Value::Object(body))
.map_err(|error| Error::InvalidBody(error.to_string()))?;
// Reject unsupported streaming before a classifier or judge can make a model call.
// Switchyard routes currently support buffered calls only.
if llm_request.stream {
return Err(Error::Route(RouteError::Unsupported("streaming")));
}
let extensions = llm_request.extensions.clone();
let mut metadata = Metadata::from_headers(&headers);
metadata.http_headers = Some(headers);
metadata.wire_format = Some(WireFormat::OpenAiChat);
metadata.wire_format = None;
let request = Request {
llm_request,
raw_request: None,
metadata: Some(metadata),
};
let client = Arc::new(LiteLlmClient {
route,
deployments: Arc::clone(&self.deployments),
cache_policy,
routes: ProviderRoutes::new(gateway, &cache_options),
targets: Arc::clone(&self.targets),
cache_policy: cache_options.policy,
});
let (_, response) = switchyard_llm_client::run(
let (selected_model, response) = switchyard_llm_client::run(
Arc::clone(&self.algorithm),
ClientRouter::single(client),
request,
@ -125,6 +133,7 @@ impl Route {
)
.await
.map_err(map_libsy_error)?;
let served_model = response.served_model().cloned().unwrap_or(selected_model);
let upstream_headers = response.upstream_headers;
let response = response
.llm_response
@ -133,18 +142,48 @@ impl Route {
.map_err(map_client_error)?;
let body = encode_aggregated_response_with_extensions(
&response,
WireFormat::OpenAiChat,
None,
wire_format,
Some(served_model.as_str()),
&extensions,
)
.map_err(|error| Error::Internal(error.to_string()))?;
Ok((body, upstream_headers))
let mut response = Json(body).into_response();
response.headers_mut().extend(upstream_headers);
Ok(response)
}
}
struct Target {
deployment: Deployment,
wire_format: WireFormat,
}
struct ProviderRoutes {
chat: ChatCompletionsRoute,
responses: ResponsesRoute,
messages: MessagesRoute,
}
impl ProviderRoutes {
fn new(gateway: &Gateway, options: &CacheOptions) -> Self {
let mut routes = Self {
chat: gateway.chat_completions.clone(),
responses: gateway.responses.clone(),
messages: gateway.messages.clone(),
};
if let Some(cache) = &gateway.cache {
let scoped = || ScopedCache::new(Arc::clone(cache), options.scope.clone());
routes.chat = routes.chat.with_cache(scoped());
routes.responses = routes.responses.with_cache(scoped());
routes.messages = routes.messages.with_cache(scoped());
}
routes
}
}
struct LiteLlmClient {
route: ChatCompletionsRoute,
deployments: Arc<ModelRouter>,
routes: ProviderRoutes,
targets: Arc<HashMap<ModelId, Target>>,
cache_policy: CachePolicy,
}
@ -156,67 +195,169 @@ impl RoutedLlmClient for LiteLlmClient {
message: "routed request has no model".to_string(),
});
};
let deployment =
self.deployments
.get(model)
.ok_or_else(|| LlmClientError::Configuration {
message: format!("unknown LiteLLM deployment {model}"),
})?;
let target = self
.targets
.get(model)
.ok_or_else(|| LlmClientError::Configuration {
message: format!("unknown LiteLLM deployment {model}"),
})?;
let Request {
llm_request,
metadata,
..
} = request;
let Value::Object(body) = encode_request(&llm_request, WireFormat::OpenAiChat)
let Value::Object(body) = encode_request(&llm_request, target.wire_format)
.map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?
else {
return Err(LlmClientError::RequestEncoding(
"OpenAI Chat request is not an object".to_string(),
));
return Err(LlmClientError::RequestEncoding(format!(
"{} request is not an object",
target.wire_format
)));
};
let cache_headers = CacheHeaders::default();
let response = self
.route
.execute(
provider_request(deployment, body),
&cache_headers,
self.cache_policy,
)
.await
.map_err(map_route_error)?;
let body = serde_json::to_value(response)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
let response = decode_aggregated_response(&body, WireFormat::OpenAiChat)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
let response = match target.wire_format {
WireFormat::OpenAiChat => {
let response = self
.routes
.chat
.execute(
chat_completions::provider_request(&target.deployment, body),
&cache_headers,
self.cache_policy,
)
.await
.map_err(map_route_error)?;
decode_complete(response, target.wire_format)?
}
WireFormat::OpenAiResponses => {
let response = self
.routes
.responses
.execute(
responses::provider_request(&target.deployment, body),
&cache_headers,
self.cache_policy,
)
.await
.map_err(map_route_error)?;
decode_output(response, target.wire_format)?
}
WireFormat::AnthropicMessages => {
let empty_headers = HeaderMap::new();
let headers = metadata
.as_ref()
.and_then(|metadata| metadata.http_headers.as_ref())
.unwrap_or(&empty_headers);
let call = messages::provider_request(&target.deployment, body, headers)
.map_err(map_route_error)?;
let response = self
.routes
.messages
.execute(call, &cache_headers, self.cache_policy)
.await
.map_err(map_route_error)?;
decode_output(response, target.wire_format)?
}
};
Ok(Response {
llm_response: LlmResponse::Agg(response),
llm_response: response,
metadata,
upstream_headers: cache_headers.headers(),
})
}
}
fn decode_complete(
response: impl serde::Serialize,
wire_format: WireFormat,
) -> Result<LlmResponse, LlmClientError> {
let body = serde_json::to_value(response)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
decode_aggregated_response(&body, wire_format)
.map(LlmResponse::Agg)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))
}
fn decode_output<T, H>(
response: CallOutput<T, H, bytes::Bytes, RouteError>,
wire_format: WireFormat,
) -> Result<LlmResponse, LlmClientError>
where
T: serde::Serialize,
{
match response {
CallOutput::Complete(response) => decode_complete(response, wire_format),
CallOutput::Stream { .. } => Err(LlmClientError::ResponseTranslation(
"LiteLLM streamed a response for a buffered Switchyard call".to_string(),
)),
}
}
struct Targets {
models: BTreeMap<String, ModelId>,
by_model: HashMap<ModelId, Target>,
}
fn targets(
route: &Model,
config: &AlgorithmSpec,
model_list: &[Model],
deployments: &ModelRouter,
) -> Result<BTreeMap<String, ModelId>, SwitchyardConfigError> {
config
.callable_target_names()
.into_iter()
.map(|target| {
let deployment = deployments
.get(target)
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
if deployment.model == SWITCHYARD_MODEL {
return Err(invalid(
route,
format!("target {target:?} is another Switchyard route"),
));
}
Ok((target.to_string(), ModelId::from(target)))
})
.collect()
) -> Result<Targets, SwitchyardConfigError> {
let mut models = BTreeMap::new();
let mut by_model = HashMap::new();
for target in config.callable_target_names() {
let configured = model_list
.iter()
.rev()
.find(|model| model.model_name == target)
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
let deployment = deployments
.get(target)
.ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?;
if deployment.model == SWITCHYARD_MODEL {
return Err(invalid(
route,
format!("target {target:?} is another Switchyard route"),
));
}
let model = ModelId::from(target);
models.insert(target.to_string(), model.clone());
by_model.insert(
model,
Target {
deployment: deployment.clone(),
wire_format: target_wire_format(route, configured)?,
},
);
}
Ok(Targets { models, by_model })
}
// Caller format and target execution format are independent, so each target declares its own.
fn target_wire_format(route: &Model, target: &Model) -> Result<WireFormat, SwitchyardConfigError> {
let value = target
.litellm_params
.extra
.get("switchyard_wire_format")
.ok_or_else(|| {
invalid(
route,
format!(
"target {:?} requires switchyard_wire_format",
target.model_name
),
)
})?;
WireFormat::deserialize(value).map_err(|error| {
invalid(
route,
format!(
"target {:?} has invalid switchyard_wire_format: {error}",
target.model_name,
),
)
})
}
fn invalid(model: &Model, message: impl Into<String>) -> SwitchyardConfigError {
@ -308,4 +449,16 @@ mod tests {
Error::Route(RouteError::Auth(_))
));
}
#[test]
fn targets_require_an_explicit_wire_format() {
let config = litellm_config::Config::from_yaml(
"model_list:\n - model_name: target\n litellm_params: { model: upstream }\n - model_name: route\n litellm_params: { model: switchyard/algorithm }\n",
)
.expect("config");
let error = target_wire_format(&config.model_list[1], &config.model_list[0])
.expect_err("ambiguous target format");
assert!(error.reason.contains("requires switchyard_wire_format"));
}
}

View file

@ -73,18 +73,21 @@ model_list:
api_base: {base}/v1
api_key: test-key
custom_llm_provider: openai_like
switchyard_wire_format: openai_chat
- model_name: strong
litellm_params:
model: strong-model
api_base: {base}/v1
api_key: test-key
custom_llm_provider: openai_like
switchyard_wire_format: openai_chat
- model_name: judge
litellm_params:
model: judge-model
api_base: {base}/v1
api_key: test-key
custom_llm_provider: openai_like
switchyard_wire_format: openai_chat
- model_name: coding-router
litellm_params:
model: switchyard/algorithm
@ -104,20 +107,8 @@ model_list:
base = upstream.uri()
))
.expect("valid config");
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
let resources = CoreResources::new(pool);
let http = Resolution::from(&HttpSettings::default()).config;
let gateway = Gateway::new(
resources,
http,
Arc::new(support::NoSecrets),
ModelRouter::from_model_list(&config.model_list),
)
.expect("gateway")
.with_switchyard_models(&config.model_list)
.expect("Switchyard routes");
let app = support::authenticated_app(
gateway,
let app = switchyard_app(
&config,
Permissions::Only(Arc::from([
AccessRequest::Model {
name: "coding-router".into(),
@ -128,7 +119,6 @@ model_list:
deployment: "switchyard/algorithm".into(),
},
])),
None,
);
let response = support::post(
@ -155,7 +145,7 @@ model_list:
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "strong-model");
assert_eq!(body["model"], "strong");
assert_eq!(body["choices"][0]["message"]["content"], "fallback answer");
let response = support::post(
@ -171,3 +161,136 @@ model_list:
let body = support::json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "OK");
}
// The caller API and selected target API are independent and use the same host bridge.
#[rstest]
#[tokio::test]
async fn configured_switchyard_routes_translate_responses_and_messages() {
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/responses"))
.and(body_partial_json(json!({"model": "responses-model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "resp_1",
"model": "responses-model",
"status": "completed",
"output": [{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "responses answer"}]
}],
"provider_extra": true,
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
})))
.expect(2)
.mount(&upstream)
.await;
Mock::given(method("POST"))
.and(path("/v1/messages"))
.and(body_partial_json(json!({"model": "messages-model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "messages-model",
"content": [{"type": "text", "text": "messages answer"}],
"provider_extra": true,
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1}
})))
.expect(1)
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
r#"
model_list:
- model_name: responses-target
litellm_params:
model: openai/responses-model
api_base: {base}
api_key: test-key
custom_llm_provider: openai
switchyard_wire_format: openai_responses
- model_name: messages-target
litellm_params:
model: anthropic/messages-model
api_base: {base}
api_key: test-key
custom_llm_provider: anthropic
switchyard_wire_format: anthropic_messages
- model_name: responses-router
litellm_params:
model: switchyard/algorithm
switchyard_config:
type: passthrough
target: responses-target
- model_name: messages-router
litellm_params:
model: switchyard/algorithm
switchyard_config:
type: passthrough
target: messages-target
"#,
base = upstream.uri()
))
.expect("valid config");
let app = switchyard_app(&config, Permissions::All);
let response = support::post(
app.clone(),
"/v1/responses",
json!({"model": "responses-router", "input": "hello"}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "responses-target");
assert_eq!(body["output"][0]["content"][0]["text"], "responses answer");
assert_eq!(body["provider_extra"], true);
let response = support::post(
app.clone(),
"/v1/messages",
json!({
"model": "messages-router",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 16
}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "messages-target");
assert_eq!(body["content"][0]["text"], "messages answer");
assert_eq!(body["provider_extra"], true);
let response = support::post(
app,
"/v1/chat/completions",
json!({
"model": "responses-router",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), 200);
let body = support::json(response).await;
assert_eq!(body["model"], "responses-target");
assert_eq!(body["choices"][0]["message"]["content"], "responses answer");
}
fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router {
let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))));
let http = Resolution::from(&HttpSettings::default()).config;
let gateway = Gateway::new(
resources,
http,
Arc::new(support::NoSecrets),
ModelRouter::from_model_list(&config.model_list),
)
.expect("gateway")
.with_switchyard_models(&config.model_list)
.expect("Switchyard routes");
support::authenticated_app(gateway, permissions, None)
}