diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index bd33e31bdc9..c920b874786 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -10,6 +10,8 @@ use axum::{ }; use litellm_inference_chat::types::{ChatCompletionsCall, ChatCompletionsRequest}; use serde_json::{Map, Value}; +#[cfg(feature = "switchyard")] +use switchyard_protocol::WireFormat; use crate::{Deployment, Error, Gateway, JsonObject, request}; @@ -67,6 +69,18 @@ async fn handle( .and_then(Value::as_str) .and_then(|model| gateway.switchyard_routes.get(model)); let (body, cache_options) = crate::caching::prepare(identity, body)?; + #[cfg(feature = "switchyard")] + if let Some(switchyard_route) = switchyard_route { + return switchyard_route + .execute( + gateway, + cache_options, + WireFormat::OpenAiChat, + body, + inbound_headers, + ) + .await; + } let route = gateway.chat_completions.clone(); let route = match &gateway.cache { Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( @@ -75,16 +89,6 @@ async fn handle( )), None => route, }; - - #[cfg(feature = "switchyard")] - if let Some(switchyard_route) = switchyard_route { - let (body, upstream_headers) = switchyard_route - .execute(route, cache_options.policy, body, inbound_headers) - .await?; - let mut response = Json(body).into_response(); - response.headers_mut().extend(upstream_headers); - return Ok(response); - } let headers = crate::caching::CacheHeaders::default(); let response = litellm_host_http::serve_unary( route.machine( diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 2b0c0a5429f..06d8aa5949d 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -101,8 +101,7 @@ impl Gateway { mut self, models: &[litellm_config::Model], ) -> Result { - self.switchyard_routes = - switchyard::Routes::from_models(models, Arc::new(self.models.clone()))?; + self.switchyard_routes = switchyard::Routes::from_models(models, &self.models)?; Ok(self) } } diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index d612a042e03..8e9850d769a 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -14,6 +14,8 @@ use litellm_host_http::Sse; use litellm_inference_messages::{MessagesCall, messages_body, route::Messages}; use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders}; use serde_json::{Map, Value}; +#[cfg(feature = "switchyard")] +use switchyard_protocol::WireFormat; use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request}; @@ -34,7 +36,7 @@ pub async fn create( body: Result, ) -> impl IntoResponse { let result = match body { - Ok(JsonObject(body)) => handle(&gateway, &identity, &headers, body).await, + Ok(JsonObject(body)) => handle(&gateway, &identity, headers, body).await, Err(error) => Err(error), }; result.map_err(|error| (error.status(), Json(error.body(request_id.as_deref())))) @@ -43,12 +45,29 @@ pub async fn create( async fn handle( gateway: &Gateway, identity: &AuthenticatedRequest, - headers: &HeaderMap, + headers: HeaderMap, body: Map, ) -> Result { let deployment = request::resolve_deployment(gateway, &body)?; request::authorize_model(identity, deployment, &body).await?; + #[cfg(feature = "switchyard")] + let switchyard_route = body + .get("model") + .and_then(Value::as_str) + .and_then(|model| gateway.switchyard_routes.get(model)); let (body, cache_options) = crate::caching::prepare(identity, body)?; + #[cfg(feature = "switchyard")] + if let Some(switchyard_route) = switchyard_route { + return switchyard_route + .execute( + gateway, + cache_options, + WireFormat::AnthropicMessages, + body, + headers, + ) + .await; + } let route = gateway.messages.clone(); let route = match &gateway.cache { Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( @@ -58,7 +77,7 @@ async fn handle( None => route, }; - let call = project(deployment, body, headers)?; + let call = provider_request(deployment, body, &headers)?; let machine = route.machine(call, cache_options.policy); let stream = Sse::::new(Json, |error| Bytes::from(Error::from(error).sse_frame())); @@ -67,18 +86,14 @@ async fn handle( Ok(headers.apply(response)) } -fn project( +pub(crate) fn provider_request( deployment: &Deployment, - body: Map, + mut body: Map, headers: &HeaderMap, -) -> Result { - let body = body - .into_iter() - .map(|(name, value)| match name.as_str() { - "model" => (name, Value::from(deployment.model.as_str())), - _ => (name, value), - }) - .collect(); +) -> Result { + if let Some(model) = body.get_mut("model") { + *model = Value::from(deployment.model.as_str()); + } Ok(MessagesCall { body: messages_body(body)?, api_key: deployment.api_key.clone(), diff --git a/litellm-rust/crates/gateway-inference/src/responses.rs b/litellm-rust/crates/gateway-inference/src/responses.rs index 77bebf610e1..0379d4fe584 100644 --- a/litellm-rust/crates/gateway-inference/src/responses.rs +++ b/litellm-rust/crates/gateway-inference/src/responses.rs @@ -1,21 +1,43 @@ use std::sync::Arc; +#[cfg(feature = "switchyard")] +use axum::http::HeaderMap; use axum::{Json, body::Bytes, extract::State, response::Response}; use litellm_gateway_auth::AuthenticatedRequest; use litellm_host_http::Sse; use litellm_inference_responses::{route::Responses, types::ResponsesCall}; -use serde_json::json; +use serde_json::{Map, Value, json}; +#[cfg(feature = "switchyard")] +use switchyard_protocol::WireFormat; -use crate::{Error, Gateway, JsonObject, request}; +use crate::{Deployment, Error, Gateway, JsonObject, request}; pub(crate) async fn create( State(gateway): State>, identity: AuthenticatedRequest, + #[cfg(feature = "switchyard")] headers: HeaderMap, JsonObject(body): JsonObject, ) -> Result { let deployment = request::resolve_deployment(&gateway, &body)?; request::authorize_model(&identity, deployment, &body).await?; + #[cfg(feature = "switchyard")] + let switchyard_route = body + .get("model") + .and_then(Value::as_str) + .and_then(|model| gateway.switchyard_routes.get(model)); let (body, cache_options) = crate::caching::prepare(&identity, body)?; + #[cfg(feature = "switchyard")] + if let Some(switchyard_route) = switchyard_route { + return switchyard_route + .execute( + &gateway, + cache_options, + WireFormat::OpenAiResponses, + body, + headers, + ) + .await; + } let route = gateway.responses.clone(); let route = match &gateway.cache { Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( @@ -25,19 +47,7 @@ pub(crate) async fn create( None => route, }; - let call = ResponsesCall { - model: deployment.model.clone(), - input: body.get("input").cloned().unwrap_or_default(), - optional_params: body - .into_iter() - .filter(|(name, _)| !matches!(name.as_str(), "model" | "input")) - .collect(), - api_key: deployment.api_key.clone(), - api_base: deployment.api_base.clone(), - custom_llm_provider: deployment.custom_llm_provider.clone(), - extra_headers: None, - timeout: deployment.timeout, - }; + let call = provider_request(deployment, body); let machine = route.machine(call, cache_options.policy); let stream = Sse::::new(Json, |error| { let error = Error::from(error); @@ -50,3 +60,21 @@ pub(crate) async fn create( let response = litellm_host_http::serve(machine, (), headers.clone(), stream, None).await?; Ok(headers.apply(response)) } + +pub(crate) fn provider_request( + deployment: &Deployment, + mut body: Map, +) -> ResponsesCall { + let input = body.remove("input").unwrap_or_default(); + body.remove("model"); + ResponsesCall { + model: deployment.model.clone(), + input, + optional_params: body, + api_key: deployment.api_key.clone(), + api_base: deployment.api_base.clone(), + custom_llm_provider: deployment.custom_llm_provider.clone(), + extra_headers: None, + timeout: deployment.timeout, + } +} diff --git a/litellm-rust/crates/gateway-inference/src/switchyard.rs b/litellm-rust/crates/gateway-inference/src/switchyard.rs index 5607e6159ab..8725b196f3d 100644 --- a/litellm-rust/crates/gateway-inference/src/switchyard.rs +++ b/litellm-rust/crates/gateway-inference/src/switchyard.rs @@ -4,11 +4,18 @@ use std::{ }; use async_trait::async_trait; -use axum::http::HeaderMap; -use litellm_cache_response::CachePolicy; +use axum::{ + Json, + http::HeaderMap, + response::{IntoResponse, Response as HttpResponse}, +}; +use litellm_cache_response::{CacheOptions, CachePolicy, ScopedCache}; use litellm_config::Model; +use litellm_host::call::CallOutput; use litellm_inference::RouteError; use litellm_inference_chat::ChatCompletionsRoute; +use litellm_inference_messages::MessagesRoute; +use litellm_inference_responses::ResponsesRoute; use litellm_http::transport::Error as TransportError; use serde::Deserialize; use serde_json::{Map, Value}; @@ -25,8 +32,8 @@ use switchyard_translation::{ }; use crate::{ - Error, ModelRouter, SwitchyardConfigError, caching::CacheHeaders, - chat_completions::provider_request, + Deployment, Error, Gateway, ModelRouter, SwitchyardConfigError, caching::CacheHeaders, + chat_completions, messages, responses, }; const SWITCHYARD_MODEL: &str = "switchyard/algorithm"; @@ -37,7 +44,7 @@ pub(crate) struct Routes(HashMap); impl Routes { pub(crate) fn from_models( model_list: &[Model], - deployments: Arc, + deployments: &ModelRouter, ) -> Result { let mut routes = HashMap::new(); let mut seen = HashSet::new(); @@ -61,16 +68,16 @@ impl Routes { .ok_or_else(|| invalid(model, "switchyard_config is required"))?; let config = AlgorithmSpec::deserialize(config) .map_err(|error| invalid(model, error.to_string()))?; - let targets = targets(model, &config, &deployments)?; + let targets = targets(model, &config, model_list, deployments)?; let (algorithm, runtime_models) = config - .build_with_runtime_models(&model.model_name, &targets) + .build_with_runtime_models(&model.model_name, &targets.models) .map_err(|error| invalid(model, error.to_string()))?; routes.insert( model.model_name.clone(), Route { algorithm, runtime_models: Arc::new(runtime_models), - deployments: Arc::clone(&deployments), + targets: Arc::new(targets.by_model), }, ); } @@ -85,38 +92,39 @@ impl Routes { pub(crate) struct Route { algorithm: Arc, runtime_models: Arc, - deployments: Arc, + targets: Arc>, } impl Route { pub(crate) async fn execute( &self, - route: ChatCompletionsRoute, - cache_policy: CachePolicy, + gateway: &Gateway, + cache_options: CacheOptions, + wire_format: WireFormat, body: Map, headers: HeaderMap, - ) -> Result<(Value, HeaderMap), Error> { - let llm_request = decode_request(WireFormat::OpenAiChat, &Value::Object(body)) + ) -> Result { + let llm_request = decode_request(wire_format, &Value::Object(body)) .map_err(|error| Error::InvalidBody(error.to_string()))?; - // Reject unsupported streaming before a classifier or judge can make a model call. + // Switchyard routes currently support buffered calls only. if llm_request.stream { return Err(Error::Route(RouteError::Unsupported("streaming"))); } let extensions = llm_request.extensions.clone(); let mut metadata = Metadata::from_headers(&headers); metadata.http_headers = Some(headers); - metadata.wire_format = Some(WireFormat::OpenAiChat); + metadata.wire_format = None; let request = Request { llm_request, raw_request: None, metadata: Some(metadata), }; let client = Arc::new(LiteLlmClient { - route, - deployments: Arc::clone(&self.deployments), - cache_policy, + routes: ProviderRoutes::new(gateway, &cache_options), + targets: Arc::clone(&self.targets), + cache_policy: cache_options.policy, }); - let (_, response) = switchyard_llm_client::run( + let (selected_model, response) = switchyard_llm_client::run( Arc::clone(&self.algorithm), ClientRouter::single(client), request, @@ -125,6 +133,7 @@ impl Route { ) .await .map_err(map_libsy_error)?; + let served_model = response.served_model().cloned().unwrap_or(selected_model); let upstream_headers = response.upstream_headers; let response = response .llm_response @@ -133,18 +142,48 @@ impl Route { .map_err(map_client_error)?; let body = encode_aggregated_response_with_extensions( &response, - WireFormat::OpenAiChat, - None, + wire_format, + Some(served_model.as_str()), &extensions, ) .map_err(|error| Error::Internal(error.to_string()))?; - Ok((body, upstream_headers)) + let mut response = Json(body).into_response(); + response.headers_mut().extend(upstream_headers); + Ok(response) + } +} + +struct Target { + deployment: Deployment, + wire_format: WireFormat, +} + +struct ProviderRoutes { + chat: ChatCompletionsRoute, + responses: ResponsesRoute, + messages: MessagesRoute, +} + +impl ProviderRoutes { + fn new(gateway: &Gateway, options: &CacheOptions) -> Self { + let mut routes = Self { + chat: gateway.chat_completions.clone(), + responses: gateway.responses.clone(), + messages: gateway.messages.clone(), + }; + if let Some(cache) = &gateway.cache { + let scoped = || ScopedCache::new(Arc::clone(cache), options.scope.clone()); + routes.chat = routes.chat.with_cache(scoped()); + routes.responses = routes.responses.with_cache(scoped()); + routes.messages = routes.messages.with_cache(scoped()); + } + routes } } struct LiteLlmClient { - route: ChatCompletionsRoute, - deployments: Arc, + routes: ProviderRoutes, + targets: Arc>, cache_policy: CachePolicy, } @@ -156,67 +195,169 @@ impl RoutedLlmClient for LiteLlmClient { message: "routed request has no model".to_string(), }); }; - let deployment = - self.deployments - .get(model) - .ok_or_else(|| LlmClientError::Configuration { - message: format!("unknown LiteLLM deployment {model}"), - })?; + let target = self + .targets + .get(model) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("unknown LiteLLM deployment {model}"), + })?; let Request { llm_request, metadata, .. } = request; - let Value::Object(body) = encode_request(&llm_request, WireFormat::OpenAiChat) + let Value::Object(body) = encode_request(&llm_request, target.wire_format) .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))? else { - return Err(LlmClientError::RequestEncoding( - "OpenAI Chat request is not an object".to_string(), - )); + return Err(LlmClientError::RequestEncoding(format!( + "{} request is not an object", + target.wire_format + ))); }; let cache_headers = CacheHeaders::default(); - let response = self - .route - .execute( - provider_request(deployment, body), - &cache_headers, - self.cache_policy, - ) - .await - .map_err(map_route_error)?; - let body = serde_json::to_value(response) - .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; - let response = decode_aggregated_response(&body, WireFormat::OpenAiChat) - .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; + let response = match target.wire_format { + WireFormat::OpenAiChat => { + let response = self + .routes + .chat + .execute( + chat_completions::provider_request(&target.deployment, body), + &cache_headers, + self.cache_policy, + ) + .await + .map_err(map_route_error)?; + decode_complete(response, target.wire_format)? + } + WireFormat::OpenAiResponses => { + let response = self + .routes + .responses + .execute( + responses::provider_request(&target.deployment, body), + &cache_headers, + self.cache_policy, + ) + .await + .map_err(map_route_error)?; + decode_output(response, target.wire_format)? + } + WireFormat::AnthropicMessages => { + let empty_headers = HeaderMap::new(); + let headers = metadata + .as_ref() + .and_then(|metadata| metadata.http_headers.as_ref()) + .unwrap_or(&empty_headers); + let call = messages::provider_request(&target.deployment, body, headers) + .map_err(map_route_error)?; + let response = self + .routes + .messages + .execute(call, &cache_headers, self.cache_policy) + .await + .map_err(map_route_error)?; + decode_output(response, target.wire_format)? + } + }; Ok(Response { - llm_response: LlmResponse::Agg(response), + llm_response: response, metadata, upstream_headers: cache_headers.headers(), }) } } +fn decode_complete( + response: impl serde::Serialize, + wire_format: WireFormat, +) -> Result { + let body = serde_json::to_value(response) + .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; + decode_aggregated_response(&body, wire_format) + .map(LlmResponse::Agg) + .map_err(|error| LlmClientError::ResponseTranslation(error.to_string())) +} + +fn decode_output( + response: CallOutput, + wire_format: WireFormat, +) -> Result +where + T: serde::Serialize, +{ + match response { + CallOutput::Complete(response) => decode_complete(response, wire_format), + CallOutput::Stream { .. } => Err(LlmClientError::ResponseTranslation( + "LiteLLM streamed a response for a buffered Switchyard call".to_string(), + )), + } +} + +struct Targets { + models: BTreeMap, + by_model: HashMap, +} + fn targets( route: &Model, config: &AlgorithmSpec, + model_list: &[Model], deployments: &ModelRouter, -) -> Result, SwitchyardConfigError> { - config - .callable_target_names() - .into_iter() - .map(|target| { - let deployment = deployments - .get(target) - .ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?; - if deployment.model == SWITCHYARD_MODEL { - return Err(invalid( - route, - format!("target {target:?} is another Switchyard route"), - )); - } - Ok((target.to_string(), ModelId::from(target))) - }) - .collect() +) -> Result { + let mut models = BTreeMap::new(); + let mut by_model = HashMap::new(); + for target in config.callable_target_names() { + let configured = model_list + .iter() + .rev() + .find(|model| model.model_name == target) + .ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?; + let deployment = deployments + .get(target) + .ok_or_else(|| invalid(route, format!("target {target:?} is not configured")))?; + if deployment.model == SWITCHYARD_MODEL { + return Err(invalid( + route, + format!("target {target:?} is another Switchyard route"), + )); + } + let model = ModelId::from(target); + models.insert(target.to_string(), model.clone()); + by_model.insert( + model, + Target { + deployment: deployment.clone(), + wire_format: target_wire_format(route, configured)?, + }, + ); + } + Ok(Targets { models, by_model }) +} + +// Caller format and target execution format are independent, so each target declares its own. +fn target_wire_format(route: &Model, target: &Model) -> Result { + let value = target + .litellm_params + .extra + .get("switchyard_wire_format") + .ok_or_else(|| { + invalid( + route, + format!( + "target {:?} requires switchyard_wire_format", + target.model_name + ), + ) + })?; + WireFormat::deserialize(value).map_err(|error| { + invalid( + route, + format!( + "target {:?} has invalid switchyard_wire_format: {error}", + target.model_name, + ), + ) + }) } fn invalid(model: &Model, message: impl Into) -> SwitchyardConfigError { @@ -308,4 +449,16 @@ mod tests { Error::Route(RouteError::Auth(_)) )); } + + #[test] + fn targets_require_an_explicit_wire_format() { + let config = litellm_config::Config::from_yaml( + "model_list:\n - model_name: target\n litellm_params: { model: upstream }\n - model_name: route\n litellm_params: { model: switchyard/algorithm }\n", + ) + .expect("config"); + + let error = target_wire_format(&config.model_list[1], &config.model_list[0]) + .expect_err("ambiguous target format"); + assert!(error.reason.contains("requires switchyard_wire_format")); + } } diff --git a/litellm-rust/crates/gateway-inference/tests/switchyard.rs b/litellm-rust/crates/gateway-inference/tests/switchyard.rs index 4985f549bd6..10f87b680d7 100644 --- a/litellm-rust/crates/gateway-inference/tests/switchyard.rs +++ b/litellm-rust/crates/gateway-inference/tests/switchyard.rs @@ -73,18 +73,21 @@ model_list: api_base: {base}/v1 api_key: test-key custom_llm_provider: openai_like + switchyard_wire_format: openai_chat - model_name: strong litellm_params: model: strong-model api_base: {base}/v1 api_key: test-key custom_llm_provider: openai_like + switchyard_wire_format: openai_chat - model_name: judge litellm_params: model: judge-model api_base: {base}/v1 api_key: test-key custom_llm_provider: openai_like + switchyard_wire_format: openai_chat - model_name: coding-router litellm_params: model: switchyard/algorithm @@ -104,20 +107,8 @@ model_list: base = upstream.uri() )) .expect("valid config"); - let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); - let resources = CoreResources::new(pool); - let http = Resolution::from(&HttpSettings::default()).config; - let gateway = Gateway::new( - resources, - http, - Arc::new(support::NoSecrets), - ModelRouter::from_model_list(&config.model_list), - ) - .expect("gateway") - .with_switchyard_models(&config.model_list) - .expect("Switchyard routes"); - let app = support::authenticated_app( - gateway, + let app = switchyard_app( + &config, Permissions::Only(Arc::from([ AccessRequest::Model { name: "coding-router".into(), @@ -128,7 +119,6 @@ model_list: deployment: "switchyard/algorithm".into(), }, ])), - None, ); let response = support::post( @@ -155,7 +145,7 @@ model_list: .await; assert_eq!(response.status(), 200); let body = support::json(response).await; - assert_eq!(body["model"], "strong-model"); + assert_eq!(body["model"], "strong"); assert_eq!(body["choices"][0]["message"]["content"], "fallback answer"); let response = support::post( @@ -171,3 +161,136 @@ model_list: let body = support::json(response).await; assert_eq!(body["choices"][0]["message"]["content"], "OK"); } + +// The caller API and selected target API are independent and use the same host bridge. +#[rstest] +#[tokio::test] +async fn configured_switchyard_routes_translate_responses_and_messages() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/responses")) + .and(body_partial_json(json!({"model": "responses-model"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "model": "responses-model", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "responses answer"}] + }], + "provider_extra": true, + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .expect(2) + .mount(&upstream) + .await; + Mock::given(method("POST")) + .and(path("/v1/messages")) + .and(body_partial_json(json!({"model": "messages-model"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "messages-model", + "content": [{"type": "text", "text": "messages answer"}], + "provider_extra": true, + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + }))) + .expect(1) + .mount(&upstream) + .await; + + let config = Config::from_yaml(&format!( + r#" +model_list: + - model_name: responses-target + litellm_params: + model: openai/responses-model + api_base: {base} + api_key: test-key + custom_llm_provider: openai + switchyard_wire_format: openai_responses + - model_name: messages-target + litellm_params: + model: anthropic/messages-model + api_base: {base} + api_key: test-key + custom_llm_provider: anthropic + switchyard_wire_format: anthropic_messages + - model_name: responses-router + litellm_params: + model: switchyard/algorithm + switchyard_config: + type: passthrough + target: responses-target + - model_name: messages-router + litellm_params: + model: switchyard/algorithm + switchyard_config: + type: passthrough + target: messages-target +"#, + base = upstream.uri() + )) + .expect("valid config"); + let app = switchyard_app(&config, Permissions::All); + + let response = support::post( + app.clone(), + "/v1/responses", + json!({"model": "responses-router", "input": "hello"}), + ) + .await; + assert_eq!(response.status(), 200); + let body = support::json(response).await; + assert_eq!(body["model"], "responses-target"); + assert_eq!(body["output"][0]["content"][0]["text"], "responses answer"); + assert_eq!(body["provider_extra"], true); + + let response = support::post( + app.clone(), + "/v1/messages", + json!({ + "model": "messages-router", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 16 + }), + ) + .await; + assert_eq!(response.status(), 200); + let body = support::json(response).await; + assert_eq!(body["model"], "messages-target"); + assert_eq!(body["content"][0]["text"], "messages answer"); + assert_eq!(body["provider_extra"], true); + + let response = support::post( + app, + "/v1/chat/completions", + json!({ + "model": "responses-router", + "messages": [{"role": "user", "content": "hello"}] + }), + ) + .await; + assert_eq!(response.status(), 200); + let body = support::json(response).await; + assert_eq!(body["model"], "responses-target"); + assert_eq!(body["choices"][0]["message"]["content"], "responses answer"); +} + +fn switchyard_app(config: &Config, permissions: Permissions) -> axum::Router { + let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))); + let http = Resolution::from(&HttpSettings::default()).config; + let gateway = Gateway::new( + resources, + http, + Arc::new(support::NoSecrets), + ModelRouter::from_model_list(&config.model_list), + ) + .expect("gateway") + .with_switchyard_models(&config.model_list) + .expect("Switchyard routes"); + support::authenticated_app(gateway, permissions, None) +}