diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs index eadedca7d3f..02182d23ae3 100644 --- a/litellm-rust/crates/inference-responses/src/handler.rs +++ b/litellm-rust/crates/inference-responses/src/handler.rs @@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender; use std::time::Duration; use futures_util::StreamExt; -use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest}; -use litellm_llms::base_llm::auth::{Authenticated, resolve_auth}; +use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}; +use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth}; +use litellm_secrets::source::Secrets; use super::{ - Error, - types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead}, + Error, ResponsesRoute, + types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead}, }; pub(super) async fn execute( - http: &litellm_http::Client, - auth: &litellm_auth::AuthServices, - request: ProviderResponsesRequest, - cache: Option, + route: &ResponsesRoute, + ResponsesProvider { identity, config }: ResponsesProvider, + call: ResponsesCall, + secrets: Secrets, cache_options: Option, interceptors: &impl Interceptors, observers: Option<&ObservationSender>, ) -> Result { - let authenticated = resolve_auth(auth, request.environment, &|_| None).await?; - let identity = litellm_host::interceptors::ProviderIdentity { - model: request.context.model.clone(), - provider: request.context.custom_llm_provider.clone(), + let lookup = |name: &str| secrets.get(name); + let environment = config.validate_environment( + litellm_http::request::string_headers("responses", call.extra_headers)?, + call.api_key.as_deref(), + &lookup, + )?; + let url = config.get_complete_url(call.api_base.as_deref(), &lookup); + let context = RequestContext { + model: identity.model.clone(), + custom_llm_provider: identity.provider.clone(), + optional_params: serde_json::Value::Object(call.optional_params.clone()), + secret_fields: Vec::new(), + api_key: match &environment.auth { + AuthScheme::Credential { secret, .. } => Some(secret.clone()), + _ => None, + }, }; + let body = config.transform_responses_api_request( + &identity.model, + call.input, + call.optional_params, + )?; + let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?; let wire = interceptors .before_provider_request( WireRequest { - url: request.url, + url, headers: authenticated.headers, - body: request.body, + body, }, - request.context, + context, ) .await?; - let cache = cache.filter(|_| authenticated.signer.is_none()); + let cache = route + .cache + .clone() + .filter(|_| authenticated.signer.is_none()); let cache_request = litellm_inference::caching::CacheRequest::from_wire( identity, cache.as_ref().map(|_| &wire), @@ -59,9 +81,9 @@ pub(super) async fn execute( }, wire.url, &wire.body, - Some(request.timeout.unwrap_or(Duration::from_secs(600))), + Some(call.timeout.unwrap_or(Duration::from_secs(600))), )?; - let response = litellm_inference::outbound::send(outbound, http) + let response = litellm_inference::outbound::send(outbound, &route.http) .await .map_err(network)?; let status = response.status().as_u16(); @@ -103,8 +125,7 @@ pub(super) async fn execute( .map_err(Error::post_call)?; let value = serde_json::from_str(&body) .map_err(|error| Error::InvalidResponse(error.to_string().into()))?; - request - .config + config .transform_response_api_response(value) .map(ResponsesOutput::Complete) .map_err(Error::from) diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index 11939823b05..6a9d5372311 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -7,14 +7,30 @@ pub mod types; pub mod websocket; mod handler; -mod prepare; use std::sync::Arc; use litellm_auth::AuthServices; +use litellm_core_utils::get_llm_provider_logic::{ + CustomLlmProvider, LlmProviders, get_custom_llm_provider, +}; use litellm_host::interceptors::Interceptors; +use litellm_host::interceptors::ProviderIdentity; +use litellm_llms::{ + base_llm::responses::transformation::BaseResponsesApiConfig, + openai::responses::transformation::OpenAiResponsesApiConfig, + openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, +}; use litellm_secrets::source::SecretSource; -use types::{ResponsesCall, ResponsesOutput}; +use types::{ResponsesCall, ResponsesOutput, ResponsesProvider}; + +fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> { + match provider { + LlmProviders::Openai => Some(&OpenAiResponsesApiConfig), + LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG), + _ => None, + } +} #[derive(Clone)] pub struct ResponsesRoute { @@ -78,34 +94,68 @@ impl ResponsesRoute { observers: Option<&ObservationSender>, ) -> Result { litellm_inference::diagnostic::call(async { - self.run_provider(call, cache_options, interceptors, observers) - .await + let explicit = call + .custom_llm_provider + .as_deref() + .filter(|provider| !provider.is_empty()); + let (provider_name, resolved_model) = + match get_custom_llm_provider(&call.model, explicit) { + Some(CustomLlmProvider { + model, + custom_llm_provider, + }) => (custom_llm_provider, model), + None => ("openai", call.model.as_str()), + }; + if resolved_model.is_empty() { + return Err(Error::InvalidProvider(call.model)); + } + let provider = provider_name + .parse::() + .map_err(|_| Error::Unsupported("native HTTP responses provider"))?; + let config = responses_provider(provider) + .ok_or(Error::Unsupported("native HTTP responses provider"))?; + let wire_model = match provider { + LlmProviders::Openai if resolved_model.contains('/') => { + return Err(Error::InvalidProvider(call.model)); + } + LlmProviders::Openai => resolved_model.into(), + LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => { + format!("openrouter/{resolved_model}") + } + LlmProviders::Openrouter => resolved_model.into(), + _ => return Err(Error::Unsupported("native HTTP responses provider")), + }; + let provider = ResponsesProvider { + identity: ProviderIdentity { + model: wire_model, + provider: provider_name.into(), + }, + config, + }; + litellm_inference::diagnostic::provider( + &provider.identity.model, + &provider.identity.provider, + ); + let secrets = self + .secrets + .resolve( + provider + .config + .secret_names(call.api_key.as_deref(), call.api_base.as_deref()), + ) + .await?; + let execute: futures_util::future::BoxFuture<'_, Result> = + Box::pin(handler::execute( + self, + provider, + call, + secrets, + cache_options, + interceptors, + observers, + )); + execute.await }) .await } - - async fn run_provider( - &self, - call: ResponsesCall, - cache_options: Option, - interceptors: &impl Interceptors, - observers: Option<&ObservationSender>, - ) -> Result { - let request = prepare::prepare(call, self.secrets.as_ref()).await?; - litellm_inference::diagnostic::provider( - &request.context.model, - &request.context.custom_llm_provider, - ); - let execute: futures_util::future::BoxFuture<'_, Result> = - Box::pin(handler::execute( - &self.http, - &self.auth, - request, - self.cache.clone(), - cache_options, - interceptors, - observers, - )); - execute.await - } } diff --git a/litellm-rust/crates/inference-responses/src/prepare.rs b/litellm-rust/crates/inference-responses/src/prepare.rs deleted file mode 100644 index 05a9772abd4..00000000000 --- a/litellm-rust/crates/inference-responses/src/prepare.rs +++ /dev/null @@ -1,162 +0,0 @@ -use litellm_core_utils::get_llm_provider_logic::{ - CustomLlmProvider, LlmProviders, get_custom_llm_provider, -}; -use litellm_host::interceptors::{ProviderIdentity, RequestContext}; -use litellm_llms::{ - base_llm::responses::transformation::BaseResponsesApiConfig, - openai::responses::transformation::OpenAiResponsesApiConfig, - openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, -}; -use litellm_secrets::source::SecretSource; -use serde_json::Value; - -use super::{ - Error, - types::{ProviderResponsesRequest, ResponsesCall}, -}; - -#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)] -pub(super) async fn prepare( - call: ResponsesCall, - secrets: &dyn SecretSource, -) -> Result { - let ResponsesProvider { identity, config } = - resolve_provider(&call.model, call.custom_llm_provider.as_deref())?; - let provider = identity.provider.as_str(); - let model = identity.model.as_str(); - let snapshot = secrets - .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref())) - .await?; - let lookup = |name: &str| snapshot.get(name); - let environment = config.validate_environment( - litellm_http::request::string_headers("responses", call.extra_headers)?, - call.api_key.as_deref(), - &lookup, - )?; - let context = RequestContext { - model: model.into(), - custom_llm_provider: provider.into(), - optional_params: Value::Object(call.optional_params.clone()), - secret_fields: Vec::new(), - api_key: match &environment.auth { - litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => { - Some(secret.clone()) - } - _ => None, - }, - }; - let body = config.transform_responses_api_request(model, call.input, call.optional_params)?; - Ok(ProviderResponsesRequest { - url: config.get_complete_url(call.api_base.as_deref(), &lookup), - config, - environment, - body, - context, - timeout: call.timeout, - }) -} - -pub(super) struct ResponsesProvider { - pub identity: ProviderIdentity, - pub config: &'static dyn BaseResponsesApiConfig, -} - -/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter -/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id. -pub(super) fn resolve_provider( - model: &str, - custom_llm_provider: Option<&str>, -) -> Result { - let explicit = custom_llm_provider.filter(|provider| !provider.is_empty()); - let (provider, resolved) = match get_custom_llm_provider(model, explicit) { - Some(CustomLlmProvider { - model, - custom_llm_provider, - }) => (custom_llm_provider, model), - None => ("openai", model), - }; - if resolved.is_empty() { - return Err(Error::InvalidProvider(model.into())); - } - let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) = - match provider.parse::() { - Ok(LlmProviders::Openai) if resolved.contains('/') => { - return Err(Error::InvalidProvider(model.into())); - } - Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()), - Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => ( - &OPENROUTER_RESPONSES_CONFIG, - format!("openrouter/{resolved}"), - ), - Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()), - _ => return Err(Error::Unsupported("native HTTP responses provider")), - }; - Ok(ResponsesProvider { - identity: ProviderIdentity { - model: wire_model, - provider: provider.into(), - }, - config, - }) -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::*; - - /// Pinned against Python's `get_llm_provider` on 2026-10-11. - #[rstest] - #[case::bare("gpt-5", None, "openai", "gpt-5")] - #[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")] - #[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")] - #[case::openrouter_prefix( - "openrouter/anthropic/claude", - None, - "openrouter", - "anthropic/claude" - )] - #[case::openrouter_explicit( - "openrouter/anthropic/claude", - Some("openrouter"), - "openrouter", - "anthropic/claude" - )] - #[case::openrouter_explicit_unprefixed( - "anthropic/claude", - Some("openrouter"), - "openrouter", - "anthropic/claude" - )] - #[case::openrouter_native_id( - "openrouter/auto", - Some("openrouter"), - "openrouter", - "openrouter/auto" - )] - #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")] - fn providers_and_wire_models_match_python( - #[case] model: &str, - #[case] custom_llm_provider: Option<&str>, - #[case] provider: &str, - #[case] wire_model: &str, - ) { - let resolved = resolve_provider(model, custom_llm_provider).unwrap(); - assert_eq!(resolved.identity.provider, provider); - assert_eq!(resolved.identity.model, wire_model); - } - - #[rstest] - #[case::other_prefix("other/model", None)] - #[case::other_explicit("model", Some("other"))] - #[case::openai_with_vendor_segment("openai/other/model", None)] - #[case::empty_model("openai/", None)] - #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))] - fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) { - assert!(matches!( - resolve_provider(model, custom_llm_provider), - Err(Error::InvalidProvider(_) | Error::Unsupported(_)) - )); - } -} diff --git a/litellm-rust/crates/inference-responses/src/types.rs b/litellm-rust/crates/inference-responses/src/types.rs index 18c64dc8178..e24cec801ba 100644 --- a/litellm-rust/crates/inference-responses/src/types.rs +++ b/litellm-rust/crates/inference-responses/src/types.rs @@ -2,9 +2,8 @@ use std::time::Duration; use bytes::Bytes; use litellm_host::call::CallOutput; -use litellm_llms::base_llm::{ - auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig, -}; +use litellm_host::interceptors::ProviderIdentity; +use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig; use litellm_llms_types::formats::responses::ResponsesApiResponse; use serde_json::{Map, Value}; @@ -21,17 +20,13 @@ pub struct ResponsesCall { pub timeout: Option, } +pub(crate) struct ResponsesProvider { + pub identity: ProviderIdentity, + pub config: &'static dyn BaseResponsesApiConfig, +} + pub struct ResponsesStreamHead { pub headers: Vec<(String, String)>, } pub type ResponsesOutput = CallOutput; - -pub(super) struct ProviderResponsesRequest { - pub config: &'static dyn BaseResponsesApiConfig, - pub environment: ValidatedEnvironment, - pub url: String, - pub body: Value, - pub context: litellm_host::interceptors::RequestContext, - pub timeout: Option, -} diff --git a/litellm-rust/crates/inference-responses/tests/responses.rs b/litellm-rust/crates/inference-responses/tests/responses.rs index f5181aca2b1..ae07a71dabd 100644 --- a/litellm-rust/crates/inference-responses/tests/responses.rs +++ b/litellm-rust/crates/inference-responses/tests/responses.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use futures_util::TryStreamExt; use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; use litellm_inference_responses::{ + Error, route::Responses, types::{ResponsesCall, ResponsesOutput}, }; @@ -32,6 +33,110 @@ fn call() -> ResponsesCall { } } +#[rstest] +#[case::bare_openai("gpt-5", None, "gpt-5")] +#[case::openai_prefix("openai/gpt-5", None, "gpt-5")] +#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")] +#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")] +#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")] +#[case::explicit_openrouter_prefix( + "openrouter/anthropic/claude", + Some("openrouter"), + "anthropic/claude" +)] +#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")] +#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")] +#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")] +#[tokio::test] +async fn provider_resolution_preserves_the_wire_model( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] expected_wire_model: &str, +) { + let response = json!({ + "id": "response-1", + "model": expected_wire_model, + "output": [{"type": "message", "content": []}], + "usage": {"total_tokens": 7} + }); + let upstream = upstream([json_response(response)]).await; + responses_route(no_secrets()) + .execute( + ResponsesCall { + model: model.into(), + api_key: Some("test-key".into()), + api_base: Some(upstream.uri()), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..call() + }, + &(), + None, + ) + .await + .unwrap(); + + assert_eq!( + only_request(&upstream).await.json()["model"], + expected_wire_model + ); +} + +#[rstest] +#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))] +#[case::unsupported_responses_provider("test-model", Some("anthropic"))] +#[tokio::test] +async fn unsupported_providers_fail_before_transport( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, +) { + let upstream = upstream([]).await; + let result = responses_route(no_secrets()) + .execute( + ResponsesCall { + model: model.into(), + api_base: Some(upstream.uri()), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..call() + }, + &(), + None, + ) + .await; + + assert!(matches!( + result, + Err(Error::Unsupported("native HTTP responses provider")) + )); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[case::openai_with_vendor_segment("openai/other/model", None)] +#[case::empty_model("openai/", None)] +#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))] +#[tokio::test] +async fn invalid_provider_models_fail_before_transport( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, +) { + let upstream = upstream([]).await; + let result = responses_route(no_secrets()) + .execute( + ResponsesCall { + model: model.into(), + api_base: Some(upstream.uri()), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..call() + }, + &(), + None, + ) + .await; + + assert!(matches!(result, Err(Error::InvalidProvider(_)))); + assert!(received(&upstream).await.is_empty()); +} + #[rstest] #[case::direct(false)] #[case::hosted(true)]