refactor(rust): separate responses provider resolution (#47215)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yujonglee 2026-10-19 09:03:43 -07:00 • committed by GitHub
parent 63b740532b
commit ede6d97785
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 232 additions and 223 deletions

View file

@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
use std::time::Duration;
use futures_util::StreamExt;
use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
use litellm_secrets::source::Secrets;
use super::{
Error,
types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
Error, ResponsesRoute,
types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
};
pub(super) async fn execute(
http: &litellm_http::Client,
auth: &litellm_auth::AuthServices,
request: ProviderResponsesRequest,
cache: Option<litellm_cache_response::ScopedCache>,
route: &ResponsesRoute,
ResponsesProvider { identity, config }: ResponsesProvider,
call: ResponsesCall,
secrets: Secrets,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ResponsesOutput, Error> {
let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
let identity = litellm_host::interceptors::ProviderIdentity {
model: request.context.model.clone(),
provider: request.context.custom_llm_provider.clone(),
let lookup = |name: &str| secrets.get(name);
let environment = config.validate_environment(
litellm_http::request::string_headers("responses", call.extra_headers)?,
call.api_key.as_deref(),
&lookup,
)?;
let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
let context = RequestContext {
model: identity.model.clone(),
custom_llm_provider: identity.provider.clone(),
optional_params: serde_json::Value::Object(call.optional_params.clone()),
secret_fields: Vec::new(),
api_key: match &environment.auth {
AuthScheme::Credential { secret, .. } => Some(secret.clone()),
_ => None,
},
};
let body = config.transform_responses_api_request(
&identity.model,
call.input,
call.optional_params,
)?;
let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
let wire = interceptors
.before_provider_request(
WireRequest {
url: request.url,
url,
headers: authenticated.headers,
body: request.body,
body,
},
request.context,
context,
)
.await?;
let cache = cache.filter(|_| authenticated.signer.is_none());
let cache = route
.cache
.clone()
.filter(|_| authenticated.signer.is_none());
let cache_request = litellm_inference::caching::CacheRequest::from_wire(
identity,
cache.as_ref().map(|_| &wire),
@ -59,9 +81,9 @@ pub(super) async fn execute(
},
wire.url,
&wire.body,
Some(request.timeout.unwrap_or(Duration::from_secs(600))),
Some(call.timeout.unwrap_or(Duration::from_secs(600))),
)?;
let response = litellm_inference::outbound::send(outbound, http)
let response = litellm_inference::outbound::send(outbound, &route.http)
.await
.map_err(network)?;
let status = response.status().as_u16();
@ -103,8 +125,7 @@ pub(super) async fn execute(
.map_err(Error::post_call)?;
let value = serde_json::from_str(&body)
.map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
request
.config
config
.transform_response_api_response(value)
.map(ResponsesOutput::Complete)
.map_err(Error::from)

View file

@ -7,14 +7,30 @@ pub mod types;
pub mod websocket;
mod handler;
mod prepare;
use std::sync::Arc;
use litellm_auth::AuthServices;
use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use litellm_host::interceptors::Interceptors;
use litellm_host::interceptors::ProviderIdentity;
use litellm_llms::{
base_llm::responses::transformation::BaseResponsesApiConfig,
openai::responses::transformation::OpenAiResponsesApiConfig,
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
};
use litellm_secrets::source::SecretSource;
use types::{ResponsesCall, ResponsesOutput};
use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
match provider {
LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
_ => None,
}
}
#[derive(Clone)]
pub struct ResponsesRoute {
@ -78,34 +94,68 @@ impl ResponsesRoute {
observers: Option<&ObservationSender>,
) -> Result<ResponsesOutput, Error> {
litellm_inference::diagnostic::call(async {
self.run_provider(call, cache_options, interceptors, observers)
.await
let explicit = call
.custom_llm_provider
.as_deref()
.filter(|provider| !provider.is_empty());
let (provider_name, resolved_model) =
match get_custom_llm_provider(&call.model, explicit) {
Some(CustomLlmProvider {
model,
custom_llm_provider,
}) => (custom_llm_provider, model),
None => ("openai", call.model.as_str()),
};
if resolved_model.is_empty() {
return Err(Error::InvalidProvider(call.model));
}
let provider = provider_name
.parse::<LlmProviders>()
.map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
let config = responses_provider(provider)
.ok_or(Error::Unsupported("native HTTP responses provider"))?;
let wire_model = match provider {
LlmProviders::Openai if resolved_model.contains('/') => {
return Err(Error::InvalidProvider(call.model));
}
LlmProviders::Openai => resolved_model.into(),
LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
format!("openrouter/{resolved_model}")
}
LlmProviders::Openrouter => resolved_model.into(),
_ => return Err(Error::Unsupported("native HTTP responses provider")),
};
let provider = ResponsesProvider {
identity: ProviderIdentity {
model: wire_model,
provider: provider_name.into(),
},
config,
};
litellm_inference::diagnostic::provider(
&provider.identity.model,
&provider.identity.provider,
);
let secrets = self
.secrets
.resolve(
provider
.config
.secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
)
.await?;
let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
Box::pin(handler::execute(
self,
provider,
call,
secrets,
cache_options,
interceptors,
observers,
));
execute.await
})
.await
}
async fn run_provider(
&self,
call: ResponsesCall,
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ResponsesOutput, Error> {
let request = prepare::prepare(call, self.secrets.as_ref()).await?;
litellm_inference::diagnostic::provider(
&request.context.model,
&request.context.custom_llm_provider,
);
let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
Box::pin(handler::execute(
&self.http,
&self.auth,
request,
self.cache.clone(),
cache_options,
interceptors,
observers,
));
execute.await
}
}

View file

@ -1,162 +0,0 @@
use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use litellm_host::interceptors::{ProviderIdentity, RequestContext};
use litellm_llms::{
base_llm::responses::transformation::BaseResponsesApiConfig,
openai::responses::transformation::OpenAiResponsesApiConfig,
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
};
use litellm_secrets::source::SecretSource;
use serde_json::Value;
use super::{
Error,
types::{ProviderResponsesRequest, ResponsesCall},
};
#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
pub(super) async fn prepare(
call: ResponsesCall,
secrets: &dyn SecretSource,
) -> Result<ProviderResponsesRequest, Error> {
let ResponsesProvider { identity, config } =
resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
let provider = identity.provider.as_str();
let model = identity.model.as_str();
let snapshot = secrets
.resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
.await?;
let lookup = |name: &str| snapshot.get(name);
let environment = config.validate_environment(
litellm_http::request::string_headers("responses", call.extra_headers)?,
call.api_key.as_deref(),
&lookup,
)?;
let context = RequestContext {
model: model.into(),
custom_llm_provider: provider.into(),
optional_params: Value::Object(call.optional_params.clone()),
secret_fields: Vec::new(),
api_key: match &environment.auth {
litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
Some(secret.clone())
}
_ => None,
},
};
let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
Ok(ProviderResponsesRequest {
url: config.get_complete_url(call.api_base.as_deref(), &lookup),
config,
environment,
body,
context,
timeout: call.timeout,
})
}
pub(super) struct ResponsesProvider {
pub identity: ProviderIdentity,
pub config: &'static dyn BaseResponsesApiConfig,
}
/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
pub(super) fn resolve_provider(
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<ResponsesProvider, Error> {
let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
Some(CustomLlmProvider {
model,
custom_llm_provider,
}) => (custom_llm_provider, model),
None => ("openai", model),
};
if resolved.is_empty() {
return Err(Error::InvalidProvider(model.into()));
}
let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
match provider.parse::<LlmProviders>() {
Ok(LlmProviders::Openai) if resolved.contains('/') => {
return Err(Error::InvalidProvider(model.into()));
}
Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
&OPENROUTER_RESPONSES_CONFIG,
format!("openrouter/{resolved}"),
),
Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
_ => return Err(Error::Unsupported("native HTTP responses provider")),
};
Ok(ResponsesProvider {
identity: ProviderIdentity {
model: wire_model,
provider: provider.into(),
},
config,
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
/// Pinned against Python's `get_llm_provider` on 2026-10-11.
#[rstest]
#[case::bare("gpt-5", None, "openai", "gpt-5")]
#[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
#[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
#[case::openrouter_prefix(
"openrouter/anthropic/claude",
None,
"openrouter",
"anthropic/claude"
)]
#[case::openrouter_explicit(
"openrouter/anthropic/claude",
Some("openrouter"),
"openrouter",
"anthropic/claude"
)]
#[case::openrouter_explicit_unprefixed(
"anthropic/claude",
Some("openrouter"),
"openrouter",
"anthropic/claude"
)]
#[case::openrouter_native_id(
"openrouter/auto",
Some("openrouter"),
"openrouter",
"openrouter/auto"
)]
#[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
fn providers_and_wire_models_match_python(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
#[case] provider: &str,
#[case] wire_model: &str,
) {
let resolved = resolve_provider(model, custom_llm_provider).unwrap();
assert_eq!(resolved.identity.provider, provider);
assert_eq!(resolved.identity.model, wire_model);
}
#[rstest]
#[case::other_prefix("other/model", None)]
#[case::other_explicit("model", Some("other"))]
#[case::openai_with_vendor_segment("openai/other/model", None)]
#[case::empty_model("openai/", None)]
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
assert!(matches!(
resolve_provider(model, custom_llm_provider),
Err(Error::InvalidProvider(_) | Error::Unsupported(_))
));
}
}

View file

@ -2,9 +2,8 @@ use std::time::Duration;
use bytes::Bytes;
use litellm_host::call::CallOutput;
use litellm_llms::base_llm::{
auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
};
use litellm_host::interceptors::ProviderIdentity;
use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
use litellm_llms_types::formats::responses::ResponsesApiResponse;
use serde_json::{Map, Value};
@ -21,17 +20,13 @@ pub struct ResponsesCall {
pub timeout: Option<Duration>,
}
pub(crate) struct ResponsesProvider {
pub identity: ProviderIdentity,
pub config: &'static dyn BaseResponsesApiConfig,
}
pub struct ResponsesStreamHead {
pub headers: Vec<(String, String)>,
}
pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
pub(super) struct ProviderResponsesRequest {
pub config: &'static dyn BaseResponsesApiConfig,
pub environment: ValidatedEnvironment,
pub url: String,
pub body: Value,
pub context: litellm_host::interceptors::RequestContext,
pub timeout: Option<Duration>,
}

View file

@ -7,6 +7,7 @@ use std::sync::Arc;
use futures_util::TryStreamExt;
use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
use litellm_inference_responses::{
Error,
route::Responses,
types::{ResponsesCall, ResponsesOutput},
};
@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
}
}
#[rstest]
#[case::bare_openai("gpt-5", None, "gpt-5")]
#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
#[case::explicit_openrouter_prefix(
"openrouter/anthropic/claude",
Some("openrouter"),
"anthropic/claude"
)]
#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
#[tokio::test]
async fn provider_resolution_preserves_the_wire_model(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
#[case] expected_wire_model: &str,
) {
let response = json!({
"id": "response-1",
"model": expected_wire_model,
"output": [{"type": "message", "content": []}],
"usage": {"total_tokens": 7}
});
let upstream = upstream([json_response(response)]).await;
responses_route(no_secrets())
.execute(
ResponsesCall {
model: model.into(),
api_key: Some("test-key".into()),
api_base: Some(upstream.uri()),
custom_llm_provider: custom_llm_provider.map(str::to_owned),
..call()
},
&(),
None,
)
.await
.unwrap();
assert_eq!(
only_request(&upstream).await.json()["model"],
expected_wire_model
);
}
#[rstest]
#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
#[tokio::test]
async fn unsupported_providers_fail_before_transport(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
) {
let upstream = upstream([]).await;
let result = responses_route(no_secrets())
.execute(
ResponsesCall {
model: model.into(),
api_base: Some(upstream.uri()),
custom_llm_provider: custom_llm_provider.map(str::to_owned),
..call()
},
&(),
None,
)
.await;
assert!(matches!(
result,
Err(Error::Unsupported("native HTTP responses provider"))
));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::openai_with_vendor_segment("openai/other/model", None)]
#[case::empty_model("openai/", None)]
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
#[tokio::test]
async fn invalid_provider_models_fail_before_transport(
#[case] model: &str,
#[case] custom_llm_provider: Option<&str>,
) {
let upstream = upstream([]).await;
let result = responses_route(no_secrets())
.execute(
ResponsesCall {
model: model.into(),
api_base: Some(upstream.uri()),
custom_llm_provider: custom_llm_provider.map(str::to_owned),
..call()
},
&(),
None,
)
.await;
assert!(matches!(result, Err(Error::InvalidProvider(_))));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::direct(false)]
#[case::hosted(true)]