mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
feat(rust): add native OpenRouter Responses (#46000)
The Responses route now resolves openrouter alongside openai. The OpenRouter config reuses the OpenAI request and response transforms and owns only its bearer credential (OPENROUTER_API_KEY, then OR_API_KEY) and base URL (OPENROUTER_API_BASE). Wire model ids follow Python's get_llm_provider, including the openrouter/auto native id with an explicit provider. The Python route host no longer hands litellm.openai_key to OpenRouter. The OpenRouter live test lives in tests/live/openrouter.rs and returns early when the provider key is not exported, so cargo test stays offline by default instead of hiding it behind --ignored. Co-authored-by: nate-berri <nate@berri.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
fbc2e358bc
commit
c0d7264e8b
15 changed files with 596 additions and 37 deletions
|
|
@ -15,6 +15,7 @@ Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `st
|
|||
- `crates/<crate>/tests/` is only for tests that drive several modules together end to end, such as a full request through a route. A test of one type or function never goes there
|
||||
- A helper shared by several inline test modules goes in a `#[cfg(test)] mod test_support;` at the crate root, and only once a second module needs it
|
||||
- A test for another crate's item belongs in that crate, not in a downstream one
|
||||
- Tests that call a real provider go in `crates/<crate>/tests/live/<provider>.rs` behind a `tests/live/main.rs`. Each reads the provider's key from the environment through an `Option` fixture and returns early when it is unset, never `#[ignore]`, so `cargo test` stays offline until the key is exported
|
||||
- Never set `autotests = false` or hand-list `[[test]]` targets. Every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests/<name>/mod.rs` or `tests/<subject>/support.rs` so it is not picked up as a test crate of its own
|
||||
|
||||
## Test fixtures and cases
|
||||
|
|
|
|||
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -4089,6 +4089,7 @@ dependencies = [
|
|||
"litellm-cache",
|
||||
"litellm-cache-memory",
|
||||
"litellm-cache-response",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-native",
|
||||
"litellm-http",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ bytes.workspace = true
|
|||
futures-util.workspace = true
|
||||
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
|
||||
litellm-cache-response.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-host.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-inference.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
use litellm_host::interceptors::RequestContext;
|
||||
use litellm_core_utils::get_llm_provider_logic::{
|
||||
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
|
||||
};
|
||||
use litellm_host::interceptors::{ProviderIdentity, RequestContext};
|
||||
use litellm_llms::{
|
||||
base_llm::responses::transformation::BaseResponsesApiConfig,
|
||||
openai::responses::transformation::OpenAiResponsesApiConfig,
|
||||
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
|
||||
};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use serde_json::Value;
|
||||
|
|
@ -16,10 +20,10 @@ pub(super) async fn prepare(
|
|||
call: ResponsesCall,
|
||||
secrets: &dyn SecretSource,
|
||||
) -> Result<ProviderResponsesRequest, Error> {
|
||||
let identity = resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
|
||||
let ResponsesProvider { identity, config } =
|
||||
resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
|
||||
let provider = identity.provider.as_str();
|
||||
let model = identity.model.as_str();
|
||||
let config: &'static dyn BaseResponsesApiConfig = &OpenAiResponsesApiConfig;
|
||||
let snapshot = secrets
|
||||
.resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
|
||||
.await?;
|
||||
|
|
@ -52,20 +56,107 @@ pub(super) async fn prepare(
|
|||
})
|
||||
}
|
||||
|
||||
pub(super) struct ResponsesProvider {
|
||||
pub identity: ProviderIdentity,
|
||||
pub config: &'static dyn BaseResponsesApiConfig,
|
||||
}
|
||||
|
||||
/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
|
||||
/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
|
||||
pub(super) fn resolve_provider(
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<litellm_host::interceptors::ProviderIdentity, Error> {
|
||||
let provider = custom_llm_provider.unwrap_or("openai");
|
||||
if provider != "openai" {
|
||||
return Err(Error::Unsupported("native HTTP responses provider"));
|
||||
}
|
||||
let resolved = model.strip_prefix("openai/").unwrap_or(model);
|
||||
if resolved.is_empty() || resolved.contains('/') {
|
||||
) -> Result<ResponsesProvider, Error> {
|
||||
let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
|
||||
let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
|
||||
Some(CustomLlmProvider {
|
||||
model,
|
||||
custom_llm_provider,
|
||||
}) => (custom_llm_provider, model),
|
||||
None => ("openai", model),
|
||||
};
|
||||
if resolved.is_empty() {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
Ok(litellm_host::interceptors::ProviderIdentity {
|
||||
model: resolved.into(),
|
||||
provider: provider.into(),
|
||||
let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
|
||||
match provider.parse::<LlmProviders>() {
|
||||
Ok(LlmProviders::Openai) if resolved.contains('/') => {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
|
||||
Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
|
||||
&OPENROUTER_RESPONSES_CONFIG,
|
||||
format!("openrouter/{resolved}"),
|
||||
),
|
||||
Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
|
||||
_ => return Err(Error::Unsupported("native HTTP responses provider")),
|
||||
};
|
||||
Ok(ResponsesProvider {
|
||||
identity: ProviderIdentity {
|
||||
model: wire_model,
|
||||
provider: provider.into(),
|
||||
},
|
||||
config,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Pinned against Python's `get_llm_provider` on 2026-10-11.
|
||||
#[rstest]
|
||||
#[case::bare("gpt-5", None, "openai", "gpt-5")]
|
||||
#[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
|
||||
#[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
|
||||
#[case::openrouter_prefix(
|
||||
"openrouter/anthropic/claude",
|
||||
None,
|
||||
"openrouter",
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::openrouter_explicit(
|
||||
"openrouter/anthropic/claude",
|
||||
Some("openrouter"),
|
||||
"openrouter",
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::openrouter_explicit_unprefixed(
|
||||
"anthropic/claude",
|
||||
Some("openrouter"),
|
||||
"openrouter",
|
||||
"anthropic/claude"
|
||||
)]
|
||||
#[case::openrouter_native_id(
|
||||
"openrouter/auto",
|
||||
Some("openrouter"),
|
||||
"openrouter",
|
||||
"openrouter/auto"
|
||||
)]
|
||||
#[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
|
||||
fn providers_and_wire_models_match_python(
|
||||
#[case] model: &str,
|
||||
#[case] custom_llm_provider: Option<&str>,
|
||||
#[case] provider: &str,
|
||||
#[case] wire_model: &str,
|
||||
) {
|
||||
let resolved = resolve_provider(model, custom_llm_provider).unwrap();
|
||||
assert_eq!(resolved.identity.provider, provider);
|
||||
assert_eq!(resolved.identity.model, wire_model);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::other_prefix("other/model", None)]
|
||||
#[case::other_explicit("model", Some("other"))]
|
||||
#[case::openai_with_vendor_segment("openai/other/model", None)]
|
||||
#[case::empty_model("openai/", None)]
|
||||
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
|
||||
fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
|
||||
assert!(matches!(
|
||||
resolve_provider(model, custom_llm_provider),
|
||||
Err(Error::InvalidProvider(_) | Error::Unsupported(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
//! Real provider calls, one module per provider. Each test skips when its provider's key is not
|
||||
//! in the environment, so `cargo test` stays offline by default.
|
||||
|
||||
#[path = "../support/mod.rs"]
|
||||
mod support;
|
||||
|
||||
mod openrouter;
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use futures_util::TryStreamExt;
|
||||
use litellm_inference_responses::types::{ResponsesCall, ResponsesOutput};
|
||||
use litellm_inference_testing::RecordingSecrets;
|
||||
use litellm_llms::base_llm::responses::streaming::decode;
|
||||
use litellm_llms_types::formats::responses::streaming::ResponsesEventType;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::support::responses_route;
|
||||
|
||||
const DEFAULT_MODEL: &str = "openrouter/openai/gpt-5.5";
|
||||
|
||||
/// `OPENROUTER_API_KEY` as a secret source; tests skip when it is unset.
|
||||
#[fixture]
|
||||
fn secrets() -> Option<Arc<RecordingSecrets>> {
|
||||
let key = std::env::var("OPENROUTER_API_KEY").ok()?;
|
||||
Some(Arc::new(RecordingSecrets::new([(
|
||||
"OPENROUTER_API_KEY",
|
||||
key.as_str(),
|
||||
)])))
|
||||
}
|
||||
|
||||
fn call(stream: bool) -> ResponsesCall {
|
||||
ResponsesCall {
|
||||
model: std::env::var("OPENROUTER_LIVE_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()),
|
||||
input: json!("Reply with the single word: pong"),
|
||||
optional_params: json!({"max_output_tokens": 32, "stream": stream})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
timeout: Some(Duration::from_secs(60)),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_complete_response_carries_output_and_usage(secrets: Option<Arc<RecordingSecrets>>) {
|
||||
let Some(secrets) = secrets else { return };
|
||||
let output = responses_route(secrets)
|
||||
.execute(call(false), &(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let ResponsesOutput::Complete(response) = output else {
|
||||
panic!("expected a complete response");
|
||||
};
|
||||
let body = serde_json::to_value(response).unwrap();
|
||||
let text = body["output"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|item| item["content"].as_array())
|
||||
.flatten()
|
||||
.filter_map(|part| part["text"].as_str())
|
||||
.collect::<String>();
|
||||
assert!(text.to_ascii_lowercase().contains("pong"), "{body}");
|
||||
assert!(
|
||||
body["usage"]["total_tokens"].as_u64().unwrap() > 0,
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_stream_ends_with_response_completed(secrets: Option<Arc<RecordingSecrets>>) {
|
||||
let Some(secrets) = secrets else { return };
|
||||
let output = responses_route(secrets)
|
||||
.execute(call(true), &(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let ResponsesOutput::Stream { chunks, .. } = output else {
|
||||
panic!("expected SSE");
|
||||
};
|
||||
let wire = chunks.try_collect::<Vec<_>>().await.unwrap().concat();
|
||||
let events = decode(&wire).unwrap();
|
||||
assert_eq!(
|
||||
events.first().map(|event| &event.event_type),
|
||||
Some(&ResponsesEventType::ResponseCreated)
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event.event_type == ResponsesEventType::OutputTextDelta)
|
||||
);
|
||||
let last = events.last().unwrap();
|
||||
assert_eq!(last.event_type, ResponsesEventType::ResponseCompleted);
|
||||
assert!(
|
||||
last.data["response"]["usage"]["total_tokens"]
|
||||
.as_u64()
|
||||
.unwrap()
|
||||
> 0
|
||||
);
|
||||
}
|
||||
150
litellm-rust/crates/inference-responses/tests/openrouter.rs
Normal file
150
litellm-rust/crates/inference-responses/tests/openrouter.rs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use futures_util::TryStreamExt;
|
||||
use litellm_inference_responses::{
|
||||
Error,
|
||||
types::{ResponsesCall, ResponsesOutput},
|
||||
};
|
||||
use litellm_inference_testing::{RecordingSecrets, no_secrets};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
use wiremock::ResponseTemplate;
|
||||
|
||||
mod support;
|
||||
use support::*;
|
||||
|
||||
#[fixture]
|
||||
fn call() -> ResponsesCall {
|
||||
ResponsesCall {
|
||||
model: "openrouter/anthropic/test-model".into(),
|
||||
input: json!("hello"),
|
||||
optional_params: Default::default(),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn response_body() -> serde_json::Value {
|
||||
json!({"id": "resp_1", "model": "anthropic/test-model", "output": [{"type": "message", "content": []}], "usage": {"total_tokens": 7, "cost": 0.00017}})
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::explicit_key(Some("explicit"), &[], "Bearer explicit")]
|
||||
#[case::managed_key(None, &[("OPENROUTER_API_KEY", "managed")], "Bearer managed")]
|
||||
#[case::legacy_name(None, &[("OR_API_KEY", "legacy")], "Bearer legacy")]
|
||||
#[tokio::test]
|
||||
async fn the_payload_goes_to_the_openrouter_responses_endpoint_with_a_bearer(
|
||||
call: ResponsesCall,
|
||||
#[case] api_key: Option<&str>,
|
||||
#[case] secrets: &[(&str, &str)],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let upstream = upstream([json_response(response_body())]).await;
|
||||
let output = responses_route(Arc::new(RecordingSecrets::new(secrets.iter().copied())))
|
||||
.execute(
|
||||
ResponsesCall {
|
||||
api_key: api_key.map(str::to_owned),
|
||||
api_base: Some(format!("{}/api/v1", upstream.uri())),
|
||||
optional_params: json!({"max_output_tokens": 16})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
..call
|
||||
},
|
||||
&(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let ResponsesOutput::Complete(response) = output else {
|
||||
panic!("expected a complete response");
|
||||
};
|
||||
assert_eq!(serde_json::to_value(response).unwrap(), response_body());
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.url.path(), "/api/v1/responses");
|
||||
assert_eq!(request.header("authorization"), Some(expected));
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"max_output_tokens": 16, "model": "anthropic/test-model", "input": "hello"})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn the_secret_manager_supplies_the_base_and_key(call: ResponsesCall) {
|
||||
let upstream = upstream([json_response(response_body())]).await;
|
||||
let base = format!("{}/api/v1/", upstream.uri());
|
||||
let secrets = Arc::new(RecordingSecrets::new([
|
||||
("OPENROUTER_API_KEY", "managed"),
|
||||
("OPENROUTER_API_BASE", base.as_str()),
|
||||
]));
|
||||
responses_route(secrets.clone())
|
||||
.execute(call, &(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
secrets.requested(),
|
||||
["OPENROUTER_API_KEY", "OR_API_KEY", "OPENROUTER_API_BASE"]
|
||||
);
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.url.path(), "/api/v1/responses");
|
||||
assert_eq!(request.header("authorization"), Some("Bearer managed"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_missing_key_fails_before_transport(call: ResponsesCall) {
|
||||
let upstream = upstream([json_response(response_body())]).await;
|
||||
let result = responses_route(no_secrets())
|
||||
.execute(
|
||||
ResponsesCall {
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
},
|
||||
&(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "OpenRouter",
|
||||
..
|
||||
}))
|
||||
));
|
||||
assert!(received(&upstream).await.is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn streaming_relays_every_openrouter_event_untouched(call: ResponsesCall) {
|
||||
let sse = "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_1\"}}\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"usage\":{\"cost\":0.0002}}}\n\n";
|
||||
let upstream =
|
||||
upstream([ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")]).await;
|
||||
let output = responses_route(no_secrets())
|
||||
.execute(
|
||||
ResponsesCall {
|
||||
api_key: Some("key".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
optional_params: json!({"stream": true}).as_object().unwrap().clone(),
|
||||
..call
|
||||
},
|
||||
&(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let ResponsesOutput::Stream { chunks, .. } = output else {
|
||||
panic!("expected SSE");
|
||||
};
|
||||
assert_eq!(
|
||||
chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
|
||||
sse.as_bytes()
|
||||
);
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.json()["stream"], true);
|
||||
assert_eq!(request.json()["model"], "anthropic/test-model");
|
||||
}
|
||||
|
|
@ -58,6 +58,7 @@
|
|||
|
||||
- `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts
|
||||
- `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation
|
||||
- OpenAI-compatible Responses hosts (`openrouter`) reuse the OpenAI request and response transforms as free functions and own only their credentials and base URL
|
||||
- Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper
|
||||
- Auth resolution is split between configs and route preparation in `inference-<format>`
|
||||
|
||||
|
|
|
|||
|
|
@ -165,34 +165,46 @@ impl BaseResponsesApiConfig for OpenAiResponsesApiConfig {
|
|||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
if !input.is_string() && !input.is_array() {
|
||||
return Err(Error::InvalidRequest(
|
||||
"responses input must be a string or an array".into(),
|
||||
));
|
||||
}
|
||||
if params
|
||||
.get("stream")
|
||||
.is_some_and(|stream| !stream.is_boolean())
|
||||
{
|
||||
return Err(Error::InvalidRequest("stream must be a boolean".into()));
|
||||
}
|
||||
Ok(Value::Object(
|
||||
params
|
||||
.into_iter()
|
||||
.chain([
|
||||
("model".into(), Value::String(model.into())),
|
||||
("input".into(), input),
|
||||
])
|
||||
.collect(),
|
||||
))
|
||||
responses_request(model, input, params)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
serde_json::from_value(body)
|
||||
.map_err(|error| Error::InvalidResponse(error.to_string().into()))
|
||||
responses_response(body)
|
||||
}
|
||||
}
|
||||
|
||||
/// The OpenAI Responses request body, shared by every OpenAI-compatible host.
|
||||
pub fn responses_request(
|
||||
model: &str,
|
||||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
if !input.is_string() && !input.is_array() {
|
||||
return Err(Error::InvalidRequest(
|
||||
"responses input must be a string or an array".into(),
|
||||
));
|
||||
}
|
||||
if params
|
||||
.get("stream")
|
||||
.is_some_and(|stream| !stream.is_boolean())
|
||||
{
|
||||
return Err(Error::InvalidRequest("stream must be a boolean".into()));
|
||||
}
|
||||
Ok(Value::Object(
|
||||
params
|
||||
.into_iter()
|
||||
.chain([
|
||||
("model".into(), Value::String(model.into())),
|
||||
("input".into(), input),
|
||||
])
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn responses_response(body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
serde_json::from_value(body).map_err(|error| Error::InvalidResponse(error.to_string().into()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
pub mod messages;
|
||||
pub mod responses;
|
||||
|
|
|
|||
10
litellm-rust/crates/llms/src/openrouter/responses/AGENTS.md
Normal file
10
litellm-rust/crates/llms/src/openrouter/responses/AGENTS.md
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# rules
|
||||
|
||||
- Send the OpenAI Responses payload to `/api/v1/responses` and relay the OpenAI SSE events
|
||||
- Authenticate with a bearer token from `OPENROUTER_API_KEY`, falling back to `OR_API_KEY`
|
||||
- Only `OPENROUTER_API_BASE` overrides the default base, never the OpenAI base variables
|
||||
- No native WebSocket mode
|
||||
|
||||
# references
|
||||
|
||||
- https://openrouter.ai/docs/api_reference/responses/overview
|
||||
1
litellm-rust/crates/llms/src/openrouter/responses/mod.rs
Normal file
1
litellm-rust/crates/llms/src/openrouter/responses/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod transformation;
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
use litellm_llms_types::formats::responses::ResponsesApiResponse;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::{
|
||||
auth::{AuthScheme, ValidatedEnvironment},
|
||||
responses::transformation::BaseResponsesApiConfig,
|
||||
},
|
||||
openai::responses::transformation::{responses_request, responses_response},
|
||||
};
|
||||
|
||||
const API_KEY_ENVS: [&str; 2] = ["OPENROUTER_API_KEY", "OR_API_KEY"];
|
||||
const API_BASE_ENV: &str = "OPENROUTER_API_BASE";
|
||||
const DEFAULT_BASE: &str = "https://openrouter.ai/api/v1";
|
||||
const RESPONSES_PATH: &str = "/responses";
|
||||
|
||||
pub struct OpenRouterResponsesApiConfig;
|
||||
pub const OPENROUTER_RESPONSES_CONFIG: OpenRouterResponsesApiConfig = OpenRouterResponsesApiConfig;
|
||||
|
||||
impl BaseResponsesApiConfig for OpenRouterResponsesApiConfig {
|
||||
fn secret_names(
|
||||
&self,
|
||||
api_key: Option<&str>,
|
||||
api_base: Option<&str>,
|
||||
) -> &'static [&'static str] {
|
||||
match (
|
||||
api_key.is_some_and(|key| !key.trim().is_empty()),
|
||||
api_base.is_some_and(|base| !base.trim().is_empty()),
|
||||
) {
|
||||
(true, true) => &[],
|
||||
(true, false) => &[API_BASE_ENV],
|
||||
(false, true) => &API_KEY_ENVS,
|
||||
(false, false) => &["OPENROUTER_API_KEY", "OR_API_KEY", API_BASE_ENV],
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_environment(
|
||||
&self,
|
||||
headers: Vec<(String, String)>,
|
||||
api_key: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
let key = resolve_non_empty(api_key, lookup, &API_KEY_ENVS).ok_or(
|
||||
litellm_auth::Error::MissingApiKey {
|
||||
provider: "OpenRouter",
|
||||
environment_variable: API_KEY_ENVS[0],
|
||||
},
|
||||
)?;
|
||||
Ok(ValidatedEnvironment {
|
||||
headers: litellm_http::request::with_default_headers(
|
||||
headers,
|
||||
&[("content-type", "application/json")],
|
||||
),
|
||||
auth: AuthScheme::Credential {
|
||||
placement: CredentialPlacement::Bearer,
|
||||
secret: SecretValue::new(key),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> String {
|
||||
let base = resolve_non_empty(api_base, lookup, &[API_BASE_ENV])
|
||||
.unwrap_or_else(|| DEFAULT_BASE.into());
|
||||
format!("{}{RESPONSES_PATH}", base.trim_end_matches('/'))
|
||||
}
|
||||
|
||||
fn transform_responses_api_request(
|
||||
&self,
|
||||
model: &str,
|
||||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
responses_request(model, input, params)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
responses_response(body)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn env(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
values
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| (*value).to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default(None, &[], "https://openrouter.ai/api/v1/responses")]
|
||||
#[case::explicit(Some("https://proxy.test/api/v1"), &[], "https://proxy.test/api/v1/responses")]
|
||||
#[case::trailing_slash(Some("https://proxy.test/api/v1/"), &[], "https://proxy.test/api/v1/responses")]
|
||||
#[case::blank_falls_to_secret(Some(" "), &[("OPENROUTER_API_BASE", "https://secret.test/v1")], "https://secret.test/v1/responses")]
|
||||
#[case::openai_base_is_ignored(None, &[("OPENAI_BASE_URL", "https://openai.test/v1")], "https://openrouter.ai/api/v1/responses")]
|
||||
fn url_is_the_base_plus_responses(
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] secrets: &'static [(&'static str, &'static str)],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
OPENROUTER_RESPONSES_CONFIG.get_complete_url(api_base, &env(secrets)),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::explicit(Some("explicit"), &[("OPENROUTER_API_KEY", "managed")], "explicit")]
|
||||
#[case::managed(None, &[("OPENROUTER_API_KEY", "managed"), ("OR_API_KEY", "legacy")], "managed")]
|
||||
#[case::legacy_name(None, &[("OR_API_KEY", "legacy")], "legacy")]
|
||||
#[case::blank_explicit(Some(" "), &[("OR_API_KEY", "legacy")], "legacy")]
|
||||
fn bearer_comes_from_the_call_then_either_secret_name(
|
||||
#[case] api_key: Option<&str>,
|
||||
#[case] secrets: &'static [(&'static str, &'static str)],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let environment = OPENROUTER_RESPONSES_CONFIG
|
||||
.validate_environment(vec![], api_key, &env(secrets))
|
||||
.unwrap();
|
||||
let AuthScheme::Credential { placement, secret } = environment.auth else {
|
||||
panic!("expected a credential");
|
||||
};
|
||||
assert_eq!(placement, CredentialPlacement::Bearer);
|
||||
assert_eq!(secret.expose(), expected);
|
||||
assert_eq!(
|
||||
environment.headers,
|
||||
vec![("content-type".to_owned(), "application/json".to_owned())]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::nothing(&[])]
|
||||
#[case::openai_key_only(&[("OPENAI_API_KEY", "openai")])]
|
||||
fn a_missing_key_names_the_primary_variable(
|
||||
#[case] secrets: &'static [(&'static str, &'static str)],
|
||||
) {
|
||||
assert!(matches!(
|
||||
OPENROUTER_RESPONSES_CONFIG.validate_environment(vec![], None, &env(secrets)),
|
||||
Err(Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "OpenRouter",
|
||||
environment_variable: "OPENROUTER_API_KEY",
|
||||
}))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::both_given(Some("key"), Some("base"), &[])]
|
||||
#[case::key_given(Some("key"), None, &["OPENROUTER_API_BASE"])]
|
||||
#[case::base_given(None, Some("base"), &["OPENROUTER_API_KEY", "OR_API_KEY"])]
|
||||
#[case::neither(None, None, &["OPENROUTER_API_KEY", "OR_API_KEY", "OPENROUTER_API_BASE"])]
|
||||
fn only_missing_settings_are_looked_up(
|
||||
#[case] api_key: Option<&str>,
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] expected: &[&str],
|
||||
) {
|
||||
assert_eq!(
|
||||
OPENROUTER_RESPONSES_CONFIG.secret_names(api_key, api_base),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -12,8 +12,9 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, Respons
|
|||
PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__)
|
||||
|
||||
|
||||
def connection_defaults(_provider: str) -> tuple[str | None, str | None]:
|
||||
return litellm.api_key or litellm.openai_key, litellm.api_base
|
||||
def connection_defaults(provider: str) -> tuple[str | None, str | None]:
|
||||
provider_key: Final = litellm.openai_key if provider == "openai" else None
|
||||
return litellm.api_key or provider_key, litellm.api_base
|
||||
|
||||
|
||||
def response(value: Mapping[str, object]) -> ResponsesAPIResponse:
|
||||
|
|
|
|||
|
|
@ -59,6 +59,16 @@ def test_connection_defaults_preserve_openai_precedence(
|
|||
assert connection_defaults("openai") == (expected, litellm.api_base)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("global_key", "expected"), (("global", "global"), (None, None)))
|
||||
def test_connection_defaults_keep_the_openai_key_away_from_openrouter(
|
||||
monkeypatch: pytest.MonkeyPatch, global_key: str | None, expected: str | None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "api_key", global_key)
|
||||
monkeypatch.setattr(litellm, "openai_key", "openai-only")
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
assert connection_defaults("openrouter") == (expected, None)
|
||||
|
||||
|
||||
class _UpstreamFailure(Exception):
|
||||
headers: Final = ()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue