scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-12 03:48:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 57

feat(rust): add native OpenRouter Messages

Browse source
This commit is contained in:
Yujong Lee 2026-10-10 18:49:03 -07:00
parent 462962c094
commit af2b17e084
12 changed files with 461 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.greptile/files.json
View file

@ -336,6 +336,11 @@
"litellm-rust/crates/llms/src/vertex_ai/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/openrouter/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/openrouter/messages/",
"scope": ["litellm-rust/crates/llms/src/openrouter/messages/**"]
},
{
"path": "litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/ocr/",

2
litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs
View file

@ -28,6 +28,8 @@ pub enum LlmProviders {
Openai,
#[strum(serialize = "openai_like")]
OpenaiLike,
#[strum(serialize = "openrouter")]
Openrouter,
#[strum(serialize = "reducto")]
Reducto,
#[strum(serialize = "vertex_ai")]

38
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -19,7 +19,12 @@ use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
/// Client headers Python forwards to Anthropic-speaking providers on every call.
const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"];
const ANTHROPIC_API_HEADER_PROVIDERS: &str = "anthropic,bedrock,bedrock_mantle,vertex_ai";
const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-metadata",
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
pub async fn create(
State(gateway): State<Arc<Gateway>>,
@ -88,17 +93,28 @@ fn project(
}
fn anthropic_api_headers(headers: &HeaderMap) -> Option<ProviderSpecificHeaders> {
let extra_headers: Map<String, Value> = ANTHROPIC_API_HEADERS
let scopes = [
(
ANTHROPIC_API_HEADER_PROVIDERS,
ANTHROPIC_API_HEADERS.as_slice(),
),
("openrouter", OPENROUTER_API_HEADERS.as_slice()),
];
let forwarded: Vec<ProviderSpecificHeader> = scopes
.into_iter()
.filter_map(|name| {
let value = headers.get(name)?.to_str().ok()?;
Some((name.to_owned(), Value::from(value)))
.filter_map(|(provider, names)| {
let extra_headers: Map<String, Value> = names
.iter()
.filter_map(|name| {
let value = headers.get(*name)?.to_str().ok()?;
Some(((*name).to_owned(), Value::from(value)))
})
.collect();
(!extra_headers.is_empty()).then(|| ProviderSpecificHeader {
custom_llm_provider: provider.into(),
extra_headers,
})
})
.collect();
(!extra_headers.is_empty()).then(|| {
ProviderSpecificHeaders::One(ProviderSpecificHeader {
custom_llm_provider: ANTHROPIC_API_HEADER_PROVIDERS.into(),
extra_headers,
})
})
(!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded))
}

49
litellm-rust/crates/gateway-inference/tests/messages.rs
View file

@ -192,3 +192,52 @@ async fn model_permissions_prevent_provider_calls(#[case] path: &str) {
assert_eq!(response.status(), 403);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[case::completion(false)]
#[case::streaming(true)]
#[tokio::test]
async fn openrouter_receives_native_messages_headers_and_its_deployment_bearer(
#[case] streaming: bool,
) {
let upstream = MockServer::start().await;
let message = json!({"id": "msg_test", "type": "message", "role": "assistant", "model": "anthropic/test-model", "content": [{"type": "text", "text": "hello"}], "stop_reason": "end_turn", "stop_sequence": null, "usage": {"input_tokens": 1, "output_tokens": 1, "cost": 0.001, "cost_details": "unexpected"}, "openrouter_metadata": {"provider": "test-provider"}});
let sse = "event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"cost\":0.001}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
let template = if streaming {
ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")
} else {
ResponseTemplate::new(200).set_body_json(message.clone())
};
Mock::given(method("POST"))
.and(path("/api/v1/messages"))
.and(header("authorization", "Bearer test-key"))
.and(header("anthropic-beta", "test-feature"))
.and(header("anthropic-version", "test-version"))
.and(header("x-openrouter-metadata", "enabled"))
.and(body_json(json!({"model": "anthropic/test-model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": streaming, "provider": {"order": ["test-provider"]}})))
.respond_with(template).expect(1).mount(&upstream).await;
let request = Request::post("/v1/messages")
.header("content-type", "application/json")
.header("anthropic-beta", "test-feature")
.header("anthropic-version", "test-version")
.header("x-openrouter-metadata", "enabled")
.header("x-api-key", "proxy-key")
.header("authorization", "Bearer proxy-key")
.body(Body::from(json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": streaming, "provider": {"order": ["test-provider"]}}).to_string())).unwrap();
let response = support::app(
"openrouter/anthropic/test-model",
&format!("{}/api/v1", upstream.uri()),
)
.oneshot(request)
.await
.unwrap();
assert_eq!(response.status(), 200);
if streaming {
assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), sse);
} else {
assert_eq!(support::json(response).await, message);
}
let requests = upstream.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert!(!requests[0].headers.contains_key("x-api-key"));
}

44
litellm-rust/crates/http/src/request.rs
View file

@ -138,7 +138,9 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
}
let value = value.trim();
value.len() > 7
&& value[..7].eq_ignore_ascii_case("bearer ")
&& value
.get(..7)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("bearer "))
&& !value[7..].trim().is_empty()
})
}
@ -279,31 +281,19 @@ mod tests {
assert!(!has_header(&headers, "authorization"));
}
#[test]
fn bearer_detection_requires_a_non_empty_token() {
assert!(has_bearer_auth(&[(
"Authorization".to_string(),
"Bearer abc".to_string()
)]));
assert!(has_bearer_auth(&[(
"authorization".to_string(),
"bearer abc".to_string()
)]));
assert!(!has_bearer_auth(&[(
"Authorization".to_string(),
"Bearer ".to_string()
)]));
assert!(!has_bearer_auth(&[(
"authorization".to_string(),
String::new()
)]));
assert!(!has_bearer_auth(&[(
"Authorization".to_string(),
"Basic abc".to_string()
)]));
assert!(!has_bearer_auth(&[(
"x-api-key".to_string(),
"abc".to_string()
)]));
#[rstest::rstest]
#[case::bearer("Authorization", "Bearer abc", true)]
#[case::lowercase("authorization", "bearer abc", true)]
#[case::blank_token("Authorization", "Bearer ", false)]
#[case::empty("authorization", "", false)]
#[case::basic("Authorization", "Basic abc", false)]
#[case::different_header("x-api-key", "abc", false)]
#[case::non_ascii("authorization", "invali\u{00e9}auth", false)]
fn bearer_detection_requires_a_non_empty_token(
#[case] name: &str,
#[case] value: &str,
#[case] expected: bool,
) {
assert_eq!(has_bearer_auth(&[(name.into(), value.into())]), expected);
}
}

3
litellm-rust/crates/inference-messages/src/common_utils.rs
View file

@ -35,6 +35,8 @@ impl MessagesProvider {
/// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
/// partner models on this route.
pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<MessagesProvider> {
use litellm_llms::openrouter::messages::transformation::OPENROUTER_MESSAGES_CONFIG;
let config: &'static dyn BaseMessagesConfig = match provider {
LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG,
@ -44,6 +46,7 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<M
LlmProviders::VertexAi if model.to_ascii_lowercase().contains("claude") => {
&VERTEX_ANTHROPIC_MESSAGES_CONFIG
}
LlmProviders::Openrouter => &OPENROUTER_MESSAGES_CONFIG,
_ => return None,
};
Some(MessagesProvider { provider, config })

147
litellm-rust/crates/inference-messages/tests/openrouter/main.rs Normal file
View file

@ -0,0 +1,147 @@
use std::{sync::Arc, time::Duration};
use futures_util::TryStreamExt;
use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesShaping};
use litellm_inference_testing::{RecordingSecrets, no_secrets};
use rstest::{fixture, rstest};
use serde_json::json;
use wiremock::ResponseTemplate;
#[path = "../support/mod.rs"]
mod support;
use support::*;
#[fixture]
fn call() -> MessagesCall {
MessagesCall {
body: serde_json::from_value(json!({"model": "openrouter/anthropic/test-model", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})).unwrap(),
api_key: None,
api_base: None,
custom_llm_provider: None,
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: Some(Duration::from_secs(5)),
shaping: MessagesShaping::default(),
}
}
fn message_response() -> ResponseTemplate {
json_response(
json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "anthropic/test-model", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}}),
)
}
#[rstest]
#[case::explicit(Some("explicit"), None, "Bearer explicit")]
#[case::managed(None, None, "Bearer managed")]
#[case::forwarded(None, Some("bEaReR caller"), "bEaReR caller")]
#[case::blank(Some("explicit"), Some("Bearer "), "Bearer explicit")]
#[case::malformed(Some("explicit"), Some("invali\u{00e9}auth"), "Bearer explicit")]
#[tokio::test]
async fn only_a_valid_bearer_can_override_the_configured_key(
call: MessagesCall,
#[case] api_key: Option<&str>,
#[case] authorization: Option<&str>,
#[case] expected: &str,
) {
let upstream = upstream([message_response()]).await;
let extra_headers = [("x-api-key".to_string(), json!("caller-anthropic-key"))]
.into_iter()
.chain(authorization.map(|value| ("Authorization".to_string(), json!(value))))
.collect();
let secrets = Arc::new(RecordingSecrets::new([("OPENROUTER_API_KEY", "managed")]));
let output = messages_route(secrets)
.execute(
MessagesCall {
api_key: api_key.map(str::to_string),
api_base: Some(format!("{}/api/v1", upstream.uri())),
extra_headers: Some(extra_headers),
..call
},
&(),
None,
)
.await
.unwrap();
assert!(matches!(output, MessagesCallResponse::Complete(_)));
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/api/v1/messages");
assert_eq!(request.header_values("authorization"), [expected]);
assert_eq!(request.header("x-api-key"), None);
assert_eq!(request.json()["model"], "anthropic/test-model");
}
#[rstest]
#[tokio::test]
async fn an_anthropic_key_alone_is_not_an_openrouter_credential(call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let result = messages_route(no_secrets())
.execute(
MessagesCall {
api_base: Some(upstream.uri()),
extra_headers: Some(
json!({"x-api-key": "caller-key"})
.as_object()
.unwrap()
.clone(),
),
..call
},
&(),
None,
)
.await;
assert!(matches!(
result,
Err(Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "OpenRouter",
..
}))
));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[tokio::test]
async fn the_secret_manager_supplies_the_native_base_and_key(call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let base = format!("{}/api/v1", upstream.uri());
let secrets = Arc::new(RecordingSecrets::new([
("OPENROUTER_API_KEY", "managed"),
("OPENROUTER_API_BASE", base.as_str()),
]));
messages_route(secrets)
.execute(call, &(), None)
.await
.unwrap();
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/api/v1/messages");
assert_eq!(request.header("authorization"), Some("Bearer managed"));
}
#[rstest]
#[tokio::test]
async fn streaming_relay_preserves_every_event_and_billed_usage(call: MessagesCall) {
let sse = "event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"cost\":\"0.001\",\"cost_details\":\"unexpected\"}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
let upstream =
upstream([ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")]).await;
let streamed = MessagesCall {
body: serde_json::from_value(json!({"model": "openrouter/anthropic/test-model", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}], "stream": true})).unwrap(),
api_key: Some("key".into()),
api_base: Some(upstream.uri()),
..call
};
let output = messages_route(no_secrets())
.execute(streamed, &(), None)
.await
.unwrap();
let MessagesCallResponse::Stream { chunks, .. } = output else {
panic!("expected SSE");
};
assert_eq!(
chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
sse.as_bytes()
);
assert_eq!(only_request(&upstream).await.json()["stream"], true);
}

1
litellm-rust/crates/llms/src/lib.rs
View file

@ -10,6 +10,7 @@ pub mod minimax;
pub mod mistral;
pub mod openai;
pub mod openai_like;
pub mod openrouter;
pub mod reducto;
pub mod vertex_ai;

10
litellm-rust/crates/llms/src/openrouter/messages/AGENTS.md Normal file
View file

@ -0,0 +1,10 @@
# rules
- Send the native Messages payload to `/api/v1/messages` and relay Anthropic SSE
- Authenticate with a bearer token, never treat `x-api-key` as an OpenRouter credential
- Keep cache TTLs, routing fields, metadata, and provider usage extensions
- Strip first-party billing system blocks using the Anthropic helper
# references
- https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages

1
litellm-rust/crates/llms/src/openrouter/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

198
litellm-rust/crates/llms/src/openrouter/messages/transformation.rs Normal file
View file

@ -0,0 +1,198 @@
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{has_bearer_auth, without_headers};
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::DEFAULT_HEADERS,
};
use litellm_router_types::LitellmParams;
use crate::{
Error,
anthropic::{
common_utils::filter_billing_headers_from_system,
messages::transformation::update_headers_with_anthropic_beta,
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
},
};
const API_KEY_ENV: &str = "OPENROUTER_API_KEY";
const API_BASE_ENV: &str = "OPENROUTER_API_BASE";
const DEFAULT_BASE: &str = "https://openrouter.ai/api/v1";
const MESSAGES_PATH: &str = "/v1/messages";
pub struct OpenRouterAnthropicMessagesConfig;
pub const OPENROUTER_MESSAGES_CONFIG: OpenRouterAnthropicMessagesConfig =
OpenRouterAnthropicMessagesConfig;
impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
_model: &str,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
let headers = without_headers(headers, &["x-api-key"]);
if has_bearer_auth(&headers) {
return Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Forwarded,
});
}
let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
litellm_auth::Error::MissingApiKey {
provider: "OpenRouter",
environment_variable: API_KEY_ENV,
},
))?;
Ok(ValidatedEnvironment {
headers: without_headers(headers, &["authorization"]),
auth: AuthScheme::Credential {
placement: CredentialPlacement::Bearer,
secret: SecretValue::new(key),
},
})
}
fn get_complete_url(
&self,
api_base: Option<&str>,
_model: &str,
params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let explicit = api_base
.filter(|base| !base.trim().is_empty())
.or(params.api_base.as_deref());
let base = resolve_non_empty(explicit, env, &[API_BASE_ENV])
.unwrap_or_else(|| DEFAULT_BASE.into());
let trimmed = base.trim_end_matches('/');
if trimmed.ends_with(MESSAGES_PATH) {
return Ok(trimmed.into());
}
Ok(format!(
"{}{MESSAGES_PATH}",
trimmed.strip_suffix("/v1").unwrap_or(trimmed)
))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
_context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
Ok(MessagesRequest {
params: MessagesOptionalParams {
system: request
.params
.system
.and_then(filter_billing_headers_from_system),
..request.params
},
..request
})
}
fn secret_names(&self) -> &'static [&'static str] {
&[API_KEY_ENV, API_BASE_ENV]
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::default(None, "https://openrouter.ai/api/v1/messages")]
#[case::root(Some("https://proxy.test"), "https://proxy.test/v1/messages")]
#[case::version(
Some("https://proxy.test/api/v1/"),
"https://proxy.test/api/v1/messages"
)]
#[case::complete(
Some("https://proxy.test/api/v1/messages/"),
"https://proxy.test/api/v1/messages"
)]
fn urls_append_messages_once(#[case] base: Option<&str>, #[case] expected: &str) {
assert_eq!(
OPENROUTER_MESSAGES_CONFIG
.get_complete_url(
base,
"anthropic/test-model",
&LitellmParams::default(),
true,
&|_| None
)
.unwrap(),
expected
);
}
#[rstest]
#[case::call(
Some("https://call.test/v1"),
Some("https://deployment.test/v1"),
"https://call.test/v1/messages"
)]
#[case::deployment(
None,
Some("https://deployment.test/v1"),
"https://deployment.test/v1/messages"
)]
#[case::blank_call(
Some(" "),
Some("https://deployment.test/v1"),
"https://deployment.test/v1/messages"
)]
#[case::environment(None, None, "https://secret.test/v1/messages")]
fn base_precedence_is_call_deployment_then_secrets(
#[case] base: Option<&str>,
#[case] deployment: Option<&str>,
#[case] expected: &str,
) {
let params = LitellmParams {
api_base: deployment.map(str::to_string),
..Default::default()
};
assert_eq!(
OPENROUTER_MESSAGES_CONFIG
.get_complete_url(base, "test-model", &params, false, &|name| (name
== API_BASE_ENV)
.then(|| "https://secret.test/v1".into()))
.unwrap(),
expected
);
}
#[rstest]
fn native_payload_is_preserved_except_first_party_billing_blocks() {
let input = json!({"model": "anthropic/test-model", "max_tokens": 128, "messages": [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}], "system": [{"type": "text", "text": "x-anthropic-billing-header: private"}, {"type": "text", "text": "instructions"}], "metadata": {"user_id": "user", "custom": "keep"}, "provider": {"order": ["test-provider"]}, "thinking": {"type": "enabled", "budget_tokens": 64}});
let request = serde_json::from_value(input.clone()).unwrap();
let actual = OPENROUTER_MESSAGES_CONFIG
.transform_anthropic_messages_request(request, &MessagesTransformContext::default())
.unwrap();
let expected = json!({"system": [{"type": "text", "text": "instructions"}]});
let body = serde_json::to_value(actual).unwrap();
assert_eq!(body["system"], expected["system"]);
assert_eq!(body["messages"], input["messages"]);
assert_eq!(body["metadata"], input["metadata"]);
assert_eq!(body["provider"], input["provider"]);
assert_eq!(body["thinking"], input["thinking"]);
}
}

1
litellm-rust/crates/llms/src/openrouter/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;