From 5f2d8adaa35eeadc6f5dd535de43f60c0e57de34 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 10 Oct 2026 17:32:51 -0700 Subject: [PATCH] feat(rust): support Tencent Messages API --- .greptile/files.json | 7 + .../core-utils/src/get_llm_provider_logic.rs | 2 + .../inference-messages/src/common_utils.rs | 3 + .../inference-messages/tests/messages/main.rs | 1 + .../tests/messages/tencent.rs | 260 ++++++++++++++++++ litellm-rust/crates/llms/src/lib.rs | 1 + .../llms/src/tencent/messages/AGENTS.md | 11 + .../crates/llms/src/tencent/messages/mod.rs | 1 + .../src/tencent/messages/transformation.rs | 241 ++++++++++++++++ litellm-rust/crates/llms/src/tencent/mod.rs | 1 + 10 files changed, 528 insertions(+) create mode 100644 litellm-rust/crates/inference-messages/tests/messages/tencent.rs create mode 100644 litellm-rust/crates/llms/src/tencent/messages/AGENTS.md create mode 100644 litellm-rust/crates/llms/src/tencent/messages/mod.rs create mode 100644 litellm-rust/crates/llms/src/tencent/messages/transformation.rs create mode 100644 litellm-rust/crates/llms/src/tencent/mod.rs diff --git a/.greptile/files.json b/.greptile/files.json index c150947746c..0f36c03b62d 100644 --- a/.greptile/files.json +++ b/.greptile/files.json @@ -329,6 +329,13 @@ "litellm-rust/crates/llms/src/reducto/ocr/**" ] }, + { + "path": "litellm-rust/crates/llms/src/tencent/messages/AGENTS.md", + "description": "Conventions for code under litellm-rust/crates/llms/src/tencent/messages/", + "scope": [ + "litellm-rust/crates/llms/src/tencent/messages/**" + ] + }, { "path": "litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md", "description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/messages/", diff --git a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index 2b37fd062b9..8436020bce2 100644 --- a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -30,6 +30,8 @@ pub enum LlmProviders { OpenaiLike, #[strum(serialize = "reducto")] Reducto, + #[strum(serialize = "tencent")] + Tencent, #[strum(serialize = "vertex_ai")] VertexAi, } diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index 4439b09a68c..ea5af89bf82 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -1,6 +1,7 @@ use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_http::request::string_headers as shared_string_headers; pub(super) use litellm_http::request::truncate_error_body; +use litellm_llms::tencent::messages::transformation::TENCENT_MESSAGES_CONFIG; use litellm_llms::{ anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, @@ -36,6 +37,7 @@ impl MessagesProvider { /// partner models on this route. pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option { let config: &'static dyn BaseMessagesConfig = match provider { + LlmProviders::Tencent => &TENCENT_MESSAGES_CONFIG, LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG, LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG, LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG, @@ -66,6 +68,7 @@ mod tests { use litellm_core_utils::get_llm_provider_logic::LlmProviders; #[rstest] + #[case::tencent("tencent")] #[case::anthropic("anthropic")] #[case::azure_ai("azure_ai")] #[case::bedrock("bedrock")] diff --git a/litellm-rust/crates/inference-messages/tests/messages/main.rs b/litellm-rust/crates/inference-messages/tests/messages/main.rs index 8f92b31c999..fcc432acb76 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/main.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/main.rs @@ -24,6 +24,7 @@ mod request; mod response; mod secrets; mod stream; +mod tencent; const MODEL: &str = "claude-sonnet-4-5"; diff --git a/litellm-rust/crates/inference-messages/tests/messages/tencent.rs b/litellm-rust/crates/inference-messages/tests/messages/tencent.rs new file mode 100644 index 00000000000..dce187f4ea2 --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/tencent.rs @@ -0,0 +1,260 @@ +use std::ops::ControlFlow; + +use bytes::Bytes; +use litellm_llms::base_llm::messages::context::MessagesModelCapabilities; +use litellm_llms_types::providers::anthropic::DEFAULT_HEADERS; +use rstest::{fixture, rstest}; + +use super::*; + +#[fixture] +fn tencent_call(call: MessagesCall) -> MessagesCall { + MessagesCall { + custom_llm_provider: Some("tencent".into()), + api_key: Some("deployment-key".into()), + ..call + } +} + +#[rstest] +#[case::deployment_key(&[], "x-api-key", "deployment-key")] +#[case::forwarded_key(&[("X-Api-Key", "caller-key")], "x-api-key", "caller-key")] +#[case::forwarded_bearer(&[("Authorization", "Bearer caller")], "authorization", "Bearer caller")] +#[tokio::test] +async fn credentials_reach_the_selected_provider( + tencent_call: MessagesCall, + #[case] forwarded: &[(&str, &str)], + #[case] header: &str, + #[case] credential: &str, +) { + let upstream = upstream([message_response()]).await; + run_message(MessagesCall { + api_base: Some(upstream.uri()), + extra_headers: headers(forwarded.iter().copied()), + ..tencent_call + }) + .await; + let sent = only_request(&upstream).await; + assert_eq!(sent.header_values(header), [credential]); + let version = DEFAULT_HEADERS + .iter() + .find_map(|(name, value)| (*name == "anthropic-version").then_some(*value)) + .unwrap(); + assert_eq!(sent.header_values("anthropic-version"), [version]); + assert_eq!(sent.json()["model"], MODEL); +} + +#[rstest] +#[tokio::test] +async fn credentials_and_base_resolve_from_the_secret_source(tencent_call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let base = format!("{}/v1/chat/completions/", upstream.uri()); + let secrets = Arc::new(RecordingSecrets::new([ + ("TENCENT_API_KEY", "secret-key"), + ("TENCENT_ANTHROPIC_API_BASE", base.as_str()), + ("TENCENT_API_BASE", UNREACHABLE_BASE), + ])); + run_with( + secrets, + MessagesCall { + api_key: None, + ..tencent_call + }, + ) + .await + .unwrap(); + let sent = only_request(&upstream).await; + assert_eq!(sent.url.path(), "/v1/messages"); + assert_eq!(sent.header("x-api-key"), Some("secret-key")); +} + +#[rstest] +#[tokio::test] +async fn missing_credentials_fail_before_sending(tencent_call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let error = run(MessagesCall { + api_key: None, + api_base: Some(upstream.uri()), + ..tencent_call + }) + .await + .unwrap_err(); + assert!(matches!( + error, + Error::Auth(litellm_auth::Error::MissingApiKey { + environment_variable: "TENCENT_API_KEY", + .. + }) + )); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[case::missing_max_tokens(json!({"max_tokens": null}))] +#[case::invalid_metadata(json!({"metadata": {"user_id": 7}}))] +#[case::invalid_reasoning_effort(json!({"reasoning_effort": "unknown"}))] +#[tokio::test] +async fn malformed_requests_fail_before_sending(tencent_call: MessagesCall, #[case] fields: Value) { + let upstream = upstream([message_response()]).await; + let error = run(with_fields( + MessagesCall { + api_base: Some(upstream.uri()), + ..tencent_call + }, + fields, + )) + .await + .unwrap_err(); + assert!(error.is_request(), "{error:?}"); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[case::legacy(false, json!({"type": "enabled", "budget_tokens": 4095}))] +#[case::adaptive(true, json!({"type": "adaptive", "display": "summarized"}))] +#[tokio::test] +async fn reasoning_effort_becomes_model_appropriate_thinking( + tencent_call: MessagesCall, + #[case] adaptive: bool, + #[case] thinking: Value, +) { + let upstream = upstream([message_response()]).await; + run_message(with_fields( + MessagesCall { + api_base: Some(upstream.uri()), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: adaptive, + ..Default::default() + }, + ..Default::default() + }, + ..tencent_call + }, + json!({"max_tokens": 4096, "reasoning_effort": "high"}), + )) + .await; + let sent = only_request(&upstream).await.json(); + assert_eq!(sent["thinking"], thinking); + assert_eq!(sent.get("reasoning_effort"), None); + assert_eq!( + sent.get("output_config"), + adaptive.then(|| json!({"effort": "high"})).as_ref() + ); +} + +#[rstest] +#[case::encrypted_reasoning(json!({"type": "thinking", "thinking": "private", "signature": "litellm_encrypted_reasoning:private"}))] +#[case::orphaned_advisor(json!({"type": "server_tool_use", "name": "advisor", "id": "srvtoolu_advisor", "input": {}}))] +#[tokio::test] +async fn replayed_history_is_cleaned_without_losing_text( + tencent_call: MessagesCall, + #[case] block: Value, +) { + let upstream = upstream([message_response()]).await; + run_message(with_fields( + MessagesCall { + api_base: Some(upstream.uri()), + ..tencent_call + }, + json!({"messages": [ + {"role": "assistant", "content": [block, {"type": "text", "text": "keep"}]}, + {"role": "user", "content": "continue"} + ]}), + )) + .await; + assert_eq!( + only_request(&upstream).await.json()["messages"], + json!([ + {"role": "assistant", "content": [{"type": "text", "text": "keep"}]}, + {"role": "user", "content": "continue"} + ]) + ); +} + +#[rstest] +#[case::billing_string(json!("x-anthropic-billing-header: private"), None)] +#[case::mixed_blocks(json!([ + {"type": "text", "text": "x-anthropic-billing-header: private"}, + {"type": "text", "text": "keep"} +]), Some(json!([{ "type": "text", "text": "keep" }])))] +#[case::ordinary_system(json!("keep"), Some(json!("keep")))] +#[tokio::test] +async fn billing_is_removed_and_tools_and_cache_hints_are_preserved( + tencent_call: MessagesCall, + #[case] system: Value, + #[case] expected: Option, +) { + let upstream = upstream([message_response()]).await; + let fields = json!({ + "system": system, + "tools": [{"name": "echo", "input_schema": {"type": "object"}}], + "messages": [{"role": "user", "content": [{ + "type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"} + }]}] + }); + run_message(with_fields( + MessagesCall { + api_base: Some(upstream.uri()), + ..tencent_call + }, + fields.clone(), + )) + .await; + let sent = only_request(&upstream).await.json(); + assert_eq!(sent.get("system"), expected.as_ref()); + assert_eq!(sent["tools"], fields["tools"]); + assert_eq!(sent["messages"], fields["messages"]); +} + +#[derive(Default)] +struct StreamSink(Mutex>); + +impl litellm_host_native::in_process::StreamConsumer for StreamSink { + async fn open_stream( + &self, + _: litellm_inference_messages::route::MessagesStreamHead, + ) -> Result, Error> { + Ok(ControlFlow::Continue(())) + } + + async fn send_chunk(&self, chunk: Bytes) -> Result, Error> { + self.0.lock().unwrap().extend_from_slice(&chunk); + Ok(ControlFlow::Continue(())) + } +} + +#[rstest] +#[tokio::test] +async fn streaming_relays_provider_events_unchanged(tencent_call: MessagesCall) { + let events = concat!( + "event: message_start\ndata: {\"type\":\"message_start\"}\n\n", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"OK\"}}\n\n", + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" + ); + let upstream = + upstream([ResponseTemplate::new(200).set_body_raw(events, "text/event-stream")]).await; + let host = LocalMessagesHost::new(with_fields( + MessagesCall { + api_base: Some(upstream.uri()), + ..tencent_call + }, + json!({"stream": true}), + )); + let sink = StreamSink::default(); + let outcome = litellm_host_native::in_process::run_hosted( + machine(Arc::new(RecordingSecrets::empty()))(host.request().unwrap()), + litellm_host_native::in_process::Host { + services: &(), + interceptors: &host, + stream: &sink, + observers: None, + }, + ) + .await + .unwrap(); + assert!(matches!(outcome, MessagesOutput::StreamEnded)); + assert_eq!(sink.0.into_inner().unwrap(), events.as_bytes()); + assert_eq!(only_request(&upstream).await.json()["stream"], true); +} diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs index 2fac586ffb3..193a671b5c8 100644 --- a/litellm-rust/crates/llms/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -11,6 +11,7 @@ pub mod mistral; pub mod openai; pub mod openai_like; pub mod reducto; +pub mod tencent; pub mod vertex_ai; pub use error::{Error, ErrorDetail, ErrorSource}; diff --git a/litellm-rust/crates/llms/src/tencent/messages/AGENTS.md b/litellm-rust/crates/llms/src/tencent/messages/AGENTS.md new file mode 100644 index 00000000000..5908ba1f9b5 --- /dev/null +++ b/litellm-rust/crates/llms/src/tencent/messages/AGENTS.md @@ -0,0 +1,11 @@ +# rules + +- Reuse Anthropic Messages shaping, reasoning translation, and history cleanup +- Use Tencent credentials and endpoints, and strip billing system blocks +- Accept a root, versioned base, chat endpoint, or complete Messages endpoint + +# references + +- Python counterpart `litellm/llms/tencent/messages/transformation.py` +- https://www.tencentcloud.com/document/product/1300/80632 +- https://cloud.tencent.com/document/product/1823/132246 diff --git a/litellm-rust/crates/llms/src/tencent/messages/mod.rs b/litellm-rust/crates/llms/src/tencent/messages/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/tencent/messages/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs new file mode 100644 index 00000000000..e96025952dd --- /dev/null +++ b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs @@ -0,0 +1,241 @@ +use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_core_utils::settings::resolve_non_empty; +use litellm_llms_types::{ + formats::messages::{MessagesOptionalParams, MessagesRequest}, + providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH}, +}; +use litellm_router_types::LitellmParams; + +use crate::{ + Error, + anthropic::{ + common_utils::{filter_billing_headers_from_system, has_anthropic_credential}, + messages::{ + handler::shape_anthropic_messages_request, + transformation::{transform_messages_request, update_headers_with_anthropic_beta}, + }, + }, + base_llm::{ + auth::{AuthScheme, Headers, ValidatedEnvironment}, + messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + }, +}; + +const API_KEY_ENV: &str = "TENCENT_API_KEY"; +const API_BASE_ENV: &str = "TENCENT_API_BASE"; +const ANTHROPIC_API_BASE_ENV: &str = "TENCENT_ANTHROPIC_API_BASE"; +const DEFAULT_BASE: &str = "https://tokenhub-intl.tencentcloudmaas.com"; + +pub struct TencentAnthropicMessagesConfig; +pub const TENCENT_MESSAGES_CONFIG: TencentAnthropicMessagesConfig = TencentAnthropicMessagesConfig; + +impl BaseMessagesConfig for TencentAnthropicMessagesConfig { + fn shape_request( + &self, + request: MessagesRequest, + reasoning_auto_summary: bool, + ) -> Result { + shape_anthropic_messages_request(request, reasoning_auto_summary) + } + + fn validate_environment( + &self, + headers: Headers, + api_key: Option<&str>, + _model: &str, + _params: &LitellmParams, + env: &dyn Fn(&str) -> Option, + ) -> Result { + if has_anthropic_credential(&headers) { + return Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Forwarded, + }); + } + let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth( + litellm_auth::Error::MissingApiKey { + provider: "Tencent", + environment_variable: API_KEY_ENV, + }, + ))?; + Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Header("x-api-key"), + secret: SecretValue::new(key), + }, + }) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + _model: &str, + _params: &LitellmParams, + _stream: bool, + env: &dyn Fn(&str) -> Option, + ) -> Result { + let base = resolve_non_empty(api_base, env, &[ANTHROPIC_API_BASE_ENV, API_BASE_ENV]) + .unwrap_or_else(|| DEFAULT_BASE.into()); + let base = base.trim_end_matches('/'); + if base.ends_with(MESSAGES_PATH) { + return Ok(base.into()); + } + let base = base + .strip_suffix("/v1/chat/completions") + .or_else(|| base.strip_suffix("/v1")) + .unwrap_or(base); + Ok(format!("{base}{MESSAGES_PATH}")) + } + + fn transform_anthropic_messages_request( + &self, + request: MessagesRequest, + context: &MessagesTransformContext, + ) -> Result { + let request = transform_messages_request(request, context)?; + Ok(MessagesRequest { + params: MessagesOptionalParams { + system: request + .params + .system + .and_then(filter_billing_headers_from_system), + ..request.params + }, + ..request + }) + } + + fn secret_names(&self) -> &'static [&'static str] { + &[API_KEY_ENV, ANTHROPIC_API_BASE_ENV, API_BASE_ENV] + } + + fn default_headers(&self) -> &'static [(&'static str, &'static str)] { + DEFAULT_HEADERS + } + + fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers { + update_headers_with_anthropic_beta(headers, request) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn no_env(_: &str) -> Option { + None + } + + #[rstest] + #[case::root("https://proxy.test", "https://proxy.test/v1/messages")] + #[case::root_slash("https://proxy.test/", "https://proxy.test/v1/messages")] + #[case::full_endpoint("https://proxy.test/v1/messages", "https://proxy.test/v1/messages")] + #[case::full_endpoint_slash( + "https://proxy.test/v1/messages/", + "https://proxy.test/v1/messages" + )] + #[case::custom_path( + "https://proxy.test/anthropic", + "https://proxy.test/anthropic/v1/messages" + )] + #[case::versioned("https://proxy.test/v1", "https://proxy.test/v1/messages")] + #[case::chat_endpoint( + "https://proxy.test/v1/chat/completions/", + "https://proxy.test/v1/messages" + )] + fn endpoint_is_appended_once(#[case] base: &str, #[case] expected: &str) { + assert_eq!( + TENCENT_MESSAGES_CONFIG.get_complete_url( + Some(base), + "test-model", + &Default::default(), + false, + &no_env, + ), + Ok(expected.into()) + ); + } + + #[rstest] + #[case::default(None, None, None, &format!("{DEFAULT_BASE}{MESSAGES_PATH}"))] + #[case::explicit( + Some("https://explicit.test"), + Some("https://anthropic.test"), + Some("https://env.test"), + "https://explicit.test/v1/messages" + )] + #[case::anthropic_base( + None, + Some("https://anthropic.test"), + Some("https://env.test"), + "https://anthropic.test/v1/messages" + )] + #[case::generic_base( + None, + None, + Some("https://env.test/v1"), + "https://env.test/v1/messages" + )] + #[case::blank_anthropic_base( + None, + Some(" "), + Some("https://env.test"), + "https://env.test/v1/messages" + )] + #[case::blank_explicit( + Some(" "), + Some("https://anthropic.test/v1/messages/"), + None, + "https://anthropic.test/v1/messages" + )] + fn base_precedence( + #[case] base: Option<&str>, + #[case] anthropic_base: Option<&str>, + #[case] generic_base: Option<&str>, + #[case] expected: &str, + ) { + let env = |name: &str| match name { + ANTHROPIC_API_BASE_ENV => anthropic_base.map(str::to_owned), + API_BASE_ENV => generic_base.map(str::to_owned), + _ => None, + }; + assert_eq!( + TENCENT_MESSAGES_CONFIG.get_complete_url( + base, + "test-model", + &Default::default(), + false, + &env, + ), + Ok(expected.into()) + ); + } + + #[rstest] + #[case::explicit(Some("explicit"), Some("environment"), "explicit")] + #[case::environment(None, Some("environment"), "environment")] + #[case::blank_explicit(Some(" "), Some("environment"), "environment")] + fn key_precedence( + #[case] key: Option<&str>, + #[case] env_key: Option<&str>, + #[case] expected: &str, + ) { + let env = |name: &str| { + (name == API_KEY_ENV) + .then(|| env_key.map(str::to_owned)) + .flatten() + }; + let environment = TENCENT_MESSAGES_CONFIG + .validate_environment(vec![], key, "test-model", &Default::default(), &env) + .unwrap(); + assert!(matches!( + environment.auth, + AuthScheme::Credential { + placement: CredentialPlacement::Header("x-api-key"), ref secret + } if secret.expose() == expected + )); + } +} diff --git a/litellm-rust/crates/llms/src/tencent/mod.rs b/litellm-rust/crates/llms/src/tencent/mod.rs new file mode 100644 index 00000000000..ba63992f3cb --- /dev/null +++ b/litellm-rust/crates/llms/src/tencent/mod.rs @@ -0,0 +1 @@ +pub mod messages;