From b31b55075b3820ccc33f86f65e592dca1909cea7 Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Thu, 15 Oct 2026 02:06:29 +0000 Subject: [PATCH] feat(rust): serve xAI and hosted vLLM responses natively The Responses route now resolves xai and hosted_vllm. The Python host fills in the key and base get_llm_provider returns before the provider config reads its own defaults, as Python's responses entry point does. xAI drops metadata, reshapes code_interpreter, web_search and x_search tools, keeps max_output_tokens as sent and restates usage.cost from cost_in_usd_ticks. Hosted vLLM sends to /v1/responses with the fake-api-key fallback. xAI OAuth and hosted vLLM without an api_base decline to Python --- .../core-utils/src/get_llm_provider_logic.rs | 4 + .../crates/inference-responses/src/prepare.rs | 11 + litellm-rust/crates/llms/AGENTS.md | 2 +- .../crates/llms/src/hosted_vllm/mod.rs | 1 + .../llms/src/hosted_vllm/responses/AGENTS.md | 9 + .../llms/src/hosted_vllm/responses/mod.rs | 1 + .../hosted_vllm/responses/transformation.rs | 166 ++++++++++ litellm-rust/crates/llms/src/lib.rs | 2 + .../llms/src/openai/responses/request.rs | 11 +- litellm-rust/crates/llms/src/xai/mod.rs | 1 + .../crates/llms/src/xai/responses/AGENTS.md | 11 + .../crates/llms/src/xai/responses/mod.rs | 1 + .../llms/src/xai/responses/transformation.rs | 295 ++++++++++++++++++ .../src/routes/responses/host.rs | 25 +- .../python-bridge/src/routes/responses/mod.rs | 2 +- litellm/rust_bridge/host/routes/responses.py | 39 ++- .../sdk/responses/test_native_parity.py | 36 +++ .../rust_bridge/host/routes/test_responses.py | 53 +++- 18 files changed, 661 insertions(+), 9 deletions(-) create mode 100644 litellm-rust/crates/llms/src/hosted_vllm/mod.rs create mode 100644 litellm-rust/crates/llms/src/hosted_vllm/responses/AGENTS.md create mode 100644 litellm-rust/crates/llms/src/hosted_vllm/responses/mod.rs create mode 100644 litellm-rust/crates/llms/src/hosted_vllm/responses/transformation.rs create mode 100644 litellm-rust/crates/llms/src/xai/mod.rs create mode 100644 litellm-rust/crates/llms/src/xai/responses/AGENTS.md create mode 100644 litellm-rust/crates/llms/src/xai/responses/mod.rs create mode 100644 litellm-rust/crates/llms/src/xai/responses/transformation.rs diff --git a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index 0f191de1d0c..c3ee369a4d8 100644 --- a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -26,6 +26,8 @@ pub enum LlmProviders { Deepseek, #[strum(serialize = "edenai")] Edenai, + #[strum(serialize = "hosted_vllm")] + HostedVllm, #[strum(serialize = "minimax")] Minimax, #[strum(serialize = "mistral")] @@ -42,6 +44,8 @@ pub enum LlmProviders { Tencent, #[strum(serialize = "vertex_ai")] VertexAi, + #[strum(serialize = "xai")] + Xai, } pub fn get_custom_llm_provider<'a>( diff --git a/litellm-rust/crates/inference-responses/src/prepare.rs b/litellm-rust/crates/inference-responses/src/prepare.rs index 436550bb8f9..8bcf828106d 100644 --- a/litellm-rust/crates/inference-responses/src/prepare.rs +++ b/litellm-rust/crates/inference-responses/src/prepare.rs @@ -5,8 +5,10 @@ use litellm_host::interceptors::{ProviderIdentity, RequestContext}; use litellm_llms::{ azure::responses::transformation::AZURE_RESPONSES_CONFIG, base_llm::responses::transformation::BaseResponsesApiConfig, + hosted_vllm::responses::transformation::HOSTED_VLLM_RESPONSES_CONFIG, openai::responses::transformation::OpenAiResponsesApiConfig, openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, + xai::responses::transformation::XAI_RESPONSES_CONFIG, }; use litellm_secrets::source::SecretSource; use serde_json::Value; @@ -95,6 +97,8 @@ pub(super) fn resolve_provider( ), Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()), Ok(LlmProviders::Azure) => (&AZURE_RESPONSES_CONFIG, resolved.into()), + Ok(LlmProviders::Xai) => (&XAI_RESPONSES_CONFIG, resolved.into()), + Ok(LlmProviders::HostedVllm) => (&HOSTED_VLLM_RESPONSES_CONFIG, resolved.into()), _ => return Err(Error::Unsupported("native HTTP responses provider")), }; Ok(ResponsesProvider { @@ -144,6 +148,13 @@ mod tests { #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")] #[case::azure_prefix("azure/my-deployment", None, "azure", "my-deployment")] #[case::azure_explicit("my-deployment", Some("azure"), "azure", "my-deployment")] + #[case::xai_prefix("xai/grok-4", None, "xai", "grok-4")] + #[case::hosted_vllm_keeps_the_vendor( + "hosted_vllm/meta-llama/Llama-3.1-8B", + None, + "hosted_vllm", + "meta-llama/Llama-3.1-8B" + )] fn providers_and_wire_models_match_python( #[case] model: &str, #[case] custom_llm_provider: Option<&str>, diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md index dd48abb81d2..d68e488486c 100644 --- a/litellm-rust/crates/llms/AGENTS.md +++ b/litellm-rust/crates/llms/AGENTS.md @@ -58,7 +58,7 @@ - `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts - `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation -- OpenAI-compatible Responses hosts (`openrouter`, `azure`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules +- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules - Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper - Auth resolution is split between configs and route preparation in `inference-` diff --git a/litellm-rust/crates/llms/src/hosted_vllm/mod.rs b/litellm-rust/crates/llms/src/hosted_vllm/mod.rs new file mode 100644 index 00000000000..b396b037bc5 --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/mod.rs @@ -0,0 +1 @@ +pub mod responses; diff --git a/litellm-rust/crates/llms/src/hosted_vllm/responses/AGENTS.md b/litellm-rust/crates/llms/src/hosted_vllm/responses/AGENTS.md new file mode 100644 index 00000000000..a98a9d3d6ff --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/responses/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- Send the OpenAI Responses payload to `{api_base}/v1/responses`, or `{api_base}/responses` when the base already ends with `/v1`. The base comes from `api_base` or `HOSTED_VLLM_API_BASE` and is required +- Authenticate with a bearer token from `api_key` or `HOSTED_VLLM_API_KEY`, falling back to `fake-api-key` since vLLM does not require one +- No native WebSocket mode + +# references + +- https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html diff --git a/litellm-rust/crates/llms/src/hosted_vllm/responses/mod.rs b/litellm-rust/crates/llms/src/hosted_vllm/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/hosted_vllm/responses/transformation.rs b/litellm-rust/crates/llms/src/hosted_vllm/responses/transformation.rs new file mode 100644 index 00000000000..03e15b6c32a --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/responses/transformation.rs @@ -0,0 +1,166 @@ +//! Hosted vLLM Responses, matching `litellm/llms/hosted_vllm/responses/transformation.py` + +use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use serde_json::{Map, Value}; + +use crate::{ + Error, + base_llm::{ + auth::{AuthScheme, ValidatedEnvironment}, + responses::transformation::BaseResponsesApiConfig, + }, + openai::responses::{ + request::ResponsesHost, + transformation::{responses_request, responses_response}, + }, +}; + +const API_KEY_ENV: &str = "HOSTED_VLLM_API_KEY"; +const API_BASE_ENV: &str = "HOSTED_VLLM_API_BASE"; +/// vLLM does not require authentication by default +const PLACEHOLDER_API_KEY: &str = "fake-api-key"; + +pub struct HostedVllmResponsesApiConfig; +pub const HOSTED_VLLM_RESPONSES_CONFIG: HostedVllmResponsesApiConfig = HostedVllmResponsesApiConfig; + +impl BaseResponsesApiConfig for HostedVllmResponsesApiConfig { + fn secret_names( + &self, + api_key: Option<&str>, + api_base: Option<&str>, + ) -> &'static [&'static str] { + match ( + api_key.is_some_and(|key| !key.is_empty()), + api_base.is_some_and(|base| !base.is_empty()), + ) { + (true, true) => &[], + (true, false) => &[API_BASE_ENV], + (false, true) => &[API_KEY_ENV], + (false, false) => &[API_KEY_ENV, API_BASE_ENV], + } + } + + fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let key = + resolved(api_key, lookup, API_KEY_ENV).unwrap_or_else(|| PLACEHOLDER_API_KEY.into()); + Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: SecretValue::new(key), + }, + }) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + _api_version: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let api_base = resolved(api_base, lookup, API_BASE_ENV).ok_or( + litellm_auth::Error::MissingApiBase { + provider: "Hosted VLLM", + guidance: "Set `api_base` or the HOSTED_VLLM_API_BASE environment variable", + }, + )?; + let api_base = api_base.trim_end_matches('/'); + Ok(match api_base.ends_with("/v1") { + true => format!("{api_base}/responses"), + false => format!("{api_base}/v1/responses"), + }) + } + + fn transform_responses_api_request( + &self, + model: &str, + input: Value, + params: Map, + ) -> Result { + responses_request(model, input, params, ResponsesHost::Compatible) + } + + fn transform_response_api_response(&self, body: Value) -> Result { + responses_response(body) + } +} + +/// Python's `value or get_secret_str(name)`: an empty value falls through, and nothing is trimmed +fn resolved( + value: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + name: &str, +) -> Option { + value + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .or_else(|| lookup(name).filter(|value| !value.is_empty())) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + } + } + + #[rstest] + #[case::bare_host(Some("http://vllm:8000"), &[], "http://vllm:8000/v1/responses")] + #[case::v1_suffix(Some("http://vllm:8000/v1"), &[], "http://vllm:8000/v1/responses")] + #[case::trailing_slashes(Some("http://vllm:8000/v1//"), &[], "http://vllm:8000/v1/responses")] + #[case::secret(None, &[("HOSTED_VLLM_API_BASE", "http://secret:8000")], "http://secret:8000/v1/responses")] + fn url_appends_the_v1_responses_route( + #[case] api_base: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + assert_eq!( + HOSTED_VLLM_RESPONSES_CONFIG + .get_complete_url(api_base, None, &env(secrets)) + .unwrap(), + expected + ); + } + + #[test] + fn url_needs_an_api_base() { + assert!(matches!( + HOSTED_VLLM_RESPONSES_CONFIG.get_complete_url(None, None, &env(&[])), + Err(Error::Auth(litellm_auth::Error::MissingApiBase { .. })) + )); + } + + #[rstest] + #[case::explicit(Some("explicit"), &[("HOSTED_VLLM_API_KEY", "managed")], "explicit")] + #[case::secret(None, &[("HOSTED_VLLM_API_KEY", "managed")], "managed")] + #[case::placeholder(None, &[], "fake-api-key")] + #[case::blank_explicit(Some(""), &[], "fake-api-key")] + fn bearer_falls_back_to_the_placeholder_key( + #[case] api_key: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + let environment = HOSTED_VLLM_RESPONSES_CONFIG + .validate_environment(vec![], api_key, &env(secrets)) + .unwrap(); + let AuthScheme::Credential { placement, secret } = environment.auth else { + panic!("expected a credential"); + }; + assert_eq!(placement, CredentialPlacement::Bearer); + assert_eq!(secret.expose(), expected); + } +} diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs index 23ed649ac8e..6fd1c8817a9 100644 --- a/litellm-rust/crates/llms/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -9,6 +9,7 @@ pub mod cohere; pub mod deepseek; pub mod edenai; mod error; +pub mod hosted_vllm; pub mod minimax; pub mod mistral; pub mod openai; @@ -17,5 +18,6 @@ pub mod openrouter; pub mod reducto; pub mod tencent; pub mod vertex_ai; +pub mod xai; pub use error::{Error, ErrorDetail, ErrorSource}; diff --git a/litellm-rust/crates/llms/src/openai/responses/request.rs b/litellm-rust/crates/llms/src/openai/responses/request.rs index 4ae8b458a00..f8b24bf4aa5 100644 --- a/litellm-rust/crates/llms/src/openai/responses/request.rs +++ b/litellm-rust/crates/llms/src/openai/responses/request.rs @@ -17,18 +17,24 @@ const REASONING_FIELDS_DROPPED_WHEN_NULL: [&str; 3] = ["status", "content", "enc /// The host behind the OpenAI wire format. OpenAI and Azure run OpenAI's item validator, which /// rejects reasoning it did not mint and tool-call item ids without its prefixes, and Azure also -/// rejects `status` on replayed reasoning and message items +/// rejects `status` on replayed reasoning and message items. xAI replaces OpenAI's param mapping, +/// so it keeps `max_output_tokens` as sent #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ResponsesHost { OpenAi, Azure, Compatible, + Xai, } impl ResponsesHost { fn validates_items(self) -> bool { matches!(self, Self::OpenAi | Self::Azure) } + + fn raises_max_output_tokens(self) -> bool { + !matches!(self, Self::Xai) + } } pub fn shape( @@ -36,7 +42,8 @@ pub fn shape( mut params: Map, host: ResponsesHost, ) -> Result<(Value, Map), Error> { - if let Some(max_output_tokens) = params.get_mut("max_output_tokens") + if host.raises_max_output_tokens() + && let Some(max_output_tokens) = params.get_mut("max_output_tokens") && max_output_tokens .as_i64() .is_some_and(|tokens| (1..MIN_MAX_OUTPUT_TOKENS).contains(&tokens)) diff --git a/litellm-rust/crates/llms/src/xai/mod.rs b/litellm-rust/crates/llms/src/xai/mod.rs new file mode 100644 index 00000000000..b396b037bc5 --- /dev/null +++ b/litellm-rust/crates/llms/src/xai/mod.rs @@ -0,0 +1 @@ +pub mod responses; diff --git a/litellm-rust/crates/llms/src/xai/responses/AGENTS.md b/litellm-rust/crates/llms/src/xai/responses/AGENTS.md new file mode 100644 index 00000000000..78204915e08 --- /dev/null +++ b/litellm-rust/crates/llms/src/xai/responses/AGENTS.md @@ -0,0 +1,11 @@ +# rules + +- Send the OpenAI Responses payload to `{api_base}/responses`, defaulting to `https://api.x.ai/v1`, with a bearer token from `api_key` or `XAI_API_KEY`. OAuth stays on Python +- Drop `metadata`, strip `container` from `code_interpreter`, reduce `web_search` and `x_search` to the fields xAI documents, and keep `max_output_tokens` as sent +- Restate `usage.cost` from `usage.cost_in_usd_ticks` +- No native WebSocket mode + +# references + +- https://docs.x.ai/docs/api-reference#create-new-response +- https://docs.x.ai/developers/cost-tracking diff --git a/litellm-rust/crates/llms/src/xai/responses/mod.rs b/litellm-rust/crates/llms/src/xai/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/xai/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/xai/responses/transformation.rs b/litellm-rust/crates/llms/src/xai/responses/transformation.rs new file mode 100644 index 00000000000..67dfbab913e --- /dev/null +++ b/litellm-rust/crates/llms/src/xai/responses/transformation.rs @@ -0,0 +1,295 @@ +//! xAI Responses, matching `litellm/llms/xai/responses/transformation.py` + +use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, + base_llm::{ + auth::{AuthScheme, ValidatedEnvironment}, + responses::transformation::BaseResponsesApiConfig, + }, + openai::responses::{ + request::ResponsesHost, + transformation::{responses_request, responses_response}, + }, +}; + +const API_KEY_ENV: &str = "XAI_API_KEY"; +const API_BASE_ENV: &str = "XAI_API_BASE"; +const DEFAULT_BASE: &str = "https://api.x.ai/v1"; +/// xAI bills in ticks of a dollar: https://docs.x.ai/developers/cost-tracking +const USD_TICKS_PER_DOLLAR: f64 = 10_000_000_000.0; +const WEB_SEARCH_FILTERS: [&str; 2] = ["allowed_domains", "excluded_domains"]; +const X_SEARCH_FIELDS: [&str; 6] = [ + "allowed_x_handles", + "excluded_x_handles", + "from_date", + "to_date", + "enable_image_understanding", + "enable_video_understanding", +]; + +pub struct XaiResponsesApiConfig; +pub const XAI_RESPONSES_CONFIG: XaiResponsesApiConfig = XaiResponsesApiConfig; + +impl BaseResponsesApiConfig for XaiResponsesApiConfig { + fn secret_names( + &self, + api_key: Option<&str>, + api_base: Option<&str>, + ) -> &'static [&'static str] { + match ( + api_key.is_some_and(|key| !key.is_empty()), + api_base.is_some_and(|base| !base.is_empty()), + ) { + (true, true) => &[], + (true, false) => &[API_BASE_ENV], + (false, true) => &[API_KEY_ENV], + (false, false) => &[API_KEY_ENV, API_BASE_ENV], + } + } + + fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let key = + resolved(api_key, lookup, API_KEY_ENV).ok_or(litellm_auth::Error::MissingApiKey { + provider: "XAI", + environment_variable: API_KEY_ENV, + })?; + Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: SecretValue::new(key), + }, + }) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + _api_version: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let api_base = + resolved(api_base, lookup, API_BASE_ENV).unwrap_or_else(|| DEFAULT_BASE.into()); + Ok(format!("{}/responses", api_base.trim_end_matches('/'))) + } + + fn transform_responses_api_request( + &self, + model: &str, + input: Value, + mut params: Map, + ) -> Result { + params.remove("metadata"); + if let Some(tools) = params.get_mut("tools").filter(|tools| python_truthy(tools)) { + let listed = match tools.take() { + Value::Array(tools) => tools, + tool => vec![tool], + }; + *tools = Value::Array(listed.into_iter().map(xai_tool).collect()); + } + responses_request(model, input, params, ResponsesHost::Xai) + } + + fn transform_response_api_response(&self, body: Value) -> Result { + let mut response = responses_response(body)?; + if let Some(Value::Object(usage)) = response.extra.get_mut("usage") + && let Some(cost) = reported_cost(usage.get("cost_in_usd_ticks")) + { + usage.insert("cost".into(), json!(cost)); + } + Ok(response) + } +} + +/// Python's `value or get_secret_str(name)`: an empty value falls through, and nothing is trimmed +fn resolved( + value: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + name: &str, +) -> Option { + value + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .or_else(|| lookup(name).filter(|value| !value.is_empty())) +} + +fn python_truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(value) => *value, + Value::Number(number) => number.as_f64() != Some(0.0), + Value::String(value) => !value.is_empty(), + Value::Array(values) => !values.is_empty(), + Value::Object(values) => !values.is_empty(), + } +} + +/// Python's `xai_reported_cost_in_usd`: only a non-negative integer tick count is a cost +fn reported_cost(ticks: Option<&Value>) -> Option { + ticks? + .as_u64() + .map(|ticks| ticks as f64 / USD_TICKS_PER_DOLLAR) +} + +/// xAI takes `code_interpreter` without a container, drops OpenAI's `search_context_size` from +/// `web_search`, and keeps only the `x_search` fields it documents +fn xai_tool(tool: Value) -> Value { + let Value::Object(fields) = tool else { + return tool; + }; + match fields.get("type").and_then(Value::as_str) { + Some("code_interpreter") => json!({"type": "code_interpreter"}), + Some("web_search") => web_search_tool(&fields), + Some("x_search") => { + let mut xai = Map::from_iter([("type".into(), json!("x_search"))]); + for name in X_SEARCH_FIELDS { + if let Some(value) = fields.get(name) { + xai.insert(name.into(), value.clone()); + } + } + Value::Object(xai) + } + _ => Value::Object(fields), + } +} + +/// Domains may come nested under `filters`, the documented shape, or flat on the tool +fn web_search_tool(tool: &Map) -> Value { + let domains = match tool.get("filters") { + Some(Value::Object(filters)) => filters, + _ => tool, + }; + let filters: Map = WEB_SEARCH_FILTERS + .iter() + .filter_map(|name| Some(((*name).into(), domains.get(*name)?.clone()))) + .collect(); + let mut xai = Map::from_iter([("type".into(), json!("web_search"))]); + if !filters.is_empty() { + xai.insert("filters".into(), Value::Object(filters)); + } + if let Some(value) = tool.get("enable_image_understanding") { + xai.insert("enable_image_understanding".into(), value.clone()); + } + Value::Object(xai) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + } + } + + #[rstest] + #[case::default(None, &[], "https://api.x.ai/v1/responses")] + #[case::explicit(Some("https://proxy.test/v1/"), &[], "https://proxy.test/v1/responses")] + #[case::secret(None, &[("XAI_API_BASE", "https://secret.test/v1")], "https://secret.test/v1/responses")] + #[case::openai_base_is_ignored(None, &[("OPENAI_BASE_URL", "https://openai.test/v1")], "https://api.x.ai/v1/responses")] + fn url_is_the_base_plus_responses( + #[case] api_base: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + assert_eq!( + XAI_RESPONSES_CONFIG + .get_complete_url(api_base, None, &env(secrets)) + .unwrap(), + expected + ); + } + + #[test] + fn a_missing_key_is_an_auth_error() { + assert!(matches!( + XAI_RESPONSES_CONFIG.validate_environment(vec![], None, &env(&[])), + Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. })) + )); + } + + #[test] + fn request_drops_metadata_and_keeps_max_output_tokens() { + let body = XAI_RESPONSES_CONFIG + .transform_responses_api_request( + "grok-4", + json!("hi"), + Map::from_iter([ + ("metadata".into(), json!({"user": "u"})), + ("max_output_tokens".into(), json!(4)), + ]), + ) + .unwrap(); + assert_eq!( + body, + json!({"max_output_tokens": 4, "model": "grok-4", "input": "hi"}) + ); + } + + #[rstest] + #[case::code_interpreter( + json!({"type": "code_interpreter", "container": {"type": "auto"}}), + json!({"type": "code_interpreter"}) + )] + #[case::web_search_nested( + json!({"type": "web_search", "search_context_size": "high", "filters": {"allowed_domains": ["a.com"], "other": 1}}), + json!({"type": "web_search", "filters": {"allowed_domains": ["a.com"]}}) + )] + #[case::web_search_flat( + json!({"type": "web_search", "excluded_domains": ["b.com"], "enable_image_understanding": true}), + json!({"type": "web_search", "filters": {"excluded_domains": ["b.com"]}, "enable_image_understanding": true}) + )] + #[case::x_search( + json!({"type": "x_search", "from_date": "2025-01-01", "unknown": 1}), + json!({"type": "x_search", "from_date": "2025-01-01"}) + )] + #[case::function( + json!({"type": "function", "name": "f", "parameters": {"type": "object"}}), + json!({"type": "function", "name": "f", "parameters": {"type": "object"}}) + )] + fn tools_take_the_xai_shape(#[case] tool: Value, #[case] expected: Value) { + let body = XAI_RESPONSES_CONFIG + .transform_responses_api_request( + "grok-4", + json!("hi"), + Map::from_iter([("tools".into(), json!([tool]))]), + ) + .unwrap(); + assert_eq!(body["tools"], json!([expected])); + } + + #[rstest] + #[case::ticks(json!(12_345_000_000u64), Some(1.2345))] + #[case::negative(json!(-1), None)] + #[case::float(json!(1.0), None)] + #[case::string(json!("5"), None)] + fn usage_cost_comes_from_reported_ticks(#[case] ticks: Value, #[case] expected: Option) { + let response = XAI_RESPONSES_CONFIG + .transform_response_api_response(json!({ + "id": "resp_1", + "model": "grok-4", + "output": [], + "usage": {"input_tokens": 1, "cost_in_usd_ticks": ticks}, + })) + .unwrap(); + assert_eq!( + response.extra["usage"].get("cost").and_then(Value::as_f64), + expected + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index e94baef061f..035bab7575d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -36,6 +36,23 @@ pub(super) fn project( ) -> PyResult { let mut call = host.project(py, arguments, "input")?; ids::restore_input(&mut call.input); + let raw = |name: &str| -> PyResult> { + host.argument(py, arguments, name)? + .map(|value| value.extract()) + .transpose() + }; + let (provider_key, provider_base): (Option, Option) = py + .import(super::ROUTE_HOST_MODULE)? + .getattr("provider_connection")? + .call1(( + &call.options.model, + &call.options.custom_llm_provider, + raw("api_key")?, + raw("api_base")? + .filter(|base| !base.is_empty()) + .or(raw("base_url")?), + ))? + .extract()?; let optional_params = call .params .into_iter() @@ -61,8 +78,12 @@ pub(super) fn project( model: call.options.model, input: call.input, optional_params, - api_key: call.options.api_key, - api_base: call.options.api_base, + api_key: provider_key + .filter(|key| !key.is_empty()) + .or(call.options.api_key), + api_base: provider_base + .filter(|base| !base.is_empty()) + .or(call.options.api_base), custom_llm_provider: call.options.custom_llm_provider, api_version: host .argument(py, arguments, "api_version")? diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index f4c6c687f9b..a40dfbca926 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -47,7 +47,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py None => !model.contains('/'), Some(resolved) => match resolved.custom_llm_provider { "openai" => !resolved.model.contains('/'), - "azure" => true, + "azure" | "xai" | "hosted_vllm" => true, _ => false, }, }; diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index b453f75c13b..4a8f78833c0 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -7,7 +7,9 @@ from typing import Final, cast # noqa: TID251 # narrows caller json the checke import litellm from litellm import get_llm_provider from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.rust_bridge import failures from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str from litellm.secret_managers.main import get_secret_str @@ -17,7 +19,12 @@ from litellm.types.router import GenericLiteLLMParams PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__) _NATIVE_PARAMETERS: Final = (*PARAMETERS, "api_version") _CONFIGS: Final[Mapping[str, OpenAIResponsesAPIConfig]] = MappingProxyType( - {"openai": OpenAIResponsesAPIConfig(), "azure": AzureOpenAIResponsesAPIConfig()} + { + "openai": OpenAIResponsesAPIConfig(), + "azure": AzureOpenAIResponsesAPIConfig(), + "xai": XAIResponsesAPIConfig(), + "hosted_vllm": HostedVLLMResponsesAPIConfig(), + } ) _AZURE_KEY_ENVIRONMENT: Final = ("AZURE_OPENAI_API_KEY", "AZURE_API_KEY") @@ -31,6 +38,16 @@ def connection_defaults(provider: str) -> tuple[str | None, str | None]: return litellm.api_key or provider_key, litellm.api_base +def provider_connection( + model: str, custom_llm_provider: str | None, api_key: str | None, api_base: str | None +) -> tuple[str | None, str | None]: + """The key and base `get_llm_provider` fills in before the provider config reads its own defaults""" + _, _, provider_key, provider_base = get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base + ) + return provider_key, provider_base + + def response(value: Mapping[str, object]) -> ResponsesAPIResponse: return ResponsesAPIResponse.model_validate(value) @@ -64,11 +81,29 @@ def decline_reason(request: Mapping[str, object]) -> str | None: return "native HTTP responses provider" return ( inference_decline_reason(_NATIVE_PARAMETERS, request) - or (_azure_decline_reason(model, request) if provider == "azure" else None) + or _provider_decline_reason(provider, model, request) or _model_rules_decline_reason(config, model, request) ) +def _provider_decline_reason(provider: str, model: str, request: Mapping[str, object]) -> str | None: + if provider == "azure": + return _azure_decline_reason(model, request) + if provider not in ("xai", "hosted_vllm"): + return None + provider_key, provider_base = provider_connection( + str(request["model"]), + optional_str(request.get("custom_llm_provider")), + optional_str(request.get("api_key")), + optional_str(request.get("api_base")) or optional_str(request.get("base_url")), + ) + if provider == "xai" and not (provider_key or litellm.api_key): + return "native xAI Responses does not implement OAuth" + if provider == "hosted_vllm" and not provider_base: + return "native hosted vLLM Responses needs an api_base" + return None + + def _azure_decline_reason(model: str, request: Mapping[str, object]) -> str | None: if "o_series/" in model: return "native Azure Responses does not implement the o-series config" diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 61114447ba9..16b8696a019 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -157,6 +157,27 @@ AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [ ] AZURE_MODEL: Final = "azure/my-deployment" +XAI_MODEL: Final = "xai/grok-4" +HOSTED_VLLM_MODEL: Final = "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct" + +XAI_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "model": "grok-4", + "usage": { + "input_tokens": 12, + "output_tokens": 70, + "total_tokens": 82, + "cost_in_usd_ticks": 12_345_000_000, + }, + } +) + +XAI_TOOLS: Final[list[JsonValue]] = [ + {"type": "code_interpreter", "container": {"type": "auto"}}, + {"type": "web_search", "search_context_size": "high", "filters": {"allowed_domains": ["example.com"]}}, + {"type": "x_search", "from_date": "2026-01-01", "enable_video_understanding": True}, +] WEATHER_TOOL_FUNCTION: Final[Mapping[str, JsonValue]] = MappingProxyType( {name: value for name, value in WEATHER_TOOL.items() if name != "type"} @@ -328,6 +349,21 @@ CASES: Final = ( "azure_forwarded_api_key_header", {"model": AZURE_MODEL, "input": "hi", "api_key": None, "extra_headers": {"api-key": "caller-key"}}, ), + Case("xai_reported_cost", {"model": XAI_MODEL, "input": "hi"}, upstream=XAI_RESPONSE), + Case( + "xai_tools_metadata_and_max_output_tokens", + {"model": XAI_MODEL, "input": "hi", "tools": XAI_TOOLS, "metadata": {"team": "a"}, "max_output_tokens": 8}, + upstream=XAI_RESPONSE, + ), + Case("hosted_vllm_placeholder_key", {"model": HOSTED_VLLM_MODEL, "input": "hi", "api_key": None}), + Case( + "hosted_vllm_keeps_replayed_reasoning_and_ids", + { + "model": HOSTED_VLLM_MODEL, + "input": [*MINTED_REASONING_REPLAY_INPUT, *FOREIGN_TOOL_CALL_REPLAY_INPUT], + "max_output_tokens": 4, + }, + ), Case("azure_o_series_stays_on_python", {"model": "azure/o_series/my-o3", "input": "hi"}, falls_back=True), Case( "tool_schema_repair_stays_on_python", diff --git a/tests/unit/rust_bridge/host/routes/test_responses.py b/tests/unit/rust_bridge/host/routes/test_responses.py index ce12a1d1adb..04f9f5dc5b2 100644 --- a/tests/unit/rust_bridge/host/routes/test_responses.py +++ b/tests/unit/rust_bridge/host/routes/test_responses.py @@ -5,7 +5,13 @@ import pytest from pydantic import ValidationError import litellm -from litellm.rust_bridge.host.routes.responses import connection_defaults, decline_reason, map_failure, response +from litellm.rust_bridge.host.routes.responses import ( + connection_defaults, + decline_reason, + map_failure, + provider_connection, + response, +) from litellm.types.llms.openai import ResponsesAPIResponse @@ -174,3 +180,48 @@ def test_azure_connection_defaults_read_the_azure_key(monkeypatch: pytest.Monkey monkeypatch.setattr(litellm, "azure_key", "azure-global") monkeypatch.setattr(litellm, "api_base", None) assert connection_defaults("azure") == ("azure-global", None) + + +@pytest.mark.parametrize( + ("request_fields", "xai_key", "expected"), + [ + pytest.param({"api_key": "xai-explicit"}, None, None, id="explicit_key"), + pytest.param({}, "xai-global", None, id="xai_key"), + pytest.param({}, None, "native xAI Responses does not implement OAuth", id="no_key"), + ], +) +def test_xai_without_a_key_stays_on_python( + monkeypatch: pytest.MonkeyPatch, request_fields: dict[str, object], xai_key: str | None, expected: str | None +) -> None: + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "xai_key", xai_key) + assert decline_reason({"model": "xai/grok-4", "input": "hi", **request_fields}) == expected + + +@pytest.mark.parametrize( + ("request_fields", "expected"), + [ + pytest.param({"api_base": "http://vllm:8000"}, None, id="api_base"), + pytest.param({"base_url": "http://vllm:8000"}, None, id="base_url"), + pytest.param({}, "native hosted vLLM Responses needs an api_base", id="no_api_base"), + ], +) +def test_hosted_vllm_without_an_api_base_stays_on_python( + monkeypatch: pytest.MonkeyPatch, request_fields: dict[str, object], expected: str | None +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + assert decline_reason({"model": "hosted_vllm/llama", "input": "hi", **request_fields}) == expected + + +def test_provider_connection_matches_get_llm_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("XAI_API_KEY", raising=False) + monkeypatch.delenv("XAI_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + monkeypatch.setattr(litellm, "xai_key", "xai-global") + assert provider_connection("xai/grok-4", None, None, None) == ("xai-global", "https://api.x.ai/v1") + assert provider_connection("hosted_vllm/llama", None, None, "http://vllm:8000") == ( + "fake-api-key", + "http://vllm:8000", + ) + assert provider_connection("gpt-4.1", None, None, None) == (None, None)