From bd100de072a9a9935fa813abe2cf7eff35bdb6ff Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Thu, 15 Oct 2026 01:46:15 +0000 Subject: [PATCH] feat(rust): serve Azure OpenAI responses natively The Responses route now resolves azure. The Azure config builds the URL like Python's _get_base_azure_url, with the v1 surface for the preview, latest and v1 API versions, authenticates with the api-key header from api_key, litellm.azure_key or the Azure key variables, lifts Chat Completions-style function tools to the top level and drops status from replayed reasoning and message items. api_version reaches the route through ResponsesCall. Azure AD token auth, the o-series config and context_management decline to Python --- .../core-utils/src/get_llm_provider_logic.rs | 2 + .../crates/gateway-inference/src/responses.rs | 1 + .../crates/inference-responses/src/prepare.rs | 10 +- .../crates/inference-responses/src/types.rs | 1 + .../inference-responses/tests/caching.rs | 1 + .../tests/live/openrouter.rs | 1 + .../inference-responses/tests/openrouter.rs | 1 + .../inference-responses/tests/responses.rs | 1 + litellm-rust/crates/llms/AGENTS.md | 2 +- litellm-rust/crates/llms/src/azure/mod.rs | 1 + .../crates/llms/src/azure/responses/AGENTS.md | 10 + .../crates/llms/src/azure/responses/mod.rs | 1 + .../src/azure/responses/transformation.rs | 388 ++++++++++++++++++ .../src/base_llm/responses/transformation.rs | 3 +- litellm-rust/crates/llms/src/lib.rs | 1 + .../llms/src/openai/responses/request.rs | 64 +-- .../src/openai/responses/transformation.rs | 13 +- .../openrouter/responses/transformation.rs | 13 +- .../src/routes/responses/host.rs | 4 + .../python-bridge/src/routes/responses/mod.rs | 20 +- litellm/rust_bridge/host/routes/responses.py | 77 +++- .../sdk/responses/test_native_error_parity.py | 15 +- .../sdk/responses/test_native_parity.py | 37 +- .../rust_bridge/host/routes/test_responses.py | 41 ++ 24 files changed, 640 insertions(+), 68 deletions(-) create mode 100644 litellm-rust/crates/llms/src/azure/mod.rs create mode 100644 litellm-rust/crates/llms/src/azure/responses/AGENTS.md create mode 100644 litellm-rust/crates/llms/src/azure/responses/mod.rs create mode 100644 litellm-rust/crates/llms/src/azure/responses/transformation.rs diff --git a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index acea48799c0..0f191de1d0c 100644 --- a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -12,6 +12,8 @@ pub enum LlmProviders { Anthropic, #[strum(serialize = "aws_textract")] AwsTextract, + #[strum(serialize = "azure")] + Azure, #[strum(serialize = "azure_ai")] AzureAi, #[strum(serialize = "bedrock")] diff --git a/litellm-rust/crates/gateway-inference/src/responses.rs b/litellm-rust/crates/gateway-inference/src/responses.rs index 77bebf610e1..f51066ed7f6 100644 --- a/litellm-rust/crates/gateway-inference/src/responses.rs +++ b/litellm-rust/crates/gateway-inference/src/responses.rs @@ -35,6 +35,7 @@ pub(crate) async fn create( api_key: deployment.api_key.clone(), api_base: deployment.api_base.clone(), custom_llm_provider: deployment.custom_llm_provider.clone(), + api_version: deployment.litellm_params.api_version.clone(), extra_headers: None, timeout: deployment.timeout, }; diff --git a/litellm-rust/crates/inference-responses/src/prepare.rs b/litellm-rust/crates/inference-responses/src/prepare.rs index 05a9772abd4..436550bb8f9 100644 --- a/litellm-rust/crates/inference-responses/src/prepare.rs +++ b/litellm-rust/crates/inference-responses/src/prepare.rs @@ -3,6 +3,7 @@ use litellm_core_utils::get_llm_provider_logic::{ }; use litellm_host::interceptors::{ProviderIdentity, RequestContext}; use litellm_llms::{ + azure::responses::transformation::AZURE_RESPONSES_CONFIG, base_llm::responses::transformation::BaseResponsesApiConfig, openai::responses::transformation::OpenAiResponsesApiConfig, openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, @@ -47,7 +48,11 @@ pub(super) async fn prepare( }; let body = config.transform_responses_api_request(model, call.input, call.optional_params)?; Ok(ProviderResponsesRequest { - url: config.get_complete_url(call.api_base.as_deref(), &lookup), + url: config.get_complete_url( + call.api_base.as_deref(), + call.api_version.as_deref(), + &lookup, + )?, config, environment, body, @@ -89,6 +94,7 @@ pub(super) fn resolve_provider( format!("openrouter/{resolved}"), ), Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()), + Ok(LlmProviders::Azure) => (&AZURE_RESPONSES_CONFIG, resolved.into()), _ => return Err(Error::Unsupported("native HTTP responses provider")), }; Ok(ResponsesProvider { @@ -136,6 +142,8 @@ mod tests { "openrouter/auto" )] #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")] + #[case::azure_prefix("azure/my-deployment", None, "azure", "my-deployment")] + #[case::azure_explicit("my-deployment", Some("azure"), "azure", "my-deployment")] fn providers_and_wire_models_match_python( #[case] model: &str, #[case] custom_llm_provider: Option<&str>, diff --git a/litellm-rust/crates/inference-responses/src/types.rs b/litellm-rust/crates/inference-responses/src/types.rs index 18c64dc8178..473e28c1103 100644 --- a/litellm-rust/crates/inference-responses/src/types.rs +++ b/litellm-rust/crates/inference-responses/src/types.rs @@ -17,6 +17,7 @@ pub struct ResponsesCall { pub api_key: Option, pub api_base: Option, pub custom_llm_provider: Option, + pub api_version: Option, pub extra_headers: Option>, pub timeout: Option, } diff --git a/litellm-rust/crates/inference-responses/tests/caching.rs b/litellm-rust/crates/inference-responses/tests/caching.rs index f45cefdc13b..0c887248da5 100644 --- a/litellm-rust/crates/inference-responses/tests/caching.rs +++ b/litellm-rust/crates/inference-responses/tests/caching.rs @@ -284,6 +284,7 @@ async fn responses_cache_identity_follows_resolved_configuration_and_request_cal api_key: None, api_base: None, custom_llm_provider: None, + api_version: None, extra_headers: None, timeout: None, }, diff --git a/litellm-rust/crates/inference-responses/tests/live/openrouter.rs b/litellm-rust/crates/inference-responses/tests/live/openrouter.rs index 364dd213ec1..b1fcf3084a5 100644 --- a/litellm-rust/crates/inference-responses/tests/live/openrouter.rs +++ b/litellm-rust/crates/inference-responses/tests/live/openrouter.rs @@ -33,6 +33,7 @@ fn call(stream: bool) -> ResponsesCall { api_key: None, api_base: None, custom_llm_provider: None, + api_version: None, extra_headers: None, timeout: Some(Duration::from_secs(60)), } diff --git a/litellm-rust/crates/inference-responses/tests/openrouter.rs b/litellm-rust/crates/inference-responses/tests/openrouter.rs index e52cdba8ce8..0833cb0fce9 100644 --- a/litellm-rust/crates/inference-responses/tests/openrouter.rs +++ b/litellm-rust/crates/inference-responses/tests/openrouter.rs @@ -22,6 +22,7 @@ fn call() -> ResponsesCall { api_key: None, api_base: None, custom_llm_provider: None, + api_version: None, extra_headers: None, timeout: None, } diff --git a/litellm-rust/crates/inference-responses/tests/responses.rs b/litellm-rust/crates/inference-responses/tests/responses.rs index f5181aca2b1..acd2fd72e81 100644 --- a/litellm-rust/crates/inference-responses/tests/responses.rs +++ b/litellm-rust/crates/inference-responses/tests/responses.rs @@ -27,6 +27,7 @@ fn call() -> ResponsesCall { api_key: Some("test-key".into()), api_base: None, custom_llm_provider: None, + api_version: None, extra_headers: None, timeout: None, } diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md index 3cf916b1721..dd48abb81d2 100644 --- a/litellm-rust/crates/llms/AGENTS.md +++ b/litellm-rust/crates/llms/AGENTS.md @@ -58,7 +58,7 @@ - `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts - `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation -- OpenAI-compatible Responses hosts (`openrouter`) reuse the OpenAI request and response transforms as free functions and own only their credentials and base URL +- OpenAI-compatible Responses hosts (`openrouter`, `azure`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules - Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper - Auth resolution is split between configs and route preparation in `inference-` diff --git a/litellm-rust/crates/llms/src/azure/mod.rs b/litellm-rust/crates/llms/src/azure/mod.rs new file mode 100644 index 00000000000..b396b037bc5 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/mod.rs @@ -0,0 +1 @@ +pub mod responses; diff --git a/litellm-rust/crates/llms/src/azure/responses/AGENTS.md b/litellm-rust/crates/llms/src/azure/responses/AGENTS.md new file mode 100644 index 00000000000..f83982bce79 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/responses/AGENTS.md @@ -0,0 +1,10 @@ +# rules + +- Build the URL like Python's `BaseAzureLLM._get_base_azure_url`: the `preview`, `latest` and `v1` API versions go to `/openai/v1/responses`, and dated versions go to `/openai/responses`. `AZURE_DEFAULT_RESPONSES_API_VERSION` sets the default, which is `preview` +- Authenticate with the `api-key` header from `api_key`, `AZURE_OPENAI_API_KEY` or `AZURE_API_KEY`, and keep a forwarded `api-key` header. Azure AD token flows stay on Python +- Lift Chat Completions-style `function` tools to the top level, and drop `status` from replayed reasoning and message items +- No native WebSocket mode + +# references + +- https://learn.microsoft.com/azure/ai-foundry/openai/how-to/responses diff --git a/litellm-rust/crates/llms/src/azure/responses/mod.rs b/litellm-rust/crates/llms/src/azure/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/azure/responses/transformation.rs b/litellm-rust/crates/llms/src/azure/responses/transformation.rs new file mode 100644 index 00000000000..d54682b1432 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/responses/transformation.rs @@ -0,0 +1,388 @@ +//! Azure OpenAI Responses, matching `litellm/llms/azure/responses/transformation.py` + +use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use serde_json::{Map, Value}; +use url::Url; + +use crate::{ + Error, ErrorDetail, + base_llm::{ + auth::{AuthScheme, ValidatedEnvironment}, + responses::transformation::BaseResponsesApiConfig, + }, + openai::responses::{ + request::ResponsesHost, + transformation::{responses_request, responses_response}, + }, +}; + +const API_KEY_ENVS: [&str; 2] = ["AZURE_OPENAI_API_KEY", "AZURE_API_KEY"]; +const API_BASE_ENV: &str = "AZURE_API_BASE"; +const DEFAULT_API_VERSION_ENV: &str = "AZURE_DEFAULT_RESPONSES_API_VERSION"; +const DEFAULT_API_VERSION: &str = "preview"; +const API_KEY_HEADER: &str = "api-key"; +const ROUTE: &str = "/openai/responses"; +const MODEL_PREFIXES: [&str; 3] = ["responses/", "o_series/", "azure_ai/"]; +/// API versions served from Azure's `/openai/v1` surface +const V1_API_VERSIONS: [&str; 3] = ["preview", "latest", "v1"]; + +pub struct AzureResponsesApiConfig; +pub const AZURE_RESPONSES_CONFIG: AzureResponsesApiConfig = AzureResponsesApiConfig; + +impl BaseResponsesApiConfig for AzureResponsesApiConfig { + fn secret_names( + &self, + api_key: Option<&str>, + api_base: Option<&str>, + ) -> &'static [&'static str] { + match ( + api_key.is_some_and(|key| !key.is_empty()), + api_base.is_some_and(|base| !base.is_empty()), + ) { + (true, true) => &[DEFAULT_API_VERSION_ENV], + (true, false) => &[API_BASE_ENV, DEFAULT_API_VERSION_ENV], + (false, true) => &[ + "AZURE_OPENAI_API_KEY", + "AZURE_API_KEY", + DEFAULT_API_VERSION_ENV, + ], + (false, false) => &[ + "AZURE_OPENAI_API_KEY", + "AZURE_API_KEY", + API_BASE_ENV, + DEFAULT_API_VERSION_ENV, + ], + } + } + + fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + if headers.iter().any(|(name, _)| name == API_KEY_HEADER) { + return Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Forwarded, + }); + } + let key = non_empty(api_key) + .map(str::to_owned) + .or_else(|| { + API_KEY_ENVS + .iter() + .find_map(|name| lookup(name).filter(|key| !key.is_empty())) + }) + .ok_or(litellm_auth::Error::MissingApiKey { + provider: "Azure", + environment_variable: API_KEY_ENVS[0], + })?; + Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Header(API_KEY_HEADER), + secret: SecretValue::new(key), + }, + }) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + api_version: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let api_base = non_empty(api_base) + .map(str::to_owned) + .or_else(|| lookup(API_BASE_ENV).filter(|base| !base.is_empty())) + .ok_or(litellm_auth::Error::MissingApiBase { + provider: "Azure", + guidance: "Set `api_base` or the AZURE_API_BASE environment variable", + })?; + let api_version = non_empty(api_version).map_or_else( + || lookup(DEFAULT_API_VERSION_ENV).unwrap_or_else(|| DEFAULT_API_VERSION.into()), + str::to_owned, + ); + complete_url(&api_base, &api_version) + } + + fn transform_responses_api_request( + &self, + model: &str, + input: Value, + mut params: Map, + ) -> Result { + if let Some(Value::Array(tools)) = params.get_mut("tools") { + tools.iter_mut().for_each(flatten_function_tool); + } + let model = MODEL_PREFIXES + .iter() + .fold(model.to_owned(), |model, prefix| model.replace(prefix, "")); + responses_request(&model, input, params, ResponsesHost::Azure) + } + + fn transform_response_api_response(&self, body: Value) -> Result { + responses_response(body) + } +} + +fn non_empty(value: Option<&str>) -> Option<&str> { + value.filter(|value| !value.is_empty()) +} + +/// Azure takes Chat Completions-style `{"type": "function", "function": {...}}` tools only with +/// the function's fields lifted to the top level +fn flatten_function_tool(tool: &mut Value) { + let Some(fields) = tool.as_object_mut() else { + return; + }; + let Some(function) = fields.remove("function") else { + return; + }; + match function { + Value::Object(function) => fields.extend(function), + other => { + fields.insert("function".into(), other); + } + } +} + +/// Python's `BaseAzureLLM._get_base_azure_url` for the Responses route +fn complete_url(api_base: &str, api_version: &str) -> Result { + let mut url = Url::parse(api_base) + .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("Azure api_base", error)))?; + let mut query: Vec<(String, String)> = Vec::new(); + for (name, value) in url.query_pairs() { + if !query.iter().any(|(seen, _)| *seen == name) { + query.push((name.into_owned(), value.into_owned())); + } + } + if !api_version.is_empty() && !query.iter().any(|(name, _)| name == "api-version") { + query.push(("api-version".into(), api_version.into())); + } + if !api_base.contains(ROUTE) { + let path = joined_path(url.path(), ROUTE); + url.set_path(&path); + } + if V1_API_VERSIONS.contains(&api_version) && !url.as_str().contains("/openai/v1") { + let path = url.path().replace("/openai", "/openai/v1"); + url.set_path(&path); + } + url.set_query(None); + if !query.is_empty() { + url.query_pairs_mut().extend_pairs(&query); + } + Ok(url.into()) +} + +/// Python's `add_path_to_api_base`: appends `ending` without repeating segments the base already +/// ends with +fn joined_path(base: &str, ending: &str) -> String { + let base: Vec<&str> = base + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + let ending: Vec<&str> = ending + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + let segments = (0..base.len()) + .find(|&index| base[index..] == ending[..(base.len() - index).min(ending.len())]) + .map_or_else( + || [base.as_slice(), ending.as_slice()].concat(), + |index| [&base[..index], ending.as_slice()].concat(), + ); + format!("/{}", segments.join("/")) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + } + } + + #[rstest] + #[case::resource_default_preview( + "https://res.openai.azure.com", + None, + "https://res.openai.azure.com/openai/v1/responses?api-version=preview" + )] + #[case::dated_version( + "https://res.openai.azure.com", + Some("2025-03-01-preview"), + "https://res.openai.azure.com/openai/responses?api-version=2025-03-01-preview" + )] + #[case::trailing_openai_segment( + "https://res.openai.azure.com/openai", + Some("2025-03-01-preview"), + "https://res.openai.azure.com/openai/responses?api-version=2025-03-01-preview" + )] + #[case::full_route_keeps_its_version( + "https://res.openai.azure.com/openai/responses?api-version=2024-05-01-preview", + Some("2025-03-01-preview"), + "https://res.openai.azure.com/openai/responses?api-version=2024-05-01-preview" + )] + #[case::full_route_moves_to_v1( + "https://res.openai.azure.com/openai/responses", + Some("latest"), + "https://res.openai.azure.com/openai/v1/responses?api-version=latest" + )] + #[case::v1_base( + "https://res.openai.azure.com/openai/v1", + Some("v1"), + "https://res.openai.azure.com/openai/v1/openai/responses?api-version=v1" + )] + #[case::proxy_path( + "http://127.0.0.1:8080/azure/", + Some("2025-03-01-preview"), + "http://127.0.0.1:8080/azure/openai/responses?api-version=2025-03-01-preview" + )] + fn urls_match_python( + #[case] api_base: &str, + #[case] api_version: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!( + AZURE_RESPONSES_CONFIG + .get_complete_url(Some(api_base), api_version, &env(&[])) + .unwrap(), + expected + ); + } + + #[rstest] + #[case::configured_default(&[(DEFAULT_API_VERSION_ENV, "2025-04-01-preview")], "https://res.openai.azure.com/openai/responses?api-version=2025-04-01-preview")] + #[case::empty_default_sends_no_version(&[(DEFAULT_API_VERSION_ENV, "")], "https://res.openai.azure.com/openai/responses")] + fn default_version_comes_from_the_environment( + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + assert_eq!( + AZURE_RESPONSES_CONFIG + .get_complete_url(Some("https://res.openai.azure.com"), None, &env(secrets)) + .unwrap(), + expected + ); + } + + #[test] + fn api_base_falls_back_to_the_environment() { + let lookup = env(&[(API_BASE_ENV, "https://env.openai.azure.com")]); + assert_eq!( + AZURE_RESPONSES_CONFIG + .get_complete_url(None, Some("v1"), &lookup) + .unwrap(), + "https://env.openai.azure.com/openai/v1/responses?api-version=v1" + ); + assert!( + AZURE_RESPONSES_CONFIG + .get_complete_url(None, None, &env(&[])) + .is_err() + ); + } + + #[rstest] + #[case::explicit(Some("explicit"), &[("AZURE_OPENAI_API_KEY", "managed")], "explicit")] + #[case::openai_key_env(None, &[("AZURE_OPENAI_API_KEY", "managed"), ("AZURE_API_KEY", "other")], "managed")] + #[case::api_key_env(Some(""), &[("AZURE_API_KEY", "other")], "other")] + fn api_key_goes_in_the_api_key_header( + #[case] api_key: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + let environment = AZURE_RESPONSES_CONFIG + .validate_environment(Vec::new(), api_key, &env(secrets)) + .unwrap(); + let AuthScheme::Credential { placement, secret } = environment.auth else { + panic!("expected a credential") + }; + assert_eq!( + (placement.header_name(), secret.expose()), + (API_KEY_HEADER, expected) + ); + } + + #[test] + fn a_forwarded_api_key_header_is_kept() { + let headers = vec![(API_KEY_HEADER.to_owned(), "caller".to_owned())]; + let environment = AZURE_RESPONSES_CONFIG + .validate_environment(headers.clone(), Some("deployment"), &env(&[])) + .unwrap(); + assert!(matches!(environment.auth, AuthScheme::Forwarded)); + assert_eq!(environment.headers, headers); + } + + #[test] + fn missing_key_is_an_auth_error() { + assert!( + AZURE_RESPONSES_CONFIG + .validate_environment(Vec::new(), None, &env(&[])) + .is_err() + ); + } + + #[test] + fn request_flattens_tools_strips_model_prefixes_and_status() { + let Value::Object(params) = json!({"tools": [ + {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}, + {"type": "web_search"} + ]}) else { + unreachable!() + }; + let body = AZURE_RESPONSES_CONFIG + .transform_responses_api_request( + "responses/o_series/my-deployment", + json!([ + {"type": "message", "role": "user", "content": "hi", "status": "completed"}, + {"type": "reasoning", "id": "rs_1", "summary": [], "status": "completed"}, + {"type": "reasoning", "summary": [], "status": "completed"} + ]), + params, + ) + .unwrap(); + assert_eq!( + body, + json!({ + "tools": [ + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + {"type": "web_search"} + ], + "model": "my-deployment", + "input": [ + {"type": "message", "role": "user", "content": "hi"}, + {"type": "reasoning", "id": "rs_1", "summary": []}, + {"type": "reasoning", "summary": [], "status": "completed"} + ] + }) + ); + } + + #[rstest] + #[case::overlap("/a/openai", "/openai/responses", "/a/openai/responses")] + #[case::no_overlap("/a", "/openai/responses", "/a/openai/responses")] + #[case::root("/", "/openai/responses", "/openai/responses")] + #[case::partial_overlap( + "/openai/responses/x", + "/openai/responses", + "/openai/responses/x/openai/responses" + )] + fn joined_path_matches_add_path_to_api_base( + #[case] base: &str, + #[case] ending: &str, + #[case] expected: &str, + ) { + assert_eq!(joined_path(base, ending), expected); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs index 93eaad0ebc0..74981df4d9b 100644 --- a/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs @@ -23,8 +23,9 @@ pub trait BaseResponsesApiConfig: Sync { fn get_complete_url( &self, api_base: Option<&str>, + api_version: Option<&str>, lookup: &dyn Fn(&str) -> Option, - ) -> String; + ) -> Result; fn transform_responses_api_request( &self, diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs index eb232404d93..23ed649ac8e 100644 --- a/litellm-rust/crates/llms/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -1,5 +1,6 @@ pub mod anthropic; pub mod aws_textract; +pub mod azure; pub mod azure_ai; pub mod base_llm; pub mod bedrock; diff --git a/litellm-rust/crates/llms/src/openai/responses/request.rs b/litellm-rust/crates/llms/src/openai/responses/request.rs index b8fe592c0e2..4ae8b458a00 100644 --- a/litellm-rust/crates/llms/src/openai/responses/request.rs +++ b/litellm-rust/crates/llms/src/openai/responses/request.rs @@ -15,18 +15,26 @@ const REASONING_ITEM_ID_PREFIX: &str = "rs_"; const MAX_STRIP_DEPTH: usize = 100; const REASONING_FIELDS_DROPPED_WHEN_NULL: [&str; 3] = ["status", "content", "encrypted_content"]; -/// Whether the host runs OpenAI's own item validator, which rejects reasoning it did not mint -/// and tool-call item ids without its prefixes +/// The host behind the OpenAI wire format. OpenAI and Azure run OpenAI's item validator, which +/// rejects reasoning it did not mint and tool-call item ids without its prefixes, and Azure also +/// rejects `status` on replayed reasoning and message items #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ItemValidation { +pub enum ResponsesHost { OpenAi, - Lenient, + Azure, + Compatible, +} + +impl ResponsesHost { + fn validates_items(self) -> bool { + matches!(self, Self::OpenAi | Self::Azure) + } } pub fn shape( input: Value, mut params: Map, - validation: ItemValidation, + host: ResponsesHost, ) -> Result<(Value, Map), Error> { if let Some(max_output_tokens) = params.get_mut("max_output_tokens") && max_output_tokens @@ -50,17 +58,17 @@ pub fn shape( }; let items = items .into_iter() - .filter(|item| validation == ItemValidation::Lenient || !is_minted_reasoning(item)) - .map(without_null_reasoning_fields) + .filter(|item| !host.validates_items() || !is_minted_reasoning(item)) + .map(|item| without_null_reasoning_fields(item, host)) .map(|mut item| { if let Some(fields) = item.as_object_mut() { strip_cache_control(fields, 0); } item }) - .map(|item| match validation { - ItemValidation::OpenAi => without_foreign_tool_call_id(item), - ItemValidation::Lenient => item, + .map(|item| match host.validates_items() { + true => without_foreign_tool_call_id(item), + false => item, }) .collect(); Ok((Value::Array(items), params)) @@ -154,15 +162,23 @@ fn carries_thinking_blocks(encrypted_content: &str) -> bool { /// OpenAI rejects `status: null` on a replayed reasoning item. Python round-trips a valid item /// through the SDK model with `exclude_none`, and otherwise drops only the nullable fields -fn without_null_reasoning_fields(item: Value) -> Value { +fn without_null_reasoning_fields(item: Value, host: ResponsesHost) -> Value { let Value::Object(mut fields) = item else { return item; }; - if fields.get("type").and_then(Value::as_str) != Some("reasoning") { - return Value::Object(fields); + match fields.get("type").and_then(Value::as_str) { + Some("message") if host == ResponsesHost::Azure => { + fields.remove("status"); + return Value::Object(fields); + } + Some("reasoning") => {} + _ => return Value::Object(fields), } if is_sdk_reasoning_item(&fields) { drop_nulls(&mut fields); + if host == ResponsesHost::Azure { + fields.remove("status"); + } fields .iter_mut() .filter(|(name, _)| matches!(name.as_str(), "summary" | "content")) @@ -233,11 +249,11 @@ mod tests { use super::*; - fn shaped(input: Value, params: Value, validation: ItemValidation) -> (Value, Value) { + fn shaped(input: Value, params: Value, host: ResponsesHost) -> (Value, Value) { let Value::Object(params) = params else { panic!("params must be an object") }; - let (input, params) = shape(input, params, validation).unwrap(); + let (input, params) = shape(input, params, host).unwrap(); (input, Value::Object(params)) } @@ -252,7 +268,7 @@ mod tests { let (_, params) = shaped( json!("hi"), json!({"max_output_tokens": requested}), - ItemValidation::Lenient, + ResponsesHost::Compatible, ); assert_eq!(params, json!({"max_output_tokens": sent})); } @@ -267,7 +283,7 @@ mod tests { {"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b", "cache_control": mark}]} ]), json!({"tools": [{"type": "function", "name": "a", "cache_control": mark, "parameters": {"type": "object"}}]}), - ItemValidation::Lenient, + ResponsesHost::Compatible, ); assert_eq!( (input, params), @@ -292,7 +308,7 @@ mod tests { {"type": "function", "name": "c", "parameters": null}, {"type": "web_search"} ]}), - ItemValidation::Lenient, + ResponsesHost::Compatible, ); assert_eq!( params, @@ -319,7 +335,7 @@ mod tests { unreachable!() }; assert_eq!( - shape(json!("hi"), params, ItemValidation::Lenient) + shape(json!("hi"), params, ResponsesHost::Compatible) .unwrap_err() .to_string(), format!( @@ -346,7 +362,7 @@ mod tests { json!({"type": "message", "role": "user", "content": "hi", "status": null}) )] fn drops_null_reasoning_fields(#[case] item: Value, #[case] expected: Value) { - let (input, _) = shaped(json!([item]), json!({}), ItemValidation::Lenient); + let (input, _) = shaped(json!([item]), json!({}), ResponsesHost::Compatible); assert_eq!(input, json!([expected])); } @@ -365,8 +381,8 @@ mod tests { #[case::uuid_v1(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-11d0-b42d-00c04fc964ff", "summary": []}), false)] fn drops_minted_reasoning_for_openai(#[case] item: Value, #[case] minted: bool) { let input = json!([item, {"role": "user", "content": "hi"}]); - let (openai, _) = shaped(input.clone(), json!({}), ItemValidation::OpenAi); - let (lenient, _) = shaped(input.clone(), json!({}), ItemValidation::Lenient); + let (openai, _) = shaped(input.clone(), json!({}), ResponsesHost::OpenAi); + let (lenient, _) = shaped(input.clone(), json!({}), ResponsesHost::Compatible); assert_eq!(openai.as_array().unwrap().len(), if minted { 1 } else { 2 }); assert_eq!(lenient, input); } @@ -377,8 +393,8 @@ mod tests { #[case::foreign_custom_tool_call(json!({"type": "custom_tool_call", "id": "x", "call_id": "c"}), json!({"type": "custom_tool_call", "call_id": "c"}))] #[case::other_item(json!({"type": "message", "id": "x", "role": "user", "content": "hi"}), json!({"type": "message", "id": "x", "role": "user", "content": "hi"}))] fn drops_foreign_tool_call_ids_for_openai(#[case] item: Value, #[case] expected: Value) { - let (openai, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::OpenAi); - let (lenient, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::Lenient); + let (openai, _) = shaped(json!([item.clone()]), json!({}), ResponsesHost::OpenAi); + let (lenient, _) = shaped(json!([item.clone()]), json!({}), ResponsesHost::Compatible); assert_eq!((openai, lenient), (json!([expected]), json!([item]))); } } diff --git a/litellm-rust/crates/llms/src/openai/responses/transformation.rs b/litellm-rust/crates/llms/src/openai/responses/transformation.rs index 42c12b6ac58..de010be3ac9 100644 --- a/litellm-rust/crates/llms/src/openai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/responses/transformation.rs @@ -5,7 +5,7 @@ use serde_json::{Map, Value}; use litellm_auth::{CredentialPlacement, SecretValue}; -use super::request::{self, ItemValidation}; +use super::request::{self, ResponsesHost}; use crate::{ Error, base_llm::{ @@ -149,15 +149,16 @@ impl BaseResponsesApiConfig for OpenAiResponsesApiConfig { fn get_complete_url( &self, api_base: Option<&str>, + _api_version: Option<&str>, lookup: &dyn Fn(&str) -> Option, - ) -> String { + ) -> Result { let base = api_base .filter(|base| !base.is_empty()) .map(str::to_owned) .or_else(|| lookup("OPENAI_BASE_URL")) .or_else(|| lookup("OPENAI_API_BASE")) .unwrap_or_else(|| OPENAI_RESPONSES_DEFAULT_API_BASE.into()); - format!("{}/responses", base.trim_end_matches('/')) + Ok(format!("{}/responses", base.trim_end_matches('/'))) } fn transform_responses_api_request( @@ -166,7 +167,7 @@ impl BaseResponsesApiConfig for OpenAiResponsesApiConfig { input: Value, params: Map, ) -> Result { - responses_request(model, input, params, ItemValidation::OpenAi) + responses_request(model, input, params, ResponsesHost::OpenAi) } fn transform_response_api_response(&self, body: Value) -> Result { @@ -179,7 +180,7 @@ pub fn responses_request( model: &str, input: Value, params: Map, - validation: ItemValidation, + host: ResponsesHost, ) -> Result { if !input.is_string() && !input.is_array() { return Err(Error::InvalidRequest( @@ -192,7 +193,7 @@ pub fn responses_request( { return Err(Error::InvalidRequest("stream must be a boolean".into())); } - let (input, params) = request::shape(input, params, validation)?; + let (input, params) = request::shape(input, params, host)?; Ok(Value::Object( params .into_iter() diff --git a/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs b/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs index b51158571d0..3d122b33104 100644 --- a/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs @@ -10,7 +10,7 @@ use crate::{ responses::transformation::BaseResponsesApiConfig, }, openai::responses::{ - request::ItemValidation, + request::ResponsesHost, transformation::{responses_request, responses_response}, }, }; @@ -67,11 +67,12 @@ impl BaseResponsesApiConfig for OpenRouterResponsesApiConfig { fn get_complete_url( &self, api_base: Option<&str>, + _api_version: Option<&str>, lookup: &dyn Fn(&str) -> Option, - ) -> String { + ) -> Result { let base = resolve_non_empty(api_base, lookup, &[API_BASE_ENV]) .unwrap_or_else(|| DEFAULT_BASE.into()); - format!("{}{RESPONSES_PATH}", base.trim_end_matches('/')) + Ok(format!("{}{RESPONSES_PATH}", base.trim_end_matches('/'))) } fn transform_responses_api_request( @@ -80,7 +81,7 @@ impl BaseResponsesApiConfig for OpenRouterResponsesApiConfig { input: Value, params: Map, ) -> Result { - responses_request(model, input, params, ItemValidation::Lenient) + responses_request(model, input, params, ResponsesHost::Compatible) } fn transform_response_api_response(&self, body: Value) -> Result { @@ -115,7 +116,9 @@ mod tests { #[case] expected: &str, ) { assert_eq!( - OPENROUTER_RESPONSES_CONFIG.get_complete_url(api_base, &env(secrets)), + OPENROUTER_RESPONSES_CONFIG + .get_complete_url(api_base, None, &env(secrets)) + .unwrap(), expected ); } diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index 20c08a26de2..e94baef061f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -64,6 +64,10 @@ pub(super) fn project( api_key: call.options.api_key, api_base: call.options.api_base, custom_llm_provider: call.options.custom_llm_provider, + api_version: host + .argument(py, arguments, "api_version")? + .map(|value| value.extract()) + .transpose()?, extra_headers: call.options.extra_headers, timeout: call.options.timeout, }) diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index 9caeae6c3da..f4c6c687f9b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -40,14 +40,18 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py let provider = argument("custom_llm_provider")? .map(|value| value.extract::()) .transpose()?; - if provider - .as_deref() - .is_some_and(|provider| provider != "openai") - || model - .strip_prefix("openai/") - .unwrap_or(&model) - .contains('/') - { + let native_provider = match litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider( + &model, + provider.as_deref(), + ) { + None => !model.contains('/'), + Some(resolved) => match resolved.custom_llm_provider { + "openai" => !resolved.model.contains('/'), + "azure" => true, + _ => false, + }, + }; + if !native_provider { return Err(RustBridgeDeclined::new_err( "native HTTP responses provider", )); diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index 153431675f2..b453f75c13b 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -1,18 +1,25 @@ from __future__ import annotations from collections.abc import Mapping +from types import MappingProxyType from typing import Final, cast # noqa: TID251 # narrows caller json the checker cannot see into import litellm from litellm import get_llm_provider +from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.rust_bridge import failures from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__) -_OPENAI_CONFIG: Final = OpenAIResponsesAPIConfig() +_NATIVE_PARAMETERS: Final = (*PARAMETERS, "api_version") +_CONFIGS: Final[Mapping[str, OpenAIResponsesAPIConfig]] = MappingProxyType( + {"openai": OpenAIResponsesAPIConfig(), "azure": AzureOpenAIResponsesAPIConfig()} +) +_AZURE_KEY_ENVIRONMENT: Final = ("AZURE_OPENAI_API_KEY", "AZURE_API_KEY") def body_params(request: Mapping[str, object]) -> Mapping[str, object]: @@ -20,7 +27,7 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]: def connection_defaults(provider: str) -> tuple[str | None, str | None]: - provider_key: Final = litellm.openai_key if provider == "openai" else None + provider_key: Final = {"openai": litellm.openai_key, "azure": litellm.azure_key}.get(provider) return litellm.api_key or provider_key, litellm.api_base @@ -29,7 +36,7 @@ def response(value: Mapping[str, object]) -> ResponsesAPIResponse: def map_failure(error: Exception, request: Mapping[str, object]) -> Exception: - provider: Final = optional_str(request.get("custom_llm_provider")) or "openai" + _, provider = _resolved_model(request) return failures.map_native_failure( error, str(request["model"]), @@ -40,47 +47,79 @@ def map_failure(error: Exception, request: Mapping[str, object]) -> Exception: ) +def _resolved_model(request: Mapping[str, object]) -> tuple[str, str]: + model, provider, _, _ = get_llm_provider( + model=str(request["model"]), custom_llm_provider=optional_str(request.get("custom_llm_provider")) + ) + return model, provider + + def decline_reason(request: Mapping[str, object]) -> str | None: - if optional_str(request.get("custom_llm_provider")) is None and "/" not in str(request["model"]): - try: - _, provider, _, _ = get_llm_provider(model=str(request["model"])) - except litellm.exceptions.BadRequestError: - return "native Responses could not resolve the provider" - if provider != "openai": - return "native HTTP responses provider" - return inference_decline_reason(PARAMETERS, request) or _model_rules_decline_reason(request) + try: + model, provider = _resolved_model(request) + except litellm.exceptions.BadRequestError: + return "native Responses could not resolve the provider" + config: Final = _CONFIGS.get(provider) + if config is None: + return "native HTTP responses provider" + return ( + inference_decline_reason(_NATIVE_PARAMETERS, request) + or (_azure_decline_reason(model, request) if provider == "azure" else None) + or _model_rules_decline_reason(config, model, request) + ) -def _model_rules_decline_reason(request: Mapping[str, object]) -> str | None: +def _azure_decline_reason(model: str, request: Mapping[str, object]) -> str | None: + if "o_series/" in model: + return "native Azure Responses does not implement the o-series config" + if request.get("context_management") is not None: + return "native Azure Responses does not implement context_management" + extra_headers: Final = request.get("extra_headers") + forwarded_key: Final = isinstance(extra_headers, Mapping) and "api-key" in extra_headers + key: Final = ( + optional_str(request.get("api_key")) + or litellm.api_key + or litellm.azure_key + or next(filter(None, (get_secret_str(name) for name in _AZURE_KEY_ENVIRONMENT)), None) + ) + if not forwarded_key and not key: + return "native Azure Responses does not implement Azure AD token auth" + return None + + +def _model_rules_decline_reason( + config: OpenAIResponsesAPIConfig, model: str, request: Mapping[str, object] +) -> str | None: """Request rewrites that read the model cost map or Python's regex engine stay on Python""" - model: Final = str(request["model"]).removeprefix("openai/") params: Final = {name: request[name] for name in PARAMETERS if request.get(name) is not None} try: - _OPENAI_CONFIG.map_openai_params(params, model, drop_params=False) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the same rules the Python route applies + config.map_openai_params(params, model, drop_params=False) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the same rules the Python route applies except litellm.UnsupportedParamsError: return "native Responses does not implement the model's sampling rules" litellm_params: Final = GenericLiteLLMParams( api_base=optional_str(request.get("api_base")) or optional_str(request.get("base_url")), model_info=request.get("model_info"), ) - if _repairs_tool_schemas(model, request.get("tools"), litellm_params): + if _repairs_tool_schemas(config, model, request.get("tools"), litellm_params): return "native Responses does not implement tool schema validation and repair" - if _carries_cache_control(request.get("input")) and _OPENAI_CONFIG._reads_prompt_cache_breakpoints( # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies + if _carries_cache_control(request.get("input")) and config._reads_prompt_cache_breakpoints( # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies model=model, litellm_params=litellm_params, prompt_cache_options=request.get("prompt_cache_options") ): return "native Responses does not implement prompt cache breakpoints" return None -def _repairs_tool_schemas(model: str, tools: object, litellm_params: GenericLiteLLMParams) -> bool: +def _repairs_tool_schemas( + config: OpenAIResponsesAPIConfig, model: str, tools: object, litellm_params: GenericLiteLLMParams +) -> bool: if not isinstance(tools, list): return False try: - decoded: Final = _OPENAI_CONFIG._tools_with_object_parameters(model=model, tools=cast(list[object], tools)) # pyright: ignore[reportPrivateUsage, reportArgumentType] # cast-ok: caller json + decoded: Final = config._tools_with_object_parameters(model=model, tools=cast(list[object], tools)) # pyright: ignore[reportPrivateUsage, reportArgumentType] # cast-ok: caller json except litellm.BadRequestError: return True return ( - _OPENAI_CONFIG._sanitized_tool_schemas_for_openai(model=model, tools=decoded, litellm_params=litellm_params) # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies + config._sanitized_tool_schemas_for_openai(model=model, tools=decoded, litellm_params=litellm_params) # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies is not decoded ) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_error_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_error_parity.py index 915e491e524..e48be2f5850 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_error_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_error_parity.py @@ -61,6 +61,11 @@ CASES: Final = ( _upstream(429, {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"}), ), FailureCase("server_error", {"input": "hi"}, _upstream(500, {"message": "boom", "type": "server_error"})), + FailureCase( + "azure_rate_limited", + {"model": "azure/my-deployment", "input": "hi"}, + _upstream(429, {"message": "Rate limit reached", "type": "requests", "code": "429"}), + ), FailureCase( "invalid_tool_parameters", {"input": "hi", "tools": [{**WEATHER_TOOL, "parameters": "{"}]}, @@ -77,7 +82,15 @@ async def _failure(case: FailureCase, server: RecordingServer, probe: FallbackPr fallbacks_before: Final = 0 if probe is None else probe.calls call: Final = PYTHON_ARESPONSES if probe is None else PUBLIC_ARESPONSES with pytest.raises(openai.APIError) as raised: - await call(model=MODEL, api_key="sk-test", api_base=server.base_url, callbacks=[recorder], **case.kwargs) + await call( + **{ + "model": MODEL, + "api_key": "sk-test", + "api_base": server.base_url, + "callbacks": [recorder], + **case.kwargs, + } + ) await drain_logging() failures: Final = await recorder.wait_for_async("async_log_failure_event") standard: Final = _LOGGED.validate_python(_LOGGED.validate_python(failures[0].kwargs)["standard_logging_object"]) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 7dbdb41aa4e..61114447ba9 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -25,7 +25,7 @@ pytestmark = pytest.mark.requires_rust_extension MODEL: Final = "openai/gpt-6.1-sol" _OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _LOGGED: Final = TypeAdapter(dict[str, object]) -_COMPARED_HEADERS: Final = ("authorization", "content-type", "openai-organization", "openai-project") +_COMPARED_HEADERS: Final = ("api-key", "authorization", "content-type", "openai-organization", "openai-project") _COMPARED_LOG_FIELDS: Final = ( "call_type", "custom_llm_provider", @@ -156,6 +156,18 @@ AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [ {"role": "user", "content": "And Germany?"}, ] +AZURE_MODEL: Final = "azure/my-deployment" + +WEATHER_TOOL_FUNCTION: Final[Mapping[str, JsonValue]] = MappingProxyType( + {name: value for name, value in WEATHER_TOOL.items() if name != "type"} +) + +AZURE_STATUS_REPLAY_INPUT: Final[list[JsonValue]] = [ + {"type": "message", "role": "user", "content": "Weather in Paris?", "status": "completed"}, + {"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": "completed"}, + {"role": "user", "content": "And Germany?"}, +] + MINTED_REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ {"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []}, { @@ -190,6 +202,7 @@ REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ @dataclass(frozen=True, slots=True) class Leg: + request_path: str request_body: Mapping[str, JsonValue] request_headers: Mapping[str, str] response: Mapping[str, JsonValue] @@ -301,6 +314,21 @@ CASES: Final = ( "foreign_tool_call_ids_are_dropped", {"input": FOREIGN_TOOL_CALL_REPLAY_INPUT, "tools": [dict(WEATHER_TOOL)]}, ), + Case("azure_default_v1_surface", {"model": AZURE_MODEL, "input": "hi"}), + Case("azure_dated_api_version", {"model": AZURE_MODEL, "input": "hi", "api_version": "2025-03-01-preview"}), + Case( + "azure_replay_without_status", + { + "model": AZURE_MODEL, + "input": AZURE_STATUS_REPLAY_INPUT, + "tools": [{"type": "function", "function": dict(WEATHER_TOOL_FUNCTION)}], + }, + ), + Case( + "azure_forwarded_api_key_header", + {"model": AZURE_MODEL, "input": "hi", "api_key": None, "extra_headers": {"api-key": "caller-key"}}, + ), + Case("azure_o_series_stays_on_python", {"model": "azure/o_series/my-o3", "input": "hi"}, falls_back=True), Case( "tool_schema_repair_stays_on_python", { @@ -348,6 +376,7 @@ async def _isolated_leg(case: Case, server: RecordingServer, probe: FallbackProb sent: Final = server.requests[-1] standard: Final = _LOGGED.validate_python(_LOGGED.validate_python(success[0].kwargs)["standard_logging_object"]) return Leg( + request_path=sent.path, request_body=_OBJECT.validate_python(sent.body), request_headers=MappingProxyType( {name: value for name, value in sent.headers.items() if name in _COMPARED_HEADERS} @@ -377,7 +406,11 @@ async def test_native_responses_match_python( case "dispatch": assert native.fell_back == case.falls_back case "request": - assert (native.request_body, native.request_headers) == (python.request_body, python.request_headers) + assert (native.request_path, native.request_body, native.request_headers) == ( + python.request_path, + python.request_body, + python.request_headers, + ) case "response": assert native.response == python.response case "logged": diff --git a/tests/unit/rust_bridge/host/routes/test_responses.py b/tests/unit/rust_bridge/host/routes/test_responses.py index 641931c8324..ce12a1d1adb 100644 --- a/tests/unit/rust_bridge/host/routes/test_responses.py +++ b/tests/unit/rust_bridge/host/routes/test_responses.py @@ -133,3 +133,44 @@ _PLAIN_TOOL: Final = MappingProxyType( ) def test_model_rules_stay_on_python(request_fields: dict[str, object], expected: str | None) -> None: assert decline_reason(MappingProxyType({"model": "openai/gpt-6.1-sol", **request_fields})) == expected + + +@pytest.mark.parametrize( + ("request_fields", "expected"), + ( + pytest.param({"api_key": "azure-key"}, None, id="api_key"), + pytest.param( + {"api_key": None, "extra_headers": {"api-key": "caller-key"}}, None, id="forwarded_api_key_header" + ), + pytest.param( + {"api_key": None}, "native Azure Responses does not implement Azure AD token auth", id="azure_ad_token_auth" + ), + pytest.param( + {"api_key": "azure-key", "model": "azure/o_series/my-o3"}, + "native Azure Responses does not implement the o-series config", + id="o_series", + ), + pytest.param( + {"api_key": "azure-key", "context_management": [{"type": "compaction"}]}, + "native Azure Responses does not implement context_management", + id="context_management", + ), + pytest.param({"api_key": "azure-key", "api_version": "2025-03-01-preview"}, None, id="api_version"), + ), +) +def test_azure_requests_outside_the_native_config_stay_on_python( + monkeypatch: pytest.MonkeyPatch, request_fields: dict[str, object], expected: str | None +) -> None: + for name in ("AZURE_OPENAI_API_KEY", "AZURE_API_KEY"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "azure_key", None) + request: Final = {"model": "azure/my-deployment", "input": "hi", **request_fields} + assert decline_reason(MappingProxyType(request)) == expected + + +def test_azure_connection_defaults_read_the_azure_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "azure_key", "azure-global") + monkeypatch.setattr(litellm, "api_base", None) + assert connection_defaults("azure") == ("azure-global", None)