diff --git a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index acea48799c0..eb352ea3ba1 100644 --- a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -12,6 +12,9 @@ pub enum LlmProviders { Anthropic, #[strum(serialize = "aws_textract")] AwsTextract, + /// Azure OpenAI (`azure/`), distinct from Azure AI Foundry's `azure_ai`. + #[strum(serialize = "azure")] + Azure, #[strum(serialize = "azure_ai")] AzureAi, #[strum(serialize = "bedrock")] diff --git a/litellm-rust/crates/gateway-hooks/src/pricing.rs b/litellm-rust/crates/gateway-hooks/src/pricing.rs index be1bca921ba..b2f91d3c91e 100644 --- a/litellm-rust/crates/gateway-hooks/src/pricing.rs +++ b/litellm-rust/crates/gateway-hooks/src/pricing.rs @@ -18,6 +18,12 @@ use crate::usage::{ServiceTier, Usage}; pub struct Prices { plans: HashMap, PricingPlan>, max_output_tokens: HashMap, u64>, + /// Deployments priced by the model their response names, with the provider route the + /// cost map keys that model under. The proxy prices an Azure deployment that sets neither + /// `base_model` nor prices of its own this way. + by_response_model: HashMap, Arc>, + /// Cost map prices by model name, for the deployments priced by their response's model. + models: HashMap, } impl Prices { @@ -27,18 +33,29 @@ impl Prices { if let Some(limit) = fields.get("max_output_tokens").and_then(Value::as_u64) { self.max_output_tokens.insert(id.clone(), limit); } - let parsed = ParsedRates::parse(fields); - if parsed.standard == Rates::EMPTY && parsed.thresholds.is_empty() { - return; + if let Some(plan) = plan(&id, fields) { + self.plans.insert(id, plan); } - match parsed.compile() { - Ok(plan) => { - self.plans.insert(id, plan); - } - Err(error) => tracing::warn!(deployment = %id, ?error, "ignoring invalid prices"), + } + + /// Prices `deployment` by the model its response names, looked up as + /// `{provider}/{model}`, before its own prices. + pub fn price_by_response_model(&mut self, deployment: Arc, provider: &str) { + self.by_response_model.insert(deployment, provider.into()); + } + + /// Adds the cost map prices of `model`, for deployments priced by their response's model. + pub fn insert_model(&mut self, model: &str, fields: &Map) { + if let Some(plan) = plan(model, fields) { + self.models.insert(model.to_owned(), plan); } } + /// Whether some deployment is priced by its response's model. + pub fn prices_by_response_model(&self) -> bool { + !self.by_response_model.is_empty() + } + pub fn len(&self) -> usize { self.plans.len() } @@ -54,41 +71,78 @@ impl Prices { /// The cost in dollars of `usage` on a deployment, or `None` when its prices are unknown. pub fn cost(&self, deployment: &str, usage: &Usage) -> Option { - let plan = self.plans.get(deployment)?; - let one_hour = usage.cache_write_1h_tokens.min(usage.cache_write_tokens); - let split = (one_hour > 0).then_some(()); - let request = Request { - usage: litellm_cost::Usage { - prompt_tokens: usage.prompt_tokens(), - completion_tokens: usage.output_tokens, - cache_read_tokens: usage.cache_read_tokens, - cache_write_tokens: usage.cache_write_tokens, - cache_write_5m_tokens: split.map(|()| usage.cache_write_tokens - one_hour), - cache_write_1h_tokens: split.map(|()| one_hour), - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: match usage.service_tier { - ServiceTier::Priority => CostTier::Priority, - ServiceTier::Flex => CostTier::Flex, - ServiceTier::Standard | ServiceTier::Batch => CostTier::Standard, - }, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - }; - match plan.calculate(&request) { - Ok(cost) => Some(cost.total()), - Err(error) => { - tracing::debug!(deployment, ?error, "no cost for response"); - None - } + plan_cost(deployment, self.plans.get(deployment)?, usage) + } + + /// [`Self::cost`], first by `response_model` for a deployment priced by its response's + /// model, as the proxy tries `{provider}/{response model}` ahead of the deployment. + pub fn response_cost( + &self, + deployment: &str, + response_model: Option<&str>, + usage: &Usage, + ) -> Option { + self.by_response_model + .get(deployment) + .zip(response_model.filter(|model| !model.is_empty())) + .and_then(|(provider, model)| { + let name = match model.starts_with(&format!("{provider}/")) { + true => model.to_owned(), + false => format!("{provider}/{model}"), + }; + plan_cost(deployment, self.models.get(&name)?, usage) + }) + .or_else(|| self.cost(deployment, usage)) + } +} + +/// The plan `fields` price, or `None` when they hold no usable prices. +fn plan(name: &str, fields: &Map) -> Option { + let parsed = ParsedRates::parse(fields); + if parsed.standard == Rates::EMPTY && parsed.thresholds.is_empty() { + return None; + } + parsed + .compile() + .inspect_err(|error| tracing::warn!(deployment = %name, ?error, "ignoring invalid prices")) + .ok() +} + +fn plan_cost(deployment: &str, plan: &PricingPlan, usage: &Usage) -> Option { + let one_hour = usage.cache_write_1h_tokens.min(usage.cache_write_tokens); + let split = (one_hour > 0).then_some(()); + let request = Request { + usage: litellm_cost::Usage { + prompt_tokens: usage.prompt_tokens(), + completion_tokens: usage.output_tokens, + cache_read_tokens: usage.cache_read_tokens, + cache_write_tokens: usage.cache_write_tokens, + cache_write_5m_tokens: split.map(|()| usage.cache_write_tokens - one_hour), + cache_write_1h_tokens: split.map(|()| one_hour), + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: match usage.service_tier { + ServiceTier::Priority => CostTier::Priority, + ServiceTier::Flex => CostTier::Flex, + ServiceTier::Standard | ServiceTier::Batch => CostTier::Standard, + }, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + }; + match plan.calculate(&request) { + Ok(cost) => Some(cost.total()), + Err(error) => { + tracing::debug!(deployment, ?error, "no cost for response"); + None } } } /// A deployment's cost map fields: the cost map entry for its model, overlaid by the prices its /// config sets. The entry is found by `base_model`, then the model name, then the name without -/// its provider route. +/// its provider route. An Azure deployment's names are keyed under `azure/` first, and its bare +/// deployment name is never looked up, since it names a resource of the user's, not a model. pub fn deployment_fields( catalog: Option<&Catalog>, model: &str, @@ -97,14 +151,10 @@ pub fn deployment_fields( ) -> Map { let mut fields = catalog .and_then(|catalog| { - let candidates = base_model.into_iter().chain([ - model, - model.split_once('/').map_or(model, |(_, rest)| rest), - model.rsplit_once('/').map_or(model, |(_, rest)| rest), - ]); - candidates + candidates(model, base_model) + .into_iter() .filter(|name| !name.is_empty()) - .find_map(|name| catalog.lookup(name)) + .find_map(|name| catalog.lookup(&name)) }) .map(|found| found.entry.fields().clone()) .unwrap_or_default(); @@ -118,6 +168,30 @@ pub fn deployment_fields( fields } +fn candidates(model: &str, base_model: Option<&str>) -> Vec { + if model.starts_with(AZURE_ROUTE) { + return base_model + .into_iter() + .flat_map(|base| match base.starts_with(AZURE_ROUTE) { + true => vec![base.to_owned()], + false => vec![format!("{AZURE_ROUTE}{base}"), base.to_owned()], + }) + .chain([model.to_owned()]) + .collect(); + } + base_model + .into_iter() + .chain([ + model, + model.split_once('/').map_or(model, |(_, rest)| rest), + model.rsplit_once('/').map_or(model, |(_, rest)| rest), + ]) + .map(str::to_owned) + .collect() +} + +const AZURE_ROUTE: &str = "azure/"; + #[derive(Clone, Copy)] enum Slot { Input, @@ -382,4 +456,81 @@ mod tests { assert_eq!(fields["output_cost_per_token"], json!(2e-6)); assert!(!fields.contains_key("api_key")); } + + fn azure_catalog() -> Catalog { + Catalog::parse( + br#"{ + "gpt-4o": {"input_cost_per_token": 1e-6, "output_cost_per_token": 1e-6}, + "azure/gpt-4o": {"input_cost_per_token": 2e-6, "output_cost_per_token": 2e-6}, + "gpt-4o-mini": {"input_cost_per_token": 3e-6, "output_cost_per_token": 3e-6}, + "azure/my-dep": {"input_cost_per_token": 4e-6, "output_cost_per_token": 4e-6}, + "chat-dep": {"input_cost_per_token": 5e-6, "output_cost_per_token": 5e-6} + }"#, + Default::default(), + ) + .unwrap() + } + + #[rstest] + #[case::base_model_under_azure("azure/chat-dep", Some("gpt-4o"), Some(2e-6))] + #[case::base_model_already_routed("azure/chat-dep", Some("azure/gpt-4o"), Some(2e-6))] + #[case::bare_base_model("azure/chat-dep", Some("gpt-4o-mini"), Some(3e-6))] + #[case::deployment_under_azure("azure/my-dep", None, Some(4e-6))] + #[case::bare_deployment_is_not_a_model("azure/chat-dep", None, None)] + fn azure_deployments_are_priced_under_the_azure_route( + #[case] model: &str, + #[case] base_model: Option<&str>, + #[case] input: Option, + ) { + let fields = deployment_fields(Some(&azure_catalog()), model, base_model, &[]); + assert_eq!( + fields.get("input_cost_per_token").and_then(Value::as_f64), + input + ); + } + + fn response_priced(own: Value) -> Prices { + let mut prices = Prices::default(); + prices.insert("d".into(), own.as_object().unwrap()); + prices.insert_model( + "azure/gpt-4o", + json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 0.0}) + .as_object() + .unwrap(), + ); + prices.price_by_response_model("d".into(), "azure"); + prices + } + + #[rstest] + #[case::response_model_under_the_route(Some("gpt-4o"), Some(2e-6))] + #[case::response_model_already_routed(Some("azure/gpt-4o"), Some(2e-6))] + #[case::unknown_response_model_uses_the_deployment(Some("gpt-9"), Some(7e-6))] + #[case::no_response_model_uses_the_deployment(None, Some(7e-6))] + fn response_model_prices_come_first( + #[case] response_model: Option<&str>, + #[case] rate: Option, + ) { + let prices = + response_priced(json!({"input_cost_per_token": 7e-6, "output_cost_per_token": 0.0})); + assert_eq!( + prices.response_cost("d", response_model, &usage(1, 0)), + rate + ); + } + + #[rstest] + fn other_deployments_ignore_the_response_model() { + let mut prices = response_priced(json!({})); + prices.insert( + "other".into(), + json!({"input_cost_per_token": 7e-6, "output_cost_per_token": 0.0}) + .as_object() + .unwrap(), + ); + assert_eq!( + prices.response_cost("other", Some("gpt-4o"), &usage(1, 0)), + Some(7e-6) + ); + } } diff --git a/litellm-rust/crates/gateway-hooks/src/usage.rs b/litellm-rust/crates/gateway-hooks/src/usage.rs index c4bcc01b431..be277b3e280 100644 --- a/litellm-rust/crates/gateway-hooks/src/usage.rs +++ b/litellm-rust/crates/gateway-hooks/src/usage.rs @@ -197,6 +197,7 @@ pub struct StreamUsage { line: Vec, state: Line, usage: Option, + model: Option, } impl StreamUsage { @@ -206,6 +207,7 @@ impl StreamUsage { line: Vec::new(), state: Line::Reading, usage: None, + model: None, } } @@ -264,6 +266,11 @@ impl StreamUsage { self.usage } + /// The model named by the chunk that carried an OpenAI stream's usage. + pub fn model(&self) -> Option<&str> { + self.model.as_deref() + } + fn may_carry_usage(&self) -> bool { let line = &self.line; if line.len() > MAX_LINE_BYTES { @@ -304,8 +311,13 @@ impl StreamUsage { if !contains(data, b"\"usage\":{") && !contains(data, b"\"usage\": {") { return; } - if let Ok(OpenAiChunk { usage: Some(usage) }) = serde_json::from_slice(data) { + if let Ok(OpenAiChunk { + usage: Some(usage), + model, + }) = serde_json::from_slice(data) + { self.usage = Some(usage.usage()); + self.model = model.filter(|model| !model.is_empty()); } } } @@ -326,6 +338,7 @@ struct AnthropicMessage { #[derive(Deserialize)] struct OpenAiChunk { usage: Option, + model: Option, } fn contains(haystack: &[u8], needle: &[u8]) -> bool { @@ -398,7 +411,7 @@ mod tests { fn openai_usage_comes_from_the_last_chunk() { let stream = concat!( "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}],\"usage\":null}\n\n", - "data: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":30,\"completion_tokens\":5,\"total_tokens\":35,\"prompt_tokens_details\":{\"cached_tokens\":10}}}\n\n", + "data: {\"id\":\"c\",\"model\":\"gpt-4o-2024-11-20\",\"choices\":[],\"usage\":{\"prompt_tokens\":30,\"completion_tokens\":5,\"total_tokens\":35,\"prompt_tokens_details\":{\"cached_tokens\":10}}}\n\n", "data: [DONE]\n\n", ); let mut meter = StreamUsage::new(StreamFormat::OpenAi); @@ -414,6 +427,7 @@ mod tests { ..Usage::default() }) ); + assert_eq!(meter.model(), Some("gpt-4o-2024-11-20")); } #[rstest] diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index e5fac1cafcc..01abf615f12 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -174,6 +174,24 @@ fn split(mut body: Map) -> (Value, Map) { (messages, body) } +/// The deployment's `litellm_params` a provider reads beyond its connection: its +/// `api_version`, `base_model` and every key the config adds. +fn provider_params(deployment: &Deployment) -> Map { + let params = &deployment.litellm_params; + let mut provider_params: Map = params + .extra + .iter() + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + if let Some(api_version) = ¶ms.api_version { + provider_params.insert("api_version".into(), api_version.clone().into()); + } + if let Some(base_model) = &deployment.base_model { + provider_params.insert("base_model".into(), base_model.clone().into()); + } + provider_params +} + async fn openai( gateway: &Gateway, deployment: &Deployment, @@ -189,6 +207,14 @@ async fn openai( custom_llm_provider: deployment.custom_llm_provider.clone(), extra_headers, timeout: deployment.timeout, + litellm_params: provider_params(deployment), + drop_params: gateway.drop_params() + || deployment + .litellm_params + .drop_params + .as_ref() + .is_some_and(|flag| flag.is_set()), + model_info: gateway.model_info(), }, None, ); diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index 3aa4b44f9aa..a2eac437ba8 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -121,12 +121,18 @@ impl Error { } _ => self.to_string(), }; - json!({"error": { + let mut error = json!({ "message": message, "type": error_type(status), "param": null, "code": status.as_u16(), - }}) + }); + if let Self::Route(RouteError::Transport(TransportError::Http { body, .. })) = self + && let Some(fields) = azure_content_filter_fields(body) + { + error["provider_specific_fields"] = fields; + } + json!({ "error": error }) } /// The Anthropic error envelope Python's `AnthropicExceptionMapping` builds: an upstream @@ -233,6 +239,44 @@ fn openai_error_message(body: &str) -> Option { .map(str::to_owned) } +/// Python's `AzureOpenAIExceptionMapping` keeps Azure's error and its content filter results +/// on a content policy violation. Only Azure nests an `inner_error`, which tells its errors +/// apart from another provider's. +fn azure_content_filter_fields(body: &str) -> Option { + const POLICY_PHRASES: [&str; 7] = [ + "content_policy_violation", + "responsibleaipolicyviolation", + "the response was filtered due to the prompt triggering azure openai's content management", + "your task failed as a result of our safety system", + "the model produced invalid content", + "content_filter_policy", + "your request was rejected as a result of our safety system", + ]; + let parsed: Value = serde_json::from_str(body).ok()?; + let azure_error = parsed + .get("error") + .filter(|error| error.is_object()) + .unwrap_or(&parsed); + let inner_error = ["inner_error", "innererror"].into_iter().find_map(|key| { + azure_error + .get(key) + .or_else(|| parsed.get(key)) + .filter(|inner| !inner.is_null()) + })?; + let lowered = body.to_lowercase(); + let violation = azure_error.get("code").and_then(Value::as_str) + == Some("content_policy_violation") + || inner_error.get("code").and_then(Value::as_str) == Some("ResponsibleAIPolicyViolation") + || POLICY_PHRASES.iter().any(|phrase| lowered.contains(phrase)); + violation.then(|| { + json!({ + "innererror": inner_error, + "inner_error": inner_error, + "azure_error": azure_error, + }) + }) +} + fn provider_message(object: &Map) -> Option<&str> { if let Some(detail) = object.get("detail").and_then(Value::as_object) { return detail.get("message").and_then(Value::as_str); @@ -326,6 +370,36 @@ mod tests { assert_eq!(frame, format!("data: {expected}\n\n")); } + #[rstest] + #[case::content_filter( + r#"{"error":{"message":"The response was filtered due to the prompt triggering Azure OpenAI's content management policy.","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":true,"severity":"high"}}}}}"#, + true, + )] + #[case::inner_error_spelling( + r#"{"error":{"message":"blocked","code":"content_policy_violation","inner_error":{"code":"ResponsibleAIPolicyViolation"}}}"#, + true, + )] + #[case::other_azure_error( + r#"{"error":{"message":"bad value","code":"invalid_value","innererror":{"code":"Other"}}}"#, + false + )] + #[case::openai_policy_error( + r#"{"error":{"message":"rejected","code":"content_policy_violation","type":"invalid_request_error"}}"#, + false, + )] + fn azure_content_filter_errors_keep_the_azure_error(#[case] body: &str, #[case] kept: bool) { + let parsed: Value = serde_json::from_str(body).unwrap(); + let error = upstream(400, body).openai_body(); + let fields = &error["error"]["provider_specific_fields"]; + assert_eq!(!fields.is_null(), kept); + if kept { + assert_eq!(fields["azure_error"], parsed["error"]); + assert_eq!(fields["innererror"], fields["inner_error"]); + assert_eq!(fields["innererror"]["code"], "ResponsibleAIPolicyViolation"); + } + assert_eq!(error["error"]["message"], parsed["error"]["message"]); + } + #[rstest] #[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)] #[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)] diff --git a/litellm-rust/crates/gateway-inference/src/hooks.rs b/litellm-rust/crates/gateway-inference/src/hooks.rs index f6f41210300..68c3714bd44 100644 --- a/litellm-rust/crates/gateway-inference/src/hooks.rs +++ b/litellm-rust/crates/gateway-inference/src/hooks.rs @@ -306,12 +306,29 @@ impl Finish { } } - fn usage(&self) -> Option { - match &self.meter { - Meter::Off => None, - Meter::Stream(meter) => meter.usage(), - Meter::Body(chunks) => body_usage(self.route, &chunks.concat()), - } + /// The usage the response reported, with its cost when the deployment is priced by the + /// model its response names, which only a chat completion carries. + fn usage_and_cost(&self) -> (Option, Option) { + let by_model = self.pipeline.prices().prices_by_response_model() + && self.route == Route::ChatCompletions; + let (usage, model) = match &self.meter { + Meter::Off => (None, None), + Meter::Stream(meter) => (meter.usage(), meter.model().map(str::to_owned)), + Meter::Body(chunks) => { + let body = chunks.concat(); + let model = by_model.then(|| body_model(&body)).flatten(); + (body_usage(self.route, &body), model) + } + }; + let cost = by_model + .then_some(()) + .zip(self.deployment.as_ref().zip(usage.as_ref())) + .and_then(|((), (deployment, usage))| { + self.pipeline + .prices() + .response_cost(&deployment.id, model.as_deref(), usage) + }); + (usage, cost) } } @@ -320,11 +337,12 @@ impl Drop for Finish { let Some(admission) = self.admission.take() else { return; }; + let (usage, cost) = self.usage_and_cost(); let outcome = Outcome { deployment: self.deployment.take(), status: self.status, - usage: self.usage(), - cost: None, + usage, + cost, time_to_first_byte: self.first_byte, upstream_started_at: self.upstream_started_at, ended_at: SystemTime::now(), @@ -350,6 +368,17 @@ fn body_usage(route: Route, body: &[u8]) -> Option { } } +/// The `model` a complete chat completion names, found by its first key like [`body_usage`]. +fn body_model(body: &[u8]) -> Option { + const KEY: &[u8] = b"\"model\":"; + let start = memchr::memmem::find(body, KEY)? + KEY.len(); + serde_json::Deserializer::from_slice(&body[start..]) + .into_iter::() + .next()? + .ok() + .filter(|model| !model.is_empty()) +} + struct Tapped { inner: BodyDataStream, finish: Finish, @@ -401,4 +430,16 @@ mod tests { body_usage(Route::ChatCompletions, &serde_json::to_vec(&chat).unwrap()).unwrap(); assert_eq!((usage.prompt_tokens(), usage.output_tokens), (5, 6)); } + + #[rstest] + #[case::top_level( + br#"{"id":"c","model":"gpt-4o-2024-11-20","choices":[]}"#, + Some("gpt-4o-2024-11-20") + )] + #[case::spaced(br#"{"model": "gpt-4o", "choices":[]}"#, Some("gpt-4o"))] + #[case::inside_content(br#"{"choices":[{"message":{"content":"\"model\":\"x\""}}]}"#, None)] + #[case::empty(br#"{"model":"","choices":[]}"#, None)] + fn body_model_reads_the_response_model(#[case] body: &[u8], #[case] expected: Option<&str>) { + assert_eq!(body_model(body).as_deref(), expected); + } } diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 56b62d83d77..65aa2d72779 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -25,7 +25,10 @@ use litellm_inference_messages::MessagesRoute; use litellm_inference_ocr::OcrRoute; use litellm_inference_responses::ResponsesRoute; use litellm_inference_transcription::AudioTranscriptionRoute; -use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings}; +use litellm_llms::base_llm::{ + model_info::ModelInfo, + ocr::{handler::OcrClient, settings::OcrSettings}, +}; use litellm_secrets::source::SecretSource; pub use error::Error; @@ -36,6 +39,8 @@ pub use request::{JsonBody, JsonObject, RequestId}; pub struct Gateway { cache: Option>, hooks: OnceLock, + model_info: OnceLock>, + drop_params: bool, allow_client_side_credentials: bool, pub audio_transcription: AudioTranscriptionRoute, pub chat_completions: ChatCompletionsRoute, @@ -67,6 +72,29 @@ impl Gateway { self.hooks.get() } + /// Gives providers whose param rules read model capabilities the model cost map. Only the + /// first call takes effect. + pub fn install_model_info(&self, model_info: Arc) { + let _ = self.model_info.set(model_info); + } + + pub(crate) fn model_info(&self) -> Option> { + self.model_info.get().cloned() + } + + /// `litellm_settings.drop_params`: drops params a provider does not support instead of + /// refusing the request. + pub fn with_drop_params(self, drop_params: bool) -> Self { + Self { + drop_params, + ..self + } + } + + pub(crate) fn drop_params(&self) -> bool { + self.drop_params + } + /// `general_settings.allow_client_side_credentials`: lets a request body name the /// connection and credential params the proxy otherwise refuses. pub fn with_client_side_credentials(self, allowed: bool) -> Self { @@ -98,6 +126,8 @@ impl Gateway { Ok(Self { cache: None, hooks: OnceLock::new(), + model_info: OnceLock::new(), + drop_params: false, allow_client_side_credentials: false, audio_transcription: AudioTranscriptionRoute::new( provider.clone(), diff --git a/litellm-rust/crates/gateway/src/hooks.rs b/litellm-rust/crates/gateway/src/hooks.rs index 02e9180c771..6c393816cbf 100644 --- a/litellm-rust/crates/gateway/src/hooks.rs +++ b/litellm-rust/crates/gateway/src/hooks.rs @@ -1,13 +1,14 @@ //! Builds the hooks and observers inference requests run through, and the per-deployment prices //! observers that read cost need. -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; -use litellm_config::Config; +use litellm_config::{Config, Model}; use litellm_gateway_hooks::{ DEFAULT_QUEUE, Facts, Hook, Observer, Observers, Pipeline, Prices, deployment_fields, }; use litellm_gateway_inference::{Gateway, Hooks}; +use litellm_llms::base_llm::model_info::ModelInfo; use litellm_model_catalog::{Catalog, Provenance}; use litellm_secrets::source::SecretSource; use serde_json::{Map, Value}; @@ -29,18 +30,26 @@ pub async fn install( hooks: Vec>, observers: Vec>, ) -> Result<(), Error> { - if hooks.is_empty() && observers.is_empty() { + let serves_azure = config.model_list.iter().any(is_azure); + if hooks.is_empty() && observers.is_empty() && !serves_azure { return Ok(()); } let needs_cost = observers .iter() .any(|observer| observer.reads().contains(Facts::COST)) || hooks.iter().any(|hook| hook.reads().contains(Facts::COST)); + let catalog = match needs_cost || serves_azure { + true => load_catalog(inference).await.map(Arc::new), + false => None, + }; + if let Some(catalog) = catalog.as_ref().filter(|_| serves_azure) { + inference.install_model_info(Arc::new(CostMap::new(catalog.clone()))); + } + if hooks.is_empty() && observers.is_empty() { + return Ok(()); + } let prices = match needs_cost { - true => { - let catalog = load_catalog(inference).await; - prices(config, inference, catalog.as_ref()) - } + true => prices(config, inference, catalog.as_deref()), false => Prices::default(), }; let names: Vec<_> = hooks @@ -135,7 +144,9 @@ async fn fetch(inference: &Gateway, url: &str) -> Result, Box) -> Prices { let mut prices = Prices::default(); for (model, id) in config @@ -151,18 +162,75 @@ fn prices(config: &Config, inference: &Gateway, catalog: Option<&Catalog>) -> Pr .iter() .map(|(name, value)| (name, value.clone())), ); - let base_model = model_info.get("base_model").and_then(Value::as_str); - let fields = deployment_fields( - catalog, - &model.litellm_params.model, - base_model, - &[&model_info, ¶ms], - ); + let base_model = params + .get("base_model") + .or_else(|| model_info.get("base_model")) + .and_then(Value::as_str) + .filter(|name| !name.is_empty()); + let name = match is_azure(model) && !model.litellm_params.model.starts_with(AZURE_ROUTE) { + true => format!("{AZURE_ROUTE}{}", model.litellm_params.model), + false => model.litellm_params.model.clone(), + }; + let overrides = [&model_info, ¶ms]; + if is_azure(model) + && base_model.is_none() + && deployment_fields(None, &name, None, &overrides).is_empty() + { + prices.price_by_response_model(id.into(), AZURE_PROVIDER); + } + let fields = deployment_fields(catalog, &name, base_model, &overrides); prices.insert(id.into(), &fields); } + if let Some(catalog) = catalog.filter(|_| prices.prices_by_response_model()) { + for name in catalog + .model_names() + .filter(|name| name.starts_with(AZURE_ROUTE)) + { + if let Some(found) = catalog.lookup(name) { + prices.insert_model(name, found.entry.fields()); + } + } + } prices } +const AZURE_PROVIDER: &str = "azure"; +const AZURE_ROUTE: &str = "azure/"; + +fn is_azure(model: &Model) -> bool { + match model.litellm_params.custom_llm_provider.as_deref() { + Some(provider) => provider == AZURE_PROVIDER, + None => model.litellm_params.model.starts_with(AZURE_ROUTE), + } +} + +/// The model cost map as the capabilities Azure's param rules read. +struct CostMap { + catalog: Arc, + keys: HashSet, +} + +impl CostMap { + fn new(catalog: Arc) -> Self { + let keys = catalog + .model_names() + .filter_map(|name| catalog.lookup(name)) + .flat_map(|found| found.entry.fields().keys().cloned().collect::>()) + .collect(); + Self { catalog, keys } + } +} + +impl ModelInfo for CostMap { + fn entry(&self, model: &str) -> Option<&Map> { + self.catalog.lookup(model).map(|found| found.entry.fields()) + } + + fn declares(&self, key: &str) -> bool { + self.keys.contains(key) + } +} + fn json_object<'a, V: serde::Serialize + 'a>( entries: impl Iterator, ) -> Map { diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index d351afb2061..82279726f27 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -79,7 +79,14 @@ pub fn build_inference(config: &Config) -> Result, Error> { let resources = CoreResources::new(pool); Ok(Arc::new( Gateway::new(resources, http, secrets, ModelRouter::from_config(config)?)? - .with_client_side_credentials(allows_client_side_credentials(config)), + .with_client_side_credentials(allows_client_side_credentials(config)) + .with_drop_params( + config + .litellm_settings + .drop_params + .as_ref() + .is_some_and(|flag| flag.is_set()), + ), )) } diff --git a/litellm-rust/crates/gateway/tests/azure_chat.rs b/litellm-rust/crates/gateway/tests/azure_chat.rs new file mode 100644 index 00000000000..c17f6cd8a19 --- /dev/null +++ b/litellm-rust/crates/gateway/tests/azure_chat.rs @@ -0,0 +1,293 @@ +//! Azure OpenAI chat completions through the gateway: the deployment URL, `api-version` and +//! `api-key` the upstream sees, and the usage and cost the request is recorded with. + +use std::{ + io::Write, + sync::{Arc, Mutex, Once}, +}; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::{Request, header::CONTENT_TYPE}, + response::IntoResponse, +}; +use futures_util::future::BoxFuture; +use litellm_config::Config; +use litellm_gateway_hooks::{Facts, Observer, Record}; +use rstest::rstest; +use serde_json::{Value, json}; +use tokio::net::TcpListener; +use tower::ServiceExt; + +const RESPONSE_MODEL: &str = "gpt-4o-2024-11-20"; +const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-4o-2024-11-20","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":6,"total_tokens":11}}"#; +const COMPLETION_STREAM: &str = concat!( + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4o-2024-11-20\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}],\"usage\":null}\n\n", + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4o-2024-11-20\",\"choices\":[],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3,\"total_tokens\":10}}\n\n", + "data: [DONE]\n\n", +); + +/// Prices the cost map gives the model a response names, and the `base_model` one deployment +/// declares, so the two pricing routes give different costs. +const RESPONSE_MODEL_RATES: (f64, f64) = (1.0, 2.0); +const BASE_MODEL_RATES: (f64, f64) = (10.0, 20.0); + +/// The proxy reads its cost map from this variable, and every test sets it before building a +/// gateway, so no test reads the environment while it changes. +fn use_local_cost_map() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let map = json!({ + format!("azure/{RESPONSE_MODEL}"): { + "litellm_provider": "azure", + "mode": "chat", + "input_cost_per_token": RESPONSE_MODEL_RATES.0, + "output_cost_per_token": RESPONSE_MODEL_RATES.1, + }, + "azure/gpt-4o": { + "litellm_provider": "azure", + "mode": "chat", + "input_cost_per_token": BASE_MODEL_RATES.0, + "output_cost_per_token": BASE_MODEL_RATES.1, + }, + }); + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(map.to_string().as_bytes()).unwrap(); + let (_, path) = file.keep().unwrap(); + // SAFETY: set once, before any test builds a gateway or spawns a thread that reads the + // environment. + unsafe { std::env::set_var("LITELLM_MODEL_COST_MAP_PATH", path) }; + }); +} + +#[derive(Clone, Debug)] +struct Sent { + path: String, + query: Option, + api_key: Option, + authorization: Option, + body: Value, +} + +type Calls = Arc>>; + +async fn upstream(calls: Calls) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().fallback(move |request: Request| { + let calls = calls.clone(); + async move { + let (parts, body) = request.into_parts(); + let header = |name: &str| { + parts + .headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let (api_key, authorization) = (header("api-key"), header("authorization")); + let path = parts.uri.path().to_owned(); + let query = parts.uri.query().map(str::to_owned); + let body: Value = + serde_json::from_slice(&to_bytes(body, 1 << 20).await.unwrap()).unwrap(); + let stream = body["stream"].as_bool().unwrap_or(false); + calls.lock().unwrap().push(Sent { + path, + query, + api_key, + authorization, + body, + }); + match stream { + true => ([(CONTENT_TYPE, "text/event-stream")], COMPLETION_STREAM).into_response(), + false => ([(CONTENT_TYPE, "application/json")], COMPLETION).into_response(), + } + } + }); + tokio::spawn(async move { axum::serve(listener, app).await }); + format!("http://{address}") +} + +#[derive(Debug)] +struct Seen { + deployment: Option, + usage: Option<(u64, u64)>, + cost: Option, +} + +struct Collect(Mutex>); + +impl Observer for Collect { + fn name(&self) -> &'static str { + "collect" + } + + fn reads(&self) -> Facts { + Facts::DEPLOYMENT | Facts::USAGE | Facts::COST + } + + fn observe<'a>(&'a self, record: &'a Record) -> BoxFuture<'a, ()> { + Box::pin(async move { + self.0.lock().unwrap().push(Seen { + deployment: record + .outcome + .deployment + .as_ref() + .map(|deployment| deployment.id.to_string()), + usage: record + .outcome + .usage + .map(|usage| (usage.prompt_tokens(), usage.output_tokens)), + cost: record.outcome.cost, + }); + }) + } +} + +struct Harness { + app: Router, + inference: Arc, + calls: Calls, + seen: Arc, +} + +async fn harness() -> Harness { + use_local_cost_map(); + let calls = Calls::default(); + let upstream = upstream(calls.clone()).await; + let config = Config::from_yaml(&format!( + "model_list: + - model_name: chat + litellm_params: + model: azure/chat-prod + api_base: {upstream} + api_key: sk-azure + api_version: 2024-10-21 + model_info: + id: azure-1 + - model_name: chat-base + litellm_params: + model: azure/chat-base + api_base: {upstream}/ + api_key: sk-azure + api_version: 2024-10-21 + base_model: gpt-4o + model_info: + id: azure-2 +router_settings: + num_retries: 0 +general_settings: + master_key: sk-master +" + )) + .unwrap(); + let inference = litellm_gateway::build_inference(&config).unwrap(); + let seen = Arc::new(Collect(Mutex::new(Vec::new()))); + litellm_gateway::hooks::install(&inference, &config, Vec::new(), vec![seen.clone()]) + .await + .unwrap(); + let app = litellm_gateway::router(inference.clone(), &config, None, None); + Harness { + app, + inference, + calls, + seen, + } +} + +async fn send(app: &Router, body: Value) -> (u16, String) { + let request = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .header("authorization", "Bearer sk-master") + .body(Body::from(body.to_string())) + .unwrap(); + let response = app.clone().oneshot(request).await.unwrap(); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1 << 20).await.unwrap(); + (status, String::from_utf8(body.to_vec()).unwrap()) +} + +fn close(left: f64, right: f64) -> bool { + (left - right).abs() < 1e-9 +} + +fn cost((input, output): (u64, u64), (input_rate, output_rate): (f64, f64)) -> f64 { + input as f64 * input_rate + output as f64 * output_rate +} + +#[rstest] +#[case::unary_by_response_model("chat", false, "chat-prod", "azure-1", (5, 6), RESPONSE_MODEL_RATES)] +#[case::stream_by_response_model("chat", true, "chat-prod", "azure-1", (7, 3), RESPONSE_MODEL_RATES)] +#[case::unary_by_base_model("chat-base", false, "chat-base", "azure-2", (5, 6), BASE_MODEL_RATES)] +#[case::stream_by_base_model("chat-base", true, "chat-base", "azure-2", (7, 3), BASE_MODEL_RATES)] +#[tokio::test] +async fn azure_chat_reaches_the_deployment_and_is_priced( + #[case] model: &str, + #[case] stream: bool, + #[case] deployment: &str, + #[case] id: &str, + #[case] usage: (u64, u64), + #[case] rates: (f64, f64), +) { + let harness = harness().await; + let mut request = json!({ + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 8, + "user": "end-user-1", + }); + if stream { + request["stream"] = json!(true); + request["stream_options"] = json!({"include_usage": true}); + } + let (status, body) = send(&harness.app, request).await; + assert_eq!(status, 200, "{body}"); + match stream { + true => assert!(body.contains("\"content\":\"hi\""), "{body}"), + false => assert_eq!( + serde_json::from_str::(&body).unwrap()["model"], + RESPONSE_MODEL + ), + } + + let sent = harness.calls.lock().unwrap()[0].clone(); + assert_eq!( + sent.path, + format!("/openai/deployments/{deployment}/chat/completions") + ); + assert_eq!(sent.query.as_deref(), Some("api-version=2024-10-21")); + assert_eq!(sent.api_key.as_deref(), Some("sk-azure")); + assert_eq!(sent.authorization, None); + assert_eq!(sent.body["model"], deployment); + assert_eq!(sent.body["max_tokens"], 8); + assert_eq!(sent.body["user"], "end-user-1"); + assert_eq!(sent.body["stream"].as_bool().unwrap_or(false), stream); + + litellm_gateway::hooks::close(&harness.inference).await; + let seen = harness.seen.0.lock().unwrap(); + let [seen] = seen.as_slice() else { + panic!("{seen:?}") + }; + assert_eq!(seen.deployment.as_deref(), Some(id)); + assert_eq!(seen.usage, Some(usage)); + assert!(close(seen.cost.unwrap(), cost(usage, rates)), "{seen:?}"); +} + +#[rstest] +#[tokio::test] +async fn an_unsupported_param_is_refused_before_the_call() { + let harness = harness().await; + let (status, body) = send( + &harness.app, + json!({ + "model": "chat", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "high", + }), + ) + .await; + assert_eq!(status, 400, "{body}"); + assert!(body.contains("reasoning_effort"), "{body}"); + assert!(harness.calls.lock().unwrap().is_empty()); +} diff --git a/litellm-rust/crates/inference-chat/src/azure.rs b/litellm-rust/crates/inference-chat/src/azure.rs new file mode 100644 index 00000000000..2f2a59c39b3 --- /dev/null +++ b/litellm-rust/crates/inference-chat/src/azure.rs @@ -0,0 +1,475 @@ +//! Azure OpenAI chat completions through the OpenAI passthrough: `litellm/llms/azure/azure.py` +//! with the client `BaseAzureLLM.get_azure_openai_client` builds. The deployment's settings +//! and `os.environ/` references resolve through the secret source, and an Entra ID token is +//! acquired through `litellm_auth_azure` when the deployment has no key. + +use std::collections::BTreeMap; + +use litellm_auth::azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAMES}; +use litellm_core_utils::get_llm_provider_logic::LlmProviders; +use litellm_llms::{ + Error as LlmError, + azure::{ + chat::transformation::{AzureChatBody, AzureChatRequest}, + common_utils::{self, AZURE_API_KEY_ENV, AzureCredential}, + }, + base_llm::auth::resolve_auth, +}; +use serde_json::{Map, Value}; + +use crate::{ + ChatCompletionsRoute, Error, + common_utils::string_headers, + openai::{OpenAiChatCall, PreparedCall}, +}; + +const ENV_REFERENCE: &str = "os.environ/"; + +/// The `litellm_params` an `os.environ/` reference may stand in for. +const REFERABLE_PARAMS: &[&str] = &[ + "api_version", + "base_model", + "azure_ad_token", + "tenant_id", + "client_id", + "client_secret", + "azure_scope", + "azure_authority_host", + "azure_credential", + "azure_federated_token_file", +]; + +impl ChatCompletionsRoute { + pub(crate) async fn prepare_azure( + &self, + deployment: String, + call: OpenAiChatCall, + ) -> Result { + let OpenAiChatCall { + body, + api_key, + api_base, + extra_headers, + litellm_params, + drop_params, + model_info, + .. + } = call; + let references: Vec = [api_key.as_deref(), api_base.as_deref()] + .into_iter() + .flatten() + .chain( + REFERABLE_PARAMS + .iter() + .filter_map(|name| litellm_params.get(*name).and_then(Value::as_str)), + ) + .filter_map(|value| value.strip_prefix(ENV_REFERENCE)) + .map(str::to_owned) + .collect(); + let names: Vec<&str> = common_utils::SECRET_NAMES + .iter() + .chain(AZURE_AUTH_SECRET_NAMES) + .copied() + .chain(references.iter().map(String::as_str)) + .collect(); + let secrets = self.secrets.resolve(&names).await?; + let lookup = |key: &str| secrets.get(key); + // Python's router reads every `os.environ/` setting from the environment when it loads + // the deployment. A reference to an unset variable counts as unset. + let resolve = |value: &str| match value.strip_prefix(ENV_REFERENCE) { + Some(name) => lookup(name), + None => Some(value.to_owned()), + }; + let params: Map = litellm_params + .into_iter() + .map(|(name, value)| match value { + Value::String(text) if REFERABLE_PARAMS.contains(&name.as_str()) => { + let value = resolve(&text).map_or(Value::Null, Value::String); + (name, value) + } + other => (name, other), + }) + .collect(); + let configured = |name: &str| params.get(name).and_then(Value::as_str); + + let api_version = common_utils::resolve_api_version(configured("api_version"), &lookup); + let api_base = common_utils::resolve_api_base( + api_base.as_deref().and_then(resolve).as_deref(), + &lookup, + )?; + let api_key = + common_utils::resolve_api_key(api_key.as_deref().and_then(resolve).as_deref(), &lookup); + let model_info = model_info.as_deref(); + let AzureChatBody { + deployment: url_deployment, + body, + } = AzureChatRequest { + deployment: &deployment, + base_model: configured("base_model"), + api_version: &api_version, + drop_params, + model_info, + } + .transform(body)?; + let url = common_utils::complete_url(&api_base, &url_deployment, &api_version)?; + + let credential = self + .azure_credential(api_key.clone(), ¶ms, &lookup) + .await?; + let environment = common_utils::environment( + string_headers(extra_headers)?, + credential, + common_utils::is_v1_api_version(&api_version), + ); + let authenticated = resolve_auth(&self.auth, environment, &lookup).await?; + Ok(PreparedCall { + model: url_deployment, + provider: <&str>::from(LlmProviders::Azure), + url, + headers: authenticated.headers, + body, + api_key, + }) + } + + /// The SDK sends a configured `azure_ad_token` ahead of a key. Without either, LiteLLM + /// gets a token from Entra ID: client credentials, an OIDC exchange, or managed identity + /// when `azure_credential` or `enable_azure_ad_token_refresh` asks for it. + async fn azure_credential( + &self, + api_key: Option, + params: &Map, + lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let configured_token = params + .get("azure_ad_token") + .and_then(Value::as_str) + .is_some_and(|token| !token.is_empty()); + if let Some(key) = &api_key + && !configured_token + { + return Ok(AzureCredential::ApiKey(litellm_auth::SecretValue::new( + key.clone(), + ))); + } + let inputs = AzureAuthInputs::from_sourced_optional_params(params, &BTreeMap::new())?; + let token = self.auth.azure.get_azure_ad_token(&inputs, lookup).await?; + match (token, api_key) { + (Some(token), _) => Ok(AzureCredential::AdToken( + token.into_value().secret().clone(), + )), + (None, Some(key)) => Ok(AzureCredential::ApiKey(litellm_auth::SecretValue::new(key))), + (None, None) => Err(Error::from(LlmError::Auth( + litellm_auth::Error::MissingApiKey { + provider: "Azure", + environment_variable: AZURE_API_KEY_ENV, + }, + ))), + } + } +} + +#[cfg(test)] +mod tests { + use std::{sync::Arc, time::Duration}; + + use bytes::Bytes; + use futures_util::TryStreamExt; + use litellm_host::call::CallOutput; + use litellm_http::transport; + use litellm_inference_testing::RecordingSecrets; + use rstest::rstest; + use serde_json::json; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_partial_json, header, method, path, query_param}, + }; + + use super::*; + use crate::test_support::{Recording, route, run}; + + const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-4o-2024-08-06","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}"#; + const STREAM: &str = "data: {\"id\":\"c\",\"model\":\"gpt-4o-2024-08-06\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\ndata: {\"id\":\"c\",\"model\":\"gpt-4o-2024-08-06\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1,\"total_tokens\":4}}\n\ndata: [DONE]\n\n"; + const DEPLOYMENT_PATH: &str = "/openai/deployments/gpt-4o/chat/completions"; + + fn call(api_base: &str, body: Value) -> OpenAiChatCall { + let Value::Object(body) = body else { + panic!("the body is an object"); + }; + OpenAiChatCall { + model: "azure/gpt-4o".into(), + body, + api_key: Some("sk-azure".into()), + api_base: Some(format!("{api_base}/")), + litellm_params: Map::from_iter([("api_version".into(), json!("2024-10-21"))]), + timeout: Some(Duration::from_secs(5)), + ..OpenAiChatCall::default() + } + } + + fn hi() -> Value { + json!({"model": "public", "messages": [{"role": "user", "content": "hi"}]}) + } + + fn with_secrets(route: ChatCompletionsRoute, values: &[(&str, &str)]) -> ChatCompletionsRoute { + ChatCompletionsRoute { + secrets: Arc::new(RecordingSecrets::new(values.iter().copied())), + ..route + } + } + + async fn complete(output: crate::openai::OpenAiChatOutput) -> Bytes { + match output { + CallOutput::Complete(body) => body, + CallOutput::Stream { .. } => panic!("expected a complete body"), + } + } + + #[rstest] + #[tokio::test] + async fn a_completion_goes_to_the_deployment_with_its_key(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path(DEPLOYMENT_PATH)) + .and(query_param("api-version", "2024-10-21")) + .and(header("api-key", "sk-azure")) + .and(body_partial_json(json!({"model": "gpt-4o"}))) + .respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json")) + .expect(1) + .mount(&upstream) + .await; + let interceptors = Recording::default(); + + let output = run(&route, call(&upstream.uri(), hi()), &interceptors) + .await + .unwrap(); + + assert_eq!(complete(output).await, COMPLETION.as_bytes()); + let requests = upstream.received_requests().await.unwrap(); + assert!(!requests[0].headers.contains_key("authorization")); + let contexts = interceptors.contexts.lock().unwrap(); + assert_eq!( + ( + contexts[0].model.as_str(), + contexts[0].custom_llm_provider.as_str() + ), + ("gpt-4o", "azure") + ); + assert_eq!(*interceptors.raw.lock().unwrap(), [COMPLETION]); + assert_eq!( + interceptors.results.lock().unwrap()[0].provider.provider, + "azure" + ); + } + + #[rstest] + #[tokio::test] + async fn a_stream_relays_every_byte(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + Mock::given(path(DEPLOYMENT_PATH)) + .and(body_partial_json( + json!({"stream": true, "stream_options": {"include_usage": true}}), + )) + .respond_with(ResponseTemplate::new(200).set_body_raw(STREAM, "text/event-stream")) + .expect(1) + .mount(&upstream) + .await; + let mut body = hi(); + body["stream"] = json!(true); + body["stream_options"] = json!({"include_usage": true}); + + let output = run(&route, call(&upstream.uri(), body), &Recording::default()) + .await + .unwrap(); + + let CallOutput::Stream { chunks, .. } = output else { + panic!("a streaming call streams"); + }; + let relayed = chunks.try_collect::>().await.unwrap().concat(); + assert_eq!(relayed, STREAM.as_bytes()); + } + + #[rstest] + #[tokio::test] + async fn the_v1_surface_takes_the_key_as_a_bearer(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + Mock::given(path("/openai/v1/chat/completions")) + .and(header("authorization", "Bearer sk-azure")) + .respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json")) + .expect(1) + .mount(&upstream) + .await; + let mut v1 = call(&upstream.uri(), hi()); + v1.litellm_params + .insert("api_version".into(), json!("latest")); + + run(&route, v1, &Recording::default()).await.unwrap(); + + let requests = upstream.received_requests().await.unwrap(); + assert_eq!(requests[0].url.query(), None); + assert!(!requests[0].headers.contains_key("api-key")); + } + + #[rstest] + #[tokio::test] + async fn base_key_and_version_fall_back_to_the_environment(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + Mock::given(path(DEPLOYMENT_PATH)) + .and(query_param("api-version", "2024-06-01")) + .and(header("api-key", "sk-env")) + .respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json")) + .expect(1) + .mount(&upstream) + .await; + let base = upstream.uri(); + let route = with_secrets( + route, + &[ + ("AZURE_API_BASE", &base), + ("AZURE_API_KEY", "sk-env"), + ("AZURE_API_VERSION", "2024-06-01"), + ], + ); + let bare = OpenAiChatCall { + api_key: None, + api_base: None, + litellm_params: Map::new(), + ..call(&base, hi()) + }; + + run(&route, bare, &Recording::default()).await.unwrap(); + } + + #[rstest] + #[tokio::test] + async fn the_default_api_version_is_pythons(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + Mock::given(query_param( + "api-version", + common_utils::AZURE_DEFAULT_API_VERSION, + )) + .respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json")) + .expect(1) + .mount(&upstream) + .await; + let mut unversioned = call(&upstream.uri(), hi()); + unversioned.litellm_params.clear(); + + run(&route, unversioned, &Recording::default()) + .await + .unwrap(); + } + + #[rstest] + #[tokio::test] + async fn environment_references_resolve(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + Mock::given(query_param("api-version", "2025-01-01-preview")) + .and(header("api-key", "sk-referenced")) + .respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json")) + .expect(1) + .mount(&upstream) + .await; + let route = with_secrets( + route, + &[ + ("MY_AZURE_KEY", "sk-referenced"), + ("MY_AZURE_VERSION", "2025-01-01-preview"), + ], + ); + let mut referenced = call(&upstream.uri(), hi()); + referenced.api_key = Some("os.environ/MY_AZURE_KEY".into()); + referenced + .litellm_params + .insert("api_version".into(), json!("os.environ/MY_AZURE_VERSION")); + + run(&route, referenced, &Recording::default()) + .await + .unwrap(); + } + + #[rstest] + #[case::configured_token_outranks_the_key(Some("sk-azure"), &[("azure_ad_token", "configured-token")], &[])] + #[case::environment_token(None, &[], &[("AZURE_AD_TOKEN", "configured-token")])] + #[tokio::test] + async fn an_ad_token_goes_as_a_bearer( + route: ChatCompletionsRoute, + #[case] api_key: Option<&str>, + #[case] params: &[(&str, &str)], + #[case] environment: &[(&str, &str)], + ) { + let upstream = MockServer::start().await; + Mock::given(header("authorization", "Bearer configured-token")) + .respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json")) + .expect(1) + .mount(&upstream) + .await; + let route = with_secrets(route, environment); + let mut tokened = call(&upstream.uri(), hi()); + tokened.api_key = api_key.map(str::to_owned); + tokened.litellm_params.extend( + params + .iter() + .map(|(name, value)| ((*name).to_owned(), json!(value))), + ); + + run(&route, tokened, &Recording::default()).await.unwrap(); + + let requests = upstream.received_requests().await.unwrap(); + assert!(!requests[0].headers.contains_key("api-key")); + } + + #[rstest] + #[tokio::test] + async fn a_missing_credential_fails_before_any_request(route: ChatCompletionsRoute) { + let mut keyless = call("http://127.0.0.1:1", hi()); + keyless.api_key = None; + assert_eq!( + run(&route, keyless, &Recording::default()).await.err(), + Some(Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Azure", + environment_variable: "AZURE_API_KEY", + })) + ); + } + + #[rstest] + #[tokio::test] + async fn an_unsupported_param_fails_before_any_request(route: ChatCompletionsRoute) { + let mut body = hi(); + body["reasoning_effort"] = json!("high"); + assert!(matches!( + run( + &route, + call("http://127.0.0.1:1", body), + &Recording::default() + ) + .await, + Err(Error::InvalidRequest(_)) + )); + } + + #[rstest] + #[tokio::test] + async fn a_content_filter_error_keeps_azures_body(route: ChatCompletionsRoute) { + let upstream = MockServer::start().await; + let body = r#"{"error":{"message":"The response was filtered","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":true,"severity":"high"}}}}}"#; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_raw(body, "application/json")) + .mount(&upstream) + .await; + + let error = run(&route, call(&upstream.uri(), hi()), &Recording::default()) + .await + .err() + .expect("the upstream failure fails the call"); + + assert_eq!( + error, + Error::Transport(transport::Error::Http { + status: 400, + body: body.into() + }) + ); + } +} diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index f3187e67ab2..cd4b9e33da7 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -1,5 +1,8 @@ use litellm_host::observation::ObservationSender; +mod azure; pub mod route; +#[cfg(test)] +mod test_support; pub mod types; pub use litellm_inference::RouteError as Error; mod common_utils; diff --git a/litellm-rust/crates/inference-chat/src/openai.rs b/litellm-rust/crates/inference-chat/src/openai.rs index 9fd389966fa..bfc82c4b09a 100644 --- a/litellm-rust/crates/inference-chat/src/openai.rs +++ b/litellm-rust/crates/inference-chat/src/openai.rs @@ -1,4 +1,4 @@ -use std::{convert::Infallible, time::Duration}; +use std::{convert::Infallible, sync::Arc, time::Duration}; use bytes::Bytes; use futures_util::{StreamExt, stream::BoxStream}; @@ -15,7 +15,10 @@ use litellm_host::{ use litellm_http::{outbound::OutboundRequest, request::truncate_error_body, transport}; use litellm_inference::{CallOptions, context::CallContext, provider::resolve_llm_provider}; use litellm_llms::{ - base_llm::auth::resolve_auth, + base_llm::{ + auth::{Headers, resolve_auth}, + model_info::ModelInfo, + }, openai::chat::transformation::{OPENAI_CHAT_COMPLETIONS_CONFIG, is_stream}, }; use serde_json::{Map, Value}; @@ -39,8 +42,10 @@ impl Protocol for OpenAiChatCompletions { pub type OpenAiChatOutput = CallOutput; -/// A Chat Completions body bound for an OpenAI deployment. `body` is the caller's request; -/// `model` is the deployment's, with or without its `openai/` prefix. +/// A Chat Completions body bound for an OpenAI or Azure OpenAI deployment. `body` is the +/// caller's request; `model` is the deployment's, with or without its `openai/` or `azure/` +/// prefix. +#[derive(Default)] pub struct OpenAiChatCall { pub model: String, pub body: Map, @@ -49,12 +54,35 @@ pub struct OpenAiChatCall { pub custom_llm_provider: Option, pub extra_headers: Option>, pub timeout: Option, + /// The deployment's other `litellm_params`: Azure reads its `api_version`, `base_model` + /// and Entra ID credentials from them. + pub litellm_params: Map, + /// `drop_params` from the deployment or the proxy settings. + pub drop_params: bool, + /// The model cost map, which Azure's param rules read. + pub model_info: Option>, +} + +/// A provider request ready to go out: where, with which headers and body, and who serves it. +pub(crate) struct PreparedCall { + pub(crate) model: String, + pub(crate) provider: &'static str, + pub(crate) url: String, + pub(crate) headers: Headers, + pub(crate) body: Map, + pub(crate) api_key: Option, } impl ChatCompletionsRoute { + /// Whether the deployment is served by the OpenAI wire passthrough: OpenAI itself, or + /// Azure OpenAI. pub fn serves_openai(model: &str, custom_llm_provider: Option<&str>) -> bool { - resolve_llm_provider(model, custom_llm_provider, "chat completions") - .is_ok_and(|resolved| resolved.provider == LlmProviders::Openai) + resolve_llm_provider(model, custom_llm_provider, "chat completions").is_ok_and(|resolved| { + matches!( + resolved.provider, + LlmProviders::Openai | LlmProviders::Azure + ) + }) } pub fn openai_machine( @@ -106,39 +134,35 @@ impl ChatCompletionsRoute { stream, outcome ))] - async fn run_openai( + pub(crate) async fn run_openai( &self, call: OpenAiChatCall, context: CallContext<'_, impl Interceptors>, ) -> Result { + let timeout = call + .timeout + .unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS)); litellm_inference::diagnostic::call(async { let resolved = resolve_llm_provider( &call.model, call.custom_llm_provider.as_deref(), "chat completions", )?; - if resolved.provider != LlmProviders::Openai { - return Err(Error::InvalidProvider( - <&str>::from(resolved.provider).to_string(), - )); - } let model = resolved.model.to_string(); - let provider = <&str>::from(LlmProviders::Openai); + let prepared = match resolved.provider { + LlmProviders::Openai => self.prepare_openai(model, call).await?, + LlmProviders::Azure => self.prepare_azure(model, call).await?, + other => return Err(Error::InvalidProvider(<&str>::from(other).to_string())), + }; + let PreparedCall { + model, + provider, + url, + headers, + body, + api_key, + } = prepared; litellm_inference::diagnostic::provider(&model, provider); - let config = &OPENAI_CHAT_COMPLETIONS_CONFIG; - let secrets = self - .secrets - .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref())) - .await?; - let lookup = |key: &str| secrets.get(key); - let body = config.transform_request(&model, call.body)?; - let environment = config.validate_environment( - string_headers(call.extra_headers)?, - call.api_key.as_deref(), - &lookup, - )?; - let url = config.get_complete_url(call.api_base.as_deref(), &lookup); - let authenticated = resolve_auth(&self.auth, environment, &lookup).await?; let request_context = RequestContext { model: model.clone(), custom_llm_provider: provider.to_string(), @@ -149,18 +173,16 @@ impl ChatCompletionsRoute { .collect(), ), secret_fields: Vec::new(), - api_key: call.api_key.map(litellm_auth::SecretValue::new), + api_key: api_key.map(litellm_auth::SecretValue::new), + }; + let wire = WireRequest { + url, + headers, + body: Value::Object(body), }; let wire = context .interceptors - .before_provider_request( - WireRequest { - url, - headers: authenticated.headers, - body: Value::Object(body), - }, - request_context, - ) + .before_provider_request(wire, request_context) .await?; let stream = wire .body @@ -168,9 +190,6 @@ impl ChatCompletionsRoute { .map(is_stream) .transpose()? .unwrap_or(false); - let timeout = call - .timeout - .unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS)); let response = send(&self.http, wire, stream, timeout).await?; let output = if stream { CallOutput::Stream { @@ -200,6 +219,37 @@ impl ChatCompletionsRoute { } } +impl ChatCompletionsRoute { + async fn prepare_openai( + &self, + model: String, + call: OpenAiChatCall, + ) -> Result { + let config = &OPENAI_CHAT_COMPLETIONS_CONFIG; + let secrets = self + .secrets + .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref())) + .await?; + let lookup = |key: &str| secrets.get(key); + let body = config.transform_request(&model, call.body)?; + let environment = config.validate_environment( + string_headers(call.extra_headers)?, + call.api_key.as_deref(), + &lookup, + )?; + let url = config.get_complete_url(call.api_base.as_deref(), &lookup); + let authenticated = resolve_auth(&self.auth, environment, &lookup).await?; + Ok(PreparedCall { + model, + provider: <&str>::from(LlmProviders::Openai), + url, + headers: authenticated.headers, + body, + api_key: call.api_key, + }) + } +} + /// A non-streaming call keeps the usual total timeout. A stream has no total limit, so a /// 30 minute generation is not cut off: `timeout` bounds the wait for the response head /// here and each read after it in [`chunks`]. @@ -254,11 +304,10 @@ fn silent_provider(timeout: Duration) -> Error { #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use futures_util::TryStreamExt; - use litellm_host::interceptors::RawResponse; - use rstest::{fixture, rstest}; + use rstest::rstest; use serde_json::json; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use wiremock::{ @@ -267,47 +316,11 @@ mod tests { }; use super::*; + use crate::test_support::{Recording, route, run}; const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-6-astra","system_fingerprint":"fp","choices":[{"index":0,"message":{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"f","arguments":"{}"}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}"#; const STREAM: &str = "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\ndata: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1,\"total_tokens\":4}}\n\ndata: [DONE]\n\n"; - #[derive(Default)] - struct Recording { - contexts: Mutex>, - raw: Mutex>, - results: Mutex>, - } - - impl Interceptors for Recording { - async fn before_provider_request( - &self, - wire: WireRequest, - context: RequestContext, - ) -> Result { - self.contexts.lock().unwrap().push(context); - Ok(wire) - } - - async fn after_provider_response(&self, raw: RawResponse) -> Result<(), Error> { - self.raw.lock().unwrap().push(raw.body); - Ok(()) - } - - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { - self.results.lock().unwrap().push(facts); - Ok(()) - } - } - - #[fixture] - fn route() -> ChatCompletionsRoute { - ChatCompletionsRoute::new( - litellm_http::Client::plain_for_test(), - Arc::new(litellm_auth::AuthServices::default()), - litellm_inference_testing::no_secrets(), - ) - } - fn call(api_base: &str, body: Value) -> OpenAiChatCall { let Value::Object(body) = body else { panic!("the body is an object"); @@ -320,19 +333,10 @@ mod tests { custom_llm_provider: None, extra_headers: None, timeout: Some(Duration::from_secs(5)), + ..OpenAiChatCall::default() } } - async fn run( - route: &ChatCompletionsRoute, - call: OpenAiChatCall, - interceptors: &Recording, - ) -> Result { - route - .run_openai(call, CallContext::new(interceptors, CallOptions::default())) - .await - } - async fn drain(chunks: BoxStream<'static, Result>) -> Result, Error> { Ok(chunks.try_collect::>().await?.concat()) } @@ -347,6 +351,8 @@ mod tests { #[rstest] #[case::openai_prefix("openai/gpt-6-astra", None, true)] #[case::explicit_provider("gpt-6-astra", Some("openai"), true)] + #[case::azure("azure/gpt-4o", None, true)] + #[case::azure_ai("azure_ai/gpt-4o", None, false)] #[case::openai_like("openai_like/gpt-6-astra", None, false)] #[case::anthropic("anthropic/claude", None, false)] #[case::unprefixed("gpt-6-astra", None, false)] diff --git a/litellm-rust/crates/inference-chat/src/test_support.rs b/litellm-rust/crates/inference-chat/src/test_support.rs new file mode 100644 index 00000000000..653b0981015 --- /dev/null +++ b/litellm-rust/crates/inference-chat/src/test_support.rs @@ -0,0 +1,61 @@ +//! Helpers the passthrough tests of several providers share. + +use std::sync::{Arc, Mutex}; + +use litellm_host::interceptors::{ + ExecutionFacts, Interceptors, RawResponse, RequestContext, WireRequest, +}; +use litellm_inference::{CallOptions, context::CallContext}; +use rstest::fixture; + +use crate::{ + ChatCompletionsRoute, Error, + openai::{OpenAiChatCall, OpenAiChatOutput}, +}; + +#[derive(Default)] +pub(crate) struct Recording { + pub(crate) contexts: Mutex>, + pub(crate) raw: Mutex>, + pub(crate) results: Mutex>, +} + +impl Interceptors for Recording { + async fn before_provider_request( + &self, + wire: WireRequest, + context: RequestContext, + ) -> Result { + self.contexts.lock().unwrap().push(context); + Ok(wire) + } + + async fn after_provider_response(&self, raw: RawResponse) -> Result<(), Error> { + self.raw.lock().unwrap().push(raw.body); + Ok(()) + } + + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { + self.results.lock().unwrap().push(facts); + Ok(()) + } +} + +#[fixture] +pub(crate) fn route() -> ChatCompletionsRoute { + ChatCompletionsRoute::new( + litellm_http::Client::plain_for_test(), + Arc::new(litellm_auth::AuthServices::default()), + litellm_inference_testing::no_secrets(), + ) +} + +pub(crate) async fn run( + route: &ChatCompletionsRoute, + call: OpenAiChatCall, + interceptors: &Recording, +) -> Result { + route + .run_openai(call, CallContext::new(interceptors, CallOptions::default())) + .await +} diff --git a/litellm-rust/crates/inference-chat/tests/live/openai.rs b/litellm-rust/crates/inference-chat/tests/live/openai.rs index f764b1b87fe..bf78c257661 100644 --- a/litellm-rust/crates/inference-chat/tests/live/openai.rs +++ b/litellm-rust/crates/inference-chat/tests/live/openai.rs @@ -47,6 +47,7 @@ fn call(extra: Value) -> OpenAiChatCall { custom_llm_provider: None, extra_headers: None, timeout: Some(Duration::from_secs(60)), + ..OpenAiChatCall::default() } } diff --git a/litellm-rust/crates/llms/src/azure/chat/mod.rs b/litellm-rust/crates/llms/src/azure/chat/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/chat/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/azure/chat/transformation.rs b/litellm-rust/crates/llms/src/azure/chat/transformation.rs new file mode 100644 index 00000000000..7c6c1056f10 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/chat/transformation.rs @@ -0,0 +1,1432 @@ +//! `litellm/llms/azure/chat/gpt_transformation.py`, `o_series_transformation.py` and +//! `gpt_5_transformation.py` over an OpenAI-shaped body: Python's `get_optional_params` sorts +//! the request's OpenAI params from the rest, refuses or drops the ones the deployment's model +//! family does not take, maps the rest by that family's rules, and passes every other key to +//! Azure through `extra_body`. The response reaches the caller as Azure sent it. + +use serde_json::{Map, Value}; + +use crate::{ + Error, ErrorDetail, + base_llm::model_info::{self, ModelInfo}, + openai::chat::transformation::{is_litellm_owned, take_extra_body}, +}; + +/// The keys of `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` a request carries as OpenAI params, in +/// that table's order. Every other key goes to Azure through `extra_body`. +const OPENAI_PARAMS: &[&str] = &[ + "functions", + "function_call", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "audio", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "response_format", + "seed", + "tools", + "tool_choice", + "max_retries", + "logprobs", + "top_logprobs", + "extra_headers", + "parallel_tool_calls", + "reasoning_effort", + "verbosity", + "thinking", + "web_search_options", + "include_server_side_tool_invocations", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "metadata", + "context_management", +]; + +/// `PROVIDER_UNVALIDATED_PARAMS`: never refused, only left out when unsupported. +const UNVALIDATED_PARAMS: &[&str] = &["user", "stream_options", "stream", "max_retries"]; + +/// `AzureOpenAIConfig.get_supported_openai_params`. +const AZURE_GPT_PARAMS: &[&str] = &[ + "temperature", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "max_completion_tokens", + "tools", + "tool_choice", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "function_call", + "functions", + "top_p", + "logprobs", + "top_logprobs", + "response_format", + "seed", + "extra_headers", + "parallel_tool_calls", + "prediction", + "modalities", + "audio", + "web_search_options", + "prompt_cache_key", + "store", +]; + +/// The `base_params` of `OpenAIGPTConfig.get_supported_openai_params`, which the o-series and +/// gpt-5 configs start from. +const OPENAI_GPT_PARAMS: &[&str] = &[ + "frequency_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "n", + "presence_penalty", + "seed", + "stop", + "stream", + "stream_options", + "temperature", + "top_p", + "tools", + "tool_choice", + "function_call", + "functions", + "max_retries", + "extra_headers", + "parallel_tool_calls", + "audio", + "web_search_options", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", +]; + +/// `AzureOpenAIO1Config`'s `non_supported_params`. +const O_SERIES_UNSUPPORTED: &[&str] = &[ + "logprobs", + "top_p", + "presence_penalty", + "frequency_penalty", + "top_logprobs", +]; + +/// `OpenAIGPT5Config`'s `non_supported_params`. +const GPT_5_UNSUPPORTED: &[&str] = &[ + "presence_penalty", + "frequency_penalty", + "stop", + "logit_bias", + "modalities", + "prediction", + "audio", + "web_search_options", +]; + +/// The params of a gpt-5 search model. +const GPT_5_SEARCH_PARAMS: &[&str] = &[ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "web_search_options", + "service_tier", + "safety_identifier", + "response_format", + "user", + "store", + "verbosity", + "max_retries", + "extra_headers", +]; + +const GPT_5_SAMPLING_PARAMS: &[&str] = &["logprobs", "top_logprobs", "top_p"]; +const REASONING_SUMMARY_ALIASES: &[&str] = &["reasoningSummary", "reasoning_summary"]; +const O_SERIES_ROUTE: &str = "o_series/"; +const GPT_5_SERIES_ROUTE: &str = "gpt5_series/"; +const RESPONSES_ROUTE: &str = "responses/"; +/// `API_VERSION_YEAR_SUPPORTED_RESPONSE_FORMAT` and its month. +const RESPONSE_FORMAT_SINCE: (u32, u32) = (2024, 8); +const DROP_HINT: &str = "Set `drop_params: true` under litellm_settings to drop it. "; + +/// Which of Python's Azure configs serves a deployment, picked from `base_model` when the +/// deployment sets one and from the deployment name otherwise. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AzureModelFamily { + /// `AzureOpenAIConfig`. + Gpt, + /// `AzureOpenAIO1Config`. + OSeries, + /// `AzureOpenAIGPT5Config`. + Gpt5, +} + +impl AzureModelFamily { + pub fn detect(model: &str) -> Self { + if is_o_series_model(model) { + Self::OSeries + } else if is_gpt_5_model(model) { + Self::Gpt5 + } else { + Self::Gpt + } + } +} + +/// `AzureOpenAIO1Config.is_o_series_model`. +fn is_o_series_model(model: &str) -> bool { + ["o1", "o3", "o4", O_SERIES_ROUTE] + .iter() + .any(|marker| model.contains(marker)) +} + +/// `is_gpt_reasoning_series_name`: a gpt-5 or gpt-6 name, except the gpt-5-chat models. +fn is_gpt_reasoning_series_name(model: &str) -> bool { + let name = model.rsplit('/').next().unwrap_or(model); + (model.contains("gpt-5") || model.contains("gpt-6")) && !name.contains("gpt-5-chat") +} + +/// `AzureOpenAIGPT5Config.is_model_gpt_5_model`. +fn is_gpt_5_model(model: &str) -> bool { + is_gpt_reasoning_series_name(model) || model.contains("gpt5_series") +} + +fn is_gpt_5_search_model(model: &str) -> bool { + model.contains("gpt-5") && model.contains("search") +} + +/// `is_model_gpt_5_2_model`: gpt-5.2 and gpt-5.4, pro included. +fn is_gpt_5_2_model(model: &str) -> bool { + let name = model.rsplit('/').next().unwrap_or(model); + name.starts_with("gpt-5.2") || name.starts_with("gpt-5.4") +} + +/// `_gpt_series_version`: the `(major, minor)` of a `gpt-N[.M]` name, as +/// `^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)` reads it. +fn gpt_series_version(model: &str) -> Option<(u32, u32)> { + let name = model.rsplit('/').next().unwrap_or(model); + let rest = name.strip_prefix("gpt-")?; + let digits = |text: &str| { + text.find(|c: char| !c.is_ascii_digit()) + .unwrap_or(text.len()) + }; + let boundary = |text: &str| text.is_empty() || text.starts_with(['.', '-']); + let major_len = digits(rest); + let major = rest[..major_len].parse().ok()?; + let after_major = &rest[major_len..]; + if let Some(minor) = after_major.strip_prefix('.') { + let minor_len = digits(minor); + if minor_len > 0 && boundary(&minor[minor_len..]) { + return Some((major, minor[..minor_len].parse().ok()?)); + } + } + boundary(after_major).then_some((major, 0)) +} + +/// `is_model_gpt_5_4_plus_model`. +fn is_gpt_5_4_plus_model(model: &str) -> bool { + gpt_series_version(model).is_some_and(|version| version >= (5, 4)) +} + +/// `AzureOpenAIGPT5Config._model_map_lookup_name`: the cost map key of a gpt-5 deployment. +fn gpt_5_lookup_name(model: &str) -> String { + match model.strip_prefix(GPT_5_SERIES_ROUTE) { + Some(rest) => format!("azure/{rest}"), + None if model.starts_with("azure/") => model.to_string(), + None => format!("azure/{model}"), + } +} + +/// `get_effort_level`: a `reasoning_effort` string, or the `effort` of its object form. +fn effort_level(value: Option<&Value>) -> Option<&str> { + match value? { + Value::String(effort) => Some(effort), + Value::Object(object) => object.get("effort").and_then(Value::as_str), + _ => None, + } +} + +/// What a request to one Azure deployment is mapped with. +pub struct AzureChatRequest<'a> { + /// The deployment, without its `azure/` prefix. + pub deployment: &'a str, + /// The deployment's `base_model`, which picks the config and the cost map entry when the + /// deployment name says nothing about the model. + pub base_model: Option<&'a str>, + pub api_version: &'a str, + /// `drop_params` from the deployment or the proxy settings. The request's own + /// `drop_params` is honoured too. + pub drop_params: bool, + pub model_info: Option<&'a dyn ModelInfo>, +} + +/// The body to send and the deployment the SDK puts in its URL. +#[derive(Debug, PartialEq)] +pub struct AzureChatBody { + pub deployment: String, + pub body: Map, +} + +impl AzureChatRequest<'_> { + /// `base_model or model`, the name Python picks the config from. + fn detection_model(&self) -> &str { + self.base_model + .filter(|model| !model.is_empty()) + .unwrap_or(self.deployment) + } + + pub fn family(&self) -> AzureModelFamily { + AzureModelFamily::detect(self.detection_model()) + } + + fn info(&self) -> Option<&dyn ModelInfo> { + self.model_info + } + + pub fn transform(&self, body: Map) -> Result { + let mut body = body; + body.remove("model"); + let messages = body.remove("messages").unwrap_or(Value::Array(Vec::new())); + let mut extra_body = take_extra_body(&mut body)?; + // `main.py` takes a token from here as the call's credential; it never reaches Azure. + extra_body.remove("azure_ad_token"); + extra_body.remove("model"); + let drop_params = self.drop_params || request_drop_params(body.remove("drop_params")); + let allowed = string_list(body.remove("allowed_openai_params")); + let additional_drop = string_list(body.remove("additional_drop_params")); + let family = self.family(); + if family == AzureModelFamily::Gpt5 { + for alias in REASONING_SUMMARY_ALIASES { + extra_body.remove(*alias); + } + } + if let Some(reason) = self.responses_bridge(&body, &extra_body) { + return Err(Error::Unsupported(reason)); + } + + let mut non_default = Map::new(); + for (name, value) in body { + if value.is_null() + || is_litellm_owned(&name) + || additional_drop.contains(&name) + || (family == AzureModelFamily::Gpt5 + && REASONING_SUMMARY_ALIASES.contains(&name.as_str())) + { + continue; + } + match OPENAI_PARAMS.contains(&name.as_str()) { + true => { + non_default.insert(name, value); + } + false => { + extra_body.insert(name, value); + } + } + } + self.check_supported(family, &mut non_default, &allowed, drop_params)?; + + let model = self.detection_model(); + let mut optional = Map::new(); + match family { + AzureModelFamily::Gpt => self.map_gpt(&non_default, &mut optional, drop_params)?, + AzureModelFamily::OSeries => { + self.map_o_series(&mut non_default, &mut optional, model, drop_params)? + } + AzureModelFamily::Gpt5 => { + self.map_gpt_5(&mut non_default, &mut optional, model, drop_params)? + } + } + // `apply_openai_param_overrides`: what the caller allowed goes as sent. + for name in &allowed { + if !optional.contains_key(name) + && let Some(value) = non_default.remove(name) + { + optional.insert(name.clone(), value); + } + } + + let (deployment, messages) = match family { + AzureModelFamily::OSeries => { + let deployment = self.deployment.replace(O_SERIES_ROUTE, ""); + let messages = self.o_series_messages(&deployment, messages); + (deployment, messages) + } + AzureModelFamily::Gpt5 => ( + self.deployment.replace(GPT_5_SERIES_ROUTE, ""), + azure_messages(messages), + ), + AzureModelFamily::Gpt => (self.deployment.to_string(), azure_messages(messages)), + }; + let keeps_tool_choice = family == AzureModelFamily::OSeries + || ["tools", "functions"] + .iter() + .any(|name| optional.get(*name).is_some_and(truthy)); + let mut sent = Map::new(); + sent.insert("model".into(), Value::String(deployment.clone())); + sent.insert("messages".into(), messages); + sent.extend( + optional + .into_iter() + .filter(|(name, _)| name != "tool_choice" || keeps_tool_choice), + ); + // The SDK merges `extra_body` over the body it builds. + sent.extend(extra_body); + Ok(AzureChatBody { + deployment, + body: sent, + }) + } + + /// `responses_api_bridge_check`: the requests Python sends to the Responses API instead, + /// which this passthrough does not do. + fn responses_bridge( + &self, + body: &Map, + extra_body: &Map, + ) -> Option<&'static str> { + let model = self.deployment; + if model.starts_with(RESPONSES_ROUTE) + || model_info::field(self.info(), &format!("azure/{model}"), "mode") + .and_then(Value::as_str) + == Some("responses") + { + return Some("Azure deployments served through the Responses API"); + } + if !is_gpt_reasoning_series_name(model) || is_gpt_5_search_model(model) { + return None; + } + let effort = body + .get("reasoning_effort") + .filter(|value| !value.is_null()); + let summary = REASONING_SUMMARY_ALIASES + .iter() + .any(|alias| body.contains_key(*alias) || extra_body.contains_key(*alias)); + let reasoning_active = match effort { + Some(Value::Object(effort)) => { + effort.get("effort").and_then(Value::as_str) != Some("none") + || effort.get("summary").is_some_and(|value| !value.is_null()) + } + other => effort_level(other) != Some("none"), + }; + let tools = body + .get("tools") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let tool_type = |tool: &Value| tool.get("type").and_then(Value::as_str).map(str::to_owned); + let has_function_tool = tools.iter().any(|tool| { + tool_type(tool).as_deref() == Some("function") + && (tool.get("function").is_some_and(Value::is_object) + || tool.get("name").is_some()) + }); + let has_tool_search = tools + .iter() + .any(|tool| tool_type(tool).as_deref() == Some("tool_search")); + let rejects_function_tools = + has_function_tool && reasoning_active && is_gpt_5_4_plus_model(model); + ((effort.is_some() && summary) || rejects_function_tools || has_tool_search).then_some( + "gpt-5 Azure requests Python bridges to the Responses API (function tools with reasoning on gpt-5.4+, tool_search, or a reasoning summary)", + ) + } + + /// `_check_valid_arg`: an OpenAI param the family does not take is dropped under + /// `drop_params` and refused otherwise. It reads the deployment name, as Python asks the + /// config for its params with the deployment rather than the base model. + fn check_supported( + &self, + family: AzureModelFamily, + non_default: &mut Map, + allowed: &[String], + drop_params: bool, + ) -> Result<(), Error> { + let supported = supported_params(family, self.deployment, self.info()); + let unsupported: Vec = non_default + .iter() + .filter(|(name, value)| { + !supported.contains(&name.as_str()) + && !allowed.iter().any(|allowed| allowed == *name) + && !UNVALIDATED_PARAMS.contains(&name.as_str()) + && !(name.as_str() == "n" && value.as_f64() == Some(1.0)) + }) + .map(|(name, _)| name.clone()) + .collect(); + if unsupported.is_empty() { + return Ok(()); + } + if !drop_params { + return Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { + provider: "azure", + params: unsupported, + model: self.deployment.to_string(), + })); + } + for name in unsupported { + non_default.remove(&name); + } + Ok(()) + } + + /// `AzureOpenAIConfig.map_openai_params`. + fn map_gpt( + &self, + non_default: &Map, + optional: &mut Map, + drop_params: bool, + ) -> Result<(), Error> { + let model = self.detection_model(); + let renames_max_tokens = + model.contains("gpt-5") || model.contains("gpt-6") || model.contains("gpt5_series"); + let parts: Vec<&str> = self.api_version.split('-').collect(); + let date = (parts.len() >= 3).then(|| (parts[0], parts[1], parts[2])); + for (name, value) in ordered(non_default) { + match name { + "max_tokens" if renames_max_tokens => { + optional + .entry("max_completion_tokens") + .or_insert_with(|| value.clone()); + } + "tool_choice" => match date { + Some((year, month, day)) + if year < "2023" + || (year == "2023" && month < "12") + || (year == "2023" && month == "12" && day < "01") => + { + if !drop_params { + return Err(self.unsupported_value( + "tool_choice", + value, + "tool_choice needs api_version 2023-12-01-preview or later. ", + )); + } + } + Some((year, month, _)) + if value.as_str() == Some("required") + && year == "2024" + && month <= "05" => + { + if !drop_params { + return Err(self.unsupported_value( + "tool_choice", + value, + "tool_choice='required' needs an api_version after 2024-05. ", + )); + } + } + _ => { + optional.insert(name.into(), value.clone()); + } + }, + "response_format" if value.is_object() => { + let supported = response_format_model(model) + && date.is_none_or(|(year, month, _)| { + match (year.parse::(), month.parse::()) { + (Ok(year), Ok(month)) => (year, month) >= RESPONSE_FORMAT_SINCE, + _ => true, + } + }); + let schema = value + .get("response_schema") + .or_else(|| { + value + .get("json_schema") + .and_then(|schema| schema.get("schema")) + }) + .filter(|schema| truthy(schema)); + match (supported, schema) { + (true, _) => { + optional.insert(name.into(), value.clone()); + } + (false, Some(_)) => { + return Err(Error::Unsupported( + "a JSON schema response_format on an Azure model or api_version without it, which Python answers through a forced tool call", + )); + } + // Python leaves a schemaless format out when Azure cannot take it. + (false, None) => {} + } + } + "tools" if value.is_array() => { + let tools = optional + .entry("tools") + .or_insert_with(|| Value::Array(Vec::new())); + if let (Value::Array(tools), Value::Array(added)) = (tools, value) { + tools.extend(added.iter().cloned()); + } + } + _ if AZURE_GPT_PARAMS.contains(&name) => { + optional.insert(name.into(), value.clone()); + } + _ => {} + } + } + Ok(()) + } + + /// `OpenAIOSeriesConfig.map_openai_params` with the Azure o-series params. + fn map_o_series( + &self, + non_default: &mut Map, + optional: &mut Map, + model: &str, + drop_params: bool, + ) -> Result<(), Error> { + if let Some(max_tokens) = non_default.remove("max_tokens") { + optional.insert("max_completion_tokens".into(), max_tokens); + } + if let Some(temperature) = non_default.remove("temperature") { + if temperature.as_f64() == Some(1.0) { + optional.insert("temperature".into(), temperature); + } else if !drop_params { + return Err(self.unsupported_value( + "temperature", + &temperature, + "O-series models only support temperature=1. ", + )); + } + } + copy_supported( + non_default, + optional, + &supported_params(AzureModelFamily::OSeries, model, self.info()), + ); + Ok(()) + } + + /// `AzureOpenAIGPT5Config.map_openai_params` over `OpenAIGPT5Config`'s. + fn map_gpt_5( + &self, + non_default: &mut Map, + optional: &mut Map, + model: &str, + drop_params: bool, + ) -> Result<(), Error> { + let info = self.info(); + let lookup = gpt_5_lookup_name(model); + let supports_none = model_info::supports(info, &lookup, "supports_none_reasoning_effort"); + if effort_level(non_default.get("reasoning_effort")) == Some("none") && !supports_none { + if !drop_params { + return Err(self.unsupported_value( + "reasoning_effort", + &Value::from("none"), + "Azure OpenAI takes reasoning_effort='none' only on models that support it. ", + )); + } + non_default.remove("reasoning_effort"); + } + let supported = supported_params(AzureModelFamily::Gpt5, model, info); + if is_gpt_5_search_model(model) { + if let Some(max_tokens) = non_default.remove("max_tokens") { + optional.insert("max_completion_tokens".into(), max_tokens); + } + copy_supported(non_default, optional, &supported); + return Ok(()); + } + let effective = effort_level(non_default.get("reasoning_effort")).map(str::to_owned); + if let Some(Value::Object(effort)) = non_default.get("reasoning_effort") + && let Some(level) = effort.get("effort").cloned() + { + non_default.insert("reasoning_effort".into(), level); + } + let refused_effort = match effective.as_deref() { + Some("xhigh") => { + !model_info::supports(info, &lookup, "supports_xhigh_reasoning_effort") + } + Some(level @ ("minimal" | "low")) => model_info::explicitly_disabled( + info, + &lookup, + &format!("supports_{level}_reasoning_effort"), + ), + _ => false, + }; + if refused_effort { + if !drop_params { + return Err(self.unsupported_value( + "reasoning_effort", + &Value::from(effective.clone().unwrap_or_default()), + "", + )); + } + non_default.remove("reasoning_effort"); + } + if let Some(max_tokens) = non_default.remove("max_tokens") { + optional.insert("max_completion_tokens".into(), max_tokens); + } + let resolves_to_none = self.effort_resolves_to_none(&lookup, effective.as_deref()); + if supports_none + && GPT_5_SAMPLING_PARAMS + .iter() + .any(|name| non_default.contains_key(*name)) + && !resolves_to_none + { + if !drop_params { + let name = GPT_5_SAMPLING_PARAMS + .iter() + .find(|name| non_default.contains_key(**name)) + .copied() + .unwrap_or_default(); + return Err(self.unsupported_value( + name, + &non_default[name], + "logprobs, top_p and top_logprobs need reasoning_effort to resolve to 'none'. ", + )); + } + for name in GPT_5_SAMPLING_PARAMS { + non_default.remove(*name); + } + } + if let Some(temperature) = non_default.remove("temperature") { + if (supports_none && resolves_to_none) || temperature.as_f64() == Some(1.0) { + optional.insert("temperature".into(), temperature); + } else if !drop_params { + return Err(self.unsupported_value( + "temperature", + &temperature, + "Only temperature=1 is supported while reasoning is active. ", + )); + } + } + copy_supported(non_default, optional, &supported); + if effort_level(optional.get("reasoning_effort")) == Some("none") && !supports_none { + optional.remove("reasoning_effort"); + } + Ok(()) + } + + /// `OpenAIGPT5Config.effort_resolves_to_none`. + fn effort_resolves_to_none(&self, lookup: &str, effective: Option<&str>) -> bool { + if let Some(effort) = effective { + return effort == "none"; + } + let info = self.info(); + if let Some(declared) = model_info::field(info, lookup, "default_reasoning_effort") { + return declared.as_str() == Some("none"); + } + if !info.is_some_and(|info| info.declares("default_reasoning_effort")) { + return model_info::supports(info, lookup, "supports_none_reasoning_effort"); + } + false + } + + /// `OpenAIOSeriesConfig._transform_messages`: a system message becomes a user message + /// unless the cost map says the model takes system messages. + fn o_series_messages(&self, model: &str, messages: Value) -> Value { + if model_info::supports(self.info(), model, "supports_system_messages") { + return messages; + } + match messages { + Value::Array(messages) => Value::Array( + messages + .into_iter() + .map( + |message| match message.get("role").and_then(Value::as_str) { + Some("system") => { + let content = + message.get("content").cloned().unwrap_or(Value::Null); + serde_json::json!({"content": content, "role": "user"}) + } + _ => message, + }, + ) + .collect(), + ), + other => other, + } + } + + fn unsupported_value(&self, param: &str, value: &Value, hint: &str) -> Error { + Error::InvalidRequest(ErrorDetail::UnsupportedParameter { + model: format!("azure/{}", self.detection_model()), + param: param.into(), + value: match value { + Value::String(text) => text.clone(), + other => other.to_string(), + }, + hint: format!("{hint}{DROP_HINT}"), + }) + } +} + +/// The OpenAI params a family takes, as its config's `get_supported_openai_params(model)` +/// lists them. +fn supported_params( + family: AzureModelFamily, + model: &str, + info: Option<&dyn ModelInfo>, +) -> Vec<&'static str> { + match family { + AzureModelFamily::Gpt => AZURE_GPT_PARAMS.to_vec(), + AzureModelFamily::OSeries => { + let mut params = openai_gpt_params(model, info); + // A model the cost map knows takes `reasoning_effort` only if it reasons; an + // unknown name, often a custom deployment name, is assumed to. + if !model_info::known(info, model) + || model_info::supports(info, model, "supports_reasoning") + { + params.push("reasoning_effort"); + } + params.retain(|name| !O_SERIES_UNSUPPORTED.contains(name)); + params + } + AzureModelFamily::Gpt5 => { + let lookup = gpt_5_lookup_name(model); + let supports_none = + model_info::supports(info, &lookup, "supports_none_reasoning_effort"); + let mut params = match is_gpt_5_search_model(model) { + true => GPT_5_SEARCH_PARAMS.to_vec(), + false => { + let mut params = openai_gpt_params(model, info); + params.extend(["reasoning_effort", "verbosity"]); + params.retain(|name| { + !GPT_5_UNSUPPORTED.contains(name) + && (supports_none || !GPT_5_SAMPLING_PARAMS.contains(name)) + }); + params + } + }; + if !params.contains(&"tool_choice") { + params.push("tool_choice"); + } + if is_gpt_5_2_model(model) { + params.extend(["logprobs", "top_logprobs"]); + } else if supports_none { + params.retain(|name| !matches!(*name, "logprobs" | "top_logprobs")); + } + params + } + } +} + +/// `OpenAIGPTConfig.get_supported_openai_params`. +fn openai_gpt_params(model: &str, info: Option<&dyn ModelInfo>) -> Vec<&'static str> { + let mut params = OPENAI_GPT_PARAMS.to_vec(); + if model != "gpt-3.5-turbo-16k" && model != "gpt-4" { + params.push("response_format"); + } + // `is_openai_catalog_model`: `user` only for a model OpenAI's own catalog lists. + let catalog_name = model + .split_once(RESPONSES_ROUTE) + .map_or(model, |(_, rest)| rest); + if matches!( + model_info::field(info, catalog_name, "litellm_provider").and_then(Value::as_str), + Some("openai" | "text-completion-openai") + ) { + params.push("user"); + } + params +} + +/// `_is_response_format_supported_model`: everything but gpt-3.5. +fn response_format_model(model: &str) -> bool { + !(dotted_versions(model).contains("gpt-3.5") || model.contains("gpt-35")) +} + +/// `re.sub(r"(\d)-(\d)", r"\1.\2", model)`: `gpt-4-1` reads as `gpt-4.1`. +fn dotted_versions(model: &str) -> String { + let chars: Vec = model.chars().collect(); + let mut dotted = String::with_capacity(model.len()); + let mut index = 0; + while index < chars.len() { + if index + 2 < chars.len() + && chars[index].is_ascii_digit() + && chars[index + 1] == '-' + && chars[index + 2].is_ascii_digit() + { + dotted.extend([chars[index], '.', chars[index + 2]]); + index += 3; + } else { + dotted.push(chars[index]); + index += 1; + } + } + dotted +} + +/// `convert_to_azure_openai_messages`: Azure wants a function call's `arguments` as a string, +/// an `image_url` as an object without `format`, and a `file` without `format`. +fn azure_messages(messages: Value) -> Value { + let Value::Array(mut messages) = messages else { + return messages; + }; + for message in &mut messages { + let Some(message) = message.as_object_mut() else { + continue; + }; + match message.get("role").and_then(Value::as_str) { + Some("assistant") => { + if let Some(Value::Object(call)) = message.get_mut("function_call") + && call.get("arguments").is_none_or(Value::is_null) + { + call.insert("arguments".into(), Value::from("")); + } + } + Some("user") => { + let Some(Value::Array(parts)) = message.get_mut("content") else { + continue; + }; + for part in parts.iter_mut().filter_map(Value::as_object_mut) { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => match part.get_mut("image_url") { + Some(Value::String(url)) => { + let url = std::mem::take(url); + part.insert("image_url".into(), serde_json::json!({"url": url})); + } + Some(Value::Object(image)) => { + image.remove("format"); + } + _ => {} + }, + Some("file") => { + if let Some(Value::Object(file)) = part.get_mut("file") { + file.remove("format"); + } + } + _ => {} + } + } + } + _ => {} + } + } + Value::Array(messages) +} + +/// The request's OpenAI params in `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` order, the order +/// Python maps them in. +fn ordered(params: &Map) -> impl Iterator { + OPENAI_PARAMS + .iter() + .filter_map(|name| params.get(*name).map(|value| (*name, value))) +} + +/// `_map_openai_params`: every supported param still unmapped goes as sent. +fn copy_supported( + non_default: &mut Map, + optional: &mut Map, + supported: &[&str], +) { + for name in OPENAI_PARAMS { + if supported.contains(name) + && let Some(value) = non_default.get(*name) + { + optional.insert((*name).into(), value.clone()); + } + } +} + +/// Python's truthiness for a JSON value. +fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64() != Some(0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(object) => !object.is_empty(), + } +} + +/// `normalize_drop_params`: a boolean, or a config's `"true"`. +fn request_drop_params(value: Option) -> bool { + match value { + Some(Value::Bool(flag)) => flag, + Some(Value::String(text)) => text.trim().eq_ignore_ascii_case("true"), + _ => false, + } +} + +fn string_list(value: Option) -> Vec { + match value { + Some(Value::Array(items)) => items + .into_iter() + .filter_map(|item| match item { + Value::String(text) => Some(text), + _ => None, + }) + .collect(), + _ => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use rstest::rstest; + use serde_json::json; + + use super::*; + + /// A cost map of a few entries. + struct Entries(HashMap>); + + impl Entries { + fn new(entries: Value) -> Self { + let Value::Object(entries) = entries else { + panic!("entries are an object"); + }; + Self( + entries + .into_iter() + .map(|(name, entry)| (name, object(entry))) + .collect(), + ) + } + } + + impl ModelInfo for Entries { + fn entry(&self, model: &str) -> Option<&Map> { + self.0.get(model) + } + + fn declares(&self, key: &str) -> bool { + self.0.values().any(|entry| entry.contains_key(key)) + } + } + + fn cost_map() -> Entries { + Entries::new(json!({ + "azure/gpt-5": {"supports_reasoning": true}, + "azure/gpt-5.1": {"supports_none_reasoning_effort": true, "default_reasoning_effort": "none"}, + "azure/gpt-5.2": {"supports_reasoning": true}, + "azure/gpt-5.5": {"supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false}, + "o3-mini": {"litellm_provider": "openai", "supports_reasoning": true}, + "o1-preview": {"litellm_provider": "openai", "supports_system_messages": true}, + "gpt-4o-mini": {"litellm_provider": "openai"}, + })) + } + + fn object(value: Value) -> Map { + let Value::Object(map) = value else { + panic!("expected an object, got {value}"); + }; + map + } + + fn request<'a>( + deployment: &'a str, + base_model: Option<&'a str>, + api_version: &'a str, + info: Option<&'a dyn ModelInfo>, + ) -> AzureChatRequest<'a> { + AzureChatRequest { + deployment, + base_model, + api_version, + drop_params: false, + model_info: info, + } + } + + fn transform(request: &AzureChatRequest<'_>, body: Value) -> Result, Error> { + request.transform(object(body)).map(|sent| sent.body) + } + + fn user(text: &str) -> Value { + json!([{"role": "user", "content": text}]) + } + + fn is_unsupported_param(result: &Result, Error>) -> bool { + matches!( + result, + Err(Error::InvalidRequest( + ErrorDetail::UnsupportedParams { .. } | ErrorDetail::UnsupportedParameter { .. } + )) + ) + } + + #[rstest] + #[case::gpt_4o("gpt-4o", None, AzureModelFamily::Gpt)] + #[case::gpt_5_chat("gpt-5-chat", None, AzureModelFamily::Gpt)] + #[case::o3_mini("o3-mini", None, AzureModelFamily::OSeries)] + #[case::o4("my-o4-mini", None, AzureModelFamily::OSeries)] + #[case::o_series_route("o_series/reasoner", None, AzureModelFamily::OSeries)] + #[case::gpt_5("gpt-5.1", None, AzureModelFamily::Gpt5)] + #[case::gpt_6("gpt-6", None, AzureModelFamily::Gpt5)] + #[case::gpt_5_route("gpt5_series/prod", None, AzureModelFamily::Gpt5)] + #[case::base_model_names_the_model("prod-chat", Some("azure/gpt-5.1"), AzureModelFamily::Gpt5)] + #[case::base_model_o_series("prod-chat", Some("azure/o3"), AzureModelFamily::OSeries)] + fn the_family_follows_the_base_model_then_the_deployment( + #[case] deployment: &str, + #[case] base_model: Option<&str>, + #[case] family: AzureModelFamily, + ) { + assert_eq!( + request(deployment, base_model, "2024-10-21", None).family(), + family + ); + } + + #[rstest] + fn a_gpt_body_passes_through_with_unknown_params_and_without_litellm_keys() { + let sent = transform( + &request("gpt-4o", None, "2024-10-21", None), + json!({ + "model": "public", + "messages": user("hi"), + "temperature": 0.2, + "stream": true, + "stream_options": {"include_usage": true}, + "top_k": 5, + "seed": null, + "metadata": {"k": "v"}, + "extra_body": {"data_sources": [{"type": "x"}], "azure_ad_token": "t"}, + }), + ) + .unwrap(); + assert_eq!( + sent, + object(json!({ + "model": "gpt-4o", + "messages": user("hi"), + "temperature": 0.2, + "stream": true, + "stream_options": {"include_usage": true}, + "top_k": 5, + "data_sources": [{"type": "x"}], + })) + ); + } + + #[rstest] + #[case::refused(false, json!({}), None)] + #[case::deployment_drops(true, json!({}), Some(json!({"model": "gpt-4o", "messages": user("hi")})))] + #[case::request_drops(false, json!({"drop_params": true}), Some(json!({"model": "gpt-4o", "messages": user("hi")})))] + #[case::allowed(false, json!({"allowed_openai_params": ["reasoning_effort"]}), Some(json!({"model": "gpt-4o", "messages": user("hi"), "reasoning_effort": "low"})))] + fn an_openai_param_the_family_lacks_is_refused_unless_dropped_or_allowed( + #[case] drop_params: bool, + #[case] extra: Value, + #[case] expected: Option, + ) { + let mut body = + object(json!({"model": "m", "messages": user("hi"), "reasoning_effort": "low"})); + body.extend(object(extra)); + let request = AzureChatRequest { + drop_params, + ..request("gpt-4o", None, "2024-10-21", None) + }; + let result = request.transform(body).map(|sent| sent.body); + match expected { + Some(expected) => assert_eq!(result.unwrap(), object(expected)), + None => assert!( + matches!( + &result, + Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { params, .. })) + if params == &["reasoning_effort"] + ), + "{result:?}" + ), + } + } + + #[rstest] + #[case::gpt_4o("gpt-4o", json!({"max_tokens": 10}))] + #[case::gpt_5_chat("gpt-5-chat", json!({"max_completion_tokens": 10}))] + fn max_tokens_is_renamed_for_the_gpt_5_names( + #[case] deployment: &str, + #[case] expected: Value, + ) { + let sent = transform( + &request(deployment, None, "2024-10-21", None), + json!({"messages": user("hi"), "max_tokens": 10}), + ) + .unwrap(); + let mut body = object(json!({"model": deployment, "messages": user("hi")})); + body.extend(object(expected)); + assert_eq!(sent, body); + } + + #[rstest] + #[case::before_tools("2023-07-01-preview", "auto", None)] + #[case::required_too_early("2024-05-01-preview", "required", None)] + #[case::auto_on_may("2024-05-01-preview", "auto", Some("auto"))] + #[case::required_later("2024-10-21", "required", Some("required"))] + #[case::v1("v1", "required", Some("required"))] + fn tool_choice_follows_the_api_version( + #[case] api_version: &str, + #[case] tool_choice: &str, + #[case] expected: Option<&str>, + ) { + let tools = json!([{"type": "function", "function": {"name": "f"}}]); + let result = transform( + &request("gpt-4o", None, api_version, None), + json!({"messages": user("hi"), "tools": tools, "tool_choice": tool_choice}), + ); + match expected { + Some(expected) => assert_eq!(result.unwrap()["tool_choice"], json!(expected)), + None => assert!(is_unsupported_param(&result), "{result:?}"), + } + } + + #[rstest] + fn tool_choice_without_tools_is_left_out() { + let sent = transform( + &request("gpt-4o", None, "2024-10-21", None), + json!({"messages": user("hi"), "tool_choice": "auto"}), + ) + .unwrap(); + assert!(!sent.contains_key("tool_choice")); + } + + #[rstest] + #[case::supported("gpt-4o", "2024-10-21", json!({"type": "json_object"}), Some(true))] + #[case::old_version_drops_json_object("gpt-4o", "2024-06-01", json!({"type": "json_object"}), Some(false))] + #[case::gpt_35_drops_json_object("gpt-35-turbo", "2024-10-21", json!({"type": "json_object"}), Some(false))] + #[case::gpt_35_schema_is_refused( + "gpt-35-turbo", + "2024-10-21", + json!({"type": "json_schema", "json_schema": {"name": "s", "schema": {"type": "object"}}}), + None + )] + fn response_format_follows_the_model_and_api_version( + #[case] deployment: &str, + #[case] api_version: &str, + #[case] format: Value, + #[case] kept: Option, + ) { + let result = transform( + &request(deployment, None, api_version, None), + json!({"messages": user("hi"), "response_format": format}), + ); + match kept { + Some(kept) => assert_eq!(result.unwrap().contains_key("response_format"), kept), + None => assert!(matches!(result, Err(Error::Unsupported(_))), "{result:?}"), + } + } + + #[rstest] + fn gpt_messages_are_shaped_for_azure() { + let sent = transform( + &request("gpt-4o", None, "2024-10-21", None), + json!({"messages": [ + {"role": "user", "content": [ + {"type": "image_url", "image_url": "https://x/i.png"}, + {"type": "image_url", "image_url": {"url": "https://x/j.png", "format": "image/png"}}, + {"type": "file", "file": {"file_id": "f", "format": "application/pdf"}}, + ]}, + {"role": "assistant", "content": null, "function_call": {"name": "f", "arguments": null}}, + ]}), + ) + .unwrap(); + assert_eq!( + sent["messages"], + json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://x/i.png"}}, + {"type": "image_url", "image_url": {"url": "https://x/j.png"}}, + {"type": "file", "file": {"file_id": "f"}}, + ]}, + {"role": "assistant", "content": null, "function_call": {"name": "f", "arguments": ""}}, + ]) + ); + } + + #[rstest] + fn o_series_maps_max_tokens_and_keeps_reasoning_effort_for_custom_names() { + let sent = transform( + &request("o_series/reasoner", None, "2024-12-01-preview", None), + json!({"messages": [{"role": "system", "content": "be brief", "name": "s"}, {"role": "user", "content": "hi"}], + "max_tokens": 100, "reasoning_effort": "high", "temperature": 1}), + ) + .unwrap(); + assert_eq!( + sent, + object(json!({ + "model": "reasoner", + "messages": [{"content": "be brief", "role": "user"}, {"role": "user", "content": "hi"}], + "max_completion_tokens": 100, + "reasoning_effort": "high", + "temperature": 1, + })) + ); + } + + #[rstest] + #[case::temperature(json!({"temperature": 0.5}))] + #[case::top_p(json!({"top_p": 0.5}))] + #[case::presence_penalty(json!({"presence_penalty": 1}))] + fn o_series_refuses_sampling_params(#[case] params: Value) { + let mut body = object(json!({"messages": user("hi")})); + body.extend(object(params)); + let result = request("o3-mini", None, "2024-12-01-preview", None) + .transform(body) + .map(|sent| sent.body); + assert!(is_unsupported_param(&result), "{result:?}"); + } + + #[rstest] + #[case::unknown_model_has_no_system_messages("o3-mini", false)] + #[case::the_cost_map_says_so("o1-preview", true)] + fn o_series_system_messages_follow_the_cost_map(#[case] deployment: &str, #[case] kept: bool) { + let map = cost_map(); + let sent = transform( + &request(deployment, None, "2024-12-01-preview", Some(&map)), + json!({"messages": [{"role": "system", "content": "s"}]}), + ) + .unwrap(); + assert_eq!( + sent["messages"][0]["role"], + json!(if kept { "system" } else { "user" }) + ); + } + + #[rstest] + fn o_series_user_needs_an_openai_catalog_model() { + let map = cost_map(); + let known = transform( + &request("o3-mini", None, "2024-12-01-preview", Some(&map)), + json!({"messages": user("hi"), "user": "u"}), + ) + .unwrap(); + assert_eq!(known["user"], json!("u")); + let custom = transform( + &request("reasoner-o3", None, "2024-12-01-preview", Some(&map)), + json!({"messages": user("hi"), "user": "u"}), + ) + .unwrap(); + assert!(!custom.contains_key("user")); + } + + #[rstest] + fn gpt_5_renames_max_tokens_and_strips_its_route() { + let sent = transform( + &request("gpt5_series/prod", None, "2025-04-01-preview", None), + json!({"messages": user("hi"), "max_tokens": 64, "reasoning_effort": "low", "verbosity": "low"}), + ) + .unwrap(); + assert_eq!( + sent, + object(json!({ + "model": "prod", + "messages": user("hi"), + "max_completion_tokens": 64, + "reasoning_effort": "low", + "verbosity": "low", + })) + ); + } + + #[rstest] + #[case::gpt_5_refuses_a_low_temperature("gpt-5", None, json!({"temperature": 0}), None)] + #[case::gpt_5_takes_temperature_one("gpt-5", None, json!({"temperature": 1}), Some(json!({"temperature": 1})))] + #[case::gpt_5_1_defaults_to_no_reasoning("gpt-5.1", None, json!({"temperature": 0}), Some(json!({"temperature": 0})))] + #[case::gpt_5_1_with_reasoning_refuses_it("gpt-5.1", None, json!({"temperature": 0, "reasoning_effort": "high"}), None)] + #[case::base_model_decides("prod", Some("azure/gpt-5.1"), json!({"temperature": 0}), Some(json!({"temperature": 0})))] + #[case::gpt_5_refuses_effort_none("gpt-5", None, json!({"reasoning_effort": "none"}), None)] + #[case::gpt_5_1_takes_effort_none("gpt-5.1", None, json!({"reasoning_effort": "none"}), Some(json!({"reasoning_effort": "none"})))] + #[case::effort_object_is_flattened("gpt-5.1", None, json!({"reasoning_effort": {"effort": "high"}}), Some(json!({"reasoning_effort": "high"})))] + #[case::xhigh_needs_support("gpt-5", None, json!({"reasoning_effort": "xhigh"}), None)] + #[case::xhigh_supported("gpt-5.5", None, json!({"reasoning_effort": "xhigh"}), Some(json!({"reasoning_effort": "xhigh"})))] + #[case::minimal_disabled("gpt-5.5", None, json!({"reasoning_effort": "minimal"}), None)] + #[case::gpt_5_refuses_top_p("gpt-5", None, json!({"top_p": 0.5}), None)] + #[case::gpt_5_refuses_stop("gpt-5", None, json!({"stop": ["x"]}), None)] + #[case::gpt_5_2_takes_logprobs("gpt-5.2", None, json!({"logprobs": true}), Some(json!({"logprobs": true})))] + fn gpt_5_params_follow_the_cost_map( + #[case] deployment: &str, + #[case] base_model: Option<&str>, + #[case] params: Value, + #[case] expected: Option, + ) { + let map = cost_map(); + let mut body = object(json!({"messages": user("hi")})); + body.extend(object(params)); + let result = request(deployment, base_model, "2025-04-01-preview", Some(&map)) + .transform(body) + .map(|sent| sent.body); + match expected { + Some(expected) => { + let mut body = object(json!({"model": deployment, "messages": user("hi")})); + body.extend(object(expected)); + assert_eq!(result.unwrap(), body); + } + None => assert!(is_unsupported_param(&result), "{result:?}"), + } + } + + #[rstest] + fn drop_params_drops_what_gpt_5_refuses() { + let map = cost_map(); + let request = AzureChatRequest { + drop_params: true, + ..request("gpt-5", None, "2025-04-01-preview", Some(&map)) + }; + let sent = request + .transform(object(json!({"messages": user("hi"), "temperature": 0, "top_p": 0.5, "reasoning_effort": "none"}))) + .unwrap() + .body; + assert_eq!( + sent, + object(json!({"model": "gpt-5", "messages": user("hi")})) + ); + } + + #[rstest] + #[case::responses_route("responses/prod", json!({}))] + #[case::function_tools_with_reasoning("gpt-5.4", json!({"tools": [{"type": "function", "function": {"name": "f"}}]}))] + #[case::reasoning_summary("gpt-5.1", json!({"reasoning_effort": "high", "reasoning_summary": "auto"}))] + fn requests_python_bridges_to_responses_are_refused( + #[case] deployment: &str, + #[case] params: Value, + ) { + let mut body = object(json!({"messages": user("hi")})); + body.extend(object(params)); + assert!(matches!( + request(deployment, None, "2025-04-01-preview", None).transform(body), + Err(Error::Unsupported(_)) + )); + } + + #[rstest] + fn function_tools_without_reasoning_stay_on_chat() { + let sent = transform( + &request("gpt-5.4", None, "2025-04-01-preview", None), + json!({"messages": user("hi"), "reasoning_effort": "none", + "tools": [{"type": "function", "function": {"name": "f"}}]}), + ); + assert!(!matches!(sent, Err(Error::Unsupported(_))), "{sent:?}"); + } + + #[rstest] + #[case::bare("gpt-5", Some((5, 0)))] + #[case::minor("gpt-5.4", Some((5, 4)))] + #[case::suffix("gpt-5.4-pro", Some((5, 4)))] + #[case::routed("azure/gpt-6", Some((6, 0)))] + #[case::dash_suffix("gpt-5-mini", Some((5, 0)))] + #[case::glued("gpt-5x", None)] + #[case::other("o3", None)] + fn the_gpt_version_is_read_from_the_name( + #[case] model: &str, + #[case] expected: Option<(u32, u32)>, + ) { + assert_eq!(gpt_series_version(model), expected); + } + + #[rstest] + #[case::dashed("gpt-3-5-turbo", "gpt-3.5-turbo")] + #[case::non_overlapping("gpt-4-1-2", "gpt-4.1-2")] + fn dashed_versions_read_as_dotted(#[case] model: &str, #[case] expected: &str) { + assert_eq!(dotted_versions(model), expected); + } +} diff --git a/litellm-rust/crates/llms/src/azure/common_utils.rs b/litellm-rust/crates/llms/src/azure/common_utils.rs new file mode 100644 index 00000000000..7b7a822985e --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/common_utils.rs @@ -0,0 +1,325 @@ +//! Endpoint, API version and credential resolution for Azure OpenAI, as +//! `litellm/llms/azure/common_utils.py`, `litellm/main.py` and the `AzureOpenAI` client of the +//! OpenAI SDK resolve them. + +use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_core_utils::settings::resolve_non_empty; + +use crate::{ + Error, + base_llm::auth::{AuthScheme, Headers, ValidatedEnvironment}, +}; + +/// `litellm.AZURE_DEFAULT_API_VERSION`. +pub const AZURE_DEFAULT_API_VERSION: &str = "2025-02-01-preview"; +pub const AZURE_API_BASE_ENV: &str = "AZURE_API_BASE"; +/// The SDK's own fallback when LiteLLM passes no endpoint. +pub const AZURE_OPENAI_ENDPOINT_ENV: &str = "AZURE_OPENAI_ENDPOINT"; +pub const AZURE_API_VERSION_ENV: &str = "AZURE_API_VERSION"; +pub const AZURE_OPENAI_API_KEY_ENV: &str = "AZURE_OPENAI_API_KEY"; +pub const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY"; +const API_KEY_HEADER: &str = "api-key"; + +/// The environment variables a call may fall back to, besides the credential ones +/// `litellm_auth_azure::SECRET_NAMES` lists. +pub const SECRET_NAMES: &[&str] = &[ + AZURE_API_BASE_ENV, + AZURE_OPENAI_ENDPOINT_ENV, + AZURE_API_VERSION_ENV, + AZURE_OPENAI_API_KEY_ENV, + AZURE_API_KEY_ENV, +]; + +/// `BaseAzureLLM._is_azure_v1_api_version`: these versions use the `/openai/v1/` surface +/// through the plain OpenAI client. +pub fn is_v1_api_version(api_version: &str) -> bool { + matches!(api_version, "preview" | "latest" | "v1") +} + +/// The deployment's `api_version`, else `AZURE_API_VERSION`, else LiteLLM's default. +pub fn resolve_api_version( + api_version: Option<&str>, + lookup: &dyn Fn(&str) -> Option, +) -> String { + resolve_non_empty(api_version, lookup, &[AZURE_API_VERSION_ENV]) + .unwrap_or_else(|| AZURE_DEFAULT_API_VERSION.into()) +} + +/// The deployment's `api_base`, else `AZURE_API_BASE`, else the SDK's `AZURE_OPENAI_ENDPOINT`. +pub fn resolve_api_base( + api_base: Option<&str>, + lookup: &dyn Fn(&str) -> Option, +) -> Result { + resolve_non_empty( + api_base, + lookup, + &[AZURE_API_BASE_ENV, AZURE_OPENAI_ENDPOINT_ENV], + ) + .ok_or(Error::Auth(litellm_auth::Error::MissingApiBase { + provider: "Azure", + guidance: "Set `api_base` or the AZURE_API_BASE environment variable. Expected format: https://.openai.azure.com", + })) +} + +/// The deployment's key, else `AZURE_OPENAI_API_KEY`, else `AZURE_API_KEY`, as `main.py` +/// reads them. +pub fn resolve_api_key( + api_key: Option<&str>, + lookup: &dyn Fn(&str) -> Option, +) -> Option { + resolve_non_empty( + api_key, + lookup, + &[AZURE_OPENAI_API_KEY_ENV, AZURE_API_KEY_ENV], + ) +} + +/// The Chat Completions URL the SDK builds for `deployment`: +/// +/// - `v1`, `latest` and `preview` go to `{api_base}/openai/v1/chat/completions` with no +/// `api-version`, as the plain OpenAI client LiteLLM switches to for them does; +/// - an `api_base` already naming `/openai/deployments/...` is the client's `base_url`, so only +/// `/chat/completions` is added; +/// - any other `api_base` is the resource endpoint, under which the SDK puts +/// `/openai/deployments/{deployment}`. +/// +/// A trailing slash on `api_base` is dropped, as `azure_endpoint.rstrip('/')` does. +pub fn complete_url(api_base: &str, deployment: &str, api_version: &str) -> Result { + if api_base.contains("gateway.ai.cloudflare.com") { + return Err(Error::Unsupported( + "Azure deployments behind a Cloudflare AI Gateway api_base", + )); + } + let base = api_base.trim_end_matches('/'); + if is_v1_api_version(api_version) { + return Ok(format!("{base}/openai/v1/chat/completions")); + } + let version: String = url::form_urlencoded::byte_serialize(api_version.as_bytes()).collect(); + Ok(match base.contains("/openai/deployments") { + true => format!("{base}/chat/completions?api-version={version}"), + false => { + format!("{base}/openai/deployments/{deployment}/chat/completions?api-version={version}") + } + }) +} + +/// How a call authenticates once its credential is known. +#[derive(Clone, Debug)] +pub enum AzureCredential { + ApiKey(SecretValue), + /// An Entra ID access token, given or acquired. + AdToken(SecretValue), +} + +/// A key goes in `api-key` and a token in `Authorization: Bearer`, as `AzureOpenAI` sends +/// them. On the v1 surface LiteLLM hands either to the plain OpenAI client as its `api_key`, +/// which sends it as a bearer. +pub fn environment( + headers: Headers, + credential: AzureCredential, + v1: bool, +) -> ValidatedEnvironment { + let (placement, secret) = match credential { + AzureCredential::ApiKey(secret) if !v1 => { + (CredentialPlacement::Header(API_KEY_HEADER), secret) + } + AzureCredential::ApiKey(secret) | AzureCredential::AdToken(secret) => { + (CredentialPlacement::Bearer, secret) + } + }; + ValidatedEnvironment { + headers: litellm_http::request::with_default_headers( + headers, + &[("content-type", "application/json")], + ), + auth: AuthScheme::Credential { placement, secret }, + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn no_env(_: &str) -> Option { + None + } + + fn env(name: &str) -> Option { + match name { + AZURE_API_BASE_ENV => Some("https://env.openai.azure.com".into()), + AZURE_API_VERSION_ENV => Some("2024-10-21".into()), + AZURE_API_KEY_ENV => Some("env-key".into()), + _ => None, + } + } + + fn endpoint_only(name: &str) -> Option { + (name == AZURE_OPENAI_ENDPOINT_ENV).then(|| "https://sdk.openai.azure.com".into()) + } + + #[rstest] + #[case::resource( + "https://res.openai.azure.com", + "2024-10-21", + "https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21" + )] + #[case::trailing_slash( + "https://res.openai.azure.com/", + "2024-10-21", + "https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21" + )] + #[case::preview_version( + "https://res.openai.azure.com", + "2025-02-01-preview", + "https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2025-02-01-preview" + )] + #[case::base_url_with_deployment( + "https://res.openai.azure.com/openai/deployments/other/", + "2024-10-21", + "https://res.openai.azure.com/openai/deployments/other/chat/completions?api-version=2024-10-21" + )] + #[case::v1( + "https://res.openai.azure.com/", + "v1", + "https://res.openai.azure.com/openai/v1/chat/completions" + )] + #[case::latest( + "https://res.openai.azure.com", + "latest", + "https://res.openai.azure.com/openai/v1/chat/completions" + )] + #[case::preview( + "https://res.openai.azure.com", + "preview", + "https://res.openai.azure.com/openai/v1/chat/completions" + )] + fn the_url_follows_the_sdk( + #[case] api_base: &str, + #[case] api_version: &str, + #[case] expected: &str, + ) { + assert_eq!( + complete_url(api_base, "gpt-4o", api_version).unwrap(), + expected + ); + } + + #[rstest] + fn a_cloudflare_gateway_base_is_refused() { + assert!(matches!( + complete_url( + "https://gateway.ai.cloudflare.com/v1/acct/gw/azure-openai/res", + "gpt-4o", + "2024-10-21" + ), + Err(Error::Unsupported(_)) + )); + } + + #[rstest] + #[case::deployment(Some("2024-06-01"), env, "2024-06-01")] + #[case::environment(None, env, "2024-10-21")] + #[case::empty_falls_back(Some(""), env, "2024-10-21")] + #[case::default(None, no_env, AZURE_DEFAULT_API_VERSION)] + fn the_api_version_prefers_the_deployment_then_the_environment( + #[case] configured: Option<&str>, + #[case] lookup: fn(&str) -> Option, + #[case] expected: &str, + ) { + assert_eq!(resolve_api_version(configured, &lookup), expected); + } + + #[rstest] + #[case::deployment( + Some("https://dep.openai.azure.com"), + env, + "https://dep.openai.azure.com" + )] + #[case::environment(None, env, "https://env.openai.azure.com")] + #[case::sdk_environment(None, endpoint_only, "https://sdk.openai.azure.com")] + fn the_api_base_prefers_the_deployment_then_the_environment( + #[case] configured: Option<&str>, + #[case] lookup: fn(&str) -> Option, + #[case] expected: &str, + ) { + assert_eq!(resolve_api_base(configured, &lookup).unwrap(), expected); + } + + #[rstest] + fn a_missing_api_base_says_where_to_set_it() { + assert!(matches!( + resolve_api_base(None, &no_env), + Err(Error::Auth(litellm_auth::Error::MissingApiBase { + provider: "Azure", + .. + })) + )); + } + + #[rstest] + #[case::deployment(Some("dep-key"), env, Some("dep-key"))] + #[case::environment(None, env, Some("env-key"))] + #[case::none(None, no_env, None)] + fn the_key_prefers_the_deployment_then_the_environment( + #[case] configured: Option<&str>, + #[case] lookup: fn(&str) -> Option, + #[case] expected: Option<&str>, + ) { + assert_eq!(resolve_api_key(configured, &lookup).as_deref(), expected); + } + + #[rstest] + fn the_sdk_key_variable_outranks_litellms() { + let lookup = |name: &str| match name { + AZURE_OPENAI_API_KEY_ENV => Some("sdk-key".to_string()), + AZURE_API_KEY_ENV => Some("litellm-key".to_string()), + _ => None, + }; + assert_eq!(resolve_api_key(None, &lookup).as_deref(), Some("sdk-key")); + } + + #[rstest] + #[case::key(AzureCredential::ApiKey(SecretValue::new("k")), false, "api-key", "k")] + #[case::token( + AzureCredential::AdToken(SecretValue::new("t")), + false, + "authorization", + "Bearer t" + )] + #[case::v1_key( + AzureCredential::ApiKey(SecretValue::new("k")), + true, + "authorization", + "Bearer k" + )] + #[case::v1_token( + AzureCredential::AdToken(SecretValue::new("t")), + true, + "authorization", + "Bearer t" + )] + #[tokio::test] + async fn the_credential_lands_in_its_header( + #[case] credential: AzureCredential, + #[case] v1: bool, + #[case] header: &str, + #[case] value: &str, + ) { + let authenticated = crate::base_llm::auth::resolve_auth( + &litellm_auth::AuthServices::default(), + environment(Vec::new(), credential, v1), + &no_env, + ) + .await + .unwrap(); + assert_eq!( + authenticated.headers, + [ + ("content-type".to_string(), "application/json".to_string()), + (header.to_string(), value.to_string()) + ] + ); + } +} diff --git a/litellm-rust/crates/llms/src/azure/mod.rs b/litellm-rust/crates/llms/src/azure/mod.rs new file mode 100644 index 00000000000..10c654d3932 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure/mod.rs @@ -0,0 +1,5 @@ +//! Azure OpenAI (`azure/`), as `litellm/llms/azure/` serves it. Azure AI Foundry's +//! `azure_ai` provider lives in `crate::azure_ai`. + +pub mod chat; +pub mod common_utils; diff --git a/litellm-rust/crates/llms/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs index 4c04c23a0bb..0de88bffc62 100644 --- a/litellm-rust/crates/llms/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -3,6 +3,7 @@ pub mod auth; pub mod base_model_iterator; pub mod chat; pub mod messages; +pub mod model_info; pub mod ocr; pub mod responses; pub mod sse; diff --git a/litellm-rust/crates/llms/src/base_llm/model_info.rs b/litellm-rust/crates/llms/src/base_llm/model_info.rs new file mode 100644 index 00000000000..a2861d138c7 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/model_info.rs @@ -0,0 +1,37 @@ +//! What the model cost map says about a model, for configs whose request rules depend on it +//! the way Python's `supports_*` helpers read `litellm.model_cost`. + +use serde_json::{Map, Value}; + +pub trait ModelInfo: Send + Sync { + /// The cost map entry `model` names, if there is one. + fn entry(&self, model: &str) -> Option<&Map>; + + /// Whether any entry of the map sets `key`. + fn declares(&self, key: &str) -> bool; +} + +/// Python's `_supports_factory`: the entry sets `key` to `true`. An unknown model supports +/// nothing. +pub fn supports(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool { + field(info, model, key).and_then(Value::as_bool) == Some(true) +} + +/// Python's `is_explicitly_disabled_factory`: the entry sets `key` to `false`. An unknown +/// model or an unset key is not disabled. +pub fn explicitly_disabled(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool { + field(info, model, key).and_then(Value::as_bool) == Some(false) +} + +/// The value the entry for `model` sets for `key`. +pub fn field<'a>(info: Option<&'a dyn ModelInfo>, model: &str, key: &str) -> Option<&'a Value> { + info? + .entry(model)? + .get(key) + .filter(|value| !value.is_null()) +} + +/// Whether the map has an entry for `model`. +pub fn known(info: Option<&dyn ModelInfo>, model: &str) -> bool { + info.is_some_and(|info| info.entry(model).is_some()) +} diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index 06a31222fda..24a1ea8a969 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -60,6 +60,14 @@ pub enum ErrorDetail { value: String, hint: String, }, + #[error( + "{provider} does not support parameters: {params:?}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={params:?} in your request." + )] + UnsupportedParams { + provider: &'static str, + params: Vec, + model: String, + }, #[error( "{provider} does not support the server-side tool '{tool}' (model '{model}'). {guidance}" )] diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs index eb232404d93..23ed649ac8e 100644 --- a/litellm-rust/crates/llms/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -1,5 +1,6 @@ pub mod anthropic; pub mod aws_textract; +pub mod azure; pub mod azure_ai; pub mod base_llm; pub mod bedrock; diff --git a/litellm-rust/crates/llms/src/openai/chat/transformation.rs b/litellm-rust/crates/llms/src/openai/chat/transformation.rs index 43a3df53510..c1c19619e4c 100644 --- a/litellm-rust/crates/llms/src/openai/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/chat/transformation.rs @@ -85,17 +85,7 @@ impl OpenAiChatConfig { body: Map, ) -> Result, Error> { let mut body = body; - let extra_body = match body.remove(EXTRA_BODY) { - None | Some(Value::Null) => Map::new(), - Some(Value::Object(extra)) => extra, - Some(other) => { - return Err(Error::InvalidRequest(ErrorDetail::InvalidValue { - field: EXTRA_BODY, - expected: "an object", - actual: other, - })); - } - }; + let extra_body = take_extra_body(&mut body)?; let mut body: Map = body .into_iter() .chain(extra_body) @@ -120,7 +110,21 @@ pub fn is_stream(body: &Map) -> Result { } } -fn is_litellm_owned(name: &str) -> bool { +/// Removes the body's `extra_body`, which the OpenAI SDK merges over the body it sends. +pub fn take_extra_body(body: &mut Map) -> Result, Error> { + match body.remove(EXTRA_BODY) { + None | Some(Value::Null) => Ok(Map::new()), + Some(Value::Object(extra)) => Ok(extra), + Some(other) => Err(Error::InvalidRequest(ErrorDetail::InvalidValue { + field: EXTRA_BODY, + expected: "an object", + actual: other, + })), + } +} + +/// Whether Python consumes `name` before the provider call, so it never reaches the wire. +pub fn is_litellm_owned(name: &str) -> bool { LITELLM_OWNED_PARAMS.contains(&name) || name.starts_with(INTERNAL_PARAM_PREFIX) } diff --git a/litellm-rust/crates/router-types/src/value.rs b/litellm-rust/crates/router-types/src/value.rs index e52cf60c0ef..967140ed656 100644 --- a/litellm-rust/crates/router-types/src/value.rs +++ b/litellm-rust/crates/router-types/src/value.rs @@ -8,3 +8,31 @@ pub enum Spelled { Value(T), Text(String), } + +impl Spelled { + /// The flag, reading a `true` word the way Python's coercion does. An `os.environ/` + /// reference left unresolved is not set. + pub fn is_set(&self) -> bool { + match self { + Self::Value(value) => *value, + Self::Text(text) => text.trim().eq_ignore_ascii_case("true"), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::value(Spelled::Value(true), true)] + #[case::false_value(Spelled::Value(false), false)] + #[case::word(Spelled::Text(" True ".into()), true)] + #[case::other_word(Spelled::Text("yes".into()), false)] + #[case::reference(Spelled::Text("os.environ/DROP".into()), false)] + fn flags_read_values_and_words(#[case] flag: Spelled, #[case] expected: bool) { + assert_eq!(flag.is_set(), expected); + } +} diff --git a/litellm-rust/crates/router/src/deployment.rs b/litellm-rust/crates/router/src/deployment.rs index 5d6c2995820..d2f55061f1c 100644 --- a/litellm-rust/crates/router/src/deployment.rs +++ b/litellm-rust/crates/router/src/deployment.rs @@ -21,6 +21,9 @@ pub struct Deployment { /// `model_info.id`, which names the deployment in state shared with other replicas and /// with Python routers on the same Redis. pub id: Option, + /// The model a deployment serves under another name, from `litellm_params.base_model` or + /// `model_info.base_model`, which Azure's param rules and prices follow. + pub base_model: Option, } impl Deployment { @@ -46,6 +49,12 @@ impl Deployment { shaping: MessagesShaping::default(), team_id: None, id: None, + base_model: params + .extra + .get("base_model") + .and_then(|value| value.as_str()) + .filter(|name| !name.is_empty()) + .map(str::to_owned), }) } diff --git a/litellm-rust/crates/router/src/lib.rs b/litellm-rust/crates/router/src/lib.rs index 264555ec66b..3e491c51b68 100644 --- a/litellm-rust/crates/router/src/lib.rs +++ b/litellm-rust/crates/router/src/lib.rs @@ -87,6 +87,10 @@ impl Router { timeout: deployment.timeout.or(policy.timeout), team_id: owner_team(model), id: model_id(model), + base_model: deployment + .base_model + .clone() + .or_else(|| info_base_model(model)), ..deployment }, )) @@ -363,6 +367,13 @@ fn owner_team(model: &Model) -> Option { } } +fn info_base_model(model: &Model) -> Option { + match model.model_info.get("base_model")? { + litellm_config::Value::String(name) if !name.is_empty() => Some(name.clone()), + _ => None, + } +} + impl FromIterator<(String, Deployment)> for Router { fn from_iter>(entries: I) -> Self { Self::new(entries, RoutingPolicy::default()) @@ -559,6 +570,31 @@ mod tests { ); } + #[rstest] + #[case::params( + " litellm_params:\n model: azure/d\n base_model: gpt-4o\n", + Some("gpt-4o") + )] + #[case::model_info( + " litellm_params:\n model: azure/d\n model_info:\n base_model: gpt-4o\n", + Some("gpt-4o") + )] + #[case::params_win( + " litellm_params:\n model: azure/d\n base_model: gpt-4o\n model_info:\n base_model: gpt-4\n", + Some("gpt-4o") + )] + #[case::absent(" litellm_params:\n model: azure/d\n", None)] + fn reads_the_base_model(#[case] entry: &str, #[case] expected: Option<&str>) { + let config = + Config::from_yaml(&format!("model_list:\n - model_name: m\n{entry}")).unwrap(); + let router = Router::from_config(&config).unwrap(); + let group = router.group("m").unwrap(); + assert_eq!( + group.deployments().next().unwrap().base_model.as_deref(), + expected + ); + } + #[rstest] #[tokio::test] async fn an_unclassified_error_is_not_retried() { diff --git a/litellm-rust/crates/router/tests/router.rs b/litellm-rust/crates/router/tests/router.rs index bd75f2b61f3..d92438bf296 100644 --- a/litellm-rust/crates/router/tests/router.rs +++ b/litellm-rust/crates/router/tests/router.rs @@ -117,6 +117,7 @@ fn programmatic_deployments_preserve_overrides_and_share_their_group() { }, team_id: None, id: None, + base_model: None, }; let router = Router::from_iter([ ("public-model".into(), Deployment::default()), diff --git a/litellm-rust/docs/customer-proxy/azure-chat.svg b/litellm-rust/docs/customer-proxy/azure-chat.svg new file mode 100644 index 00000000000..9e389ac77ae --- /dev/null +++ b/litellm-rust/docs/customer-proxy/azure-chat.svg @@ -0,0 +1,87 @@ + + + + + + + + POST /v1/chat/completions to an azure/ deployment + + + + Client + OpenAI SDK body + + + + gateway-inference chat_completions + auth, hooks admit, route deployment, + litellm_params + base_model + drop_params + + + + + serves_openai? + openai | azure + + + + gateway startup (hooks::install) + an azure deployment loads the cost map and + installs it as ModelInfo for the param rules + + + + inference-chat run_openai, prepare_azure + 1. resolve os.environ/ refs and AZURE_* env through the secret source + 2. api_version: deployment, AZURE_API_VERSION, else 2025-02-01-preview + api_base: deployment, AZURE_API_BASE, AZURE_OPENAI_ENDPOINT + 3. AzureChatRequest::transform: family from base_model or deployment + (gpt, o-series, gpt-5), unsupported params refused or dropped + 4. url: {base}/openai/deployments/{dep}/chat/completions?api-version= + v1, latest, preview: {base}/openai/v1/chat/completions + 5. credential: azure_ad_token, else api-key, else Entra ID token + (client secret, OIDC, AZURE_AD_TOKEN, managed identity) + then the shared interceptors, send and SSE relay of the OpenAI path + + yes + + + + Azure OpenAI + api-key: key (deployment path) + Authorization: Bearer (AD token, v1) + application/json or text/event-stream + + + + + + Refused before the call + unsupported param without drop_params: 400 + Responses API bridge cases, Cloudflare bases: 501 + + + + usage and cost (gateway-inference Finish) + usage from the body or the include_usage chunk + config prices on the deployment win + base_model set: cost map azure/{base_model}, + then {base_model} + otherwise: azure/{model the response names}, + then azure/{deployment} + the bare deployment name is never priced, it names + a resource, not a model + + + errors + upstream 4xx/5xx: same status, OpenAI error envelope + carrying Azure's error.message + content filter (inner_error with a policy violation): + provider_specific_fields keeps innererror, + inner_error and the whole azure_error + stream cut after headers: a final + data: {"error": ...} event, then close + + +