mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
feat(gateway): Azure OpenAI chat completions through the OpenAI passthrough
Adds azure/<deployment> chat completions, streaming and not, on the same passthrough the openai/ route uses. The request goes to the deployment path with the api-version query, or to /openai/v1/ for the v1, latest and preview versions, with api_base, api_key and api_version falling back to the AZURE_* variables and os.environ references resolved as the proxy does. A key is sent as api-key, and an azure_ad_token or an Entra ID token from auth-azure goes as a Bearer token. The body follows the proxy's gpt, o-series and gpt-5 param rules, picked by base_model or the deployment name and reading model capabilities from the cost map, so an unsupported param is refused unless drop_params allows it to be dropped. Spend is priced as the proxy prices Azure, by azure/<base_model> when one is set and otherwise by the model the response names, and a content filter error keeps Azure's inner error for the client
This commit is contained in:
narmstrong-berry
GitHub
parent
2fa1d8e52b
commit
c4f3565193
29 changed files with 3402 additions and 174 deletions
|
|
@ -12,6 +12,9 @@ pub enum LlmProviders {
|
|||
Anthropic,
|
||||
#[strum(serialize = "aws_textract")]
|
||||
AwsTextract,
|
||||
/// Azure OpenAI (`azure/<deployment>`), distinct from Azure AI Foundry's `azure_ai`.
|
||||
#[strum(serialize = "azure")]
|
||||
Azure,
|
||||
#[strum(serialize = "azure_ai")]
|
||||
AzureAi,
|
||||
#[strum(serialize = "bedrock")]
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ use crate::usage::{ServiceTier, Usage};
|
|||
pub struct Prices {
|
||||
plans: HashMap<Arc<str>, PricingPlan>,
|
||||
max_output_tokens: HashMap<Arc<str>, u64>,
|
||||
/// Deployments priced by the model their response names, with the provider route the
|
||||
/// cost map keys that model under. The proxy prices an Azure deployment that sets neither
|
||||
/// `base_model` nor prices of its own this way.
|
||||
by_response_model: HashMap<Arc<str>, Arc<str>>,
|
||||
/// Cost map prices by model name, for the deployments priced by their response's model.
|
||||
models: HashMap<String, PricingPlan>,
|
||||
}
|
||||
|
||||
impl Prices {
|
||||
|
|
@ -27,18 +33,29 @@ impl Prices {
|
|||
if let Some(limit) = fields.get("max_output_tokens").and_then(Value::as_u64) {
|
||||
self.max_output_tokens.insert(id.clone(), limit);
|
||||
}
|
||||
let parsed = ParsedRates::parse(fields);
|
||||
if parsed.standard == Rates::EMPTY && parsed.thresholds.is_empty() {
|
||||
return;
|
||||
if let Some(plan) = plan(&id, fields) {
|
||||
self.plans.insert(id, plan);
|
||||
}
|
||||
match parsed.compile() {
|
||||
Ok(plan) => {
|
||||
self.plans.insert(id, plan);
|
||||
}
|
||||
Err(error) => tracing::warn!(deployment = %id, ?error, "ignoring invalid prices"),
|
||||
}
|
||||
|
||||
/// Prices `deployment` by the model its response names, looked up as
|
||||
/// `{provider}/{model}`, before its own prices.
|
||||
pub fn price_by_response_model(&mut self, deployment: Arc<str>, provider: &str) {
|
||||
self.by_response_model.insert(deployment, provider.into());
|
||||
}
|
||||
|
||||
/// Adds the cost map prices of `model`, for deployments priced by their response's model.
|
||||
pub fn insert_model(&mut self, model: &str, fields: &Map<String, Value>) {
|
||||
if let Some(plan) = plan(model, fields) {
|
||||
self.models.insert(model.to_owned(), plan);
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether some deployment is priced by its response's model.
|
||||
pub fn prices_by_response_model(&self) -> bool {
|
||||
!self.by_response_model.is_empty()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.plans.len()
|
||||
}
|
||||
|
|
@ -54,41 +71,78 @@ impl Prices {
|
|||
|
||||
/// The cost in dollars of `usage` on a deployment, or `None` when its prices are unknown.
|
||||
pub fn cost(&self, deployment: &str, usage: &Usage) -> Option<f64> {
|
||||
let plan = self.plans.get(deployment)?;
|
||||
let one_hour = usage.cache_write_1h_tokens.min(usage.cache_write_tokens);
|
||||
let split = (one_hour > 0).then_some(());
|
||||
let request = Request {
|
||||
usage: litellm_cost::Usage {
|
||||
prompt_tokens: usage.prompt_tokens(),
|
||||
completion_tokens: usage.output_tokens,
|
||||
cache_read_tokens: usage.cache_read_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
cache_write_5m_tokens: split.map(|()| usage.cache_write_tokens - one_hour),
|
||||
cache_write_1h_tokens: split.map(|()| one_hour),
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: match usage.service_tier {
|
||||
ServiceTier::Priority => CostTier::Priority,
|
||||
ServiceTier::Flex => CostTier::Flex,
|
||||
ServiceTier::Standard | ServiceTier::Batch => CostTier::Standard,
|
||||
},
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
match plan.calculate(&request) {
|
||||
Ok(cost) => Some(cost.total()),
|
||||
Err(error) => {
|
||||
tracing::debug!(deployment, ?error, "no cost for response");
|
||||
None
|
||||
}
|
||||
plan_cost(deployment, self.plans.get(deployment)?, usage)
|
||||
}
|
||||
|
||||
/// [`Self::cost`], first by `response_model` for a deployment priced by its response's
|
||||
/// model, as the proxy tries `{provider}/{response model}` ahead of the deployment.
|
||||
pub fn response_cost(
|
||||
&self,
|
||||
deployment: &str,
|
||||
response_model: Option<&str>,
|
||||
usage: &Usage,
|
||||
) -> Option<f64> {
|
||||
self.by_response_model
|
||||
.get(deployment)
|
||||
.zip(response_model.filter(|model| !model.is_empty()))
|
||||
.and_then(|(provider, model)| {
|
||||
let name = match model.starts_with(&format!("{provider}/")) {
|
||||
true => model.to_owned(),
|
||||
false => format!("{provider}/{model}"),
|
||||
};
|
||||
plan_cost(deployment, self.models.get(&name)?, usage)
|
||||
})
|
||||
.or_else(|| self.cost(deployment, usage))
|
||||
}
|
||||
}
|
||||
|
||||
/// The plan `fields` price, or `None` when they hold no usable prices.
|
||||
fn plan(name: &str, fields: &Map<String, Value>) -> Option<PricingPlan> {
|
||||
let parsed = ParsedRates::parse(fields);
|
||||
if parsed.standard == Rates::EMPTY && parsed.thresholds.is_empty() {
|
||||
return None;
|
||||
}
|
||||
parsed
|
||||
.compile()
|
||||
.inspect_err(|error| tracing::warn!(deployment = %name, ?error, "ignoring invalid prices"))
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn plan_cost(deployment: &str, plan: &PricingPlan, usage: &Usage) -> Option<f64> {
|
||||
let one_hour = usage.cache_write_1h_tokens.min(usage.cache_write_tokens);
|
||||
let split = (one_hour > 0).then_some(());
|
||||
let request = Request {
|
||||
usage: litellm_cost::Usage {
|
||||
prompt_tokens: usage.prompt_tokens(),
|
||||
completion_tokens: usage.output_tokens,
|
||||
cache_read_tokens: usage.cache_read_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
cache_write_5m_tokens: split.map(|()| usage.cache_write_tokens - one_hour),
|
||||
cache_write_1h_tokens: split.map(|()| one_hour),
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: match usage.service_tier {
|
||||
ServiceTier::Priority => CostTier::Priority,
|
||||
ServiceTier::Flex => CostTier::Flex,
|
||||
ServiceTier::Standard | ServiceTier::Batch => CostTier::Standard,
|
||||
},
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
match plan.calculate(&request) {
|
||||
Ok(cost) => Some(cost.total()),
|
||||
Err(error) => {
|
||||
tracing::debug!(deployment, ?error, "no cost for response");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A deployment's cost map fields: the cost map entry for its model, overlaid by the prices its
|
||||
/// config sets. The entry is found by `base_model`, then the model name, then the name without
|
||||
/// its provider route.
|
||||
/// its provider route. An Azure deployment's names are keyed under `azure/` first, and its bare
|
||||
/// deployment name is never looked up, since it names a resource of the user's, not a model.
|
||||
pub fn deployment_fields(
|
||||
catalog: Option<&Catalog>,
|
||||
model: &str,
|
||||
|
|
@ -97,14 +151,10 @@ pub fn deployment_fields(
|
|||
) -> Map<String, Value> {
|
||||
let mut fields = catalog
|
||||
.and_then(|catalog| {
|
||||
let candidates = base_model.into_iter().chain([
|
||||
model,
|
||||
model.split_once('/').map_or(model, |(_, rest)| rest),
|
||||
model.rsplit_once('/').map_or(model, |(_, rest)| rest),
|
||||
]);
|
||||
candidates
|
||||
candidates(model, base_model)
|
||||
.into_iter()
|
||||
.filter(|name| !name.is_empty())
|
||||
.find_map(|name| catalog.lookup(name))
|
||||
.find_map(|name| catalog.lookup(&name))
|
||||
})
|
||||
.map(|found| found.entry.fields().clone())
|
||||
.unwrap_or_default();
|
||||
|
|
@ -118,6 +168,30 @@ pub fn deployment_fields(
|
|||
fields
|
||||
}
|
||||
|
||||
fn candidates(model: &str, base_model: Option<&str>) -> Vec<String> {
|
||||
if model.starts_with(AZURE_ROUTE) {
|
||||
return base_model
|
||||
.into_iter()
|
||||
.flat_map(|base| match base.starts_with(AZURE_ROUTE) {
|
||||
true => vec![base.to_owned()],
|
||||
false => vec![format!("{AZURE_ROUTE}{base}"), base.to_owned()],
|
||||
})
|
||||
.chain([model.to_owned()])
|
||||
.collect();
|
||||
}
|
||||
base_model
|
||||
.into_iter()
|
||||
.chain([
|
||||
model,
|
||||
model.split_once('/').map_or(model, |(_, rest)| rest),
|
||||
model.rsplit_once('/').map_or(model, |(_, rest)| rest),
|
||||
])
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
const AZURE_ROUTE: &str = "azure/";
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Slot {
|
||||
Input,
|
||||
|
|
@ -382,4 +456,81 @@ mod tests {
|
|||
assert_eq!(fields["output_cost_per_token"], json!(2e-6));
|
||||
assert!(!fields.contains_key("api_key"));
|
||||
}
|
||||
|
||||
fn azure_catalog() -> Catalog {
|
||||
Catalog::parse(
|
||||
br#"{
|
||||
"gpt-4o": {"input_cost_per_token": 1e-6, "output_cost_per_token": 1e-6},
|
||||
"azure/gpt-4o": {"input_cost_per_token": 2e-6, "output_cost_per_token": 2e-6},
|
||||
"gpt-4o-mini": {"input_cost_per_token": 3e-6, "output_cost_per_token": 3e-6},
|
||||
"azure/my-dep": {"input_cost_per_token": 4e-6, "output_cost_per_token": 4e-6},
|
||||
"chat-dep": {"input_cost_per_token": 5e-6, "output_cost_per_token": 5e-6}
|
||||
}"#,
|
||||
Default::default(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::base_model_under_azure("azure/chat-dep", Some("gpt-4o"), Some(2e-6))]
|
||||
#[case::base_model_already_routed("azure/chat-dep", Some("azure/gpt-4o"), Some(2e-6))]
|
||||
#[case::bare_base_model("azure/chat-dep", Some("gpt-4o-mini"), Some(3e-6))]
|
||||
#[case::deployment_under_azure("azure/my-dep", None, Some(4e-6))]
|
||||
#[case::bare_deployment_is_not_a_model("azure/chat-dep", None, None)]
|
||||
fn azure_deployments_are_priced_under_the_azure_route(
|
||||
#[case] model: &str,
|
||||
#[case] base_model: Option<&str>,
|
||||
#[case] input: Option<f64>,
|
||||
) {
|
||||
let fields = deployment_fields(Some(&azure_catalog()), model, base_model, &[]);
|
||||
assert_eq!(
|
||||
fields.get("input_cost_per_token").and_then(Value::as_f64),
|
||||
input
|
||||
);
|
||||
}
|
||||
|
||||
fn response_priced(own: Value) -> Prices {
|
||||
let mut prices = Prices::default();
|
||||
prices.insert("d".into(), own.as_object().unwrap());
|
||||
prices.insert_model(
|
||||
"azure/gpt-4o",
|
||||
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 0.0})
|
||||
.as_object()
|
||||
.unwrap(),
|
||||
);
|
||||
prices.price_by_response_model("d".into(), "azure");
|
||||
prices
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::response_model_under_the_route(Some("gpt-4o"), Some(2e-6))]
|
||||
#[case::response_model_already_routed(Some("azure/gpt-4o"), Some(2e-6))]
|
||||
#[case::unknown_response_model_uses_the_deployment(Some("gpt-9"), Some(7e-6))]
|
||||
#[case::no_response_model_uses_the_deployment(None, Some(7e-6))]
|
||||
fn response_model_prices_come_first(
|
||||
#[case] response_model: Option<&str>,
|
||||
#[case] rate: Option<f64>,
|
||||
) {
|
||||
let prices =
|
||||
response_priced(json!({"input_cost_per_token": 7e-6, "output_cost_per_token": 0.0}));
|
||||
assert_eq!(
|
||||
prices.response_cost("d", response_model, &usage(1, 0)),
|
||||
rate
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn other_deployments_ignore_the_response_model() {
|
||||
let mut prices = response_priced(json!({}));
|
||||
prices.insert(
|
||||
"other".into(),
|
||||
json!({"input_cost_per_token": 7e-6, "output_cost_per_token": 0.0})
|
||||
.as_object()
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
prices.response_cost("other", Some("gpt-4o"), &usage(1, 0)),
|
||||
Some(7e-6)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -197,6 +197,7 @@ pub struct StreamUsage {
|
|||
line: Vec<u8>,
|
||||
state: Line,
|
||||
usage: Option<Usage>,
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
impl StreamUsage {
|
||||
|
|
@ -206,6 +207,7 @@ impl StreamUsage {
|
|||
line: Vec::new(),
|
||||
state: Line::Reading,
|
||||
usage: None,
|
||||
model: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -264,6 +266,11 @@ impl StreamUsage {
|
|||
self.usage
|
||||
}
|
||||
|
||||
/// The model named by the chunk that carried an OpenAI stream's usage.
|
||||
pub fn model(&self) -> Option<&str> {
|
||||
self.model.as_deref()
|
||||
}
|
||||
|
||||
fn may_carry_usage(&self) -> bool {
|
||||
let line = &self.line;
|
||||
if line.len() > MAX_LINE_BYTES {
|
||||
|
|
@ -304,8 +311,13 @@ impl StreamUsage {
|
|||
if !contains(data, b"\"usage\":{") && !contains(data, b"\"usage\": {") {
|
||||
return;
|
||||
}
|
||||
if let Ok(OpenAiChunk { usage: Some(usage) }) = serde_json::from_slice(data) {
|
||||
if let Ok(OpenAiChunk {
|
||||
usage: Some(usage),
|
||||
model,
|
||||
}) = serde_json::from_slice(data)
|
||||
{
|
||||
self.usage = Some(usage.usage());
|
||||
self.model = model.filter(|model| !model.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -326,6 +338,7 @@ struct AnthropicMessage {
|
|||
#[derive(Deserialize)]
|
||||
struct OpenAiChunk {
|
||||
usage: Option<OpenAiUsage>,
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
fn contains(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
|
|
@ -398,7 +411,7 @@ mod tests {
|
|||
fn openai_usage_comes_from_the_last_chunk() {
|
||||
let stream = concat!(
|
||||
"data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}],\"usage\":null}\n\n",
|
||||
"data: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":30,\"completion_tokens\":5,\"total_tokens\":35,\"prompt_tokens_details\":{\"cached_tokens\":10}}}\n\n",
|
||||
"data: {\"id\":\"c\",\"model\":\"gpt-4o-2024-11-20\",\"choices\":[],\"usage\":{\"prompt_tokens\":30,\"completion_tokens\":5,\"total_tokens\":35,\"prompt_tokens_details\":{\"cached_tokens\":10}}}\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
);
|
||||
let mut meter = StreamUsage::new(StreamFormat::OpenAi);
|
||||
|
|
@ -414,6 +427,7 @@ mod tests {
|
|||
..Usage::default()
|
||||
})
|
||||
);
|
||||
assert_eq!(meter.model(), Some("gpt-4o-2024-11-20"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -174,6 +174,24 @@ fn split(mut body: Map<String, Value>) -> (Value, Map<String, Value>) {
|
|||
(messages, body)
|
||||
}
|
||||
|
||||
/// The deployment's `litellm_params` a provider reads beyond its connection: its
|
||||
/// `api_version`, `base_model` and every key the config adds.
|
||||
fn provider_params(deployment: &Deployment) -> Map<String, Value> {
|
||||
let params = &deployment.litellm_params;
|
||||
let mut provider_params: Map<String, Value> = params
|
||||
.extra
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect();
|
||||
if let Some(api_version) = ¶ms.api_version {
|
||||
provider_params.insert("api_version".into(), api_version.clone().into());
|
||||
}
|
||||
if let Some(base_model) = &deployment.base_model {
|
||||
provider_params.insert("base_model".into(), base_model.clone().into());
|
||||
}
|
||||
provider_params
|
||||
}
|
||||
|
||||
async fn openai(
|
||||
gateway: &Gateway,
|
||||
deployment: &Deployment,
|
||||
|
|
@ -189,6 +207,14 @@ async fn openai(
|
|||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
extra_headers,
|
||||
timeout: deployment.timeout,
|
||||
litellm_params: provider_params(deployment),
|
||||
drop_params: gateway.drop_params()
|
||||
|| deployment
|
||||
.litellm_params
|
||||
.drop_params
|
||||
.as_ref()
|
||||
.is_some_and(|flag| flag.is_set()),
|
||||
model_info: gateway.model_info(),
|
||||
},
|
||||
None,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -121,12 +121,18 @@ impl Error {
|
|||
}
|
||||
_ => self.to_string(),
|
||||
};
|
||||
json!({"error": {
|
||||
let mut error = json!({
|
||||
"message": message,
|
||||
"type": error_type(status),
|
||||
"param": null,
|
||||
"code": status.as_u16(),
|
||||
}})
|
||||
});
|
||||
if let Self::Route(RouteError::Transport(TransportError::Http { body, .. })) = self
|
||||
&& let Some(fields) = azure_content_filter_fields(body)
|
||||
{
|
||||
error["provider_specific_fields"] = fields;
|
||||
}
|
||||
json!({ "error": error })
|
||||
}
|
||||
|
||||
/// The Anthropic error envelope Python's `AnthropicExceptionMapping` builds: an upstream
|
||||
|
|
@ -233,6 +239,44 @@ fn openai_error_message(body: &str) -> Option<String> {
|
|||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
/// Python's `AzureOpenAIExceptionMapping` keeps Azure's error and its content filter results
|
||||
/// on a content policy violation. Only Azure nests an `inner_error`, which tells its errors
|
||||
/// apart from another provider's.
|
||||
fn azure_content_filter_fields(body: &str) -> Option<Value> {
|
||||
const POLICY_PHRASES: [&str; 7] = [
|
||||
"content_policy_violation",
|
||||
"responsibleaipolicyviolation",
|
||||
"the response was filtered due to the prompt triggering azure openai's content management",
|
||||
"your task failed as a result of our safety system",
|
||||
"the model produced invalid content",
|
||||
"content_filter_policy",
|
||||
"your request was rejected as a result of our safety system",
|
||||
];
|
||||
let parsed: Value = serde_json::from_str(body).ok()?;
|
||||
let azure_error = parsed
|
||||
.get("error")
|
||||
.filter(|error| error.is_object())
|
||||
.unwrap_or(&parsed);
|
||||
let inner_error = ["inner_error", "innererror"].into_iter().find_map(|key| {
|
||||
azure_error
|
||||
.get(key)
|
||||
.or_else(|| parsed.get(key))
|
||||
.filter(|inner| !inner.is_null())
|
||||
})?;
|
||||
let lowered = body.to_lowercase();
|
||||
let violation = azure_error.get("code").and_then(Value::as_str)
|
||||
== Some("content_policy_violation")
|
||||
|| inner_error.get("code").and_then(Value::as_str) == Some("ResponsibleAIPolicyViolation")
|
||||
|| POLICY_PHRASES.iter().any(|phrase| lowered.contains(phrase));
|
||||
violation.then(|| {
|
||||
json!({
|
||||
"innererror": inner_error,
|
||||
"inner_error": inner_error,
|
||||
"azure_error": azure_error,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn provider_message(object: &Map<String, Value>) -> Option<&str> {
|
||||
if let Some(detail) = object.get("detail").and_then(Value::as_object) {
|
||||
return detail.get("message").and_then(Value::as_str);
|
||||
|
|
@ -326,6 +370,36 @@ mod tests {
|
|||
assert_eq!(frame, format!("data: {expected}\n\n"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::content_filter(
|
||||
r#"{"error":{"message":"The response was filtered due to the prompt triggering Azure OpenAI's content management policy.","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":true,"severity":"high"}}}}}"#,
|
||||
true,
|
||||
)]
|
||||
#[case::inner_error_spelling(
|
||||
r#"{"error":{"message":"blocked","code":"content_policy_violation","inner_error":{"code":"ResponsibleAIPolicyViolation"}}}"#,
|
||||
true,
|
||||
)]
|
||||
#[case::other_azure_error(
|
||||
r#"{"error":{"message":"bad value","code":"invalid_value","innererror":{"code":"Other"}}}"#,
|
||||
false
|
||||
)]
|
||||
#[case::openai_policy_error(
|
||||
r#"{"error":{"message":"rejected","code":"content_policy_violation","type":"invalid_request_error"}}"#,
|
||||
false,
|
||||
)]
|
||||
fn azure_content_filter_errors_keep_the_azure_error(#[case] body: &str, #[case] kept: bool) {
|
||||
let parsed: Value = serde_json::from_str(body).unwrap();
|
||||
let error = upstream(400, body).openai_body();
|
||||
let fields = &error["error"]["provider_specific_fields"];
|
||||
assert_eq!(!fields.is_null(), kept);
|
||||
if kept {
|
||||
assert_eq!(fields["azure_error"], parsed["error"]);
|
||||
assert_eq!(fields["innererror"], fields["inner_error"]);
|
||||
assert_eq!(fields["innererror"]["code"], "ResponsibleAIPolicyViolation");
|
||||
}
|
||||
assert_eq!(error["error"]["message"], parsed["error"]["message"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
|
||||
#[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
|
||||
|
|
|
|||
|
|
@ -306,12 +306,29 @@ impl Finish {
|
|||
}
|
||||
}
|
||||
|
||||
fn usage(&self) -> Option<Usage> {
|
||||
match &self.meter {
|
||||
Meter::Off => None,
|
||||
Meter::Stream(meter) => meter.usage(),
|
||||
Meter::Body(chunks) => body_usage(self.route, &chunks.concat()),
|
||||
}
|
||||
/// The usage the response reported, with its cost when the deployment is priced by the
|
||||
/// model its response names, which only a chat completion carries.
|
||||
fn usage_and_cost(&self) -> (Option<Usage>, Option<f64>) {
|
||||
let by_model = self.pipeline.prices().prices_by_response_model()
|
||||
&& self.route == Route::ChatCompletions;
|
||||
let (usage, model) = match &self.meter {
|
||||
Meter::Off => (None, None),
|
||||
Meter::Stream(meter) => (meter.usage(), meter.model().map(str::to_owned)),
|
||||
Meter::Body(chunks) => {
|
||||
let body = chunks.concat();
|
||||
let model = by_model.then(|| body_model(&body)).flatten();
|
||||
(body_usage(self.route, &body), model)
|
||||
}
|
||||
};
|
||||
let cost = by_model
|
||||
.then_some(())
|
||||
.zip(self.deployment.as_ref().zip(usage.as_ref()))
|
||||
.and_then(|((), (deployment, usage))| {
|
||||
self.pipeline
|
||||
.prices()
|
||||
.response_cost(&deployment.id, model.as_deref(), usage)
|
||||
});
|
||||
(usage, cost)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -320,11 +337,12 @@ impl Drop for Finish {
|
|||
let Some(admission) = self.admission.take() else {
|
||||
return;
|
||||
};
|
||||
let (usage, cost) = self.usage_and_cost();
|
||||
let outcome = Outcome {
|
||||
deployment: self.deployment.take(),
|
||||
status: self.status,
|
||||
usage: self.usage(),
|
||||
cost: None,
|
||||
usage,
|
||||
cost,
|
||||
time_to_first_byte: self.first_byte,
|
||||
upstream_started_at: self.upstream_started_at,
|
||||
ended_at: SystemTime::now(),
|
||||
|
|
@ -350,6 +368,17 @@ fn body_usage(route: Route, body: &[u8]) -> Option<Usage> {
|
|||
}
|
||||
}
|
||||
|
||||
/// The `model` a complete chat completion names, found by its first key like [`body_usage`].
|
||||
fn body_model(body: &[u8]) -> Option<String> {
|
||||
const KEY: &[u8] = b"\"model\":";
|
||||
let start = memchr::memmem::find(body, KEY)? + KEY.len();
|
||||
serde_json::Deserializer::from_slice(&body[start..])
|
||||
.into_iter::<String>()
|
||||
.next()?
|
||||
.ok()
|
||||
.filter(|model| !model.is_empty())
|
||||
}
|
||||
|
||||
struct Tapped {
|
||||
inner: BodyDataStream,
|
||||
finish: Finish,
|
||||
|
|
@ -401,4 +430,16 @@ mod tests {
|
|||
body_usage(Route::ChatCompletions, &serde_json::to_vec(&chat).unwrap()).unwrap();
|
||||
assert_eq!((usage.prompt_tokens(), usage.output_tokens), (5, 6));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::top_level(
|
||||
br#"{"id":"c","model":"gpt-4o-2024-11-20","choices":[]}"#,
|
||||
Some("gpt-4o-2024-11-20")
|
||||
)]
|
||||
#[case::spaced(br#"{"model": "gpt-4o", "choices":[]}"#, Some("gpt-4o"))]
|
||||
#[case::inside_content(br#"{"choices":[{"message":{"content":"\"model\":\"x\""}}]}"#, None)]
|
||||
#[case::empty(br#"{"model":"","choices":[]}"#, None)]
|
||||
fn body_model_reads_the_response_model(#[case] body: &[u8], #[case] expected: Option<&str>) {
|
||||
assert_eq!(body_model(body).as_deref(), expected);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,10 @@ use litellm_inference_messages::MessagesRoute;
|
|||
use litellm_inference_ocr::OcrRoute;
|
||||
use litellm_inference_responses::ResponsesRoute;
|
||||
use litellm_inference_transcription::AudioTranscriptionRoute;
|
||||
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
|
||||
use litellm_llms::base_llm::{
|
||||
model_info::ModelInfo,
|
||||
ocr::{handler::OcrClient, settings::OcrSettings},
|
||||
};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
||||
pub use error::Error;
|
||||
|
|
@ -36,6 +39,8 @@ pub use request::{JsonBody, JsonObject, RequestId};
|
|||
pub struct Gateway {
|
||||
cache: Option<Arc<dyn litellm_cache_response::ResponseCacheService>>,
|
||||
hooks: OnceLock<Hooks>,
|
||||
model_info: OnceLock<Arc<dyn ModelInfo>>,
|
||||
drop_params: bool,
|
||||
allow_client_side_credentials: bool,
|
||||
pub audio_transcription: AudioTranscriptionRoute,
|
||||
pub chat_completions: ChatCompletionsRoute,
|
||||
|
|
@ -67,6 +72,29 @@ impl Gateway {
|
|||
self.hooks.get()
|
||||
}
|
||||
|
||||
/// Gives providers whose param rules read model capabilities the model cost map. Only the
|
||||
/// first call takes effect.
|
||||
pub fn install_model_info(&self, model_info: Arc<dyn ModelInfo>) {
|
||||
let _ = self.model_info.set(model_info);
|
||||
}
|
||||
|
||||
pub(crate) fn model_info(&self) -> Option<Arc<dyn ModelInfo>> {
|
||||
self.model_info.get().cloned()
|
||||
}
|
||||
|
||||
/// `litellm_settings.drop_params`: drops params a provider does not support instead of
|
||||
/// refusing the request.
|
||||
pub fn with_drop_params(self, drop_params: bool) -> Self {
|
||||
Self {
|
||||
drop_params,
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn drop_params(&self) -> bool {
|
||||
self.drop_params
|
||||
}
|
||||
|
||||
/// `general_settings.allow_client_side_credentials`: lets a request body name the
|
||||
/// connection and credential params the proxy otherwise refuses.
|
||||
pub fn with_client_side_credentials(self, allowed: bool) -> Self {
|
||||
|
|
@ -98,6 +126,8 @@ impl Gateway {
|
|||
Ok(Self {
|
||||
cache: None,
|
||||
hooks: OnceLock::new(),
|
||||
model_info: OnceLock::new(),
|
||||
drop_params: false,
|
||||
allow_client_side_credentials: false,
|
||||
audio_transcription: AudioTranscriptionRoute::new(
|
||||
provider.clone(),
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
//! Builds the hooks and observers inference requests run through, and the per-deployment prices
|
||||
//! observers that read cost need.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
use litellm_config::Config;
|
||||
use litellm_config::{Config, Model};
|
||||
use litellm_gateway_hooks::{
|
||||
DEFAULT_QUEUE, Facts, Hook, Observer, Observers, Pipeline, Prices, deployment_fields,
|
||||
};
|
||||
use litellm_gateway_inference::{Gateway, Hooks};
|
||||
use litellm_llms::base_llm::model_info::ModelInfo;
|
||||
use litellm_model_catalog::{Catalog, Provenance};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -29,18 +30,26 @@ pub async fn install(
|
|||
hooks: Vec<Arc<dyn Hook>>,
|
||||
observers: Vec<Arc<dyn Observer>>,
|
||||
) -> Result<(), Error> {
|
||||
if hooks.is_empty() && observers.is_empty() {
|
||||
let serves_azure = config.model_list.iter().any(is_azure);
|
||||
if hooks.is_empty() && observers.is_empty() && !serves_azure {
|
||||
return Ok(());
|
||||
}
|
||||
let needs_cost = observers
|
||||
.iter()
|
||||
.any(|observer| observer.reads().contains(Facts::COST))
|
||||
|| hooks.iter().any(|hook| hook.reads().contains(Facts::COST));
|
||||
let catalog = match needs_cost || serves_azure {
|
||||
true => load_catalog(inference).await.map(Arc::new),
|
||||
false => None,
|
||||
};
|
||||
if let Some(catalog) = catalog.as_ref().filter(|_| serves_azure) {
|
||||
inference.install_model_info(Arc::new(CostMap::new(catalog.clone())));
|
||||
}
|
||||
if hooks.is_empty() && observers.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let prices = match needs_cost {
|
||||
true => {
|
||||
let catalog = load_catalog(inference).await;
|
||||
prices(config, inference, catalog.as_ref())
|
||||
}
|
||||
true => prices(config, inference, catalog.as_deref()),
|
||||
false => Prices::default(),
|
||||
};
|
||||
let names: Vec<_> = hooks
|
||||
|
|
@ -135,7 +144,9 @@ async fn fetch(inference: &Gateway, url: &str) -> Result<Vec<u8>, Box<dyn std::e
|
|||
}
|
||||
|
||||
/// Every deployment's prices: the cost map entry for its model, overlaid by the prices its
|
||||
/// `model_info` and `litellm_params` set, keyed by the router's deployment ids.
|
||||
/// `model_info` and `litellm_params` set, keyed by the router's deployment ids. An Azure
|
||||
/// deployment that names neither a `base_model` nor prices of its own is priced by the model
|
||||
/// its response names, as the proxy prices it.
|
||||
fn prices(config: &Config, inference: &Gateway, catalog: Option<&Catalog>) -> Prices {
|
||||
let mut prices = Prices::default();
|
||||
for (model, id) in config
|
||||
|
|
@ -151,18 +162,75 @@ fn prices(config: &Config, inference: &Gateway, catalog: Option<&Catalog>) -> Pr
|
|||
.iter()
|
||||
.map(|(name, value)| (name, value.clone())),
|
||||
);
|
||||
let base_model = model_info.get("base_model").and_then(Value::as_str);
|
||||
let fields = deployment_fields(
|
||||
catalog,
|
||||
&model.litellm_params.model,
|
||||
base_model,
|
||||
&[&model_info, ¶ms],
|
||||
);
|
||||
let base_model = params
|
||||
.get("base_model")
|
||||
.or_else(|| model_info.get("base_model"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|name| !name.is_empty());
|
||||
let name = match is_azure(model) && !model.litellm_params.model.starts_with(AZURE_ROUTE) {
|
||||
true => format!("{AZURE_ROUTE}{}", model.litellm_params.model),
|
||||
false => model.litellm_params.model.clone(),
|
||||
};
|
||||
let overrides = [&model_info, ¶ms];
|
||||
if is_azure(model)
|
||||
&& base_model.is_none()
|
||||
&& deployment_fields(None, &name, None, &overrides).is_empty()
|
||||
{
|
||||
prices.price_by_response_model(id.into(), AZURE_PROVIDER);
|
||||
}
|
||||
let fields = deployment_fields(catalog, &name, base_model, &overrides);
|
||||
prices.insert(id.into(), &fields);
|
||||
}
|
||||
if let Some(catalog) = catalog.filter(|_| prices.prices_by_response_model()) {
|
||||
for name in catalog
|
||||
.model_names()
|
||||
.filter(|name| name.starts_with(AZURE_ROUTE))
|
||||
{
|
||||
if let Some(found) = catalog.lookup(name) {
|
||||
prices.insert_model(name, found.entry.fields());
|
||||
}
|
||||
}
|
||||
}
|
||||
prices
|
||||
}
|
||||
|
||||
const AZURE_PROVIDER: &str = "azure";
|
||||
const AZURE_ROUTE: &str = "azure/";
|
||||
|
||||
fn is_azure(model: &Model) -> bool {
|
||||
match model.litellm_params.custom_llm_provider.as_deref() {
|
||||
Some(provider) => provider == AZURE_PROVIDER,
|
||||
None => model.litellm_params.model.starts_with(AZURE_ROUTE),
|
||||
}
|
||||
}
|
||||
|
||||
/// The model cost map as the capabilities Azure's param rules read.
|
||||
struct CostMap {
|
||||
catalog: Arc<Catalog>,
|
||||
keys: HashSet<String>,
|
||||
}
|
||||
|
||||
impl CostMap {
|
||||
fn new(catalog: Arc<Catalog>) -> Self {
|
||||
let keys = catalog
|
||||
.model_names()
|
||||
.filter_map(|name| catalog.lookup(name))
|
||||
.flat_map(|found| found.entry.fields().keys().cloned().collect::<Vec<_>>())
|
||||
.collect();
|
||||
Self { catalog, keys }
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelInfo for CostMap {
|
||||
fn entry(&self, model: &str) -> Option<&Map<String, Value>> {
|
||||
self.catalog.lookup(model).map(|found| found.entry.fields())
|
||||
}
|
||||
|
||||
fn declares(&self, key: &str) -> bool {
|
||||
self.keys.contains(key)
|
||||
}
|
||||
}
|
||||
|
||||
fn json_object<'a, V: serde::Serialize + 'a>(
|
||||
entries: impl Iterator<Item = (&'a String, V)>,
|
||||
) -> Map<String, Value> {
|
||||
|
|
|
|||
|
|
@ -79,7 +79,14 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
|||
let resources = CoreResources::new(pool);
|
||||
Ok(Arc::new(
|
||||
Gateway::new(resources, http, secrets, ModelRouter::from_config(config)?)?
|
||||
.with_client_side_credentials(allows_client_side_credentials(config)),
|
||||
.with_client_side_credentials(allows_client_side_credentials(config))
|
||||
.with_drop_params(
|
||||
config
|
||||
.litellm_settings
|
||||
.drop_params
|
||||
.as_ref()
|
||||
.is_some_and(|flag| flag.is_set()),
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
|
|
|
|||
293
litellm-rust/crates/gateway/tests/azure_chat.rs
Normal file
293
litellm-rust/crates/gateway/tests/azure_chat.rs
Normal file
|
|
@ -0,0 +1,293 @@
|
|||
//! Azure OpenAI chat completions through the gateway: the deployment URL, `api-version` and
|
||||
//! `api-key` the upstream sees, and the usage and cost the request is recorded with.
|
||||
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{Arc, Mutex, Once},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, header::CONTENT_TYPE},
|
||||
response::IntoResponse,
|
||||
};
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_config::Config;
|
||||
use litellm_gateway_hooks::{Facts, Observer, Record};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
use tower::ServiceExt;
|
||||
|
||||
const RESPONSE_MODEL: &str = "gpt-4o-2024-11-20";
|
||||
const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-4o-2024-11-20","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":6,"total_tokens":11}}"#;
|
||||
const COMPLETION_STREAM: &str = concat!(
|
||||
"data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4o-2024-11-20\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}],\"usage\":null}\n\n",
|
||||
"data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4o-2024-11-20\",\"choices\":[],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3,\"total_tokens\":10}}\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
);
|
||||
|
||||
/// Prices the cost map gives the model a response names, and the `base_model` one deployment
|
||||
/// declares, so the two pricing routes give different costs.
|
||||
const RESPONSE_MODEL_RATES: (f64, f64) = (1.0, 2.0);
|
||||
const BASE_MODEL_RATES: (f64, f64) = (10.0, 20.0);
|
||||
|
||||
/// The proxy reads its cost map from this variable, and every test sets it before building a
|
||||
/// gateway, so no test reads the environment while it changes.
|
||||
fn use_local_cost_map() {
|
||||
static ONCE: Once = Once::new();
|
||||
ONCE.call_once(|| {
|
||||
let map = json!({
|
||||
format!("azure/{RESPONSE_MODEL}"): {
|
||||
"litellm_provider": "azure",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": RESPONSE_MODEL_RATES.0,
|
||||
"output_cost_per_token": RESPONSE_MODEL_RATES.1,
|
||||
},
|
||||
"azure/gpt-4o": {
|
||||
"litellm_provider": "azure",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": BASE_MODEL_RATES.0,
|
||||
"output_cost_per_token": BASE_MODEL_RATES.1,
|
||||
},
|
||||
});
|
||||
let mut file = tempfile::NamedTempFile::new().unwrap();
|
||||
file.write_all(map.to_string().as_bytes()).unwrap();
|
||||
let (_, path) = file.keep().unwrap();
|
||||
// SAFETY: set once, before any test builds a gateway or spawns a thread that reads the
|
||||
// environment.
|
||||
unsafe { std::env::set_var("LITELLM_MODEL_COST_MAP_PATH", path) };
|
||||
});
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct Sent {
|
||||
path: String,
|
||||
query: Option<String>,
|
||||
api_key: Option<String>,
|
||||
authorization: Option<String>,
|
||||
body: Value,
|
||||
}
|
||||
|
||||
type Calls = Arc<Mutex<Vec<Sent>>>;
|
||||
|
||||
async fn upstream(calls: Calls) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let app = Router::new().fallback(move |request: Request<Body>| {
|
||||
let calls = calls.clone();
|
||||
async move {
|
||||
let (parts, body) = request.into_parts();
|
||||
let header = |name: &str| {
|
||||
parts
|
||||
.headers
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned)
|
||||
};
|
||||
let (api_key, authorization) = (header("api-key"), header("authorization"));
|
||||
let path = parts.uri.path().to_owned();
|
||||
let query = parts.uri.query().map(str::to_owned);
|
||||
let body: Value =
|
||||
serde_json::from_slice(&to_bytes(body, 1 << 20).await.unwrap()).unwrap();
|
||||
let stream = body["stream"].as_bool().unwrap_or(false);
|
||||
calls.lock().unwrap().push(Sent {
|
||||
path,
|
||||
query,
|
||||
api_key,
|
||||
authorization,
|
||||
body,
|
||||
});
|
||||
match stream {
|
||||
true => ([(CONTENT_TYPE, "text/event-stream")], COMPLETION_STREAM).into_response(),
|
||||
false => ([(CONTENT_TYPE, "application/json")], COMPLETION).into_response(),
|
||||
}
|
||||
}
|
||||
});
|
||||
tokio::spawn(async move { axum::serve(listener, app).await });
|
||||
format!("http://{address}")
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Seen {
|
||||
deployment: Option<String>,
|
||||
usage: Option<(u64, u64)>,
|
||||
cost: Option<f64>,
|
||||
}
|
||||
|
||||
struct Collect(Mutex<Vec<Seen>>);
|
||||
|
||||
impl Observer for Collect {
|
||||
fn name(&self) -> &'static str {
|
||||
"collect"
|
||||
}
|
||||
|
||||
fn reads(&self) -> Facts {
|
||||
Facts::DEPLOYMENT | Facts::USAGE | Facts::COST
|
||||
}
|
||||
|
||||
fn observe<'a>(&'a self, record: &'a Record) -> BoxFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
self.0.lock().unwrap().push(Seen {
|
||||
deployment: record
|
||||
.outcome
|
||||
.deployment
|
||||
.as_ref()
|
||||
.map(|deployment| deployment.id.to_string()),
|
||||
usage: record
|
||||
.outcome
|
||||
.usage
|
||||
.map(|usage| (usage.prompt_tokens(), usage.output_tokens)),
|
||||
cost: record.outcome.cost,
|
||||
});
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct Harness {
|
||||
app: Router,
|
||||
inference: Arc<litellm_gateway_inference::Gateway>,
|
||||
calls: Calls,
|
||||
seen: Arc<Collect>,
|
||||
}
|
||||
|
||||
async fn harness() -> Harness {
|
||||
use_local_cost_map();
|
||||
let calls = Calls::default();
|
||||
let upstream = upstream(calls.clone()).await;
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list:
|
||||
- model_name: chat
|
||||
litellm_params:
|
||||
model: azure/chat-prod
|
||||
api_base: {upstream}
|
||||
api_key: sk-azure
|
||||
api_version: 2024-10-21
|
||||
model_info:
|
||||
id: azure-1
|
||||
- model_name: chat-base
|
||||
litellm_params:
|
||||
model: azure/chat-base
|
||||
api_base: {upstream}/
|
||||
api_key: sk-azure
|
||||
api_version: 2024-10-21
|
||||
base_model: gpt-4o
|
||||
model_info:
|
||||
id: azure-2
|
||||
router_settings:
|
||||
num_retries: 0
|
||||
general_settings:
|
||||
master_key: sk-master
|
||||
"
|
||||
))
|
||||
.unwrap();
|
||||
let inference = litellm_gateway::build_inference(&config).unwrap();
|
||||
let seen = Arc::new(Collect(Mutex::new(Vec::new())));
|
||||
litellm_gateway::hooks::install(&inference, &config, Vec::new(), vec![seen.clone()])
|
||||
.await
|
||||
.unwrap();
|
||||
let app = litellm_gateway::router(inference.clone(), &config, None, None);
|
||||
Harness {
|
||||
app,
|
||||
inference,
|
||||
calls,
|
||||
seen,
|
||||
}
|
||||
}
|
||||
|
||||
async fn send(app: &Router, body: Value) -> (u16, String) {
|
||||
let request = Request::post("/v1/chat/completions")
|
||||
.header("content-type", "application/json")
|
||||
.header("authorization", "Bearer sk-master")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap();
|
||||
let response = app.clone().oneshot(request).await.unwrap();
|
||||
let status = response.status().as_u16();
|
||||
let body = to_bytes(response.into_body(), 1 << 20).await.unwrap();
|
||||
(status, String::from_utf8(body.to_vec()).unwrap())
|
||||
}
|
||||
|
||||
fn close(left: f64, right: f64) -> bool {
|
||||
(left - right).abs() < 1e-9
|
||||
}
|
||||
|
||||
fn cost((input, output): (u64, u64), (input_rate, output_rate): (f64, f64)) -> f64 {
|
||||
input as f64 * input_rate + output as f64 * output_rate
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unary_by_response_model("chat", false, "chat-prod", "azure-1", (5, 6), RESPONSE_MODEL_RATES)]
|
||||
#[case::stream_by_response_model("chat", true, "chat-prod", "azure-1", (7, 3), RESPONSE_MODEL_RATES)]
|
||||
#[case::unary_by_base_model("chat-base", false, "chat-base", "azure-2", (5, 6), BASE_MODEL_RATES)]
|
||||
#[case::stream_by_base_model("chat-base", true, "chat-base", "azure-2", (7, 3), BASE_MODEL_RATES)]
|
||||
#[tokio::test]
|
||||
async fn azure_chat_reaches_the_deployment_and_is_priced(
|
||||
#[case] model: &str,
|
||||
#[case] stream: bool,
|
||||
#[case] deployment: &str,
|
||||
#[case] id: &str,
|
||||
#[case] usage: (u64, u64),
|
||||
#[case] rates: (f64, f64),
|
||||
) {
|
||||
let harness = harness().await;
|
||||
let mut request = json!({
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 8,
|
||||
"user": "end-user-1",
|
||||
});
|
||||
if stream {
|
||||
request["stream"] = json!(true);
|
||||
request["stream_options"] = json!({"include_usage": true});
|
||||
}
|
||||
let (status, body) = send(&harness.app, request).await;
|
||||
assert_eq!(status, 200, "{body}");
|
||||
match stream {
|
||||
true => assert!(body.contains("\"content\":\"hi\""), "{body}"),
|
||||
false => assert_eq!(
|
||||
serde_json::from_str::<Value>(&body).unwrap()["model"],
|
||||
RESPONSE_MODEL
|
||||
),
|
||||
}
|
||||
|
||||
let sent = harness.calls.lock().unwrap()[0].clone();
|
||||
assert_eq!(
|
||||
sent.path,
|
||||
format!("/openai/deployments/{deployment}/chat/completions")
|
||||
);
|
||||
assert_eq!(sent.query.as_deref(), Some("api-version=2024-10-21"));
|
||||
assert_eq!(sent.api_key.as_deref(), Some("sk-azure"));
|
||||
assert_eq!(sent.authorization, None);
|
||||
assert_eq!(sent.body["model"], deployment);
|
||||
assert_eq!(sent.body["max_tokens"], 8);
|
||||
assert_eq!(sent.body["user"], "end-user-1");
|
||||
assert_eq!(sent.body["stream"].as_bool().unwrap_or(false), stream);
|
||||
|
||||
litellm_gateway::hooks::close(&harness.inference).await;
|
||||
let seen = harness.seen.0.lock().unwrap();
|
||||
let [seen] = seen.as_slice() else {
|
||||
panic!("{seen:?}")
|
||||
};
|
||||
assert_eq!(seen.deployment.as_deref(), Some(id));
|
||||
assert_eq!(seen.usage, Some(usage));
|
||||
assert!(close(seen.cost.unwrap(), cost(usage, rates)), "{seen:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_unsupported_param_is_refused_before_the_call() {
|
||||
let harness = harness().await;
|
||||
let (status, body) = send(
|
||||
&harness.app,
|
||||
json!({
|
||||
"model": "chat",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"reasoning_effort": "high",
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, 400, "{body}");
|
||||
assert!(body.contains("reasoning_effort"), "{body}");
|
||||
assert!(harness.calls.lock().unwrap().is_empty());
|
||||
}
|
||||
475
litellm-rust/crates/inference-chat/src/azure.rs
Normal file
475
litellm-rust/crates/inference-chat/src/azure.rs
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
//! Azure OpenAI chat completions through the OpenAI passthrough: `litellm/llms/azure/azure.py`
|
||||
//! with the client `BaseAzureLLM.get_azure_openai_client` builds. The deployment's settings
|
||||
//! and `os.environ/` references resolve through the secret source, and an Entra ID token is
|
||||
//! acquired through `litellm_auth_azure` when the deployment has no key.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_auth::azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAMES};
|
||||
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
|
||||
use litellm_llms::{
|
||||
Error as LlmError,
|
||||
azure::{
|
||||
chat::transformation::{AzureChatBody, AzureChatRequest},
|
||||
common_utils::{self, AZURE_API_KEY_ENV, AzureCredential},
|
||||
},
|
||||
base_llm::auth::resolve_auth,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
ChatCompletionsRoute, Error,
|
||||
common_utils::string_headers,
|
||||
openai::{OpenAiChatCall, PreparedCall},
|
||||
};
|
||||
|
||||
const ENV_REFERENCE: &str = "os.environ/";
|
||||
|
||||
/// The `litellm_params` an `os.environ/` reference may stand in for.
|
||||
const REFERABLE_PARAMS: &[&str] = &[
|
||||
"api_version",
|
||||
"base_model",
|
||||
"azure_ad_token",
|
||||
"tenant_id",
|
||||
"client_id",
|
||||
"client_secret",
|
||||
"azure_scope",
|
||||
"azure_authority_host",
|
||||
"azure_credential",
|
||||
"azure_federated_token_file",
|
||||
];
|
||||
|
||||
impl ChatCompletionsRoute {
|
||||
pub(crate) async fn prepare_azure(
|
||||
&self,
|
||||
deployment: String,
|
||||
call: OpenAiChatCall,
|
||||
) -> Result<PreparedCall, Error> {
|
||||
let OpenAiChatCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
extra_headers,
|
||||
litellm_params,
|
||||
drop_params,
|
||||
model_info,
|
||||
..
|
||||
} = call;
|
||||
let references: Vec<String> = [api_key.as_deref(), api_base.as_deref()]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.chain(
|
||||
REFERABLE_PARAMS
|
||||
.iter()
|
||||
.filter_map(|name| litellm_params.get(*name).and_then(Value::as_str)),
|
||||
)
|
||||
.filter_map(|value| value.strip_prefix(ENV_REFERENCE))
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
let names: Vec<&str> = common_utils::SECRET_NAMES
|
||||
.iter()
|
||||
.chain(AZURE_AUTH_SECRET_NAMES)
|
||||
.copied()
|
||||
.chain(references.iter().map(String::as_str))
|
||||
.collect();
|
||||
let secrets = self.secrets.resolve(&names).await?;
|
||||
let lookup = |key: &str| secrets.get(key);
|
||||
// Python's router reads every `os.environ/` setting from the environment when it loads
|
||||
// the deployment. A reference to an unset variable counts as unset.
|
||||
let resolve = |value: &str| match value.strip_prefix(ENV_REFERENCE) {
|
||||
Some(name) => lookup(name),
|
||||
None => Some(value.to_owned()),
|
||||
};
|
||||
let params: Map<String, Value> = litellm_params
|
||||
.into_iter()
|
||||
.map(|(name, value)| match value {
|
||||
Value::String(text) if REFERABLE_PARAMS.contains(&name.as_str()) => {
|
||||
let value = resolve(&text).map_or(Value::Null, Value::String);
|
||||
(name, value)
|
||||
}
|
||||
other => (name, other),
|
||||
})
|
||||
.collect();
|
||||
let configured = |name: &str| params.get(name).and_then(Value::as_str);
|
||||
|
||||
let api_version = common_utils::resolve_api_version(configured("api_version"), &lookup);
|
||||
let api_base = common_utils::resolve_api_base(
|
||||
api_base.as_deref().and_then(resolve).as_deref(),
|
||||
&lookup,
|
||||
)?;
|
||||
let api_key =
|
||||
common_utils::resolve_api_key(api_key.as_deref().and_then(resolve).as_deref(), &lookup);
|
||||
let model_info = model_info.as_deref();
|
||||
let AzureChatBody {
|
||||
deployment: url_deployment,
|
||||
body,
|
||||
} = AzureChatRequest {
|
||||
deployment: &deployment,
|
||||
base_model: configured("base_model"),
|
||||
api_version: &api_version,
|
||||
drop_params,
|
||||
model_info,
|
||||
}
|
||||
.transform(body)?;
|
||||
let url = common_utils::complete_url(&api_base, &url_deployment, &api_version)?;
|
||||
|
||||
let credential = self
|
||||
.azure_credential(api_key.clone(), ¶ms, &lookup)
|
||||
.await?;
|
||||
let environment = common_utils::environment(
|
||||
string_headers(extra_headers)?,
|
||||
credential,
|
||||
common_utils::is_v1_api_version(&api_version),
|
||||
);
|
||||
let authenticated = resolve_auth(&self.auth, environment, &lookup).await?;
|
||||
Ok(PreparedCall {
|
||||
model: url_deployment,
|
||||
provider: <&str>::from(LlmProviders::Azure),
|
||||
url,
|
||||
headers: authenticated.headers,
|
||||
body,
|
||||
api_key,
|
||||
})
|
||||
}
|
||||
|
||||
/// The SDK sends a configured `azure_ad_token` ahead of a key. Without either, LiteLLM
|
||||
/// gets a token from Entra ID: client credentials, an OIDC exchange, or managed identity
|
||||
/// when `azure_credential` or `enable_azure_ad_token_refresh` asks for it.
|
||||
async fn azure_credential(
|
||||
&self,
|
||||
api_key: Option<String>,
|
||||
params: &Map<String, Value>,
|
||||
lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<AzureCredential, Error> {
|
||||
let configured_token = params
|
||||
.get("azure_ad_token")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|token| !token.is_empty());
|
||||
if let Some(key) = &api_key
|
||||
&& !configured_token
|
||||
{
|
||||
return Ok(AzureCredential::ApiKey(litellm_auth::SecretValue::new(
|
||||
key.clone(),
|
||||
)));
|
||||
}
|
||||
let inputs = AzureAuthInputs::from_sourced_optional_params(params, &BTreeMap::new())?;
|
||||
let token = self.auth.azure.get_azure_ad_token(&inputs, lookup).await?;
|
||||
match (token, api_key) {
|
||||
(Some(token), _) => Ok(AzureCredential::AdToken(
|
||||
token.into_value().secret().clone(),
|
||||
)),
|
||||
(None, Some(key)) => Ok(AzureCredential::ApiKey(litellm_auth::SecretValue::new(key))),
|
||||
(None, None) => Err(Error::from(LlmError::Auth(
|
||||
litellm_auth::Error::MissingApiKey {
|
||||
provider: "Azure",
|
||||
environment_variable: AZURE_API_KEY_ENV,
|
||||
},
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::TryStreamExt;
|
||||
use litellm_host::call::CallOutput;
|
||||
use litellm_http::transport;
|
||||
use litellm_inference_testing::RecordingSecrets;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{body_partial_json, header, method, path, query_param},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::test_support::{Recording, route, run};
|
||||
|
||||
const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-4o-2024-08-06","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}"#;
|
||||
const STREAM: &str = "data: {\"id\":\"c\",\"model\":\"gpt-4o-2024-08-06\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\ndata: {\"id\":\"c\",\"model\":\"gpt-4o-2024-08-06\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1,\"total_tokens\":4}}\n\ndata: [DONE]\n\n";
|
||||
const DEPLOYMENT_PATH: &str = "/openai/deployments/gpt-4o/chat/completions";
|
||||
|
||||
fn call(api_base: &str, body: Value) -> OpenAiChatCall {
|
||||
let Value::Object(body) = body else {
|
||||
panic!("the body is an object");
|
||||
};
|
||||
OpenAiChatCall {
|
||||
model: "azure/gpt-4o".into(),
|
||||
body,
|
||||
api_key: Some("sk-azure".into()),
|
||||
api_base: Some(format!("{api_base}/")),
|
||||
litellm_params: Map::from_iter([("api_version".into(), json!("2024-10-21"))]),
|
||||
timeout: Some(Duration::from_secs(5)),
|
||||
..OpenAiChatCall::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn hi() -> Value {
|
||||
json!({"model": "public", "messages": [{"role": "user", "content": "hi"}]})
|
||||
}
|
||||
|
||||
fn with_secrets(route: ChatCompletionsRoute, values: &[(&str, &str)]) -> ChatCompletionsRoute {
|
||||
ChatCompletionsRoute {
|
||||
secrets: Arc::new(RecordingSecrets::new(values.iter().copied())),
|
||||
..route
|
||||
}
|
||||
}
|
||||
|
||||
async fn complete(output: crate::openai::OpenAiChatOutput) -> Bytes {
|
||||
match output {
|
||||
CallOutput::Complete(body) => body,
|
||||
CallOutput::Stream { .. } => panic!("expected a complete body"),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_completion_goes_to_the_deployment_with_its_key(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path(DEPLOYMENT_PATH))
|
||||
.and(query_param("api-version", "2024-10-21"))
|
||||
.and(header("api-key", "sk-azure"))
|
||||
.and(body_partial_json(json!({"model": "gpt-4o"})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let interceptors = Recording::default();
|
||||
|
||||
let output = run(&route, call(&upstream.uri(), hi()), &interceptors)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(complete(output).await, COMPLETION.as_bytes());
|
||||
let requests = upstream.received_requests().await.unwrap();
|
||||
assert!(!requests[0].headers.contains_key("authorization"));
|
||||
let contexts = interceptors.contexts.lock().unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
contexts[0].model.as_str(),
|
||||
contexts[0].custom_llm_provider.as_str()
|
||||
),
|
||||
("gpt-4o", "azure")
|
||||
);
|
||||
assert_eq!(*interceptors.raw.lock().unwrap(), [COMPLETION]);
|
||||
assert_eq!(
|
||||
interceptors.results.lock().unwrap()[0].provider.provider,
|
||||
"azure"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_stream_relays_every_byte(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(path(DEPLOYMENT_PATH))
|
||||
.and(body_partial_json(
|
||||
json!({"stream": true, "stream_options": {"include_usage": true}}),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(STREAM, "text/event-stream"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let mut body = hi();
|
||||
body["stream"] = json!(true);
|
||||
body["stream_options"] = json!({"include_usage": true});
|
||||
|
||||
let output = run(&route, call(&upstream.uri(), body), &Recording::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let CallOutput::Stream { chunks, .. } = output else {
|
||||
panic!("a streaming call streams");
|
||||
};
|
||||
let relayed = chunks.try_collect::<Vec<_>>().await.unwrap().concat();
|
||||
assert_eq!(relayed, STREAM.as_bytes());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn the_v1_surface_takes_the_key_as_a_bearer(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(path("/openai/v1/chat/completions"))
|
||||
.and(header("authorization", "Bearer sk-azure"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let mut v1 = call(&upstream.uri(), hi());
|
||||
v1.litellm_params
|
||||
.insert("api_version".into(), json!("latest"));
|
||||
|
||||
run(&route, v1, &Recording::default()).await.unwrap();
|
||||
|
||||
let requests = upstream.received_requests().await.unwrap();
|
||||
assert_eq!(requests[0].url.query(), None);
|
||||
assert!(!requests[0].headers.contains_key("api-key"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn base_key_and_version_fall_back_to_the_environment(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(path(DEPLOYMENT_PATH))
|
||||
.and(query_param("api-version", "2024-06-01"))
|
||||
.and(header("api-key", "sk-env"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let base = upstream.uri();
|
||||
let route = with_secrets(
|
||||
route,
|
||||
&[
|
||||
("AZURE_API_BASE", &base),
|
||||
("AZURE_API_KEY", "sk-env"),
|
||||
("AZURE_API_VERSION", "2024-06-01"),
|
||||
],
|
||||
);
|
||||
let bare = OpenAiChatCall {
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
litellm_params: Map::new(),
|
||||
..call(&base, hi())
|
||||
};
|
||||
|
||||
run(&route, bare, &Recording::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn the_default_api_version_is_pythons(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(query_param(
|
||||
"api-version",
|
||||
common_utils::AZURE_DEFAULT_API_VERSION,
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let mut unversioned = call(&upstream.uri(), hi());
|
||||
unversioned.litellm_params.clear();
|
||||
|
||||
run(&route, unversioned, &Recording::default())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn environment_references_resolve(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(query_param("api-version", "2025-01-01-preview"))
|
||||
.and(header("api-key", "sk-referenced"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let route = with_secrets(
|
||||
route,
|
||||
&[
|
||||
("MY_AZURE_KEY", "sk-referenced"),
|
||||
("MY_AZURE_VERSION", "2025-01-01-preview"),
|
||||
],
|
||||
);
|
||||
let mut referenced = call(&upstream.uri(), hi());
|
||||
referenced.api_key = Some("os.environ/MY_AZURE_KEY".into());
|
||||
referenced
|
||||
.litellm_params
|
||||
.insert("api_version".into(), json!("os.environ/MY_AZURE_VERSION"));
|
||||
|
||||
run(&route, referenced, &Recording::default())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::configured_token_outranks_the_key(Some("sk-azure"), &[("azure_ad_token", "configured-token")], &[])]
|
||||
#[case::environment_token(None, &[], &[("AZURE_AD_TOKEN", "configured-token")])]
|
||||
#[tokio::test]
|
||||
async fn an_ad_token_goes_as_a_bearer(
|
||||
route: ChatCompletionsRoute,
|
||||
#[case] api_key: Option<&str>,
|
||||
#[case] params: &[(&str, &str)],
|
||||
#[case] environment: &[(&str, &str)],
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(header("authorization", "Bearer configured-token"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(COMPLETION, "application/json"))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let route = with_secrets(route, environment);
|
||||
let mut tokened = call(&upstream.uri(), hi());
|
||||
tokened.api_key = api_key.map(str::to_owned);
|
||||
tokened.litellm_params.extend(
|
||||
params
|
||||
.iter()
|
||||
.map(|(name, value)| ((*name).to_owned(), json!(value))),
|
||||
);
|
||||
|
||||
run(&route, tokened, &Recording::default()).await.unwrap();
|
||||
|
||||
let requests = upstream.received_requests().await.unwrap();
|
||||
assert!(!requests[0].headers.contains_key("api-key"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_missing_credential_fails_before_any_request(route: ChatCompletionsRoute) {
|
||||
let mut keyless = call("http://127.0.0.1:1", hi());
|
||||
keyless.api_key = None;
|
||||
assert_eq!(
|
||||
run(&route, keyless, &Recording::default()).await.err(),
|
||||
Some(Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Azure",
|
||||
environment_variable: "AZURE_API_KEY",
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_unsupported_param_fails_before_any_request(route: ChatCompletionsRoute) {
|
||||
let mut body = hi();
|
||||
body["reasoning_effort"] = json!("high");
|
||||
assert!(matches!(
|
||||
run(
|
||||
&route,
|
||||
call("http://127.0.0.1:1", body),
|
||||
&Recording::default()
|
||||
)
|
||||
.await,
|
||||
Err(Error::InvalidRequest(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_content_filter_error_keeps_azures_body(route: ChatCompletionsRoute) {
|
||||
let upstream = MockServer::start().await;
|
||||
let body = r#"{"error":{"message":"The response was filtered","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":true,"severity":"high"}}}}}"#;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(400).set_body_raw(body, "application/json"))
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
|
||||
let error = run(&route, call(&upstream.uri(), hi()), &Recording::default())
|
||||
.await
|
||||
.err()
|
||||
.expect("the upstream failure fails the call");
|
||||
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::Transport(transport::Error::Http {
|
||||
status: 400,
|
||||
body: body.into()
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
mod azure;
|
||||
pub mod route;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
pub mod types;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
mod common_utils;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::{convert::Infallible, time::Duration};
|
||||
use std::{convert::Infallible, sync::Arc, time::Duration};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::{StreamExt, stream::BoxStream};
|
||||
|
|
@ -15,7 +15,10 @@ use litellm_host::{
|
|||
use litellm_http::{outbound::OutboundRequest, request::truncate_error_body, transport};
|
||||
use litellm_inference::{CallOptions, context::CallContext, provider::resolve_llm_provider};
|
||||
use litellm_llms::{
|
||||
base_llm::auth::resolve_auth,
|
||||
base_llm::{
|
||||
auth::{Headers, resolve_auth},
|
||||
model_info::ModelInfo,
|
||||
},
|
||||
openai::chat::transformation::{OPENAI_CHAT_COMPLETIONS_CONFIG, is_stream},
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -39,8 +42,10 @@ impl Protocol for OpenAiChatCompletions {
|
|||
|
||||
pub type OpenAiChatOutput = CallOutput<Bytes, (), Bytes, Error>;
|
||||
|
||||
/// A Chat Completions body bound for an OpenAI deployment. `body` is the caller's request;
|
||||
/// `model` is the deployment's, with or without its `openai/` prefix.
|
||||
/// A Chat Completions body bound for an OpenAI or Azure OpenAI deployment. `body` is the
|
||||
/// caller's request; `model` is the deployment's, with or without its `openai/` or `azure/`
|
||||
/// prefix.
|
||||
#[derive(Default)]
|
||||
pub struct OpenAiChatCall {
|
||||
pub model: String,
|
||||
pub body: Map<String, Value>,
|
||||
|
|
@ -49,12 +54,35 @@ pub struct OpenAiChatCall {
|
|||
pub custom_llm_provider: Option<String>,
|
||||
pub extra_headers: Option<Map<String, Value>>,
|
||||
pub timeout: Option<Duration>,
|
||||
/// The deployment's other `litellm_params`: Azure reads its `api_version`, `base_model`
|
||||
/// and Entra ID credentials from them.
|
||||
pub litellm_params: Map<String, Value>,
|
||||
/// `drop_params` from the deployment or the proxy settings.
|
||||
pub drop_params: bool,
|
||||
/// The model cost map, which Azure's param rules read.
|
||||
pub model_info: Option<Arc<dyn ModelInfo>>,
|
||||
}
|
||||
|
||||
/// A provider request ready to go out: where, with which headers and body, and who serves it.
|
||||
pub(crate) struct PreparedCall {
|
||||
pub(crate) model: String,
|
||||
pub(crate) provider: &'static str,
|
||||
pub(crate) url: String,
|
||||
pub(crate) headers: Headers,
|
||||
pub(crate) body: Map<String, Value>,
|
||||
pub(crate) api_key: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionsRoute {
|
||||
/// Whether the deployment is served by the OpenAI wire passthrough: OpenAI itself, or
|
||||
/// Azure OpenAI.
|
||||
pub fn serves_openai(model: &str, custom_llm_provider: Option<&str>) -> bool {
|
||||
resolve_llm_provider(model, custom_llm_provider, "chat completions")
|
||||
.is_ok_and(|resolved| resolved.provider == LlmProviders::Openai)
|
||||
resolve_llm_provider(model, custom_llm_provider, "chat completions").is_ok_and(|resolved| {
|
||||
matches!(
|
||||
resolved.provider,
|
||||
LlmProviders::Openai | LlmProviders::Azure
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn openai_machine(
|
||||
|
|
@ -106,39 +134,35 @@ impl ChatCompletionsRoute {
|
|||
stream,
|
||||
outcome
|
||||
))]
|
||||
async fn run_openai(
|
||||
pub(crate) async fn run_openai(
|
||||
&self,
|
||||
call: OpenAiChatCall,
|
||||
context: CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<OpenAiChatOutput, Error> {
|
||||
let timeout = call
|
||||
.timeout
|
||||
.unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS));
|
||||
litellm_inference::diagnostic::call(async {
|
||||
let resolved = resolve_llm_provider(
|
||||
&call.model,
|
||||
call.custom_llm_provider.as_deref(),
|
||||
"chat completions",
|
||||
)?;
|
||||
if resolved.provider != LlmProviders::Openai {
|
||||
return Err(Error::InvalidProvider(
|
||||
<&str>::from(resolved.provider).to_string(),
|
||||
));
|
||||
}
|
||||
let model = resolved.model.to_string();
|
||||
let provider = <&str>::from(LlmProviders::Openai);
|
||||
let prepared = match resolved.provider {
|
||||
LlmProviders::Openai => self.prepare_openai(model, call).await?,
|
||||
LlmProviders::Azure => self.prepare_azure(model, call).await?,
|
||||
other => return Err(Error::InvalidProvider(<&str>::from(other).to_string())),
|
||||
};
|
||||
let PreparedCall {
|
||||
model,
|
||||
provider,
|
||||
url,
|
||||
headers,
|
||||
body,
|
||||
api_key,
|
||||
} = prepared;
|
||||
litellm_inference::diagnostic::provider(&model, provider);
|
||||
let config = &OPENAI_CHAT_COMPLETIONS_CONFIG;
|
||||
let secrets = self
|
||||
.secrets
|
||||
.resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
|
||||
.await?;
|
||||
let lookup = |key: &str| secrets.get(key);
|
||||
let body = config.transform_request(&model, call.body)?;
|
||||
let environment = config.validate_environment(
|
||||
string_headers(call.extra_headers)?,
|
||||
call.api_key.as_deref(),
|
||||
&lookup,
|
||||
)?;
|
||||
let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
|
||||
let authenticated = resolve_auth(&self.auth, environment, &lookup).await?;
|
||||
let request_context = RequestContext {
|
||||
model: model.clone(),
|
||||
custom_llm_provider: provider.to_string(),
|
||||
|
|
@ -149,18 +173,16 @@ impl ChatCompletionsRoute {
|
|||
.collect(),
|
||||
),
|
||||
secret_fields: Vec::new(),
|
||||
api_key: call.api_key.map(litellm_auth::SecretValue::new),
|
||||
api_key: api_key.map(litellm_auth::SecretValue::new),
|
||||
};
|
||||
let wire = WireRequest {
|
||||
url,
|
||||
headers,
|
||||
body: Value::Object(body),
|
||||
};
|
||||
let wire = context
|
||||
.interceptors
|
||||
.before_provider_request(
|
||||
WireRequest {
|
||||
url,
|
||||
headers: authenticated.headers,
|
||||
body: Value::Object(body),
|
||||
},
|
||||
request_context,
|
||||
)
|
||||
.before_provider_request(wire, request_context)
|
||||
.await?;
|
||||
let stream = wire
|
||||
.body
|
||||
|
|
@ -168,9 +190,6 @@ impl ChatCompletionsRoute {
|
|||
.map(is_stream)
|
||||
.transpose()?
|
||||
.unwrap_or(false);
|
||||
let timeout = call
|
||||
.timeout
|
||||
.unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS));
|
||||
let response = send(&self.http, wire, stream, timeout).await?;
|
||||
let output = if stream {
|
||||
CallOutput::Stream {
|
||||
|
|
@ -200,6 +219,37 @@ impl ChatCompletionsRoute {
|
|||
}
|
||||
}
|
||||
|
||||
impl ChatCompletionsRoute {
|
||||
async fn prepare_openai(
|
||||
&self,
|
||||
model: String,
|
||||
call: OpenAiChatCall,
|
||||
) -> Result<PreparedCall, Error> {
|
||||
let config = &OPENAI_CHAT_COMPLETIONS_CONFIG;
|
||||
let secrets = self
|
||||
.secrets
|
||||
.resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
|
||||
.await?;
|
||||
let lookup = |key: &str| secrets.get(key);
|
||||
let body = config.transform_request(&model, call.body)?;
|
||||
let environment = config.validate_environment(
|
||||
string_headers(call.extra_headers)?,
|
||||
call.api_key.as_deref(),
|
||||
&lookup,
|
||||
)?;
|
||||
let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
|
||||
let authenticated = resolve_auth(&self.auth, environment, &lookup).await?;
|
||||
Ok(PreparedCall {
|
||||
model,
|
||||
provider: <&str>::from(LlmProviders::Openai),
|
||||
url,
|
||||
headers: authenticated.headers,
|
||||
body,
|
||||
api_key: call.api_key,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A non-streaming call keeps the usual total timeout. A stream has no total limit, so a
|
||||
/// 30 minute generation is not cut off: `timeout` bounds the wait for the response head
|
||||
/// here and each read after it in [`chunks`].
|
||||
|
|
@ -254,11 +304,10 @@ fn silent_provider(timeout: Duration) -> Error {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures_util::TryStreamExt;
|
||||
use litellm_host::interceptors::RawResponse;
|
||||
use rstest::{fixture, rstest};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use wiremock::{
|
||||
|
|
@ -267,47 +316,11 @@ mod tests {
|
|||
};
|
||||
|
||||
use super::*;
|
||||
use crate::test_support::{Recording, route, run};
|
||||
|
||||
const COMPLETION: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-6-astra","system_fingerprint":"fp","choices":[{"index":0,"message":{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"f","arguments":"{}"}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}"#;
|
||||
const STREAM: &str = "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\ndata: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1,\"total_tokens\":4}}\n\ndata: [DONE]\n\n";
|
||||
|
||||
#[derive(Default)]
|
||||
struct Recording {
|
||||
contexts: Mutex<Vec<RequestContext>>,
|
||||
raw: Mutex<Vec<String>>,
|
||||
results: Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<Error> for Recording {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
wire: WireRequest,
|
||||
context: RequestContext,
|
||||
) -> Result<WireRequest, Error> {
|
||||
self.contexts.lock().unwrap().push(context);
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, raw: RawResponse) -> Result<(), Error> {
|
||||
self.raw.lock().unwrap().push(raw.body);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.results.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn route() -> ChatCompletionsRoute {
|
||||
ChatCompletionsRoute::new(
|
||||
litellm_http::Client::plain_for_test(),
|
||||
Arc::new(litellm_auth::AuthServices::default()),
|
||||
litellm_inference_testing::no_secrets(),
|
||||
)
|
||||
}
|
||||
|
||||
fn call(api_base: &str, body: Value) -> OpenAiChatCall {
|
||||
let Value::Object(body) = body else {
|
||||
panic!("the body is an object");
|
||||
|
|
@ -320,19 +333,10 @@ mod tests {
|
|||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
timeout: Some(Duration::from_secs(5)),
|
||||
..OpenAiChatCall::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(
|
||||
route: &ChatCompletionsRoute,
|
||||
call: OpenAiChatCall,
|
||||
interceptors: &Recording,
|
||||
) -> Result<OpenAiChatOutput, Error> {
|
||||
route
|
||||
.run_openai(call, CallContext::new(interceptors, CallOptions::default()))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn drain(chunks: BoxStream<'static, Result<Bytes, Error>>) -> Result<Vec<u8>, Error> {
|
||||
Ok(chunks.try_collect::<Vec<_>>().await?.concat())
|
||||
}
|
||||
|
|
@ -347,6 +351,8 @@ mod tests {
|
|||
#[rstest]
|
||||
#[case::openai_prefix("openai/gpt-6-astra", None, true)]
|
||||
#[case::explicit_provider("gpt-6-astra", Some("openai"), true)]
|
||||
#[case::azure("azure/gpt-4o", None, true)]
|
||||
#[case::azure_ai("azure_ai/gpt-4o", None, false)]
|
||||
#[case::openai_like("openai_like/gpt-6-astra", None, false)]
|
||||
#[case::anthropic("anthropic/claude", None, false)]
|
||||
#[case::unprefixed("gpt-6-astra", None, false)]
|
||||
|
|
|
|||
61
litellm-rust/crates/inference-chat/src/test_support.rs
Normal file
61
litellm-rust/crates/inference-chat/src/test_support.rs
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
//! Helpers the passthrough tests of several providers share.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use litellm_host::interceptors::{
|
||||
ExecutionFacts, Interceptors, RawResponse, RequestContext, WireRequest,
|
||||
};
|
||||
use litellm_inference::{CallOptions, context::CallContext};
|
||||
use rstest::fixture;
|
||||
|
||||
use crate::{
|
||||
ChatCompletionsRoute, Error,
|
||||
openai::{OpenAiChatCall, OpenAiChatOutput},
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Recording {
|
||||
pub(crate) contexts: Mutex<Vec<RequestContext>>,
|
||||
pub(crate) raw: Mutex<Vec<String>>,
|
||||
pub(crate) results: Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<Error> for Recording {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
wire: WireRequest,
|
||||
context: RequestContext,
|
||||
) -> Result<WireRequest, Error> {
|
||||
self.contexts.lock().unwrap().push(context);
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, raw: RawResponse) -> Result<(), Error> {
|
||||
self.raw.lock().unwrap().push(raw.body);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.results.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
pub(crate) fn route() -> ChatCompletionsRoute {
|
||||
ChatCompletionsRoute::new(
|
||||
litellm_http::Client::plain_for_test(),
|
||||
Arc::new(litellm_auth::AuthServices::default()),
|
||||
litellm_inference_testing::no_secrets(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn run(
|
||||
route: &ChatCompletionsRoute,
|
||||
call: OpenAiChatCall,
|
||||
interceptors: &Recording,
|
||||
) -> Result<OpenAiChatOutput, Error> {
|
||||
route
|
||||
.run_openai(call, CallContext::new(interceptors, CallOptions::default()))
|
||||
.await
|
||||
}
|
||||
|
|
@ -47,6 +47,7 @@ fn call(extra: Value) -> OpenAiChatCall {
|
|||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
timeout: Some(Duration::from_secs(60)),
|
||||
..OpenAiChatCall::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
1
litellm-rust/crates/llms/src/azure/chat/mod.rs
Normal file
1
litellm-rust/crates/llms/src/azure/chat/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod transformation;
|
||||
1432
litellm-rust/crates/llms/src/azure/chat/transformation.rs
Normal file
1432
litellm-rust/crates/llms/src/azure/chat/transformation.rs
Normal file
File diff suppressed because it is too large
Load diff
325
litellm-rust/crates/llms/src/azure/common_utils.rs
Normal file
325
litellm-rust/crates/llms/src/azure/common_utils.rs
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
//! Endpoint, API version and credential resolution for Azure OpenAI, as
|
||||
//! `litellm/llms/azure/common_utils.py`, `litellm/main.py` and the `AzureOpenAI` client of the
|
||||
//! OpenAI SDK resolve them.
|
||||
|
||||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::auth::{AuthScheme, Headers, ValidatedEnvironment},
|
||||
};
|
||||
|
||||
/// `litellm.AZURE_DEFAULT_API_VERSION`.
|
||||
pub const AZURE_DEFAULT_API_VERSION: &str = "2025-02-01-preview";
|
||||
pub const AZURE_API_BASE_ENV: &str = "AZURE_API_BASE";
|
||||
/// The SDK's own fallback when LiteLLM passes no endpoint.
|
||||
pub const AZURE_OPENAI_ENDPOINT_ENV: &str = "AZURE_OPENAI_ENDPOINT";
|
||||
pub const AZURE_API_VERSION_ENV: &str = "AZURE_API_VERSION";
|
||||
pub const AZURE_OPENAI_API_KEY_ENV: &str = "AZURE_OPENAI_API_KEY";
|
||||
pub const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY";
|
||||
const API_KEY_HEADER: &str = "api-key";
|
||||
|
||||
/// The environment variables a call may fall back to, besides the credential ones
|
||||
/// `litellm_auth_azure::SECRET_NAMES` lists.
|
||||
pub const SECRET_NAMES: &[&str] = &[
|
||||
AZURE_API_BASE_ENV,
|
||||
AZURE_OPENAI_ENDPOINT_ENV,
|
||||
AZURE_API_VERSION_ENV,
|
||||
AZURE_OPENAI_API_KEY_ENV,
|
||||
AZURE_API_KEY_ENV,
|
||||
];
|
||||
|
||||
/// `BaseAzureLLM._is_azure_v1_api_version`: these versions use the `/openai/v1/` surface
|
||||
/// through the plain OpenAI client.
|
||||
pub fn is_v1_api_version(api_version: &str) -> bool {
|
||||
matches!(api_version, "preview" | "latest" | "v1")
|
||||
}
|
||||
|
||||
/// The deployment's `api_version`, else `AZURE_API_VERSION`, else LiteLLM's default.
|
||||
pub fn resolve_api_version(
|
||||
api_version: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> String {
|
||||
resolve_non_empty(api_version, lookup, &[AZURE_API_VERSION_ENV])
|
||||
.unwrap_or_else(|| AZURE_DEFAULT_API_VERSION.into())
|
||||
}
|
||||
|
||||
/// The deployment's `api_base`, else `AZURE_API_BASE`, else the SDK's `AZURE_OPENAI_ENDPOINT`.
|
||||
pub fn resolve_api_base(
|
||||
api_base: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
resolve_non_empty(
|
||||
api_base,
|
||||
lookup,
|
||||
&[AZURE_API_BASE_ENV, AZURE_OPENAI_ENDPOINT_ENV],
|
||||
)
|
||||
.ok_or(Error::Auth(litellm_auth::Error::MissingApiBase {
|
||||
provider: "Azure",
|
||||
guidance: "Set `api_base` or the AZURE_API_BASE environment variable. Expected format: https://<resource-name>.openai.azure.com",
|
||||
}))
|
||||
}
|
||||
|
||||
/// The deployment's key, else `AZURE_OPENAI_API_KEY`, else `AZURE_API_KEY`, as `main.py`
|
||||
/// reads them.
|
||||
pub fn resolve_api_key(
|
||||
api_key: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
resolve_non_empty(
|
||||
api_key,
|
||||
lookup,
|
||||
&[AZURE_OPENAI_API_KEY_ENV, AZURE_API_KEY_ENV],
|
||||
)
|
||||
}
|
||||
|
||||
/// The Chat Completions URL the SDK builds for `deployment`:
|
||||
///
|
||||
/// - `v1`, `latest` and `preview` go to `{api_base}/openai/v1/chat/completions` with no
|
||||
/// `api-version`, as the plain OpenAI client LiteLLM switches to for them does;
|
||||
/// - an `api_base` already naming `/openai/deployments/...` is the client's `base_url`, so only
|
||||
/// `/chat/completions` is added;
|
||||
/// - any other `api_base` is the resource endpoint, under which the SDK puts
|
||||
/// `/openai/deployments/{deployment}`.
|
||||
///
|
||||
/// A trailing slash on `api_base` is dropped, as `azure_endpoint.rstrip('/')` does.
|
||||
pub fn complete_url(api_base: &str, deployment: &str, api_version: &str) -> Result<String, Error> {
|
||||
if api_base.contains("gateway.ai.cloudflare.com") {
|
||||
return Err(Error::Unsupported(
|
||||
"Azure deployments behind a Cloudflare AI Gateway api_base",
|
||||
));
|
||||
}
|
||||
let base = api_base.trim_end_matches('/');
|
||||
if is_v1_api_version(api_version) {
|
||||
return Ok(format!("{base}/openai/v1/chat/completions"));
|
||||
}
|
||||
let version: String = url::form_urlencoded::byte_serialize(api_version.as_bytes()).collect();
|
||||
Ok(match base.contains("/openai/deployments") {
|
||||
true => format!("{base}/chat/completions?api-version={version}"),
|
||||
false => {
|
||||
format!("{base}/openai/deployments/{deployment}/chat/completions?api-version={version}")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// How a call authenticates once its credential is known.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum AzureCredential {
|
||||
ApiKey(SecretValue),
|
||||
/// An Entra ID access token, given or acquired.
|
||||
AdToken(SecretValue),
|
||||
}
|
||||
|
||||
/// A key goes in `api-key` and a token in `Authorization: Bearer`, as `AzureOpenAI` sends
|
||||
/// them. On the v1 surface LiteLLM hands either to the plain OpenAI client as its `api_key`,
|
||||
/// which sends it as a bearer.
|
||||
pub fn environment(
|
||||
headers: Headers,
|
||||
credential: AzureCredential,
|
||||
v1: bool,
|
||||
) -> ValidatedEnvironment {
|
||||
let (placement, secret) = match credential {
|
||||
AzureCredential::ApiKey(secret) if !v1 => {
|
||||
(CredentialPlacement::Header(API_KEY_HEADER), secret)
|
||||
}
|
||||
AzureCredential::ApiKey(secret) | AzureCredential::AdToken(secret) => {
|
||||
(CredentialPlacement::Bearer, secret)
|
||||
}
|
||||
};
|
||||
ValidatedEnvironment {
|
||||
headers: litellm_http::request::with_default_headers(
|
||||
headers,
|
||||
&[("content-type", "application/json")],
|
||||
),
|
||||
auth: AuthScheme::Credential { placement, secret },
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
fn env(name: &str) -> Option<String> {
|
||||
match name {
|
||||
AZURE_API_BASE_ENV => Some("https://env.openai.azure.com".into()),
|
||||
AZURE_API_VERSION_ENV => Some("2024-10-21".into()),
|
||||
AZURE_API_KEY_ENV => Some("env-key".into()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_only(name: &str) -> Option<String> {
|
||||
(name == AZURE_OPENAI_ENDPOINT_ENV).then(|| "https://sdk.openai.azure.com".into())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::resource(
|
||||
"https://res.openai.azure.com",
|
||||
"2024-10-21",
|
||||
"https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21"
|
||||
)]
|
||||
#[case::trailing_slash(
|
||||
"https://res.openai.azure.com/",
|
||||
"2024-10-21",
|
||||
"https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21"
|
||||
)]
|
||||
#[case::preview_version(
|
||||
"https://res.openai.azure.com",
|
||||
"2025-02-01-preview",
|
||||
"https://res.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2025-02-01-preview"
|
||||
)]
|
||||
#[case::base_url_with_deployment(
|
||||
"https://res.openai.azure.com/openai/deployments/other/",
|
||||
"2024-10-21",
|
||||
"https://res.openai.azure.com/openai/deployments/other/chat/completions?api-version=2024-10-21"
|
||||
)]
|
||||
#[case::v1(
|
||||
"https://res.openai.azure.com/",
|
||||
"v1",
|
||||
"https://res.openai.azure.com/openai/v1/chat/completions"
|
||||
)]
|
||||
#[case::latest(
|
||||
"https://res.openai.azure.com",
|
||||
"latest",
|
||||
"https://res.openai.azure.com/openai/v1/chat/completions"
|
||||
)]
|
||||
#[case::preview(
|
||||
"https://res.openai.azure.com",
|
||||
"preview",
|
||||
"https://res.openai.azure.com/openai/v1/chat/completions"
|
||||
)]
|
||||
fn the_url_follows_the_sdk(
|
||||
#[case] api_base: &str,
|
||||
#[case] api_version: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
complete_url(api_base, "gpt-4o", api_version).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_cloudflare_gateway_base_is_refused() {
|
||||
assert!(matches!(
|
||||
complete_url(
|
||||
"https://gateway.ai.cloudflare.com/v1/acct/gw/azure-openai/res",
|
||||
"gpt-4o",
|
||||
"2024-10-21"
|
||||
),
|
||||
Err(Error::Unsupported(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::deployment(Some("2024-06-01"), env, "2024-06-01")]
|
||||
#[case::environment(None, env, "2024-10-21")]
|
||||
#[case::empty_falls_back(Some(""), env, "2024-10-21")]
|
||||
#[case::default(None, no_env, AZURE_DEFAULT_API_VERSION)]
|
||||
fn the_api_version_prefers_the_deployment_then_the_environment(
|
||||
#[case] configured: Option<&str>,
|
||||
#[case] lookup: fn(&str) -> Option<String>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(resolve_api_version(configured, &lookup), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::deployment(
|
||||
Some("https://dep.openai.azure.com"),
|
||||
env,
|
||||
"https://dep.openai.azure.com"
|
||||
)]
|
||||
#[case::environment(None, env, "https://env.openai.azure.com")]
|
||||
#[case::sdk_environment(None, endpoint_only, "https://sdk.openai.azure.com")]
|
||||
fn the_api_base_prefers_the_deployment_then_the_environment(
|
||||
#[case] configured: Option<&str>,
|
||||
#[case] lookup: fn(&str) -> Option<String>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(resolve_api_base(configured, &lookup).unwrap(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_missing_api_base_says_where_to_set_it() {
|
||||
assert!(matches!(
|
||||
resolve_api_base(None, &no_env),
|
||||
Err(Error::Auth(litellm_auth::Error::MissingApiBase {
|
||||
provider: "Azure",
|
||||
..
|
||||
}))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::deployment(Some("dep-key"), env, Some("dep-key"))]
|
||||
#[case::environment(None, env, Some("env-key"))]
|
||||
#[case::none(None, no_env, None)]
|
||||
fn the_key_prefers_the_deployment_then_the_environment(
|
||||
#[case] configured: Option<&str>,
|
||||
#[case] lookup: fn(&str) -> Option<String>,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(resolve_api_key(configured, &lookup).as_deref(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_sdk_key_variable_outranks_litellms() {
|
||||
let lookup = |name: &str| match name {
|
||||
AZURE_OPENAI_API_KEY_ENV => Some("sdk-key".to_string()),
|
||||
AZURE_API_KEY_ENV => Some("litellm-key".to_string()),
|
||||
_ => None,
|
||||
};
|
||||
assert_eq!(resolve_api_key(None, &lookup).as_deref(), Some("sdk-key"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::key(AzureCredential::ApiKey(SecretValue::new("k")), false, "api-key", "k")]
|
||||
#[case::token(
|
||||
AzureCredential::AdToken(SecretValue::new("t")),
|
||||
false,
|
||||
"authorization",
|
||||
"Bearer t"
|
||||
)]
|
||||
#[case::v1_key(
|
||||
AzureCredential::ApiKey(SecretValue::new("k")),
|
||||
true,
|
||||
"authorization",
|
||||
"Bearer k"
|
||||
)]
|
||||
#[case::v1_token(
|
||||
AzureCredential::AdToken(SecretValue::new("t")),
|
||||
true,
|
||||
"authorization",
|
||||
"Bearer t"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn the_credential_lands_in_its_header(
|
||||
#[case] credential: AzureCredential,
|
||||
#[case] v1: bool,
|
||||
#[case] header: &str,
|
||||
#[case] value: &str,
|
||||
) {
|
||||
let authenticated = crate::base_llm::auth::resolve_auth(
|
||||
&litellm_auth::AuthServices::default(),
|
||||
environment(Vec::new(), credential, v1),
|
||||
&no_env,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
authenticated.headers,
|
||||
[
|
||||
("content-type".to_string(), "application/json".to_string()),
|
||||
(header.to_string(), value.to_string())
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
5
litellm-rust/crates/llms/src/azure/mod.rs
Normal file
5
litellm-rust/crates/llms/src/azure/mod.rs
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
//! Azure OpenAI (`azure/<deployment>`), as `litellm/llms/azure/` serves it. Azure AI Foundry's
|
||||
//! `azure_ai` provider lives in `crate::azure_ai`.
|
||||
|
||||
pub mod chat;
|
||||
pub mod common_utils;
|
||||
|
|
@ -3,6 +3,7 @@ pub mod auth;
|
|||
pub mod base_model_iterator;
|
||||
pub mod chat;
|
||||
pub mod messages;
|
||||
pub mod model_info;
|
||||
pub mod ocr;
|
||||
pub mod responses;
|
||||
pub mod sse;
|
||||
|
|
|
|||
37
litellm-rust/crates/llms/src/base_llm/model_info.rs
Normal file
37
litellm-rust/crates/llms/src/base_llm/model_info.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
//! What the model cost map says about a model, for configs whose request rules depend on it
|
||||
//! the way Python's `supports_*` helpers read `litellm.model_cost`.
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub trait ModelInfo: Send + Sync {
|
||||
/// The cost map entry `model` names, if there is one.
|
||||
fn entry(&self, model: &str) -> Option<&Map<String, Value>>;
|
||||
|
||||
/// Whether any entry of the map sets `key`.
|
||||
fn declares(&self, key: &str) -> bool;
|
||||
}
|
||||
|
||||
/// Python's `_supports_factory`: the entry sets `key` to `true`. An unknown model supports
|
||||
/// nothing.
|
||||
pub fn supports(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool {
|
||||
field(info, model, key).and_then(Value::as_bool) == Some(true)
|
||||
}
|
||||
|
||||
/// Python's `is_explicitly_disabled_factory`: the entry sets `key` to `false`. An unknown
|
||||
/// model or an unset key is not disabled.
|
||||
pub fn explicitly_disabled(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool {
|
||||
field(info, model, key).and_then(Value::as_bool) == Some(false)
|
||||
}
|
||||
|
||||
/// The value the entry for `model` sets for `key`.
|
||||
pub fn field<'a>(info: Option<&'a dyn ModelInfo>, model: &str, key: &str) -> Option<&'a Value> {
|
||||
info?
|
||||
.entry(model)?
|
||||
.get(key)
|
||||
.filter(|value| !value.is_null())
|
||||
}
|
||||
|
||||
/// Whether the map has an entry for `model`.
|
||||
pub fn known(info: Option<&dyn ModelInfo>, model: &str) -> bool {
|
||||
info.is_some_and(|info| info.entry(model).is_some())
|
||||
}
|
||||
|
|
@ -60,6 +60,14 @@ pub enum ErrorDetail {
|
|||
value: String,
|
||||
hint: String,
|
||||
},
|
||||
#[error(
|
||||
"{provider} does not support parameters: {params:?}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={params:?} in your request."
|
||||
)]
|
||||
UnsupportedParams {
|
||||
provider: &'static str,
|
||||
params: Vec<String>,
|
||||
model: String,
|
||||
},
|
||||
#[error(
|
||||
"{provider} does not support the server-side tool '{tool}' (model '{model}'). {guidance}"
|
||||
)]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
pub mod anthropic;
|
||||
pub mod aws_textract;
|
||||
pub mod azure;
|
||||
pub mod azure_ai;
|
||||
pub mod base_llm;
|
||||
pub mod bedrock;
|
||||
|
|
|
|||
|
|
@ -85,17 +85,7 @@ impl OpenAiChatConfig {
|
|||
body: Map<String, Value>,
|
||||
) -> Result<Map<String, Value>, Error> {
|
||||
let mut body = body;
|
||||
let extra_body = match body.remove(EXTRA_BODY) {
|
||||
None | Some(Value::Null) => Map::new(),
|
||||
Some(Value::Object(extra)) => extra,
|
||||
Some(other) => {
|
||||
return Err(Error::InvalidRequest(ErrorDetail::InvalidValue {
|
||||
field: EXTRA_BODY,
|
||||
expected: "an object",
|
||||
actual: other,
|
||||
}));
|
||||
}
|
||||
};
|
||||
let extra_body = take_extra_body(&mut body)?;
|
||||
let mut body: Map<String, Value> = body
|
||||
.into_iter()
|
||||
.chain(extra_body)
|
||||
|
|
@ -120,7 +110,21 @@ pub fn is_stream(body: &Map<String, Value>) -> Result<bool, Error> {
|
|||
}
|
||||
}
|
||||
|
||||
fn is_litellm_owned(name: &str) -> bool {
|
||||
/// Removes the body's `extra_body`, which the OpenAI SDK merges over the body it sends.
|
||||
pub fn take_extra_body(body: &mut Map<String, Value>) -> Result<Map<String, Value>, Error> {
|
||||
match body.remove(EXTRA_BODY) {
|
||||
None | Some(Value::Null) => Ok(Map::new()),
|
||||
Some(Value::Object(extra)) => Ok(extra),
|
||||
Some(other) => Err(Error::InvalidRequest(ErrorDetail::InvalidValue {
|
||||
field: EXTRA_BODY,
|
||||
expected: "an object",
|
||||
actual: other,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether Python consumes `name` before the provider call, so it never reaches the wire.
|
||||
pub fn is_litellm_owned(name: &str) -> bool {
|
||||
LITELLM_OWNED_PARAMS.contains(&name) || name.starts_with(INTERNAL_PARAM_PREFIX)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,3 +8,31 @@ pub enum Spelled<T> {
|
|||
Value(T),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
impl Spelled<bool> {
|
||||
/// The flag, reading a `true` word the way Python's coercion does. An `os.environ/`
|
||||
/// reference left unresolved is not set.
|
||||
pub fn is_set(&self) -> bool {
|
||||
match self {
|
||||
Self::Value(value) => *value,
|
||||
Self::Text(text) => text.trim().eq_ignore_ascii_case("true"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::value(Spelled::Value(true), true)]
|
||||
#[case::false_value(Spelled::Value(false), false)]
|
||||
#[case::word(Spelled::Text(" True ".into()), true)]
|
||||
#[case::other_word(Spelled::Text("yes".into()), false)]
|
||||
#[case::reference(Spelled::Text("os.environ/DROP".into()), false)]
|
||||
fn flags_read_values_and_words(#[case] flag: Spelled<bool>, #[case] expected: bool) {
|
||||
assert_eq!(flag.is_set(), expected);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,9 @@ pub struct Deployment {
|
|||
/// `model_info.id`, which names the deployment in state shared with other replicas and
|
||||
/// with Python routers on the same Redis.
|
||||
pub id: Option<String>,
|
||||
/// The model a deployment serves under another name, from `litellm_params.base_model` or
|
||||
/// `model_info.base_model`, which Azure's param rules and prices follow.
|
||||
pub base_model: Option<String>,
|
||||
}
|
||||
|
||||
impl Deployment {
|
||||
|
|
@ -46,6 +49,12 @@ impl Deployment {
|
|||
shaping: MessagesShaping::default(),
|
||||
team_id: None,
|
||||
id: None,
|
||||
base_model: params
|
||||
.extra
|
||||
.get("base_model")
|
||||
.and_then(|value| value.as_str())
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_owned),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,6 +87,10 @@ impl Router {
|
|||
timeout: deployment.timeout.or(policy.timeout),
|
||||
team_id: owner_team(model),
|
||||
id: model_id(model),
|
||||
base_model: deployment
|
||||
.base_model
|
||||
.clone()
|
||||
.or_else(|| info_base_model(model)),
|
||||
..deployment
|
||||
},
|
||||
))
|
||||
|
|
@ -363,6 +367,13 @@ fn owner_team(model: &Model) -> Option<String> {
|
|||
}
|
||||
}
|
||||
|
||||
fn info_base_model(model: &Model) -> Option<String> {
|
||||
match model.model_info.get("base_model")? {
|
||||
litellm_config::Value::String(name) if !name.is_empty() => Some(name.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl FromIterator<(String, Deployment)> for Router {
|
||||
fn from_iter<I: IntoIterator<Item = (String, Deployment)>>(entries: I) -> Self {
|
||||
Self::new(entries, RoutingPolicy::default())
|
||||
|
|
@ -559,6 +570,31 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::params(
|
||||
" litellm_params:\n model: azure/d\n base_model: gpt-4o\n",
|
||||
Some("gpt-4o")
|
||||
)]
|
||||
#[case::model_info(
|
||||
" litellm_params:\n model: azure/d\n model_info:\n base_model: gpt-4o\n",
|
||||
Some("gpt-4o")
|
||||
)]
|
||||
#[case::params_win(
|
||||
" litellm_params:\n model: azure/d\n base_model: gpt-4o\n model_info:\n base_model: gpt-4\n",
|
||||
Some("gpt-4o")
|
||||
)]
|
||||
#[case::absent(" litellm_params:\n model: azure/d\n", None)]
|
||||
fn reads_the_base_model(#[case] entry: &str, #[case] expected: Option<&str>) {
|
||||
let config =
|
||||
Config::from_yaml(&format!("model_list:\n - model_name: m\n{entry}")).unwrap();
|
||||
let router = Router::from_config(&config).unwrap();
|
||||
let group = router.group("m").unwrap();
|
||||
assert_eq!(
|
||||
group.deployments().next().unwrap().base_model.as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_unclassified_error_is_not_retried() {
|
||||
|
|
|
|||
|
|
@ -117,6 +117,7 @@ fn programmatic_deployments_preserve_overrides_and_share_their_group() {
|
|||
},
|
||||
team_id: None,
|
||||
id: None,
|
||||
base_model: None,
|
||||
};
|
||||
let router = Router::from_iter([
|
||||
("public-model".into(), Deployment::default()),
|
||||
|
|
|
|||
87
litellm-rust/docs/customer-proxy/azure-chat.svg
Normal file
87
litellm-rust/docs/customer-proxy/azure-chat.svg
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" width="980" height="760" viewBox="0 0 980 760" font-family="Helvetica, Arial, sans-serif" font-size="13">
|
||||
<defs>
|
||||
<marker id="a" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
||||
<path d="M0,0 L10,5 L0,10 z" fill="#333"/>
|
||||
</marker>
|
||||
</defs>
|
||||
<rect x="0" y="0" width="980" height="760" fill="#ffffff"/>
|
||||
<text x="20" y="30" font-size="17" font-weight="bold" fill="#111">POST /v1/chat/completions to an azure/ deployment</text>
|
||||
|
||||
<!-- client -->
|
||||
<rect x="20" y="60" width="150" height="56" rx="6" fill="#eef3fb" stroke="#4a6fa5"/>
|
||||
<text x="95" y="84" text-anchor="middle" fill="#111" font-weight="bold">Client</text>
|
||||
<text x="95" y="102" text-anchor="middle" fill="#333">OpenAI SDK body</text>
|
||||
|
||||
<!-- gateway handle -->
|
||||
<rect x="220" y="50" width="300" height="76" rx="6" fill="#f5f5f5" stroke="#666"/>
|
||||
<text x="370" y="72" text-anchor="middle" fill="#111" font-weight="bold">gateway-inference chat_completions</text>
|
||||
<text x="370" y="92" text-anchor="middle" fill="#333">auth, hooks admit, route deployment,</text>
|
||||
<text x="370" y="110" text-anchor="middle" fill="#333">litellm_params + base_model + drop_params</text>
|
||||
<line x1="170" y1="88" x2="218" y2="88" stroke="#333" marker-end="url(#a)"/>
|
||||
|
||||
<!-- decision -->
|
||||
<polygon points="370,150 490,190 370,230 250,190" fill="#fff8e6" stroke="#b8860b"/>
|
||||
<text x="370" y="186" text-anchor="middle" fill="#111">serves_openai?</text>
|
||||
<text x="370" y="202" text-anchor="middle" fill="#111">openai | azure</text>
|
||||
<line x1="370" y1="126" x2="370" y2="148" stroke="#333" marker-end="url(#a)"/>
|
||||
|
||||
<rect x="580" y="50" width="380" height="76" rx="6" fill="#fff8e6" stroke="#b8860b"/>
|
||||
<text x="770" y="72" text-anchor="middle" fill="#111" font-weight="bold">gateway startup (hooks::install)</text>
|
||||
<text x="770" y="92" text-anchor="middle" fill="#333">an azure deployment loads the cost map and</text>
|
||||
<text x="770" y="110" text-anchor="middle" fill="#333">installs it as ModelInfo for the param rules</text>
|
||||
|
||||
<!-- run_openai / prepare_azure -->
|
||||
<rect x="20" y="262" width="560" height="232" rx="6" fill="#eef7ee" stroke="#3c8d3c"/>
|
||||
<text x="300" y="284" text-anchor="middle" fill="#111" font-weight="bold">inference-chat run_openai, prepare_azure</text>
|
||||
<text x="36" y="308" fill="#333">1. resolve os.environ/ refs and AZURE_* env through the secret source</text>
|
||||
<text x="36" y="328" fill="#333">2. api_version: deployment, AZURE_API_VERSION, else 2025-02-01-preview</text>
|
||||
<text x="54" y="346" fill="#333">api_base: deployment, AZURE_API_BASE, AZURE_OPENAI_ENDPOINT</text>
|
||||
<text x="36" y="366" fill="#333">3. AzureChatRequest::transform: family from base_model or deployment</text>
|
||||
<text x="54" y="384" fill="#333">(gpt, o-series, gpt-5), unsupported params refused or dropped</text>
|
||||
<text x="36" y="404" fill="#333">4. url: {base}/openai/deployments/{dep}/chat/completions?api-version=</text>
|
||||
<text x="54" y="422" fill="#333">v1, latest, preview: {base}/openai/v1/chat/completions</text>
|
||||
<text x="36" y="442" fill="#333">5. credential: azure_ad_token, else api-key, else Entra ID token</text>
|
||||
<text x="54" y="460" fill="#333">(client secret, OIDC, AZURE_AD_TOKEN, managed identity)</text>
|
||||
<text x="36" y="482" fill="#555">then the shared interceptors, send and SSE relay of the OpenAI path</text>
|
||||
<line x1="370" y1="230" x2="370" y2="260" stroke="#333" marker-end="url(#a)"/>
|
||||
<text x="382" y="250" fill="#555">yes</text>
|
||||
|
||||
<!-- upstream -->
|
||||
<rect x="680" y="300" width="280" height="110" rx="6" fill="#eef3fb" stroke="#4a6fa5"/>
|
||||
<text x="820" y="326" text-anchor="middle" fill="#111" font-weight="bold">Azure OpenAI</text>
|
||||
<text x="820" y="346" text-anchor="middle" fill="#333">api-key: key (deployment path)</text>
|
||||
<text x="820" y="364" text-anchor="middle" fill="#333">Authorization: Bearer (AD token, v1)</text>
|
||||
<text x="820" y="390" text-anchor="middle" fill="#333">application/json or text/event-stream</text>
|
||||
<line x1="580" y1="340" x2="678" y2="340" stroke="#333" marker-end="url(#a)"/>
|
||||
<line x1="678" y1="380" x2="580" y2="380" stroke="#333" marker-end="url(#a)"/>
|
||||
|
||||
<!-- refusals -->
|
||||
<rect x="580" y="160" width="380" height="76" rx="6" fill="#f5f5f5" stroke="#999" stroke-dasharray="5,4"/>
|
||||
<text x="770" y="182" text-anchor="middle" fill="#111">Refused before the call</text>
|
||||
<text x="770" y="200" text-anchor="middle" fill="#555">unsupported param without drop_params: 400</text>
|
||||
<text x="770" y="218" text-anchor="middle" fill="#555">Responses API bridge cases, Cloudflare bases: 501</text>
|
||||
|
||||
<!-- bottom boxes -->
|
||||
<rect x="20" y="530" width="440" height="210" rx="6" fill="#ffffff" stroke="#b8860b"/>
|
||||
<text x="240" y="552" text-anchor="middle" fill="#111" font-weight="bold">usage and cost (gateway-inference Finish)</text>
|
||||
<text x="36" y="576" fill="#333">usage from the body or the include_usage chunk</text>
|
||||
<text x="36" y="596" fill="#333">config prices on the deployment win</text>
|
||||
<text x="36" y="616" fill="#333">base_model set: cost map azure/{base_model},</text>
|
||||
<text x="54" y="634" fill="#333">then {base_model}</text>
|
||||
<text x="36" y="654" fill="#333">otherwise: azure/{model the response names},</text>
|
||||
<text x="54" y="672" fill="#333">then azure/{deployment}</text>
|
||||
<text x="36" y="696" fill="#555">the bare deployment name is never priced, it names</text>
|
||||
<text x="36" y="714" fill="#555">a resource, not a model</text>
|
||||
|
||||
<rect x="500" y="530" width="460" height="210" rx="6" fill="#ffffff" stroke="#3c8d3c"/>
|
||||
<text x="730" y="552" text-anchor="middle" fill="#111" font-weight="bold">errors</text>
|
||||
<text x="516" y="576" fill="#333">upstream 4xx/5xx: same status, OpenAI error envelope</text>
|
||||
<text x="516" y="596" fill="#333">carrying Azure's error.message</text>
|
||||
<text x="516" y="624" fill="#333">content filter (inner_error with a policy violation):</text>
|
||||
<text x="516" y="644" fill="#333">provider_specific_fields keeps innererror,</text>
|
||||
<text x="516" y="664" fill="#333">inner_error and the whole azure_error</text>
|
||||
<text x="516" y="692" fill="#333">stream cut after headers: a final</text>
|
||||
<text x="516" y="712" fill="#333">data: {"error": ...} event, then close</text>
|
||||
<line x1="240" y1="494" x2="240" y2="528" stroke="#333" marker-end="url(#a)"/>
|
||||
<line x1="500" y1="494" x2="730" y2="528" stroke="#333" marker-end="url(#a)"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.3 KiB |
Loading…
Add table
Reference in a new issue