diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index 95d911cca29..8b6496cdf2d 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -62,6 +62,7 @@ async fn handle( ) -> Result { let body = with_model(object, model); gateway.check_body(&body)?; + request::check_chat_messages(&body)?; let group = request::resolve_group(gateway, &body)?; let name = request::model_name(&body)?.to_owned(); let tracking = crate::hooks::admit( diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index a2eac437ba8..230663e2dd9 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -16,6 +16,15 @@ pub enum Error { Auth(#[from] litellm_gateway_auth::Error), #[error("invalid request body: {0}")] InvalidBody(String), + /// A param the route requires is absent, worded as the proxy's + /// `ProxyMissingRequiredParamError`. + #[error("{route}: Missing required parameter: '{param}'.")] + MissingParam { + route: &'static str, + param: &'static str, + }, + #[error(transparent)] + InvalidMessages(#[from] MalformedMessages), #[error( "/v1/messages: Invalid model name passed in model={0}. Call `/v1/models` to view available models for your key." )] @@ -35,6 +44,23 @@ pub enum Error { Refused(Refusal), } +/// How a request's `messages` falls short of a list of messages the route can send. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum MalformedMessages { + #[error("Invalid type for 'messages': expected an array of messages.")] + NotAnArray, + #[error( + "Invalid 'messages': empty array. Expected an array with minimum length 1, but got an empty array instead." + )] + Empty, + #[error("Invalid type for 'messages[{0}]': expected an object.")] + NotAnObject(usize), + #[error("Invalid type for 'messages[{0}].role': expected a string.")] + Role(usize), + #[error("Invalid type for 'messages[{0}].content': expected a string or an array.")] + Content(usize), +} + impl From for Error { fn from(refusal: Refusal) -> Self { Self::Refused(refusal) @@ -77,7 +103,10 @@ impl Error { .http_status_code() .and_then(|status| StatusCode::from_u16(status).ok()) .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), - Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST, + Self::InvalidBody(_) + | Self::UnknownModel(_) + | Self::MissingParam { .. } + | Self::InvalidMessages(_) => StatusCode::BAD_REQUEST, Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => { StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY) } @@ -121,10 +150,15 @@ impl Error { } _ => self.to_string(), }; + let param = match self { + Self::MissingParam { param, .. } => Some(*param), + Self::InvalidMessages(_) => Some("messages"), + _ => None, + }; let mut error = json!({ "message": message, "type": error_type(status), - "param": null, + "param": param, "code": status.as_u16(), }); if let Self::Route(RouteError::Transport(TransportError::Http { body, .. })) = self @@ -370,6 +404,48 @@ mod tests { assert_eq!(frame, format!("data: {expected}\n\n")); } + #[rstest] + #[case::missing( + Error::MissingParam { route: "/chat/completions", param: "messages" }, + json!({"error": { + "message": "/chat/completions: Missing required parameter: 'messages'.", + "type": "invalid_request_error", "param": "messages", "code": 400, + }}), + )] + #[case::malformed( + Error::InvalidMessages(MalformedMessages::NotAnObject(2)), + json!({"error": { + "message": "Invalid type for 'messages[2]': expected an object.", + "type": "invalid_request_error", "param": "messages", "code": 400, + }}), + )] + fn malformed_requests_are_invalid_request_errors_naming_the_param( + #[case] error: Error, + #[case] expected: Value, + ) { + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + assert_eq!(error.openai_body(), expected); + } + + #[rstest] + #[case::missing( + Error::MissingParam { route: "anthropic_messages", param: "max_tokens" }, + "anthropic_messages: Missing required parameter: 'max_tokens'.", + )] + #[case::malformed( + Error::InvalidMessages(MalformedMessages::Content(0)), + "Invalid type for 'messages[0].content': expected a string or an array." + )] + fn malformed_messages_requests_get_the_anthropic_envelope( + #[case] error: Error, + #[case] message: &str, + ) { + assert_eq!( + error.body(Some("req_1")), + json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}, "request_id": "req_1"}), + ); + } + #[rstest] #[case::content_filter( r#"{"error":{"message":"The response was filtered due to the prompt triggering Azure OpenAI's content management policy.","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":true,"severity":"high"}}}}}"#, diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 2b210e23e9b..e0209446c60 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -25,7 +25,7 @@ use litellm_inference_messages::MessagesRoute; use litellm_llms::base_llm::model_info::ModelInfo; use litellm_secrets::source::SecretSource; -pub use error::Error; +pub use error::{Error, MalformedMessages}; pub use hooks::Hooks; pub use litellm_router::{Deployment, Error as RoutingError, Router as ModelRouter}; pub use request::{JsonBody, JsonObject, RequestId}; diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index 4eb0923871a..d06e7ed4b93 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -50,6 +50,7 @@ async fn handle( JsonBody { object, raw }: JsonBody, ) -> Result { gateway.check_body(&object)?; + request::check_messages(&object)?; let group = request::resolve_group(gateway, &object)?; let name = request::model_name(&object)?.to_owned(); let tracking = crate::hooks::admit( @@ -77,6 +78,18 @@ async fn handle( } }; edit(&mut body); + let max_tokens = request::check_max_tokens(&body, || { + let usable = request::usable(identity); + group + .deployments() + .any(|deployment| usable(deployment) && request::sets_max_tokens(deployment)) + }); + if let Err(error) = max_tokens { + if let Some(tracking) = tracking { + tracking.fail(&error); + } + return Err(error); + } let reparse = || -> Result, Error> { let mut body = crate::caching::prepare(identity, request::object(&raw)?)?.0; edit(&mut body); diff --git a/litellm-rust/crates/gateway-inference/src/request.rs b/litellm-rust/crates/gateway-inference/src/request.rs index 07730b8d7bb..f92fbab9aff 100644 --- a/litellm-rust/crates/gateway-inference/src/request.rs +++ b/litellm-rust/crates/gateway-inference/src/request.rs @@ -8,7 +8,12 @@ use futures_util::StreamExt; use litellm_router::{ModelGroup, Routed}; use serde_json::{Map, Value}; -use crate::{Deployment, Error, Gateway}; +use crate::{Deployment, Error, Gateway, MalformedMessages}; + +/// The names the proxy's `ProxyMissingRequiredParamError` gives each route, as +/// `ROUTE_ENDPOINT_MAPPING` maps them. +const CHAT_COMPLETIONS: &str = "/chat/completions"; +const MESSAGES: &str = "anthropic_messages"; pub(crate) const MAX_FILE_BYTES: usize = 50 * 1024 * 1024; pub(crate) const MAX_BODY_BYTES: usize = MAX_FILE_BYTES + 1024 * 1024; @@ -96,6 +101,94 @@ pub(crate) fn model_name(body: &Map) -> Result<&str, Error> { .ok_or_else(|| Error::InvalidBody("model is required".into())) } +/// Refuses a Chat Completions body whose `messages` no provider could take, before a hook counts +/// the request or a provider sees it. As in the proxy, `messages` must be present and not null, +/// then as Python's `completion` reads it, an array of objects. A message may leave out its +/// `role`, which `validate_and_fix_openai_messages` fills in as `assistant`, but a role it names +/// must be a string. An empty array is refused here rather than by every provider. +pub(crate) fn check_chat_messages(body: &Map) -> Result<(), Error> { + let messages = match body.get("messages") { + None | Some(Value::Null) => { + return Err(Error::MissingParam { + route: CHAT_COMPLETIONS, + param: "messages", + }); + } + Some(messages) => messages, + }; + each_message(messages, |message| match message.get("role") { + None | Some(Value::Null | Value::String(_)) => Ok(()), + Some(_) => Err(MalformedMessages::Role), + })?; + match messages.as_array().is_some_and(Vec::is_empty) { + true => Err(MalformedMessages::Empty.into()), + false => Ok(()), + } +} + +/// Refuses a `/v1/messages` body whose `messages` no provider could take, before a hook counts +/// the request or a provider sees it: the proxy requires the key, and Anthropic's shape needs +/// each message to have a string `role` and a string or array `content`. +pub(crate) fn check_messages(body: &Map) -> Result<(), Error> { + let messages = body.get("messages").ok_or(Error::MissingParam { + route: MESSAGES, + param: "messages", + })?; + each_message(messages, |message| { + match message.get("role") { + Some(Value::String(_)) => {} + _ => return Err(MalformedMessages::Role), + } + match message.get("content") { + Some(Value::String(_) | Value::Array(_)) => Ok(()), + _ => Err(MalformedMessages::Content), + } + }) +} + +/// Refuses a `/v1/messages` body without a `max_tokens` key, as the proxy does once its hooks +/// have run, since a hook may set one, as the rate limits hook does for a request it caps. A +/// `max_tokens` that some deployment the caller may use sets stands in for the caller's. +pub(crate) fn check_max_tokens( + body: &Map, + deployment_sets_max_tokens: impl FnOnce() -> bool, +) -> Result<(), Error> { + match body.contains_key("max_tokens") || deployment_sets_max_tokens() { + true => Ok(()), + false => Err(Error::MissingParam { + route: MESSAGES, + param: "max_tokens", + }), + } +} + +/// Whether `deployment` names a `max_tokens` of its own in `litellm_params`. +pub(crate) fn sets_max_tokens(deployment: &Deployment) -> bool { + deployment + .litellm_params + .extra + .get("max_tokens") + .is_some_and(|value| !value.is_null()) +} + +/// Runs `check` on each message of a `messages` array, refusing anything else in its place. +fn each_message( + messages: &Value, + check: impl Fn(&Map) -> Result<(), fn(usize) -> MalformedMessages>, +) -> Result<(), Error> { + let Value::Array(messages) = messages else { + return Err(MalformedMessages::NotAnArray.into()); + }; + messages + .iter() + .enumerate() + .try_for_each(|(index, message)| match message { + Value::Object(message) => check(message).map_err(|fault| fault(index)), + _ => Err(MalformedMessages::NotAnObject(index)), + }) + .map_err(Error::from) +} + /// The model group a request names. pub(crate) fn resolve_group<'a>( gateway: &'a Gateway, @@ -145,10 +238,102 @@ mod tests { routing::post, }; use rstest::{fixture, rstest}; + use serde_json::json; use tower::ServiceExt; use super::*; + fn body(value: Value) -> Map { + let Value::Object(fields) = value else { + unreachable!("test bodies are objects") + }; + fields + } + + fn missing(route: &'static str, param: &'static str) -> Option { + Some(Error::MissingParam { route, param }.to_string()) + } + + fn malformed(fault: MalformedMessages) -> Option { + Some(Error::from(fault).to_string()) + } + + #[rstest] + #[case::text(json!([{"role": "user", "content": "hi"}]), None)] + #[case::parts_and_tools(json!([{"role": "system", "content": "s"}, {"role": "user", "content": [{"type": "text", "text": "hi"}]}, {"role": "tool", "tool_call_id": "t", "content": "x"}]), None)] + #[case::role_left_out(json!([{"content": "hi"}]), None)] + #[case::null_role(json!([{"role": null, "content": "hi"}]), None)] + #[case::absent(Value::Null, missing("/chat/completions", "messages"))] + #[case::string(json!("invalid"), malformed(MalformedMessages::NotAnArray))] + #[case::object(json!({"role": "user", "content": "hi"}), malformed(MalformedMessages::NotAnArray))] + #[case::empty(json!([]), malformed(MalformedMessages::Empty))] + #[case::string_element(json!([{"role": "user", "content": "hi"}, "hi"]), malformed(MalformedMessages::NotAnObject(1)))] + #[case::array_element(json!([[]]), malformed(MalformedMessages::NotAnObject(0)))] + #[case::numeric_role(json!([{"role": 1, "content": "hi"}]), malformed(MalformedMessages::Role(0)))] + #[case::object_role(json!([{"role": "user", "content": "a"}, {"role": {}, "content": "b"}]), malformed(MalformedMessages::Role(1)))] + fn chat_messages_must_be_a_list_of_messages( + #[case] messages: Value, + #[case] refusal: Option, + ) { + let result = check_chat_messages(&body(json!({"model": "m", "messages": messages}))); + assert_eq!(result.err().map(|error| error.to_string()), refusal); + } + + #[rstest] + fn chat_messages_left_out_are_missing() { + let error = check_chat_messages(&body(json!({"model": "m"}))).unwrap_err(); + assert_eq!( + error.to_string(), + "/chat/completions: Missing required parameter: 'messages'." + ); + } + + #[rstest] + #[case::text(json!({"messages": [{"role": "user", "content": "hi"}]}), None)] + #[case::blocks(json!({"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]}), None)] + #[case::empty_list_is_left_to_the_provider(json!({"messages": []}), None)] + #[case::no_messages(json!({"max_tokens": 8}), missing("anthropic_messages", "messages"))] + #[case::string(json!({"messages": "invalid"}), malformed(MalformedMessages::NotAnArray))] + #[case::null(json!({"messages": null}), malformed(MalformedMessages::NotAnArray))] + #[case::string_element(json!({"messages": ["hi"]}), malformed(MalformedMessages::NotAnObject(0)))] + #[case::role_left_out(json!({"messages": [{"content": "hi"}]}), malformed(MalformedMessages::Role(0)))] + #[case::numeric_role(json!({"messages": [{"role": 1, "content": "hi"}]}), malformed(MalformedMessages::Role(0)))] + #[case::content_left_out(json!({"messages": [{"role": "user", "content": "a"}, {"role": "user"}]}), malformed(MalformedMessages::Content(1)))] + #[case::null_content(json!({"messages": [{"role": "user", "content": null}]}), malformed(MalformedMessages::Content(0)))] + fn anthropic_messages_need_a_role_and_content( + #[case] request: Value, + #[case] refusal: Option, + ) { + let result = check_messages(&body(request)); + assert_eq!(result.err().map(|error| error.to_string()), refusal); + } + + #[rstest] + #[case::set(json!({"max_tokens": 8}), false, None)] + #[case::null_is_present(json!({"max_tokens": null}), false, None)] + #[case::deployment_sets_it(json!({}), true, None)] + #[case::unset(json!({}), false, missing("anthropic_messages", "max_tokens"))] + fn anthropic_messages_need_max_tokens_from_the_caller_or_a_deployment( + #[case] request: Value, + #[case] deployment_sets_max_tokens: bool, + #[case] refusal: Option, + ) { + let result = check_max_tokens(&body(request), || deployment_sets_max_tokens); + assert_eq!(result.err().map(|error| error.to_string()), refusal); + } + + #[rstest] + #[case::set(json!({"model": "anthropic/x", "max_tokens": 8}), true)] + #[case::null(json!({"model": "anthropic/x", "max_tokens": null}), false)] + #[case::unset(json!({"model": "anthropic/x"}), false)] + fn deployments_set_max_tokens_in_their_litellm_params( + #[case] params: Value, + #[case] sets: bool, + ) { + let deployment = Deployment::from_params(&serde_json::from_value(params).unwrap()).unwrap(); + assert_eq!(sets_max_tokens(&deployment), sets); + } + #[fixture] fn limited_bodies() -> Router { Router::new() diff --git a/litellm-rust/crates/gateway-inference/tests/messages.rs b/litellm-rust/crates/gateway-inference/tests/messages.rs index 46721d07347..21f36084862 100644 --- a/litellm-rust/crates/gateway-inference/tests/messages.rs +++ b/litellm-rust/crates/gateway-inference/tests/messages.rs @@ -90,6 +90,30 @@ async fn invalid_messages_stays_an_anthropic_error() { assert_eq!(body["error"]["type"], "invalid_request_error"); } +#[rstest] +#[case::messages_left_out(json!({"model": "public/model", "max_tokens": 16}), "anthropic_messages: Missing required parameter: 'messages'.")] +#[case::max_tokens_left_out(json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}), "anthropic_messages: Missing required parameter: 'max_tokens'.")] +#[case::message_without_content(json!({"model": "public/model", "messages": [{"role": "user"}], "max_tokens": 16}), "Invalid type for 'messages[0].content': expected a string or an array.")] +#[tokio::test] +async fn malformed_messages_requests_never_reach_the_provider( + #[case] body: serde_json::Value, + #[case] message: &str, +) { + let upstream = MockServer::start().await; + let response = support::post( + support::app("anthropic/test-model", &upstream.uri()), + "/v1/messages", + body, + ) + .await; + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["type"], "error"); + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert_eq!(body["error"]["message"], message); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + /// Answers with the SSE head and one event, then drops the connection short of the /// announced body length. async fn truncating_upstream() -> String { @@ -186,7 +210,12 @@ async fn model_permissions_prevent_provider_calls(#[case] path: &str) { &upstream.uri(), litellm_gateway_auth::Permissions::None, ); - let response = support::post(app, path, json!({"model": "public/model"})).await; + let response = support::post( + app, + path, + json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ) + .await; assert_eq!(response.status(), 403); assert!(upstream.received_requests().await.unwrap().is_empty()); } diff --git a/litellm-rust/crates/gateway-inference/tests/routes.rs b/litellm-rust/crates/gateway-inference/tests/routes.rs index 6d37f57746f..f1fb8a977ab 100644 --- a/litellm-rust/crates/gateway-inference/tests/routes.rs +++ b/litellm-rust/crates/gateway-inference/tests/routes.rs @@ -1,16 +1,9 @@ mod support; -use std::sync::Arc; - use axum::{ body::{Body, Bytes}, http::Request, }; -use litellm_http::{ - ClientVariant, HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver, -}; -use litellm_inference::resources::CoreResources; -use litellm_inference_chat::{ChatCompletionsRoute, types::ChatCompletionsRequest}; use rstest::rstest; use serde_json::{Value, json}; use tower::ServiceExt; @@ -68,66 +61,37 @@ async fn chat_aliases_call_core_and_use_the_body_model_before_the_path( } #[rstest] -#[case::missing_messages(json!({}), 400)] -#[case::malformed_messages(json!({"messages": "hi"}), 400)] -#[case::invalid_streaming_request(json!({"messages": [], "stream": true}), 400)] +#[case::missing_messages(json!({}), "/chat/completions: Missing required parameter: 'messages'.")] +#[case::malformed_messages(json!({"messages": "hi"}), "Invalid type for 'messages': expected an array of messages.")] +#[case::empty_streaming_request(json!({"messages": [], "stream": true}), "Invalid 'messages': empty array. Expected an array with minimum length 1, but got an empty array instead.")] +#[case::message_not_an_object(json!({"messages": ["hi"]}), "Invalid type for 'messages[0]': expected an object.")] +#[case::role_not_a_string(json!({"messages": [{"role": 1, "content": "hi"}]}), "Invalid type for 'messages[0].role': expected a string.")] #[tokio::test] -async fn chat_errors_come_from_core( +async fn malformed_chat_requests_are_refused_before_any_provider_call( #[case] fields: Value, - #[case] status: u16, + #[case] message: &str, #[values("/v1/chat/completions", "/engines/public%2Fmodel/chat/completions")] path: &str, + #[values("anthropic/test-model", "openai/test-model")] model: &str, ) { let upstream = MockServer::start().await; - let base = upstream.uri(); - let resources = CoreResources::new(Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)))); - let http = Resolution::from(&HttpSettings::default()).config; - let provider = resources - .pool - .client(&http, ClientVariant::Provider) - .unwrap(); - let fields = fields.as_object().unwrap(); - let error = ChatCompletionsRoute::new( - provider, - resources.auth.clone(), - Arc::new(support::NoSecrets), - ) - .execute( - ChatCompletionsRequest { - model: "anthropic/test-model", - messages: fields.get("messages").cloned().unwrap_or_default(), - optional_params: fields - .iter() - .filter(|(name, _)| name.as_str() != "messages") - .map(|(name, value)| (name.clone(), value.clone())) - .collect(), - api_key: Some("test-key"), - api_base: Some(&base), - custom_llm_provider: None, - extra_headers: None, - timeout: None, - drop_params: Some(false), - model_info: None, - }, - &(), - None, - ) - .await - .unwrap_err(); let body = fields + .as_object() + .unwrap() .iter() .map(|(name, value)| (name.clone(), value.clone())) .chain([("model".into(), json!("public/model"))]) .collect(); let response = support::post( - support::app("anthropic/test-model", &base), + support::app(model, &upstream.uri()), path, Value::Object(body), ) .await; - assert_eq!(response.status(), status); + assert_eq!(response.status(), 400); let body = support::json(response).await; - assert_eq!(body["error"]["message"], error.to_string()); - assert_eq!(body["error"]["code"], status); + assert_eq!(body["error"]["message"], message); + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert_eq!(body["error"]["param"], "messages"); assert!(upstream.received_requests().await.unwrap().is_empty()); } diff --git a/litellm-rust/crates/gateway/tests/rate_limits.rs b/litellm-rust/crates/gateway/tests/rate_limits.rs index 3bb2b9f9efb..9d96450405a 100644 --- a/litellm-rust/crates/gateway/tests/rate_limits.rs +++ b/litellm-rust/crates/gateway/tests/rate_limits.rs @@ -269,6 +269,41 @@ async fn requests_over_the_key_limit_are_refused_like_the_proxy() { assert_eq!(gateway.upstream.calls.load(Ordering::SeqCst), 2); } +#[rstest] +#[case::chat_messages_text("/v1/chat/completions", json!({"model": "claude", "messages": "invalid"}))] +#[case::chat_messages_left_out("/v1/chat/completions", json!({"model": "claude"}))] +#[case::chat_messages_empty("/v1/chat/completions", json!({"model": "claude", "messages": []}))] +#[case::chat_message_text("/v1/chat/completions", json!({"model": "claude", "messages": ["hi"]}))] +#[case::chat_numeric_role("/v1/chat/completions", json!({"model": "claude", "messages": [{"role": 7, "content": "hi"}]}))] +#[case::messages_text("/v1/messages", json!({"model": "claude", "max_tokens": 8, "messages": "invalid"}))] +#[case::messages_without_content("/v1/messages", json!({"model": "claude", "max_tokens": 8, "messages": [{"role": "user"}]}))] +#[tokio::test] +async fn malformed_requests_are_refused_before_they_count_or_reach_a_provider( + #[case] path: &str, + #[case] body: Value, +) { + let limits = KeyLimits { + key: rpm(1), + ..KeyLimits::default() + }; + let gateway = gateway(vec![("sk-key", key(None, limits))], Setup::default()).await; + let refused = send(&gateway.app, path, body).await; + assert_eq!(refused.status, 400); + assert_eq!(refused.body["error"]["type"], "invalid_request_error"); + // `/v1/messages` answers in Anthropic's envelope, Chat Completions in OpenAI's. + let anthropic = path == "/v1/messages"; + assert_eq!(refused.body["type"] == "error", anthropic); + assert_eq!(gateway.upstream.calls.load(Ordering::SeqCst), 0); + // The refusal took nothing from the key's one request a minute. + let admitted = send(&gateway.app, path, message(json!({}))).await; + assert_eq!(admitted.status, 200); + assert_eq!( + header(&admitted, "x-ratelimit-api_key-remaining-requests"), + Some("0") + ); + assert_eq!(gateway.upstream.calls.load(Ordering::SeqCst), 1); +} + #[rstest] #[case::user(KeyLimits { user: rpm(1), ..KeyLimits::default() }, "user: user-1")] #[case::team(KeyLimits { team: rpm(1), ..KeyLimits::default() }, "team: team-1")] diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index bdc96ec6bab..349f8aa5c50 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -86,7 +86,11 @@ async fn authenticates_before_serving_mounted_inference_routes( .post(format!("http://{address}{path}")) .timeout(Duration::from_secs(5)) .header("x-request-id", "gateway-test") - .json(&json!({"model": "unconfigured-model"})); + .json(&json!({ + "model": "unconfigured-model", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 8, + })); let request = match authorization { Some(value) => request.header("authorization", value), None => request,