diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 710f38b7f9a..e15a1df68d1 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3775,6 +3775,7 @@ dependencies = [ "tower-sessions-moka-store", "tracing", "uuid", + "wiremock", ] [[package]] @@ -3818,6 +3819,7 @@ dependencies = [ "litellm-inference-messages", "litellm-inference-ocr", "litellm-inference-responses", + "litellm-inference-tokenize", "litellm-inference-transcription", "litellm-llms", "litellm-llms-types", diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 312d44b9a4b..cecf80a2a61 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -18,6 +18,7 @@ litellm-inference-responses.workspace = true litellm-inference-messages.workspace = true litellm-inference-chat.workspace = true litellm-inference-ocr.workspace = true +litellm-inference-tokenize.workspace = true litellm-host-http.workspace = true litellm-host.workspace = true litellm-http.workspace = true diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 4536a94e566..e2134291d8a 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -11,16 +11,19 @@ pub mod messages; mod ocr; mod request; mod responses; +mod tokenize; use std::sync::Arc; use axum::{Router, routing::post}; +use litellm_host::{interceptors::Interceptors, observation::ObservationSender}; use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy}; use litellm_inference::resources::CoreResources; use litellm_inference_chat::ChatCompletionsRoute; use litellm_inference_messages::MessagesRoute; use litellm_inference_ocr::OcrRoute; use litellm_inference_responses::ResponsesRoute; +use litellm_inference_tokenize::TokenizeRoute; use litellm_inference_transcription::AudioTranscriptionRoute; use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings}; use litellm_secrets::source::SecretSource; @@ -36,6 +39,7 @@ pub struct Gateway { pub messages: MessagesRoute, pub ocr: OcrRoute, pub responses: ResponsesRoute, + pub tokenize: TokenizeRoute, pub models: ModelRouter, pub secrets: Arc, pub resources: CoreResources, @@ -79,6 +83,12 @@ impl Gateway { auth.clone(), secrets.clone(), ), + tokenize: TokenizeRoute::new( + provider.clone(), + credential.clone(), + auth.clone(), + secrets.clone(), + ), responses: ResponsesRoute::new(provider, credential, auth.clone(), secrets.clone()), ocr: OcrRoute::new(OcrClient::new( &resources.pool, @@ -97,6 +107,17 @@ impl Gateway { } pub fn router(gateway: Arc) -> Router { + router_with_tokenize_hooks(gateway, (), None) +} + +pub fn router_with_tokenize_hooks( + gateway: Arc, + interceptors: H, + observers: Option, +) -> Router +where + H: Interceptors + Clone + 'static, +{ Router::new() .route("/v1/messages", post(messages::create)) .route("/ocr", post(ocr::create)) @@ -116,10 +137,11 @@ pub fn router(gateway: Arc) -> Router { .route("/v1/embeddings", post(request::unsupported)) .route("/completions", post(request::unsupported)) .route("/v1/completions", post(request::unsupported)) + .with_state(gateway.clone()) + .merge(tokenize::router(gateway, interceptors, observers)) .layer(axum::extract::DefaultBodyLimit::max( request::MAX_BODY_BYTES, )) - .with_state(gateway) } fn model_routes() -> Router> { diff --git a/litellm-rust/crates/gateway-inference/src/tokenize.rs b/litellm-rust/crates/gateway-inference/src/tokenize.rs new file mode 100644 index 00000000000..44f4de60a09 --- /dev/null +++ b/litellm-rust/crates/gateway-inference/src/tokenize.rs @@ -0,0 +1,118 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use axum::{Json, Router, extract::State, response::Response, routing::post}; +use litellm_auth::{InputSource, SecretValue, Sourced}; +use litellm_gateway_auth::AuthenticatedRequest; +use litellm_host::{interceptors::Interceptors, observation::ObservationSender}; +use litellm_inference_tokenize::TokenizeRequest; +use litellm_llms_types::formats::tokenize::TokenizeRequest as TokenizePayload; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::{Deployment, Error, Gateway, JsonObject, request}; + +#[derive(Clone)] +struct TokenizeState { + gateway: Arc, + interceptors: H, + observers: Option, +} + +#[derive(Deserialize)] +struct DeploymentOptions { + extra_headers: Option>, + #[serde(flatten)] + params: Map, +} + +pub(crate) fn router( + gateway: Arc, + interceptors: H, + observers: Option, +) -> Router +where + H: Interceptors + Clone + 'static, +{ + Router::new() + .route("/tokenize", post(create::)) + .route("/v1/tokenize", post(create::)) + .with_state(TokenizeState { + gateway, + interceptors, + observers, + }) +} + +async fn create( + State(state): State>, + identity: AuthenticatedRequest, + JsonObject(body): JsonObject, +) -> Result +where + H: Interceptors + Clone + 'static, +{ + let deployment = request::resolve_deployment(&state.gateway, &body)?; + request::authorize_model(&identity, deployment, &body).await?; + let payload: TokenizePayload = serde_json::from_value(Value::Object(body)) + .map_err(|error| Error::InvalidBody(error.to_string()))?; + let call = project(payload, deployment)?; + litellm_host_http::serve_unary( + state + .gateway + .tokenize + .clone() + .machine(call, state.observers.clone()), + (), + state.interceptors, + litellm_host_http::Unary::new(Json), + state.observers, + ) + .await + .map_err(Error::from) +} + +fn project(payload: TokenizePayload, deployment: &Deployment) -> Result { + let options: DeploymentOptions = serde_json::from_value(Value::Object( + deployment + .litellm_params + .extra + .iter() + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + )) + .map_err(|_| Error::InvalidBody("invalid tokenization deployment configuration".into()))?; + let params = options + .params + .into_iter() + .chain( + deployment + .litellm_params + .api_version + .as_ref() + .map(|version| ("api_version".into(), Value::String(version.clone()))), + ) + .collect(); + Ok(TokenizeRequest { + model: deployment.model.clone(), + custom_llm_provider: deployment.custom_llm_provider.clone(), + api_key: deployment + .api_key + .clone() + .map(|key| Sourced::new(SecretValue::new(key), InputSource::Deployment)), + api_base: deployment + .api_base + .clone() + .map(|base| Sourced::new(base, InputSource::Deployment)), + extra_headers: deployment_values(options.extra_headers.unwrap_or_default()), + deployment_params: deployment_values(params), + timeout: deployment.timeout, + ..payload.into() + }) +} + +fn deployment_values(values: Map) -> BTreeMap> { + values + .into_iter() + .map(|(name, value)| (name, Sourced::new(value, InputSource::Deployment))) + .collect() +} diff --git a/litellm-rust/crates/gateway-inference/tests/support/mod.rs b/litellm-rust/crates/gateway-inference/tests/support/mod.rs index 0e37bb0aff9..226c65428db 100644 --- a/litellm-rust/crates/gateway-inference/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-inference/tests/support/mod.rs @@ -10,7 +10,8 @@ use axum::{ response::Response, }; use futures_util::future::BoxFuture; -use litellm_gateway_inference::{Deployment, Gateway, router}; +use litellm_gateway_inference::{Deployment, Gateway, router, router_with_tokenize_hooks}; +use litellm_host::{interceptors::Interceptors, observation::ObservationSender}; use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; use litellm_secrets::{SecretValue, source::SecretSource}; @@ -40,6 +41,27 @@ pub fn app_with_permissions( configured_app(model, api_base, permissions, None, None) } +pub fn app_with_deployment(deployment: Deployment) -> Router { + configured_router( + gateway_with_deployment(deployment), + litellm_gateway_auth::Permissions::All, + None, + ) +} + +pub fn app_with_tokenize_hooks( + deployment: Deployment, + permissions: litellm_gateway_auth::Permissions, + interceptors: H, + observers: Option, +) -> Router +where + H: Interceptors + Clone + 'static, +{ + let gateway = gateway_with_deployment(deployment); + configured_router_with_hooks(gateway, permissions, interceptors, observers) +} + pub fn app_with_cache( model: &str, api_base: &str, @@ -76,38 +98,63 @@ fn configured_app( cache: Option>, principal: Option, ) -> Router { - let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); - let http = Resolution::from(&HttpSettings::default()).config; - let secrets = Arc::new(NoSecrets); - let resources = CoreResources::new(pool); - let gateway = Gateway::new( - resources, - http, - secrets, - [( - "public/model".into(), - Deployment { - model: model.into(), - api_base: Some(api_base.into()), - api_key: Some("test-key".into()), - timeout: Some(Duration::from_secs(5)), - ..Default::default() - }, - )] - .into_iter() - .collect(), - ) - .unwrap(); + let gateway = build_gateway(Deployment { + model: model.into(), + api_base: Some(api_base.into()), + api_key: Some("test-key".into()), + timeout: Some(Duration::from_secs(5)), + ..Default::default() + }); let gateway = match cache { Some(cache) => gateway.with_cache(cache), None => gateway, }; - router(Arc::new(gateway)).layer(axum::middleware::from_fn_with_state( + configured_router(Arc::new(gateway), permissions, principal) +} + +fn gateway_with_deployment(deployment: Deployment) -> Arc { + Arc::new(build_gateway(deployment)) +} + +fn build_gateway(deployment: Deployment) -> Gateway { + let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); + let http = Resolution::from(&HttpSettings::default()).config; + let secrets = Arc::new(NoSecrets); + let resources = CoreResources::new(pool); + Gateway::new( + resources, + http, + secrets, + [("public/model".into(), deployment)].into_iter().collect(), + ) + .unwrap() +} + +fn configured_router( + gateway: Arc, + permissions: litellm_gateway_auth::Permissions, + principal: Option, +) -> Router { + router(gateway).layer(axum::middleware::from_fn_with_state( (permissions, principal), test_identity, )) } +fn configured_router_with_hooks( + gateway: Arc, + permissions: litellm_gateway_auth::Permissions, + interceptors: H, + observers: Option, +) -> Router +where + H: Interceptors + Clone + 'static, +{ + router_with_tokenize_hooks(gateway, interceptors, observers).layer( + axum::middleware::from_fn_with_state((permissions, None), test_identity), + ) +} + async fn test_identity( axum::extract::State((permissions, principal)): axum::extract::State<( litellm_gateway_auth::Permissions, diff --git a/litellm-rust/crates/gateway-inference/tests/tokenize.rs b/litellm-rust/crates/gateway-inference/tests/tokenize.rs new file mode 100644 index 00000000000..78889ab36a7 --- /dev/null +++ b/litellm-rust/crates/gateway-inference/tests/tokenize.rs @@ -0,0 +1,534 @@ +mod support; + +use std::sync::{Arc, Mutex}; + +use litellm_gateway_inference::Deployment; +use litellm_host::{ + interceptors::{ + Cost, ExecutionFacts, Interceptors, RawResponse, RequestContext, ResultSource, WireRequest, + }, + lifecycle::{CallEvent, ExecutionEvent}, + observation::observation_channel, +}; +use rstest::rstest; +use serde_json::{Value, json}; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json, header, method, path}, +}; + +fn deployment(model: &str, api_base: Option) -> Deployment { + let mut deployment = Deployment { + model: model.into(), + api_base, + api_key: Some("deployment-key".into()), + ..Default::default() + }; + deployment.litellm_params = serde_json::from_value(json!({ + "model": model + })) + .unwrap(); + deployment +} + +async fn response_json(response: axum::response::Response) -> Value { + serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap() +} + +#[rstest] +#[case::available_omitted(json!({}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":[11,12],"tokens":[" café"," 世界 "]}))] +#[case::count_empty(json!({"include":[]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":null,"tokens":null}))] +#[case::ids(json!({"include":["token_ids"]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":[11,12],"tokens":null}))] +#[case::pieces(json!({"include":["tokens"]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":null,"tokens":[" café"," 世界 "]}))] +#[case::both(json!({"include":["token_ids","tokens"]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":[11,12],"tokens":[" café"," 世界 "]}))] +#[tokio::test] +async fn aliases_project_strict_payload_and_keep_all_response_keys( + #[case] options: Value, + #[case] upstream_request: Value, + #[case] expected: Value, + #[values("/tokenize", "/v1/tokenize")] endpoint: &str, +) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(header("authorization", "Bearer deployment-key")) + .and(body_json(upstream_request)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count": 2, "tokens": [11, 12], "token_strs": [" café", " 世界 "] + }))) + .expect(1) + .mount(&upstream) + .await; + let body = json!({"model":"public/model","prompt":" café 世界 "}) + .as_object() + .unwrap() + .iter() + .chain(options.as_object().unwrap().iter()) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + let response = support::post( + support::app_with_deployment(deployment( + "hosted_vllm/org/model", + Some(format!("{}/v1", upstream.uri())), + )), + endpoint, + Value::Object(body), + ) + .await; + assert_eq!(response.status(), 200); + let result = response_json(response).await; + assert_eq!(result["model"], "org/model"); + assert_eq!(result["count"], 2); + assert_eq!(result["token_ids"], expected["token_ids"]); + assert_eq!(result["tokens"], expected["tokens"]); + assert_eq!(result.as_object().unwrap().len(), 4); +} + +#[rstest] +#[case::unknown("messages", json!([]))] +#[case::private("api_base", json!("https://caller.invalid"))] +#[case::resource("project", json!("caller-project"))] +#[case::generation("temperature", json!(0.2))] +#[case::credentials("api_key", json!("caller-secret"))] +#[case::metadata("metadata", json!({"user":"caller"}))] +#[case::images("images", json!([]))] +#[case::tools("tools", json!([]))] +#[case::template("chat_template", json!("template"))] +#[case::output("output", json!("count"))] +#[case::available("available", json!(true))] +#[case::custom_backend("backend", json!({"type":"local"}))] +#[case::trusted_sources("trusted_sources", json!([]))] +#[case::space("space", json!("caller-space"))] +#[case::api_version("api_version", json!("caller-version"))] +#[case::credentials_map("credentials", json!({}))] +#[tokio::test] +async fn unknown_semantic_private_and_resource_fields_are_rejected( + #[case] field: &str, + #[case] value: Value, +) { + let upstream = MockServer::start().await; + let mut body = json!({"model":"public/model","prompt":"hello"}); + body[field] = value; + let response = support::post( + support::app("hosted_vllm/model", &upstream.uri()), + "/tokenize", + body, + ) + .await; + assert_eq!(response.status(), 400); + assert!(response_json(response).await["error"]["message"].is_string()); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[case::null_include(json!({"include":null}))] +#[case::scalar_include(json!({"include":"tokens"}))] +#[case::duplicate_include(json!({"include":["tokens","tokens"]}))] +#[case::unknown_include(json!({"include":["pieces"]}))] +#[case::null_special(json!({"add_special_tokens":null}))] +#[case::string_special(json!({"add_special_tokens":"false"}))] +#[case::number_special(json!({"add_special_tokens":0}))] +#[tokio::test] +async fn malformed_include_and_special_token_values_fail_before_io(#[case] extra: Value) { + let upstream = MockServer::start().await; + let mut body = json!({"model":"public/model","prompt":"hello"}); + body.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + let response = support::post( + support::app("hosted_vllm/model", &upstream.uri()), + "/v1/tokenize", + body, + ) + .await; + assert_eq!(response.status(), 400); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[case::tokens(json!(["tokens"]))] +#[case::ids(json!(["token_ids"]))] +#[case::both(json!(["token_ids", "tokens"]))] +#[tokio::test] +async fn requested_representations_are_enforced(#[case] include: Value) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count": 1, "tokens": [7], "token_strs": ["hello"] + }))) + .expect(1) + .mount(&upstream) + .await; + let response = support::post( + support::app("hosted_vllm/model", &upstream.uri()), + "/tokenize", + json!({"model":"public/model","prompt":"hello","include":include}), + ) + .await; + assert_eq!(response.status(), 200); +} + +#[rstest] +#[tokio::test] +async fn configured_provider_headers_and_special_token_override_reach_vllm_without_defaults() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(header("authorization", "Bearer deployment-key")) + .and(header("x-deployment", "configured")) + .and(body_json(json!({ + "model":"org/model", + "prompt":"", + "add_special_tokens":false, + "return_token_strs":true + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":0,"tokens":[],"token_strs":[] + }))) + .expect(1) + .mount(&upstream) + .await; + let mut deployment = deployment( + "hosted_vllm/org/model", + Some(format!("{}/v1", upstream.uri())), + ); + deployment.litellm_params = serde_json::from_value(json!({ + "model":"hosted_vllm/org/model", + "backend":{"type":"provider"}, + "extra_headers":{"x-deployment":"configured"}, + "temperature":0.7, + "max_tokens":100 + })) + .unwrap(); + let response = support::post( + support::app_with_deployment(deployment), + "/tokenize", + json!({ + "model":"public/model", + "prompt":"", + "add_special_tokens":false, + "include":["token_ids","tokens"] + }), + ) + .await; + assert_eq!(response.status(), 200); + let body = response_json(response).await; + assert_eq!(body["count"], 0); + assert_eq!(body["token_ids"], json!([])); + assert_eq!(body["tokens"], json!([])); +} + +#[rstest] +#[tokio::test] +async fn watsonx_unsupported_representation_is_rejected_before_io() { + let upstream = MockServer::start().await; + let response = support::post( + support::app("watsonx/model", &upstream.uri()), + "/tokenize", + json!({ + "model":"public/model", + "prompt":"hello", + "include":["token_ids"] + }), + ) + .await; + assert_eq!(response.status(), 400); + assert!( + response_json(response).await["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("include")) + ); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[case::enabled(true)] +#[case::disabled(false)] +#[tokio::test] +async fn watsonx_rejects_explicit_special_token_policy_before_io(#[case] add_special_tokens: bool) { + let upstream = MockServer::start().await; + let response = support::post( + support::app("watsonx/model", &upstream.uri()), + "/tokenize", + json!({ + "model":"public/model", + "prompt":"hello", + "add_special_tokens":add_special_tokens + }), + ) + .await; + assert_eq!(response.status(), 400); + assert!( + response_json(response).await["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("add_special_tokens")) + ); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[case::pieces(json!({"count":1,"tokens":[7],"token_strs":null}), json!(["tokens"]))] +#[case::ids(json!({"count":1,"tokens":null,"token_strs":["hello"]}), json!(["token_ids"]))] +#[tokio::test] +async fn provider_missing_requested_representation_is_not_fabricated( + #[case] response_body: Value, + #[case] include: Value, +) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(response_body)) + .expect(1) + .mount(&upstream) + .await; + let response = support::post( + support::app("hosted_vllm/model", &upstream.uri()), + "/tokenize", + json!({"model":"public/model","prompt":"hello","include":include}), + ) + .await; + assert_eq!(response.status(), 500); +} + +#[rstest] +#[case::rate_limit(429)] +#[case::unauthorized(401)] +#[case::unavailable(503)] +#[tokio::test] +async fn upstream_errors_keep_their_status(#[case] status: u16) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status)) + .expect(1) + .mount(&upstream) + .await; + let response = support::post( + support::app("hosted_vllm/model", &upstream.uri()), + "/tokenize", + json!({"model":"public/model","prompt":"hello"}), + ) + .await; + assert_eq!(response.status().as_u16(), status); +} + +#[rstest] +#[tokio::test] +async fn authorization_and_unknown_aliases_precede_provider_io() { + let upstream = MockServer::start().await; + let denied = support::post( + support::app_with_permissions( + "hosted_vllm/model", + &upstream.uri(), + litellm_gateway_auth::Permissions::None, + ), + "/tokenize", + json!({"model":"public/model","prompt":"hello"}), + ) + .await; + assert_eq!(denied.status(), 403); + let unknown = support::post( + support::app("hosted_vllm/model", &upstream.uri()), + "/tokenize", + json!({"model":"missing","prompt":"hello"}), + ) + .await; + assert_eq!(unknown.status(), 400); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[case::alias_grant("public/model", 200, 1)] +#[case::provider_model_grant("hosted_vllm/org/model", 403, 0)] +#[tokio::test] +async fn restricted_model_grants_match_public_alias_before_provider_io( + #[case] granted_name: &str, + #[case] expected_status: u16, + #[case] expected_requests: usize, + #[values("/tokenize", "/v1/tokenize")] endpoint: &str, +) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1,"tokens":[7],"token_strs":["hello"] + }))) + .mount(&upstream) + .await; + let permissions = litellm_gateway_auth::Permissions::Only(Arc::from([ + litellm_gateway_auth::AccessRequest::Model { + name: granted_name.into(), + deployment: "hosted_vllm/org/model".into(), + }, + ])); + let response = support::post( + support::app_with_permissions("hosted_vllm/org/model", &upstream.uri(), permissions), + endpoint, + json!({"model":"public/model","prompt":"hello"}), + ) + .await; + + assert_eq!(response.status().as_u16(), expected_status); + assert_eq!( + upstream.received_requests().await.unwrap().len(), + expected_requests + ); +} + +#[derive(Default, Clone)] +struct Hooks { + rewrite: bool, + fail_result: bool, + calls: Arc>>, +} + +impl Interceptors for Hooks { + async fn before_provider_request( + &self, + mut wire: WireRequest, + _: RequestContext, + ) -> Result { + self.calls.lock().unwrap().push("before".into()); + if self.rewrite { + wire.body["prompt"] = json!("rewritten"); + } + Ok(wire) + } + + async fn after_provider_response( + &self, + _: RawResponse, + ) -> Result<(), litellm_inference::Error> { + self.calls.lock().unwrap().push("after".into()); + Ok(()) + } + + async fn result_ready(&self, _: ExecutionFacts) -> Result<(), litellm_inference::Error> { + self.calls.lock().unwrap().push("result".into()); + if self.fail_result { + return Err(litellm_inference::Error::InvalidRequest( + "result hook failed".into(), + )); + } + Ok(()) + } +} + +#[rstest] +#[tokio::test] +async fn hooks_rewrite_wire_and_result_failure_does_not_replay() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(body_json(json!({ + "model":"org/model","prompt":"rewritten", + "add_special_tokens":true,"return_token_strs":true + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1,"tokens":[7],"token_strs":["rewritten"] + }))) + .expect(1) + .mount(&upstream) + .await; + let hooks = Hooks { + rewrite: true, + ..Default::default() + }; + let response = support::app_with_tokenize_hooks( + deployment("hosted_vllm/org/model", Some(upstream.uri())), + litellm_gateway_auth::Permissions::All, + hooks.clone(), + None, + ); + let ok = support::post( + response, + "/tokenize", + json!({"model":"public/model","prompt":"original"}), + ) + .await; + assert_eq!(ok.status(), 200); + assert_eq!( + &*hooks.calls.lock().unwrap(), + &["before", "after", "result"] + ); + + let failing_upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1,"tokens":[7],"token_strs":["original"] + }))) + .expect(1) + .mount(&failing_upstream) + .await; + let failing = Hooks { + fail_result: true, + ..Default::default() + }; + let response = support::post( + support::app_with_tokenize_hooks( + deployment("hosted_vllm/org/model", Some(failing_upstream.uri())), + litellm_gateway_auth::Permissions::All, + failing.clone(), + None, + ), + "/tokenize", + json!({"model":"public/model","prompt":"original"}), + ) + .await; + assert_eq!(response.status(), 400); + assert_eq!( + &*failing.calls.lock().unwrap(), + &["before", "after", "result"] + ); +} + +#[rstest] +#[tokio::test] +async fn provider_facts_defer_cost_and_observers_see_terminals() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1,"tokens":[7],"token_strs":["hello"] + }))) + .expect(1) + .mount(&upstream) + .await; + let (sender, mut events) = observation_channel(std::num::NonZeroUsize::new(16).unwrap()); + let hooks = Hooks::default(); + let response = support::post( + support::app_with_tokenize_hooks( + deployment("hosted_vllm/org/model", Some(upstream.uri())), + litellm_gateway_auth::Permissions::All, + hooks.clone(), + Some(sender), + ), + "/tokenize", + json!({"model":"public/model","prompt":"hello"}), + ) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + &*hooks.calls.lock().unwrap(), + &["before", "after", "result"] + ); + assert!(matches!( + events.recv().await, + Some(CallEvent::Started { .. }) + )); + let facts = loop { + match events.recv().await { + Some(CallEvent::Execution(ExecutionEvent::ResultReady { facts })) => break facts, + Some(_) => {} + None => panic!("observer closed before facts"), + } + }; + assert_eq!(facts.provider.provider, "hosted_vllm"); + assert_eq!(facts.source, ResultSource::Provider); + assert_eq!(facts.cost, Cost::Deferred); + assert!(matches!( + events.recv().await, + Some(CallEvent::Succeeded { .. }) + )); +} diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 387bcd43231..1e041347ec9 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -36,3 +36,4 @@ rstest.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["sync"] } tower = { version = "0.5", features = ["util"] } +wiremock.workspace = true diff --git a/litellm-rust/crates/gateway/tests/tokenize.rs b/litellm-rust/crates/gateway/tests/tokenize.rs new file mode 100644 index 00000000000..0259866817e --- /dev/null +++ b/litellm-rust/crates/gateway/tests/tokenize.rs @@ -0,0 +1,80 @@ +use axum::{ + body::{Body, to_bytes}, + http::Request, +}; +use litellm_config::Config; +use rstest::rstest; +use serde_json::{Value, json}; +use tower::ServiceExt; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, +}; + +fn config(api_base: &str) -> Config { + Config::from_yaml( + &json!({ + "model_list": [{ + "model_name": "public/model", + "litellm_params": { + "model": "hosted_vllm/org/model", + "api_base": api_base, + "api_key": "deployment-key" + } + }], + "general_settings": {"master_key": "gateway-key"} + }) + .to_string(), + ) + .unwrap() +} + +#[rstest] +#[case::unversioned("/tokenize")] +#[case::versioned("/v1/tokenize")] +#[tokio::test] +async fn mounted_gateway_authenticates_alias_before_provider_io(#[case] endpoint: &str) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(header("authorization", "Bearer deployment-key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count": 2, "tokens": [11, 12], "token_strs": ["hel", "lo"] + }))) + .expect(1) + .mount(&upstream) + .await; + let config = config(&upstream.uri()); + let inference = litellm_gateway::build_inference(&config).unwrap(); + let app = litellm_gateway::router(inference, &config, None, None); + let unauthenticated = app + .clone() + .oneshot( + Request::post(endpoint) + .header("content-type", "application/json") + .body(Body::from( + json!({"model":"public/model","prompt":"hello"}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unauthenticated.status(), 401); + let response = app + .oneshot( + Request::post(endpoint) + .header("authorization", "Bearer gateway-key") + .header("content-type", "application/json") + .body(Body::from( + json!({"model":"public/model","prompt":"hello"}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + let body: Value = + serde_json::from_slice(&to_bytes(response.into_body(), 1024 * 1024).await.unwrap()) + .unwrap(); + assert_eq!(body["count"], 2); +}