mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
refactor(gateway): serve provider tokenization only
This commit is contained in:
parent
45b8747efa
commit
3d88f0ee3f
8 changed files with 830 additions and 25 deletions
2
litellm-rust/Cargo.lock
generated
2
litellm-rust/Cargo.lock
generated
|
|
@ -3775,6 +3775,7 @@ dependencies = [
|
|||
"tower-sessions-moka-store",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3818,6 +3819,7 @@ dependencies = [
|
|||
"litellm-inference-messages",
|
||||
"litellm-inference-ocr",
|
||||
"litellm-inference-responses",
|
||||
"litellm-inference-tokenize",
|
||||
"litellm-inference-transcription",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ litellm-inference-responses.workspace = true
|
|||
litellm-inference-messages.workspace = true
|
||||
litellm-inference-chat.workspace = true
|
||||
litellm-inference-ocr.workspace = true
|
||||
litellm-inference-tokenize.workspace = true
|
||||
litellm-host-http.workspace = true
|
||||
litellm-host.workspace = true
|
||||
litellm-http.workspace = true
|
||||
|
|
|
|||
|
|
@ -11,16 +11,19 @@ pub mod messages;
|
|||
mod ocr;
|
||||
mod request;
|
||||
mod responses;
|
||||
mod tokenize;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{Router, routing::post};
|
||||
use litellm_host::{interceptors::Interceptors, observation::ObservationSender};
|
||||
use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy};
|
||||
use litellm_inference::resources::CoreResources;
|
||||
use litellm_inference_chat::ChatCompletionsRoute;
|
||||
use litellm_inference_messages::MessagesRoute;
|
||||
use litellm_inference_ocr::OcrRoute;
|
||||
use litellm_inference_responses::ResponsesRoute;
|
||||
use litellm_inference_tokenize::TokenizeRoute;
|
||||
use litellm_inference_transcription::AudioTranscriptionRoute;
|
||||
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
|
@ -36,6 +39,7 @@ pub struct Gateway {
|
|||
pub messages: MessagesRoute,
|
||||
pub ocr: OcrRoute,
|
||||
pub responses: ResponsesRoute,
|
||||
pub tokenize: TokenizeRoute,
|
||||
pub models: ModelRouter,
|
||||
pub secrets: Arc<dyn SecretSource>,
|
||||
pub resources: CoreResources,
|
||||
|
|
@ -79,6 +83,12 @@ impl Gateway {
|
|||
auth.clone(),
|
||||
secrets.clone(),
|
||||
),
|
||||
tokenize: TokenizeRoute::new(
|
||||
provider.clone(),
|
||||
credential.clone(),
|
||||
auth.clone(),
|
||||
secrets.clone(),
|
||||
),
|
||||
responses: ResponsesRoute::new(provider, credential, auth.clone(), secrets.clone()),
|
||||
ocr: OcrRoute::new(OcrClient::new(
|
||||
&resources.pool,
|
||||
|
|
@ -97,6 +107,17 @@ impl Gateway {
|
|||
}
|
||||
|
||||
pub fn router(gateway: Arc<Gateway>) -> Router {
|
||||
router_with_tokenize_hooks(gateway, (), None)
|
||||
}
|
||||
|
||||
pub fn router_with_tokenize_hooks<H>(
|
||||
gateway: Arc<Gateway>,
|
||||
interceptors: H,
|
||||
observers: Option<ObservationSender>,
|
||||
) -> Router
|
||||
where
|
||||
H: Interceptors<litellm_inference::Error> + Clone + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/v1/messages", post(messages::create))
|
||||
.route("/ocr", post(ocr::create))
|
||||
|
|
@ -116,10 +137,11 @@ pub fn router(gateway: Arc<Gateway>) -> Router {
|
|||
.route("/v1/embeddings", post(request::unsupported))
|
||||
.route("/completions", post(request::unsupported))
|
||||
.route("/v1/completions", post(request::unsupported))
|
||||
.with_state(gateway.clone())
|
||||
.merge(tokenize::router(gateway, interceptors, observers))
|
||||
.layer(axum::extract::DefaultBodyLimit::max(
|
||||
request::MAX_BODY_BYTES,
|
||||
))
|
||||
.with_state(gateway)
|
||||
}
|
||||
|
||||
fn model_routes() -> Router<Arc<Gateway>> {
|
||||
|
|
|
|||
118
litellm-rust/crates/gateway-inference/src/tokenize.rs
Normal file
118
litellm-rust/crates/gateway-inference/src/tokenize.rs
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use axum::{Json, Router, extract::State, response::Response, routing::post};
|
||||
use litellm_auth::{InputSource, SecretValue, Sourced};
|
||||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use litellm_host::{interceptors::Interceptors, observation::ObservationSender};
|
||||
use litellm_inference_tokenize::TokenizeRequest;
|
||||
use litellm_llms_types::formats::tokenize::TokenizeRequest as TokenizePayload;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, request};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TokenizeState<H> {
|
||||
gateway: Arc<Gateway>,
|
||||
interceptors: H,
|
||||
observers: Option<ObservationSender>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct DeploymentOptions {
|
||||
extra_headers: Option<Map<String, Value>>,
|
||||
#[serde(flatten)]
|
||||
params: Map<String, Value>,
|
||||
}
|
||||
|
||||
pub(crate) fn router<H>(
|
||||
gateway: Arc<Gateway>,
|
||||
interceptors: H,
|
||||
observers: Option<ObservationSender>,
|
||||
) -> Router
|
||||
where
|
||||
H: Interceptors<litellm_inference::Error> + Clone + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/tokenize", post(create::<H>))
|
||||
.route("/v1/tokenize", post(create::<H>))
|
||||
.with_state(TokenizeState {
|
||||
gateway,
|
||||
interceptors,
|
||||
observers,
|
||||
})
|
||||
}
|
||||
|
||||
async fn create<H>(
|
||||
State(state): State<TokenizeState<H>>,
|
||||
identity: AuthenticatedRequest,
|
||||
JsonObject(body): JsonObject,
|
||||
) -> Result<Response, Error>
|
||||
where
|
||||
H: Interceptors<litellm_inference::Error> + Clone + 'static,
|
||||
{
|
||||
let deployment = request::resolve_deployment(&state.gateway, &body)?;
|
||||
request::authorize_model(&identity, deployment, &body).await?;
|
||||
let payload: TokenizePayload = serde_json::from_value(Value::Object(body))
|
||||
.map_err(|error| Error::InvalidBody(error.to_string()))?;
|
||||
let call = project(payload, deployment)?;
|
||||
litellm_host_http::serve_unary(
|
||||
state
|
||||
.gateway
|
||||
.tokenize
|
||||
.clone()
|
||||
.machine(call, state.observers.clone()),
|
||||
(),
|
||||
state.interceptors,
|
||||
litellm_host_http::Unary::new(Json),
|
||||
state.observers,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
fn project(payload: TokenizePayload, deployment: &Deployment) -> Result<TokenizeRequest, Error> {
|
||||
let options: DeploymentOptions = serde_json::from_value(Value::Object(
|
||||
deployment
|
||||
.litellm_params
|
||||
.extra
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect(),
|
||||
))
|
||||
.map_err(|_| Error::InvalidBody("invalid tokenization deployment configuration".into()))?;
|
||||
let params = options
|
||||
.params
|
||||
.into_iter()
|
||||
.chain(
|
||||
deployment
|
||||
.litellm_params
|
||||
.api_version
|
||||
.as_ref()
|
||||
.map(|version| ("api_version".into(), Value::String(version.clone()))),
|
||||
)
|
||||
.collect();
|
||||
Ok(TokenizeRequest {
|
||||
model: deployment.model.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
api_key: deployment
|
||||
.api_key
|
||||
.clone()
|
||||
.map(|key| Sourced::new(SecretValue::new(key), InputSource::Deployment)),
|
||||
api_base: deployment
|
||||
.api_base
|
||||
.clone()
|
||||
.map(|base| Sourced::new(base, InputSource::Deployment)),
|
||||
extra_headers: deployment_values(options.extra_headers.unwrap_or_default()),
|
||||
deployment_params: deployment_values(params),
|
||||
timeout: deployment.timeout,
|
||||
..payload.into()
|
||||
})
|
||||
}
|
||||
|
||||
fn deployment_values(values: Map<String, Value>) -> BTreeMap<String, Sourced<Value>> {
|
||||
values
|
||||
.into_iter()
|
||||
.map(|(name, value)| (name, Sourced::new(value, InputSource::Deployment)))
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -10,7 +10,8 @@ use axum::{
|
|||
response::Response,
|
||||
};
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_gateway_inference::{Deployment, Gateway, router};
|
||||
use litellm_gateway_inference::{Deployment, Gateway, router, router_with_tokenize_hooks};
|
||||
use litellm_host::{interceptors::Interceptors, observation::ObservationSender};
|
||||
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
|
||||
use litellm_inference::resources::CoreResources;
|
||||
use litellm_secrets::{SecretValue, source::SecretSource};
|
||||
|
|
@ -40,6 +41,27 @@ pub fn app_with_permissions(
|
|||
configured_app(model, api_base, permissions, None, None)
|
||||
}
|
||||
|
||||
pub fn app_with_deployment(deployment: Deployment) -> Router {
|
||||
configured_router(
|
||||
gateway_with_deployment(deployment),
|
||||
litellm_gateway_auth::Permissions::All,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn app_with_tokenize_hooks<H>(
|
||||
deployment: Deployment,
|
||||
permissions: litellm_gateway_auth::Permissions,
|
||||
interceptors: H,
|
||||
observers: Option<ObservationSender>,
|
||||
) -> Router
|
||||
where
|
||||
H: Interceptors<litellm_inference::Error> + Clone + 'static,
|
||||
{
|
||||
let gateway = gateway_with_deployment(deployment);
|
||||
configured_router_with_hooks(gateway, permissions, interceptors, observers)
|
||||
}
|
||||
|
||||
pub fn app_with_cache(
|
||||
model: &str,
|
||||
api_base: &str,
|
||||
|
|
@ -76,38 +98,63 @@ fn configured_app(
|
|||
cache: Option<Arc<dyn litellm_cache_response::ResponseCacheService>>,
|
||||
principal: Option<litellm_gateway_auth::Principal>,
|
||||
) -> Router {
|
||||
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
|
||||
let http = Resolution::from(&HttpSettings::default()).config;
|
||||
let secrets = Arc::new(NoSecrets);
|
||||
let resources = CoreResources::new(pool);
|
||||
let gateway = Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
secrets,
|
||||
[(
|
||||
"public/model".into(),
|
||||
Deployment {
|
||||
model: model.into(),
|
||||
api_base: Some(api_base.into()),
|
||||
api_key: Some("test-key".into()),
|
||||
timeout: Some(Duration::from_secs(5)),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
)
|
||||
.unwrap();
|
||||
let gateway = build_gateway(Deployment {
|
||||
model: model.into(),
|
||||
api_base: Some(api_base.into()),
|
||||
api_key: Some("test-key".into()),
|
||||
timeout: Some(Duration::from_secs(5)),
|
||||
..Default::default()
|
||||
});
|
||||
let gateway = match cache {
|
||||
Some(cache) => gateway.with_cache(cache),
|
||||
None => gateway,
|
||||
};
|
||||
router(Arc::new(gateway)).layer(axum::middleware::from_fn_with_state(
|
||||
configured_router(Arc::new(gateway), permissions, principal)
|
||||
}
|
||||
|
||||
fn gateway_with_deployment(deployment: Deployment) -> Arc<Gateway> {
|
||||
Arc::new(build_gateway(deployment))
|
||||
}
|
||||
|
||||
fn build_gateway(deployment: Deployment) -> Gateway {
|
||||
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
|
||||
let http = Resolution::from(&HttpSettings::default()).config;
|
||||
let secrets = Arc::new(NoSecrets);
|
||||
let resources = CoreResources::new(pool);
|
||||
Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
secrets,
|
||||
[("public/model".into(), deployment)].into_iter().collect(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn configured_router(
|
||||
gateway: Arc<Gateway>,
|
||||
permissions: litellm_gateway_auth::Permissions,
|
||||
principal: Option<litellm_gateway_auth::Principal>,
|
||||
) -> Router {
|
||||
router(gateway).layer(axum::middleware::from_fn_with_state(
|
||||
(permissions, principal),
|
||||
test_identity,
|
||||
))
|
||||
}
|
||||
|
||||
fn configured_router_with_hooks<H>(
|
||||
gateway: Arc<Gateway>,
|
||||
permissions: litellm_gateway_auth::Permissions,
|
||||
interceptors: H,
|
||||
observers: Option<ObservationSender>,
|
||||
) -> Router
|
||||
where
|
||||
H: Interceptors<litellm_inference::Error> + Clone + 'static,
|
||||
{
|
||||
router_with_tokenize_hooks(gateway, interceptors, observers).layer(
|
||||
axum::middleware::from_fn_with_state((permissions, None), test_identity),
|
||||
)
|
||||
}
|
||||
|
||||
async fn test_identity(
|
||||
axum::extract::State((permissions, principal)): axum::extract::State<(
|
||||
litellm_gateway_auth::Permissions,
|
||||
|
|
|
|||
534
litellm-rust/crates/gateway-inference/tests/tokenize.rs
Normal file
534
litellm-rust/crates/gateway-inference/tests/tokenize.rs
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
mod support;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use litellm_gateway_inference::Deployment;
|
||||
use litellm_host::{
|
||||
interceptors::{
|
||||
Cost, ExecutionFacts, Interceptors, RawResponse, RequestContext, ResultSource, WireRequest,
|
||||
},
|
||||
lifecycle::{CallEvent, ExecutionEvent},
|
||||
observation::observation_channel,
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{body_json, header, method, path},
|
||||
};
|
||||
|
||||
fn deployment(model: &str, api_base: Option<String>) -> Deployment {
|
||||
let mut deployment = Deployment {
|
||||
model: model.into(),
|
||||
api_base,
|
||||
api_key: Some("deployment-key".into()),
|
||||
..Default::default()
|
||||
};
|
||||
deployment.litellm_params = serde_json::from_value(json!({
|
||||
"model": model
|
||||
}))
|
||||
.unwrap();
|
||||
deployment
|
||||
}
|
||||
|
||||
async fn response_json(response: axum::response::Response) -> Value {
|
||||
serde_json::from_slice(
|
||||
&axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::available_omitted(json!({}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":[11,12],"tokens":[" café"," 世界 "]}))]
|
||||
#[case::count_empty(json!({"include":[]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":null,"tokens":null}))]
|
||||
#[case::ids(json!({"include":["token_ids"]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":[11,12],"tokens":null}))]
|
||||
#[case::pieces(json!({"include":["tokens"]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":null,"tokens":[" café"," 世界 "]}))]
|
||||
#[case::both(json!({"include":["token_ids","tokens"]}), json!({"model":"org/model","prompt":" café 世界 ","add_special_tokens":true,"return_token_strs":true}), json!({"token_ids":[11,12],"tokens":[" café"," 世界 "]}))]
|
||||
#[tokio::test]
|
||||
async fn aliases_project_strict_payload_and_keep_all_response_keys(
|
||||
#[case] options: Value,
|
||||
#[case] upstream_request: Value,
|
||||
#[case] expected: Value,
|
||||
#[values("/tokenize", "/v1/tokenize")] endpoint: &str,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/tokenize"))
|
||||
.and(header("authorization", "Bearer deployment-key"))
|
||||
.and(body_json(upstream_request))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count": 2, "tokens": [11, 12], "token_strs": [" café", " 世界 "]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let body = json!({"model":"public/model","prompt":" café 世界 "})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.chain(options.as_object().unwrap().iter())
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect::<serde_json::Map<_, _>>();
|
||||
let response = support::post(
|
||||
support::app_with_deployment(deployment(
|
||||
"hosted_vllm/org/model",
|
||||
Some(format!("{}/v1", upstream.uri())),
|
||||
)),
|
||||
endpoint,
|
||||
Value::Object(body),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let result = response_json(response).await;
|
||||
assert_eq!(result["model"], "org/model");
|
||||
assert_eq!(result["count"], 2);
|
||||
assert_eq!(result["token_ids"], expected["token_ids"]);
|
||||
assert_eq!(result["tokens"], expected["tokens"]);
|
||||
assert_eq!(result.as_object().unwrap().len(), 4);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown("messages", json!([]))]
|
||||
#[case::private("api_base", json!("https://caller.invalid"))]
|
||||
#[case::resource("project", json!("caller-project"))]
|
||||
#[case::generation("temperature", json!(0.2))]
|
||||
#[case::credentials("api_key", json!("caller-secret"))]
|
||||
#[case::metadata("metadata", json!({"user":"caller"}))]
|
||||
#[case::images("images", json!([]))]
|
||||
#[case::tools("tools", json!([]))]
|
||||
#[case::template("chat_template", json!("template"))]
|
||||
#[case::output("output", json!("count"))]
|
||||
#[case::available("available", json!(true))]
|
||||
#[case::custom_backend("backend", json!({"type":"local"}))]
|
||||
#[case::trusted_sources("trusted_sources", json!([]))]
|
||||
#[case::space("space", json!("caller-space"))]
|
||||
#[case::api_version("api_version", json!("caller-version"))]
|
||||
#[case::credentials_map("credentials", json!({}))]
|
||||
#[tokio::test]
|
||||
async fn unknown_semantic_private_and_resource_fields_are_rejected(
|
||||
#[case] field: &str,
|
||||
#[case] value: Value,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
let mut body = json!({"model":"public/model","prompt":"hello"});
|
||||
body[field] = value;
|
||||
let response = support::post(
|
||||
support::app("hosted_vllm/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
assert!(response_json(response).await["error"]["message"].is_string());
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::null_include(json!({"include":null}))]
|
||||
#[case::scalar_include(json!({"include":"tokens"}))]
|
||||
#[case::duplicate_include(json!({"include":["tokens","tokens"]}))]
|
||||
#[case::unknown_include(json!({"include":["pieces"]}))]
|
||||
#[case::null_special(json!({"add_special_tokens":null}))]
|
||||
#[case::string_special(json!({"add_special_tokens":"false"}))]
|
||||
#[case::number_special(json!({"add_special_tokens":0}))]
|
||||
#[tokio::test]
|
||||
async fn malformed_include_and_special_token_values_fail_before_io(#[case] extra: Value) {
|
||||
let upstream = MockServer::start().await;
|
||||
let mut body = json!({"model":"public/model","prompt":"hello"});
|
||||
body.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(extra.as_object().unwrap().clone());
|
||||
let response = support::post(
|
||||
support::app("hosted_vllm/model", &upstream.uri()),
|
||||
"/v1/tokenize",
|
||||
body,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::tokens(json!(["tokens"]))]
|
||||
#[case::ids(json!(["token_ids"]))]
|
||||
#[case::both(json!(["token_ids", "tokens"]))]
|
||||
#[tokio::test]
|
||||
async fn requested_representations_are_enforced(#[case] include: Value) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count": 1, "tokens": [7], "token_strs": ["hello"]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let response = support::post(
|
||||
support::app("hosted_vllm/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"hello","include":include}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn configured_provider_headers_and_special_token_override_reach_vllm_without_defaults() {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/tokenize"))
|
||||
.and(header("authorization", "Bearer deployment-key"))
|
||||
.and(header("x-deployment", "configured"))
|
||||
.and(body_json(json!({
|
||||
"model":"org/model",
|
||||
"prompt":"",
|
||||
"add_special_tokens":false,
|
||||
"return_token_strs":true
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count":0,"tokens":[],"token_strs":[]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let mut deployment = deployment(
|
||||
"hosted_vllm/org/model",
|
||||
Some(format!("{}/v1", upstream.uri())),
|
||||
);
|
||||
deployment.litellm_params = serde_json::from_value(json!({
|
||||
"model":"hosted_vllm/org/model",
|
||||
"backend":{"type":"provider"},
|
||||
"extra_headers":{"x-deployment":"configured"},
|
||||
"temperature":0.7,
|
||||
"max_tokens":100
|
||||
}))
|
||||
.unwrap();
|
||||
let response = support::post(
|
||||
support::app_with_deployment(deployment),
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model":"public/model",
|
||||
"prompt":"",
|
||||
"add_special_tokens":false,
|
||||
"include":["token_ids","tokens"]
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["count"], 0);
|
||||
assert_eq!(body["token_ids"], json!([]));
|
||||
assert_eq!(body["tokens"], json!([]));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn watsonx_unsupported_representation_is_rejected_before_io() {
|
||||
let upstream = MockServer::start().await;
|
||||
let response = support::post(
|
||||
support::app("watsonx/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model":"public/model",
|
||||
"prompt":"hello",
|
||||
"include":["token_ids"]
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
assert!(
|
||||
response_json(response).await["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|message| message.contains("include"))
|
||||
);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::enabled(true)]
|
||||
#[case::disabled(false)]
|
||||
#[tokio::test]
|
||||
async fn watsonx_rejects_explicit_special_token_policy_before_io(#[case] add_special_tokens: bool) {
|
||||
let upstream = MockServer::start().await;
|
||||
let response = support::post(
|
||||
support::app("watsonx/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
json!({
|
||||
"model":"public/model",
|
||||
"prompt":"hello",
|
||||
"add_special_tokens":add_special_tokens
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
assert!(
|
||||
response_json(response).await["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|message| message.contains("add_special_tokens"))
|
||||
);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::pieces(json!({"count":1,"tokens":[7],"token_strs":null}), json!(["tokens"]))]
|
||||
#[case::ids(json!({"count":1,"tokens":null,"token_strs":["hello"]}), json!(["token_ids"]))]
|
||||
#[tokio::test]
|
||||
async fn provider_missing_requested_representation_is_not_fabricated(
|
||||
#[case] response_body: Value,
|
||||
#[case] include: Value,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(response_body))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let response = support::post(
|
||||
support::app("hosted_vllm/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"hello","include":include}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 500);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::rate_limit(429)]
|
||||
#[case::unauthorized(401)]
|
||||
#[case::unavailable(503)]
|
||||
#[tokio::test]
|
||||
async fn upstream_errors_keep_their_status(#[case] status: u16) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(status))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let response = support::post(
|
||||
support::app("hosted_vllm/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status().as_u16(), status);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn authorization_and_unknown_aliases_precede_provider_io() {
|
||||
let upstream = MockServer::start().await;
|
||||
let denied = support::post(
|
||||
support::app_with_permissions(
|
||||
"hosted_vllm/model",
|
||||
&upstream.uri(),
|
||||
litellm_gateway_auth::Permissions::None,
|
||||
),
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(denied.status(), 403);
|
||||
let unknown = support::post(
|
||||
support::app("hosted_vllm/model", &upstream.uri()),
|
||||
"/tokenize",
|
||||
json!({"model":"missing","prompt":"hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(unknown.status(), 400);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::alias_grant("public/model", 200, 1)]
|
||||
#[case::provider_model_grant("hosted_vllm/org/model", 403, 0)]
|
||||
#[tokio::test]
|
||||
async fn restricted_model_grants_match_public_alias_before_provider_io(
|
||||
#[case] granted_name: &str,
|
||||
#[case] expected_status: u16,
|
||||
#[case] expected_requests: usize,
|
||||
#[values("/tokenize", "/v1/tokenize")] endpoint: &str,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/tokenize"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count":1,"tokens":[7],"token_strs":["hello"]
|
||||
})))
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let permissions = litellm_gateway_auth::Permissions::Only(Arc::from([
|
||||
litellm_gateway_auth::AccessRequest::Model {
|
||||
name: granted_name.into(),
|
||||
deployment: "hosted_vllm/org/model".into(),
|
||||
},
|
||||
]));
|
||||
let response = support::post(
|
||||
support::app_with_permissions("hosted_vllm/org/model", &upstream.uri(), permissions),
|
||||
endpoint,
|
||||
json!({"model":"public/model","prompt":"hello"}),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status().as_u16(), expected_status);
|
||||
assert_eq!(
|
||||
upstream.received_requests().await.unwrap().len(),
|
||||
expected_requests
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct Hooks {
|
||||
rewrite: bool,
|
||||
fail_result: bool,
|
||||
calls: Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl Interceptors<litellm_inference::Error> for Hooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, litellm_inference::Error> {
|
||||
self.calls.lock().unwrap().push("before".into());
|
||||
if self.rewrite {
|
||||
wire.body["prompt"] = json!("rewritten");
|
||||
}
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(
|
||||
&self,
|
||||
_: RawResponse,
|
||||
) -> Result<(), litellm_inference::Error> {
|
||||
self.calls.lock().unwrap().push("after".into());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, _: ExecutionFacts) -> Result<(), litellm_inference::Error> {
|
||||
self.calls.lock().unwrap().push("result".into());
|
||||
if self.fail_result {
|
||||
return Err(litellm_inference::Error::InvalidRequest(
|
||||
"result hook failed".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn hooks_rewrite_wire_and_result_failure_does_not_replay() {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(body_json(json!({
|
||||
"model":"org/model","prompt":"rewritten",
|
||||
"add_special_tokens":true,"return_token_strs":true
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count":1,"tokens":[7],"token_strs":["rewritten"]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let hooks = Hooks {
|
||||
rewrite: true,
|
||||
..Default::default()
|
||||
};
|
||||
let response = support::app_with_tokenize_hooks(
|
||||
deployment("hosted_vllm/org/model", Some(upstream.uri())),
|
||||
litellm_gateway_auth::Permissions::All,
|
||||
hooks.clone(),
|
||||
None,
|
||||
);
|
||||
let ok = support::post(
|
||||
response,
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"original"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(ok.status(), 200);
|
||||
assert_eq!(
|
||||
&*hooks.calls.lock().unwrap(),
|
||||
&["before", "after", "result"]
|
||||
);
|
||||
|
||||
let failing_upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count":1,"tokens":[7],"token_strs":["original"]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&failing_upstream)
|
||||
.await;
|
||||
let failing = Hooks {
|
||||
fail_result: true,
|
||||
..Default::default()
|
||||
};
|
||||
let response = support::post(
|
||||
support::app_with_tokenize_hooks(
|
||||
deployment("hosted_vllm/org/model", Some(failing_upstream.uri())),
|
||||
litellm_gateway_auth::Permissions::All,
|
||||
failing.clone(),
|
||||
None,
|
||||
),
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"original"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
assert_eq!(
|
||||
&*failing.calls.lock().unwrap(),
|
||||
&["before", "after", "result"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn provider_facts_defer_cost_and_observers_see_terminals() {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count":1,"tokens":[7],"token_strs":["hello"]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let (sender, mut events) = observation_channel(std::num::NonZeroUsize::new(16).unwrap());
|
||||
let hooks = Hooks::default();
|
||||
let response = support::post(
|
||||
support::app_with_tokenize_hooks(
|
||||
deployment("hosted_vllm/org/model", Some(upstream.uri())),
|
||||
litellm_gateway_auth::Permissions::All,
|
||||
hooks.clone(),
|
||||
Some(sender),
|
||||
),
|
||||
"/tokenize",
|
||||
json!({"model":"public/model","prompt":"hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(
|
||||
&*hooks.calls.lock().unwrap(),
|
||||
&["before", "after", "result"]
|
||||
);
|
||||
assert!(matches!(
|
||||
events.recv().await,
|
||||
Some(CallEvent::Started { .. })
|
||||
));
|
||||
let facts = loop {
|
||||
match events.recv().await {
|
||||
Some(CallEvent::Execution(ExecutionEvent::ResultReady { facts })) => break facts,
|
||||
Some(_) => {}
|
||||
None => panic!("observer closed before facts"),
|
||||
}
|
||||
};
|
||||
assert_eq!(facts.provider.provider, "hosted_vllm");
|
||||
assert_eq!(facts.source, ResultSource::Provider);
|
||||
assert_eq!(facts.cost, Cost::Deferred);
|
||||
assert!(matches!(
|
||||
events.recv().await,
|
||||
Some(CallEvent::Succeeded { .. })
|
||||
));
|
||||
}
|
||||
|
|
@ -36,3 +36,4 @@ rstest.workspace = true
|
|||
tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
wiremock.workspace = true
|
||||
|
|
|
|||
80
litellm-rust/crates/gateway/tests/tokenize.rs
Normal file
80
litellm-rust/crates/gateway/tests/tokenize.rs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::Request,
|
||||
};
|
||||
use litellm_config::Config;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{header, method, path},
|
||||
};
|
||||
|
||||
fn config(api_base: &str) -> Config {
|
||||
Config::from_yaml(
|
||||
&json!({
|
||||
"model_list": [{
|
||||
"model_name": "public/model",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/org/model",
|
||||
"api_base": api_base,
|
||||
"api_key": "deployment-key"
|
||||
}
|
||||
}],
|
||||
"general_settings": {"master_key": "gateway-key"}
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unversioned("/tokenize")]
|
||||
#[case::versioned("/v1/tokenize")]
|
||||
#[tokio::test]
|
||||
async fn mounted_gateway_authenticates_alias_before_provider_io(#[case] endpoint: &str) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/tokenize"))
|
||||
.and(header("authorization", "Bearer deployment-key"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"count": 2, "tokens": [11, 12], "token_strs": ["hel", "lo"]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let config = config(&upstream.uri());
|
||||
let inference = litellm_gateway::build_inference(&config).unwrap();
|
||||
let app = litellm_gateway::router(inference, &config, None, None);
|
||||
let unauthenticated = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::post(endpoint)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({"model":"public/model","prompt":"hello"}).to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(unauthenticated.status(), 401);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::post(endpoint)
|
||||
.header("authorization", "Bearer gateway-key")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({"model":"public/model","prompt":"hello"}).to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let body: Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), 1024 * 1024).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(body["count"], 2);
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue