diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index b8949d176cc..710f38b7f9a 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4139,6 +4139,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-inference-tokenize" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-auth", + "litellm-core-utils", + "litellm-host", + "litellm-host-native", + "litellm-http", + "litellm-inference", + "litellm-inference-testing", + "litellm-llms", + "litellm-llms-types", + "litellm-secrets", + "rstest", + "serde_json", + "testcontainers", + "tokio", + "tracing", + "wiremock", +] + [[package]] name = "litellm-inference-transcription" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 4be6be93a49..4fcb3a3eea2 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -21,6 +21,7 @@ litellm-inference-responses = { path = "crates/inference-responses" } litellm-inference-messages = { path = "crates/inference-messages" } litellm-inference-chat = { path = "crates/inference-chat" } litellm-inference-ocr = { path = "crates/inference-ocr" } +litellm-inference-tokenize = { path = "crates/inference-tokenize" } litellm-inference-testing = { path = "crates/inference-testing" } litellm-gateway-mcp = { path = "crates/gateway-mcp" } litellm-gateway = { path = "crates/gateway" } diff --git a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index acea48799c0..dceafaab4e8 100644 --- a/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -1,4 +1,4 @@ -use strum::{EnumString, IntoStaticStr}; +use strum::{EnumString, IntoStaticStr, VariantArray}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct CustomLlmProvider<'a> { @@ -6,7 +6,7 @@ pub struct CustomLlmProvider<'a> { pub custom_llm_provider: &'a str, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumString, IntoStaticStr)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumString, IntoStaticStr, VariantArray)] pub enum LlmProviders { #[strum(serialize = "anthropic")] Anthropic, @@ -24,6 +24,8 @@ pub enum LlmProviders { Deepseek, #[strum(serialize = "edenai")] Edenai, + #[strum(serialize = "hosted_vllm")] + HostedVllm, #[strum(serialize = "minimax")] Minimax, #[strum(serialize = "mistral")] @@ -40,6 +42,8 @@ pub enum LlmProviders { Tencent, #[strum(serialize = "vertex_ai")] VertexAi, + #[strum(serialize = "watsonx")] + Watsonx, } pub fn get_custom_llm_provider<'a>( diff --git a/litellm-rust/crates/inference-tokenize/AGENTS.md b/litellm-rust/crates/inference-tokenize/AGENTS.md new file mode 100644 index 00000000000..a80b45bd743 --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/AGENTS.md @@ -0,0 +1,5 @@ +- Own tokenization call orchestration, provider dispatch, and hosted execution +- Keep public payloads in llms-types and provider policy in llms +- Resolve providers through inference::provider when execution begins +- Token strings and numeric token IDs are separate optional fields; never synthesize either from the other +- Omitted include returns every available representation; explicit include selects representations and an empty array requests count-only diff --git a/litellm-rust/crates/inference-tokenize/Cargo.toml b/litellm-rust/crates/inference-tokenize/Cargo.toml new file mode 100644 index 00000000000..d06a9b03186 --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "litellm-inference-tokenize" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +futures-util.workspace = true +litellm-auth.workspace = true +litellm-core-utils.workspace = true +litellm-host.workspace = true +litellm-http.workspace = true +litellm-inference.workspace = true +litellm-llms.workspace = true +litellm-llms-types.workspace = true +litellm-secrets.workspace = true +serde_json.workspace = true +tracing.workspace = true + +[dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } +litellm-inference-testing.workspace = true +litellm-host-native.workspace = true +rstest.workspace = true +testcontainers = { version = "0.27.3", default-features = false, features = ["device-requests", "http_wait"] } +tokio.workspace = true +wiremock.workspace = true diff --git a/litellm-rust/crates/inference-tokenize/README.md b/litellm-rust/crates/inference-tokenize/README.md new file mode 100644 index 00000000000..551bc4f8deb --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/README.md @@ -0,0 +1,23 @@ +# Native tokenization + +`TokenizeRoute` accepts `litellm_inference_tokenize::TokenizeRequest`, which is the host execution envelope. It is distinct from `litellm_llms_types::formats::tokenize::TokenizeRequest`, which is the shared raw-text API payload. The public payload uses a strict text-only schema with required `model` and `prompt`, an optional non-null `include` array, and an optional non-null boolean `add_special_tokens` + +An omitted `include` returns every representation available from the provider. An empty `include` array is count-only, and a nonempty array selects the requested representations. `TokenizeInclude::new(ids, tokens)` constructs requested representations, and `TokenizeInclude::output()` projects the internal representation enum. `From` creates the host envelope and defaults operational fields. The public payload has no `output` field, unknown fields are rejected, and explicit `null` for `include` or `add_special_tokens` is rejected. Projection happens after provider response validation, and include values are never sent as provider parameters + +The only include values are `token_ids` and `tokens`. Duplicate entries, unknown entries, wrong types, and null values are rejected. Provider-unavailable representations remain `null` according to the provider contract. The model must be nonempty while prompt may be empty or whitespace without trimming + +```json +{"model":"public-alias","prompt":"hello"} +``` + +The count-only response contains a count with null representation arrays. A representation request can ask for either or both arrays + +```json +{"model":"public-alias","prompt":"hello","include":["token_ids","tokens"]} +``` + +Omitting `add_special_tokens` preserves the provider policy. Hosted vLLM accepts explicit true and false values. Operational metadata belongs to the host envelope and is not forwarded as provider kwargs. Existing provider wire flags are unchanged, and count-only projection does not claim provider-side wire or cost savings + +Downstream issue28444 acceptance requires fresh native-build coverage for `/tokenize` and `/v1/tokenize` across count-only and requested representation modes. A registered `CustomLogger` must prove that pre-call prompt mutation reaches the executed input, post-success response replacement reaches the wire HTTP result with correct `Content-Length` and context, each hook runs exactly once, and the mutation-bypass negative check remains enforced. Native `after_provider_response` is a notification and does not rewrite responses. This is not a generic passthrough fix, and legacy `/utils/token_counter` behavior remains unchanged + +The ignored `containers` integration test runs a local vLLM container with `cargo test -p litellm-inference-tokenize --test containers --locked -- --ignored`. It requires Docker, NVIDIA Container Toolkit, a compatible GPU, network access, and permission to download the model image and weights diff --git a/litellm-rust/crates/inference-tokenize/src/handler.rs b/litellm-rust/crates/inference-tokenize/src/handler.rs new file mode 100644 index 00000000000..bc52000dfd5 --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/src/handler.rs @@ -0,0 +1,215 @@ +use std::time::Duration; + +use litellm_host::{ + interceptors::{ + Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, + WireRequest, + }, + observation::ObservationSender, +}; +use litellm_http::transport::Error as TransportError; +use litellm_inference::{context::CallContext, provider::ResolvedProvider}; +use litellm_llms::base_llm::{ + auth::{AuthScheme, resolve_auth}, + tokenize::transformation::{BaseTokenizeConfig, TokenizeContext}, +}; +use litellm_llms_types::formats::tokenize::{TokenizeOutput, TokenizeResponse}; +use litellm_secrets::source::Secrets; +use serde_json::{Map, Value}; + +use crate::{Error, TokenizeRequest, TokenizeRoute}; + +pub(crate) async fn execute( + route: &TokenizeRoute, + call: &TokenizeRequest, + config: &dyn BaseTokenizeConfig, + provider: ResolvedProvider<'_>, + secrets: Secrets, + interceptors: &impl Interceptors, + observers: Option<&ObservationSender>, +) -> Result { + let context = CallContext::new(interceptors, observers.cloned().into()); + let env_lookup = |key: &str| secrets.get(key); + let input = TokenizeContext::new( + call.api_key.as_ref(), + call.api_base.as_ref(), + &call.extra_headers, + &call.deployment_params, + &call.optional_params, + ); + config.validate_tokenize_input(&input, &env_lookup)?; + let supported_params = config.get_supported_tokenize_params(); + let mut params: Map = call + .deployment_params + .iter() + .filter(|(key, _)| supported_params.contains(&key.as_str())) + .chain(call.optional_params.iter()) + .map(|(key, value)| (key.clone(), value.value().clone())) + .collect(); + if let Some(value) = call.add_special_tokens { + params.insert("add_special_tokens".into(), Value::Bool(value)); + } + let params = config.map_tokenize_params(provider.model, ¶ms)?; + let environment = config.validate_environment(&input, ¶ms, &env_lookup)?; + let url = config.get_complete_url( + call.api_base.as_ref().map(|base| base.value().as_str()), + provider.model, + ¶ms, + &env_lookup, + )?; + let body = + config.transform_tokenize_request(provider.model, &call.prompt, ¶ms, &env_lookup)?; + let callback_api_key = match &environment.auth { + AuthScheme::Credential { secret, .. } => Some(secret.clone()), + AuthScheme::Forwarded => None, + _ => call.api_key.as_ref().map(|api_key| api_key.value().clone()), + }; + let mut authenticated = + resolve_auth(&route.auth, &route.credential, environment, &env_lookup).await?; + let wire = interceptors + .before_provider_request( + WireRequest { + url, + headers: authenticated.headers, + body, + }, + RequestContext { + model: provider.model.into(), + custom_llm_provider: <&str>::from(provider.provider).into(), + optional_params: Value::Object(params), + secret_fields: config + .secret_param_names() + .iter() + .map(|name| (*name).into()) + .collect(), + api_key: callback_api_key, + }, + ) + .await?; + authenticated.headers = wire.headers; + let outbound = litellm_inference::outbound::outbound_request( + authenticated, + wire.url, + &wire.body, + Some(call.timeout.unwrap_or(Duration::from_secs(60))), + )?; + let response = litellm_inference::outbound::send(outbound, &route.http) + .await + .map_err(TransportError::from_reqwest_before_dispatch)?; + let status = response.status(); + let text = response.text().await.map_err(TransportError::from)?; + if !status.is_success() { + return Err(TransportError::Http { + status: status.as_u16(), + body: litellm_http::request::truncate_error_body(&text), + } + .into()); + } + context.response_received(&text).await?; + let json = serde_json::from_str(&text).map_err(|error| { + Error::InvalidResponse(litellm_llms::ErrorDetail::invalid( + "tokenize response JSON", + error, + )) + })?; + let response = project_response( + config.transform_tokenize_response(provider.model, json)?, + call.output(), + )?; + context + .result_ready(ExecutionFacts { + provider: ProviderIdentity { + model: provider.model.into(), + provider: <&str>::from(provider.provider).into(), + }, + source: ResultSource::Provider, + cost: Cost::Deferred, + }) + .await?; + Ok(response) +} + +fn project_response( + response: TokenizeResponse, + output: TokenizeOutput, +) -> Result { + let require_ids = output.requires_ids(); + let require_tokens = output.requires_tokens(); + if require_ids && response.token_ids.is_none() { + return Err(Error::InvalidResponse( + litellm_llms::ErrorDetail::InvalidValue { + field: "token_ids", + expected: "a provider token ID representation", + actual: Value::Null, + }, + )); + } + if require_tokens && response.tokens.is_none() { + return Err(Error::InvalidResponse( + litellm_llms::ErrorDetail::InvalidValue { + field: "tokens", + expected: "a provider token piece representation", + actual: Value::Null, + }, + )); + } + if output == TokenizeOutput::Available { + return Ok(response); + } + Ok(TokenizeResponse { + model: response.model, + count: response.count, + token_ids: require_ids.then_some(response.token_ids).flatten(), + tokens: require_tokens.then_some(response.tokens).flatten(), + }) +} + +#[cfg(test)] +mod tests { + use litellm_llms_types::formats::tokenize::{TokenizeOutput, TokenizeResponse}; + use rstest::rstest; + + use super::project_response; + use crate::Error; + + fn response() -> TokenizeResponse { + TokenizeResponse { + model: "model".into(), + count: 0, + token_ids: Some(Vec::new()), + tokens: Some(Vec::new()), + } + } + + #[rstest] + #[case::count(TokenizeOutput::Count, None, None)] + #[case::ids(TokenizeOutput::Ids, Some(Vec::new()), None)] + #[case::tokens(TokenizeOutput::Tokens, None, Some(Vec::new()))] + #[case::both(TokenizeOutput::Both, Some(Vec::new()), Some(Vec::new()))] + fn strict_outputs_preserve_empty_arrays_and_project_other_fields( + #[case] output: TokenizeOutput, + #[case] ids: Option>, + #[case] tokens: Option>, + ) { + let projected = project_response(response(), output).unwrap(); + + assert_eq!(projected.count, 0); + assert_eq!(projected.token_ids, ids); + assert_eq!(projected.tokens, tokens); + } + + #[rstest] + #[case::ids(TokenizeOutput::Ids)] + #[case::tokens(TokenizeOutput::Tokens)] + #[case::both(TokenizeOutput::Both)] + fn strict_outputs_reject_missing_representations(#[case] output: TokenizeOutput) { + let mut provider_response = response(); + provider_response.token_ids = None; + provider_response.tokens = None; + + assert!(matches!( + project_response(provider_response, output), + Err(Error::InvalidResponse(_)) + )); + } +} diff --git a/litellm-rust/crates/inference-tokenize/src/lib.rs b/litellm-rust/crates/inference-tokenize/src/lib.rs new file mode 100644 index 00000000000..ce53e2bf85e --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/src/lib.rs @@ -0,0 +1,123 @@ +mod handler; +mod route; +mod types; + +use std::sync::Arc; + +use litellm_auth::AuthServices; +use litellm_core_utils::get_llm_provider_logic::LlmProviders; +use litellm_host::{interceptors::Interceptors, observation::ObservationSender}; +use litellm_http::Client; +use litellm_inference::provider::resolve_llm_provider; +use litellm_llms::{ + base_llm::tokenize::transformation::BaseTokenizeConfig, + hosted_vllm::tokenize::transformation::HostedVllmTokenizeConfig, +}; +use litellm_llms_types::formats::tokenize::TokenizeResponse; +use litellm_secrets::source::SecretSource; + +pub use litellm_inference::Error; +pub use route::{Tokenize, TokenizeCall, TokenizeMachine}; +pub use types::TokenizeRequest; + +#[derive(Clone)] +pub struct TokenizeRoute { + http: Client, + credential: litellm_http::CredentialClient, + auth: Arc, + secrets: Arc, +} + +impl TokenizeRoute { + pub fn new( + http: Client, + credential: litellm_http::CredentialClient, + auth: Arc, + secrets: Arc, + ) -> Self { + Self { + http, + credential, + auth, + secrets, + } + } + + pub async fn execute( + &self, + request: TokenizeRequest, + interceptors: &impl Interceptors, + observers: Option, + ) -> Result { + litellm_host::lifecycle::observe_unary( + observers.clone(), + self.run(request, interceptors, observers.as_ref()), + ) + .await + } + + #[tracing::instrument(name = "litellm.route", skip_all, fields( + route = "tokenize", model = %call.model, resolved_model, provider, + stream = false, outcome + ))] + async fn run( + &self, + call: TokenizeRequest, + interceptors: &impl Interceptors, + observers: Option<&ObservationSender>, + ) -> Result { + litellm_inference::diagnostic::unary(async { + if call.model.is_empty() { + return Err(Error::InvalidRequest("model must not be empty".into())); + } + let provider = + resolve_llm_provider(&call.model, call.custom_llm_provider.as_deref(), "tokenize")?; + let config: &dyn BaseTokenizeConfig = match provider.provider { + LlmProviders::HostedVllm => &HostedVllmTokenizeConfig, + _ => return Err(Error::Unsupported("tokenize provider")), + }; + if call.add_special_tokens.is_some() + && call + .optional_params + .contains_key("add_special_tokens") + { + return Err(Error::InvalidRequest( + "add_special_tokens cannot be specified in both the top-level request and optional_params" + .into(), + )); + } + if call.add_special_tokens.is_some() + && !config + .get_supported_tokenize_params() + .contains(&"add_special_tokens") + { + return Err(Error::InvalidRequest( + "add_special_tokens is not supported by the selected provider".into(), + )); + } + if !config + .tokenize_capabilities() + .supports(call.output()) + { + return Err(Error::InvalidRequest( + "include requests a token representation not supported by the selected provider" + .into(), + )); + } + litellm_inference::diagnostic::provider(provider.model, provider.provider.into()); + let secrets = self.secrets.resolve(&config.secret_names()).await?; + let execute: futures_util::future::BoxFuture<'_, Result> = + Box::pin(handler::execute( + self, + &call, + config, + provider, + secrets, + interceptors, + observers, + )); + execute.await + }) + .await + } +} diff --git a/litellm-rust/crates/inference-tokenize/src/route.rs b/litellm-rust/crates/inference-tokenize/src/route.rs new file mode 100644 index 00000000000..5406551c0e6 --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/src/route.rs @@ -0,0 +1,43 @@ +use std::convert::Infallible; + +use litellm_host::{ + call::{CallOutput, HostedMachine, hosted_call}, + observation::ObservationSender, + protocol::Protocol, +}; +use litellm_llms_types::formats::tokenize::TokenizeResponse; + +use crate::{Error, TokenizeRequest, TokenizeRoute}; + +pub type TokenizeCall = TokenizeRequest; + +pub struct Tokenize; + +impl Protocol for Tokenize { + type Response = TokenizeResponse; + type Error = Error; + type Request = TokenizeCall; + type HostCall = Infallible; + type Chunk = Infallible; + type StreamHead = Infallible; +} + +pub type TokenizeMachine = HostedMachine; + +impl TokenizeRoute { + pub fn machine( + self, + request: TokenizeCall, + observers: Option, + ) -> TokenizeMachine { + hosted_call( + request, + observers, + move |request, _, interceptors, observers| async move { + self.run(request, &interceptors, observers.as_ref()) + .await + .map(CallOutput::Complete) + }, + ) + } +} diff --git a/litellm-rust/crates/inference-tokenize/src/types.rs b/litellm-rust/crates/inference-tokenize/src/types.rs new file mode 100644 index 00000000000..ecc48d907ea --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/src/types.rs @@ -0,0 +1,70 @@ +use std::{collections::BTreeMap, time::Duration}; + +use litellm_auth::{SecretValue, Sourced}; +use litellm_llms_types::formats::tokenize::{ + TokenizeInclude, TokenizeOutput, TokenizeRequest as PublicTokenizeRequest, +}; +use serde_json::Value; + +#[derive(Clone, Default)] +pub struct TokenizeRequest { + pub model: String, + pub prompt: String, + pub include: Option, + pub add_special_tokens: Option, + pub custom_llm_provider: Option, + pub api_key: Option>, + pub api_base: Option>, + pub extra_headers: BTreeMap>, + pub deployment_params: BTreeMap>, + pub optional_params: BTreeMap>, + pub timeout: Option, +} + +impl TokenizeRequest { + pub fn output(&self) -> TokenizeOutput { + self.include + .as_ref() + .map_or(TokenizeOutput::Available, TokenizeInclude::output) + } +} + +impl From for TokenizeRequest { + fn from(request: PublicTokenizeRequest) -> Self { + Self { + model: request.model, + prompt: request.prompt, + include: request.include, + add_special_tokens: request.add_special_tokens, + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use litellm_llms_types::formats::tokenize::TokenizeOutput; + use rstest::rstest; + use serde_json::json; + + use super::TokenizeRequest; + + #[rstest] + fn public_request_conversion_carries_only_public_fields() { + let public: litellm_llms_types::formats::tokenize::TokenizeRequest = + serde_json::from_value(json!({ + "model": "model-alias", + "prompt": "hello", + "include": ["token_ids"], + "add_special_tokens": false + })) + .unwrap(); + let request: TokenizeRequest = public.into(); + + assert_eq!(request.model, "model-alias"); + assert_eq!(request.prompt, "hello"); + assert_eq!(request.output(), TokenizeOutput::Ids); + assert_eq!(request.add_special_tokens, Some(false)); + assert!(request.optional_params.is_empty()); + } +} diff --git a/litellm-rust/crates/inference-tokenize/tests/containers.rs b/litellm-rust/crates/inference-tokenize/tests/containers.rs new file mode 100644 index 00000000000..0f31e20f0c5 --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/tests/containers.rs @@ -0,0 +1,102 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_auth::{AuthServices, InputSource, Sourced}; +use litellm_http::Client; +use litellm_inference_testing::no_secrets; +use litellm_inference_tokenize::{TokenizeRequest, TokenizeRoute}; +use litellm_llms_types::formats::tokenize::TokenizeInclude; +use rstest::rstest; +use testcontainers::{ + GenericImage, ImageExt, + bollard::models::DeviceRequest, + core::{IntoContainerPort, WaitFor, wait::HttpWaitStrategy}, + runners::AsyncRunner, +}; + +#[rstest] +#[tokio::test] +#[ignore = "requires Docker, NVIDIA Container Toolkit, a GPU and model downloads"] +async fn hosted_vllm_returns_available_representations_and_respects_include() { + let container = GenericImage::new("vllm/vllm-openai", "v0.15.1") + .with_exposed_port(8000.tcp()) + .with_wait_for(WaitFor::http( + HttpWaitStrategy::new("/health").with_expected_status_code(200_u16), + )) + .with_device_requests(vec![DeviceRequest { + driver: Some("nvidia".into()), + count: Some(1), + capabilities: Some(vec![vec!["gpu".into()]]), + ..Default::default() + }]) + .with_shm_size(1 << 30) + .with_startup_timeout(Duration::from_secs(600)) + .with_cmd([ + "--model", + "Qwen/Qwen3-0.6B", + "--served-model-name", + "tokenizer", + "--max-model-len", + "2048", + "--enforce-eager", + ]) + .start() + .await + .unwrap(); + let api_base = format!( + "http://{}:{}", + container.get_host().await.unwrap(), + container.get_host_port_ipv4(8000).await.unwrap(), + ); + let route = TokenizeRoute::new( + Client::plain_for_test(), + litellm_http::CredentialClient::no_redirect_for_test(), + Arc::new(AuthServices::default()), + no_secrets(), + ); + let request = TokenizeRequest { + model: "hosted_vllm/tokenizer".into(), + prompt: "Hello world".into(), + api_base: Some(Sourced::new(api_base, InputSource::Request)), + ..Default::default() + }; + let available = route.execute(request.clone(), &(), None).await.unwrap(); + assert!(available.count > 0); + assert_eq!( + available.token_ids.as_ref().unwrap().len(), + available.count as usize + ); + assert_eq!( + available.tokens.as_ref().unwrap().len(), + available.count as usize + ); + + let ids = route + .execute( + TokenizeRequest { + include: Some(TokenizeInclude::new(true, false)), + ..request.clone() + }, + &(), + None, + ) + .await + .unwrap(); + assert_eq!(ids.count, available.count); + assert_eq!(ids.token_ids, available.token_ids); + assert_eq!(ids.tokens, None); + + let count = route + .execute( + TokenizeRequest { + include: Some(TokenizeInclude::new(false, false)), + ..request + }, + &(), + None, + ) + .await + .unwrap(); + assert_eq!(count.count, available.count); + assert_eq!(count.token_ids, None); + assert_eq!(count.tokens, None); +} diff --git a/litellm-rust/crates/inference-tokenize/tests/observations.rs b/litellm-rust/crates/inference-tokenize/tests/observations.rs new file mode 100644 index 00000000000..4c1d0c00baf --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/tests/observations.rs @@ -0,0 +1,155 @@ +use std::{num::NonZeroUsize, sync::Arc, time::Duration}; + +use litellm_auth::{AuthServices, InputSource, Sourced}; +use litellm_host::{ + call::HostedCompletion, + interceptors::ResultSource, + lifecycle::{CallEvent, ExecutionEvent}, + observation::observation_channel, +}; +use litellm_host_native::in_process::{Host, run_hosted}; +use litellm_http::{Client, CredentialClient}; +use litellm_inference_testing::no_secrets; +use litellm_inference_tokenize::{TokenizeRequest, TokenizeRoute}; +use litellm_llms_types::formats::tokenize::TokenizeInclude; +use rstest::{fixture, rstest}; +use serde_json::json; +use tokio::sync::Notify; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; + +fn route_with_credentials( + http: Client, + auth: Arc, + secrets: Arc, +) -> TokenizeRoute { + TokenizeRoute::new( + http, + CredentialClient::no_redirect_for_test(), + auth, + secrets, + ) +} + +#[fixture] +fn route() -> TokenizeRoute { + route_with_credentials( + Client::plain_for_test(), + Arc::new(AuthServices::default()), + no_secrets(), + ) +} + +fn request_api_base(value: String) -> Sourced { + Sourced::new(value, InputSource::Request) +} + +#[rstest] +#[case::direct(false)] +#[case::hosted(true)] +#[tokio::test] +async fn observers_receive_result_facts_before_one_terminal_event( + route: TokenizeRoute, + #[case] hosted: bool, +) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"count":1,"tokens":[42]}))) + .expect(1) + .mount(&server) + .await; + let request = TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + include: Some(TokenizeInclude::new(true, false)), + api_base: Some(request_api_base(server.uri())), + ..Default::default() + }; + let (observer, mut events) = observation_channel(NonZeroUsize::new(8).unwrap()); + let response = if hosted { + let output = run_hosted( + route.machine(request, Some(observer.clone())), + Host { + services: &(), + interceptors: &(), + stream: &(), + observers: Some(&observer), + }, + ) + .await + .unwrap(); + let HostedCompletion::Complete(response) = output else { + panic!("tokenize returns a complete response") + }; + response + } else { + route + .execute(request, &(), Some(observer.clone())) + .await + .unwrap() + }; + assert_eq!(response.token_ids, Some(vec![42])); + assert!(matches!( + events.try_recv().unwrap(), + CallEvent::Started { .. } + )); + assert!(matches!( + events.try_recv().unwrap(), + CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { .. }) + )); + let CallEvent::Execution(ExecutionEvent::ResultReady { facts }) = events.try_recv().unwrap() + else { + panic!("provider facts must precede the terminal event") + }; + assert_eq!(facts.provider.provider, "hosted_vllm"); + assert_eq!(facts.provider.model, "org/model"); + assert_eq!(facts.source, ResultSource::Provider); + assert!(matches!( + events.try_recv().unwrap(), + CallEvent::Succeeded { .. } + )); + assert!(events.try_recv().is_err()); + assert_eq!(observer.dropped_events(), 0); +} + +#[rstest] +#[tokio::test] +async fn cancellation_emits_one_cancelled_terminal_event(route: TokenizeRoute) { + let server = MockServer::start().await; + let request_received = Arc::new(Notify::new()); + let server_request_received = request_received.clone(); + Mock::given(method("POST")) + .respond_with(move |_: &wiremock::Request| { + server_request_received.notify_one(); + ResponseTemplate::new(200) + .set_body_json(json!({"count":1,"tokens":[42]})) + .set_delay(Duration::from_secs(30)) + }) + .expect(1) + .mount(&server) + .await; + let request = TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + include: Some(TokenizeInclude::new(true, false)), + api_base: Some(request_api_base(server.uri())), + ..Default::default() + }; + let (observer, mut events) = observation_channel(NonZeroUsize::new(8).unwrap()); + let call_observer = observer.clone(); + let call = tokio::spawn(async move { route.execute(request, &(), Some(call_observer)).await }); + tokio::time::timeout(Duration::from_secs(5), request_received.notified()) + .await + .expect("the provider request starts before cancellation"); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + assert!(matches!( + events.try_recv().unwrap(), + CallEvent::Started { .. } + )); + assert!(matches!( + events.try_recv().unwrap(), + CallEvent::Cancelled { .. } + )); + assert!(events.try_recv().is_err()); + assert_eq!(observer.dropped_events(), 0); +} diff --git a/litellm-rust/crates/inference-tokenize/tests/route.rs b/litellm-rust/crates/inference-tokenize/tests/route.rs new file mode 100644 index 00000000000..3174ab9eb73 --- /dev/null +++ b/litellm-rust/crates/inference-tokenize/tests/route.rs @@ -0,0 +1,735 @@ +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +use litellm_auth::{AuthServices, InputSource, SecretValue, Sourced}; +use litellm_host::{ + call::HostedCompletion, + interceptors::{ + Cost, ExecutionFacts, Interceptors, RawResponse, RequestContext, ResultSource, WireRequest, + }, +}; +use litellm_host_native::in_process::{Host, run_hosted}; +use litellm_http::{Client, CredentialClient, transport::Error as TransportError}; +use litellm_inference_testing::{RecordingSecrets, no_secrets}; +use litellm_inference_tokenize::{Error, TokenizeRequest, TokenizeRoute}; +use litellm_llms_types::formats::tokenize::TokenizeInclude; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json, header, method, path}, +}; + +fn route_with_credentials( + http: Client, + auth: Arc, + secrets: Arc, +) -> TokenizeRoute { + TokenizeRoute::new( + http, + CredentialClient::no_redirect_for_test(), + auth, + secrets, + ) +} + +#[fixture] +fn route() -> TokenizeRoute { + route_with_credentials( + Client::plain_for_test(), + Arc::new(AuthServices::default()), + no_secrets(), + ) +} + +fn sourced(value: T) -> Sourced { + Sourced::new(value, InputSource::Request) +} + +fn sourced_map(value: &Value) -> BTreeMap> { + sourced_map_at(value, InputSource::Request) +} + +fn sourced_at(value: T, source: InputSource) -> Sourced { + Sourced::new(value, source) +} + +fn sourced_map_at(value: &Value, source: InputSource) -> BTreeMap> { + value + .as_object() + .unwrap() + .iter() + .map(|(key, value)| (key.clone(), sourced_at(value.clone(), source))) + .collect() +} + +fn assert_credential_boundary_error(error: &Error) { + assert!(error.is_request()); + assert!( + error + .to_string() + .contains("request-controlled inputs cannot be combined with host credentials"), + "{error:?}" + ); +} + +#[rstest] +#[tokio::test] +async fn hosted_vllm_returns_a_typed_response(route: TokenizeRoute) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(header("authorization", "Bearer test-key")) + .and(body_json(json!({ + "model":"org/model", + "prompt":"hello", + "add_special_tokens":true, + "return_token_strs":true + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1, + "max_model_len":4096, + "tokens":[42], + "token_strs":["hello"] + }))) + .expect(1) + .mount(&server) + .await; + let result = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + include: Some(TokenizeInclude::new(true, true)), + api_base: Some(sourced(server.uri())), + api_key: Some(sourced(SecretValue::new("test-key"))), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(result).unwrap(), + json!({ + "model":"org/model", "count":1, "token_ids":[42], "tokens":["hello"] + }) + ); +} + +#[rstest] +#[tokio::test] +async fn hosted_vllm_omitted_include_returns_available_representations(route: TokenizeRoute) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count": 2, + "tokens": [42, 43], + "token_strs": ["hello", "world"] + }))) + .expect(1) + .mount(&server) + .await; + + let response = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello world".into(), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap(); + + assert_eq!(response.count, 2); + assert_eq!(response.token_ids, Some(vec![42, 43])); + assert_eq!( + response.tokens, + Some(vec!["hello".to_owned(), "world".to_owned()]) + ); +} + +#[rstest] +#[tokio::test] +async fn request_hosted_vllm_endpoint_is_rejected_with_environment_credentials() { + let server = MockServer::start().await; + let route = route_with_credentials( + Client::plain_for_test(), + Arc::new(AuthServices::default()), + Arc::new(RecordingSecrets::new([( + "HOSTED_VLLM_API_KEY", + "host-api-key", + )])), + ); + let error = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + assert_credential_boundary_error(&error); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[derive(Default)] +struct Hooks { + context: Mutex>, + raw: Mutex>, + facts: Mutex>, +} + +impl Interceptors for Hooks { + async fn before_provider_request( + &self, + mut wire: WireRequest, + context: RequestContext, + ) -> Result { + *self.context.lock().unwrap() = Some(context); + wire.body["prompt"] = json!("rewritten"); + Ok(wire) + } + async fn after_provider_response(&self, raw: RawResponse) -> Result<(), Error> { + *self.raw.lock().unwrap() = Some(raw); + Ok(()) + } + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { + *self.facts.lock().unwrap() = Some(facts); + Ok(()) + } +} + +struct CallbackCredentialCase { + environment_key: Option<&'static str>, + request_key: Option<&'static str>, + forwarded_authorization: Option<&'static str>, + expected_callback_key: Option<&'static str>, + expected_authorization: &'static str, +} + +#[rstest] +#[case::environment_credential(CallbackCredentialCase { + environment_key: Some("environment-key"), + request_key: None, + forwarded_authorization: None, + expected_callback_key: Some("environment-key"), + expected_authorization: "Bearer environment-key", +})] +#[case::explicit_credential(CallbackCredentialCase { + environment_key: None, + request_key: Some("request-key"), + forwarded_authorization: None, + expected_callback_key: Some("request-key"), + expected_authorization: "Bearer request-key", +})] +#[case::forwarded_authorization_ignores_unused_request_key(CallbackCredentialCase { + environment_key: None, + request_key: Some("unused-request-key"), + forwarded_authorization: Some("Bearer forwarded-key"), + expected_callback_key: None, + expected_authorization: "Bearer forwarded-key", +})] +#[tokio::test] +async fn callback_key_matches_the_credential_used_on_the_wire( + #[case] case: CallbackCredentialCase, +) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(header("authorization", case.expected_authorization)) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"count":1,"tokens":[7],"token_strs":["hello"]})), + ) + .expect(1) + .mount(&server) + .await; + + let secrets = Arc::new(match case.environment_key { + Some(environment_key) => RecordingSecrets::new([("HOSTED_VLLM_API_KEY", environment_key)]), + None => RecordingSecrets::empty(), + }); + let route = route_with_credentials( + Client::plain_for_test(), + Arc::new(AuthServices::default()), + secrets, + ); + let extra_headers = case + .forwarded_authorization + .map(|authorization| { + sourced_map_at( + &json!({"authorization": authorization}), + InputSource::Request, + ) + }) + .unwrap_or_default(); + let hooks = Hooks::default(); + + route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + api_key: case.request_key.map(|key| sourced(SecretValue::new(key))), + api_base: Some(sourced_at(server.uri(), InputSource::Deployment)), + extra_headers, + ..Default::default() + }, + &hooks, + None, + ) + .await + .unwrap(); + + let context = hooks.context.lock().unwrap(); + let callback_key = context + .as_ref() + .unwrap() + .api_key + .as_ref() + .map(SecretValue::expose); + assert_eq!(callback_key, case.expected_callback_key); +} + +#[rstest] +#[tokio::test] +async fn hosted_execution_is_lazy_and_applies_hooks_before_dispatch() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(body_json(json!({ + "model":"org/model", "prompt":"rewritten", "add_special_tokens":true, "return_token_strs":true + }))) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"count":1,"tokens":[7],"token_strs":["hello"]})), + ) + .expect(1) + .mount(&server) + .await; + let secrets = Arc::new(RecordingSecrets::empty()); + let route = route_with_credentials( + Client::plain_for_test(), + Arc::new(AuthServices::default()), + secrets.clone(), + ); + let machine = route.machine( + TokenizeRequest { + model: "org/model".into(), + custom_llm_provider: Some("hosted_vllm".into()), + prompt: "original".into(), + include: Some(TokenizeInclude::new(true, true)), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + None, + ); + assert!(secrets.requested().is_empty()); + assert!(server.received_requests().await.unwrap().is_empty()); + let hooks = Hooks::default(); + let completion = run_hosted( + machine, + Host { + services: &(), + interceptors: &hooks, + stream: &(), + observers: None, + }, + ) + .await + .unwrap(); + let HostedCompletion::Complete(response) = completion else { + panic!("tokenization must return a unary response") + }; + assert_eq!(response.token_ids, Some(vec![7])); + assert_eq!( + hooks + .context + .lock() + .unwrap() + .as_ref() + .unwrap() + .custom_llm_provider, + "hosted_vllm" + ); + let facts = hooks.facts.lock().unwrap().clone().unwrap(); + assert_eq!(facts.provider.model, "org/model"); + assert_eq!(facts.provider.provider, "hosted_vllm"); + assert_eq!(facts.source, ResultSource::Provider); + assert_eq!(facts.cost, Cost::Deferred); + let raw = hooks.raw.lock().unwrap().clone().unwrap(); + assert_eq!( + serde_json::from_str::(&raw.body).unwrap(), + json!({"count":1,"tokens":[7],"token_strs":["hello"]}) + ); +} + +struct FailingPostCall; + +impl Interceptors for FailingPostCall { + async fn before_provider_request( + &self, + wire: WireRequest, + _: RequestContext, + ) -> Result { + Ok(wire) + } + + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { + Err(Error::InvalidRequest("post-call callback failed".into())) + } +} + +#[rstest] +#[tokio::test] +async fn post_call_failures_are_not_request_failures_and_do_not_replay(route: TokenizeRoute) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"count":1,"tokens":[7]}))) + .expect(1) + .mount(&server) + .await; + let error = route + .execute( + TokenizeRequest { + model: "hosted_vllm/model".into(), + prompt: "hello".into(), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + &FailingPostCall, + None, + ) + .await + .unwrap_err(); + assert!(!error.is_request()); + let Error::PostCallHook(cause) = error else { + panic!("post-call errors must preserve their origin") + }; + assert!( + matches!(cause.as_ref(), Error::InvalidRequest(message) if message == &"post-call callback failed".into()) + ); +} + +#[rstest] +#[case::rate_limit(429)] +#[case::unavailable(503)] +#[tokio::test] +async fn upstream_status_is_preserved_without_replaying(route: TokenizeRoute, #[case] status: u16) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status).set_body_json(json!({"error":"busy"}))) + .expect(1) + .mount(&server) + .await; + let error = route + .execute( + TokenizeRequest { + model: "hosted_vllm/model".into(), + prompt: "hello".into(), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + assert!( + matches!(error,Error::Transport(TransportError::Http {status: actual,..}) if actual==status) + ); +} + +#[rstest] +#[tokio::test] +async fn unsupported_deployment_defaults_are_filtered_before_provider_validation() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(body_json(json!({ + "model": "model", + "prompt": "hello", + "add_special_tokens": true, + "return_token_strs": true + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"count": 1, "tokens": [1]}))) + .expect(1) + .mount(&server) + .await; + let error = route() + .execute( + TokenizeRequest { + model: "hosted_vllm/model".into(), + prompt: "hello".into(), + api_base: Some(sourced(server.uri())), + deployment_params: BTreeMap::from([( + "temperature".into(), + sourced_at(json!(0.2), InputSource::Deployment), + )]), + ..Default::default() + }, + &(), + None, + ) + .await; + assert!(error.is_ok()); +} + +#[rstest] +#[tokio::test] +async fn request_owned_values_cannot_be_smuggled_through_deployment_parameters() { + let server = MockServer::start().await; + let error = route() + .execute( + TokenizeRequest { + model: "hosted_vllm/model".into(), + api_base: Some(sourced(server.uri())), + deployment_params: BTreeMap::from([("temperature".into(), sourced(json!(0.2)))]), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + assert!(error.is_request()); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[case::unsupported_provider("gemini/model", json!({}))] +#[case::bare_vllm_is_not_hosted_vllm("vllm/model", json!({}))] +#[case::unknown_option("hosted_vllm/model", json!({"messages":[]}))] +#[tokio::test] +async fn unsupported_calls_fail_before_dispatch( + route: TokenizeRoute, + #[case] model: &str, + #[case] options: Value, +) { + let server = MockServer::start().await; + let error = route + .execute( + TokenizeRequest { + model: model.into(), + prompt: "hello".into(), + api_base: Some(sourced(server.uri())), + optional_params: sourced_map(&options), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + assert!(error.is_request()); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn directly_constructed_empty_model_is_rejected_before_dispatch(route: TokenizeRoute) { + let error = route + .execute( + TokenizeRequest { + model: String::new(), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidRequest(message) if message.to_string() == "model must not be empty") + ); +} + +#[rstest] +#[case::enabled(true)] +#[case::disabled(false)] +#[tokio::test] +async fn top_level_special_token_policy_reaches_vllm_wire( + route: TokenizeRoute, + #[case] add_special_tokens: bool, +) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(body_json(json!({ + "model":"org/model", + "prompt":"hello", + "add_special_tokens":add_special_tokens, + "return_token_strs":true + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count": 1, + "tokens": [42], + "token_strs": ["hello"] + }))) + .expect(1) + .mount(&server) + .await; + + route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + add_special_tokens: Some(add_special_tokens), + api_base: Some(sourced(server.uri())), + deployment_params: sourced_map_at( + &json!({"add_special_tokens": !add_special_tokens}), + InputSource::Deployment, + ), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap(); +} + +#[rstest] +#[tokio::test] +async fn duplicate_special_token_policy_is_rejected_before_provider_io(route: TokenizeRoute) { + let server = MockServer::start().await; + let error = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + add_special_tokens: Some(true), + api_base: Some(sourced(server.uri())), + optional_params: BTreeMap::from([( + "add_special_tokens".into(), + sourced(json!(false)), + )]), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("both")); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn count_output_is_projected_without_becoming_a_provider_parameter(route: TokenizeRoute) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .and(body_json(json!({ + "model":"org/model", + "prompt":"hello", + "add_special_tokens":true, + "return_token_strs":true + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1, + "max_model_len":4096, + "tokens":[42], + "token_strs":["hello"] + }))) + .expect(1) + .mount(&server) + .await; + let response = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + include: Some(TokenizeInclude::new(false, false)), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap(); + + assert_eq!(response.count, 1); + assert_eq!(response.token_ids, None); + assert_eq!(response.tokens, None); +} + +#[rstest] +#[tokio::test] +async fn strict_provider_output_rejects_missing_but_preserves_empty_representations( + route: TokenizeRoute, +) { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":0, + "max_model_len":4096, + "tokens":[], + "token_strs":[] + }))) + .expect(1) + .mount(&server) + .await; + let response = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "".into(), + include: Some(TokenizeInclude::new(true, true)), + api_base: Some(sourced(server.uri())), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap(); + assert_eq!(response.token_ids, Some(Vec::new())); + assert_eq!(response.tokens, Some(Vec::new())); + + let missing = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tokenize")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "count":1, + "max_model_len":4096, + "tokens":[42], + "token_strs":null + }))) + .expect(1) + .mount(&missing) + .await; + let error = route + .execute( + TokenizeRequest { + model: "hosted_vllm/org/model".into(), + prompt: "hello".into(), + include: Some(TokenizeInclude::new(false, true)), + api_base: Some(sourced(missing.uri())), + ..Default::default() + }, + &(), + None, + ) + .await + .unwrap_err(); + assert!(matches!(error, Error::InvalidResponse(_))); +} diff --git a/litellm-rust/crates/llms-types/src/formats/mod.rs b/litellm-rust/crates/llms-types/src/formats/mod.rs index 53f2577090b..b2f2077e9a7 100644 --- a/litellm-rust/crates/llms-types/src/formats/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/mod.rs @@ -4,3 +4,4 @@ pub mod chat_completions; pub mod messages; pub mod ocr; pub mod responses; +pub mod tokenize; diff --git a/litellm-rust/crates/llms-types/src/formats/tokenize/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/tokenize/AGENTS.md new file mode 100644 index 00000000000..5438f43d47d --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/tokenize/AGENTS.md @@ -0,0 +1,8 @@ +# rules + +- The unified endpoint accepts text in `prompt` and keeps numeric `token_ids` separate from string `tokens` +- Missing provider token representations serialize as null, never fabricated IDs or empty arrays + +# references + +- Provider endpoint contracts live in `../../../../llms/src/hosted_vllm/tokenize/AGENTS.md` and `../../../../llms/src/watsonx/tokenize/AGENTS.md` diff --git a/litellm-rust/crates/llms-types/src/formats/tokenize/include.rs b/litellm-rust/crates/llms-types/src/formats/tokenize/include.rs new file mode 100644 index 00000000000..dc198655b11 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/tokenize/include.rs @@ -0,0 +1,105 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use strum::VariantArray; + +use super::TokenizeOutput; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, VariantArray)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum TokenizeRepresentation { + TokenIds, + Tokens, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(transparent)] +pub struct TokenizeInclude( + #[serde(deserialize_with = "deserialize_unique")] + #[cfg_attr( + feature = "schema", + schemars(with = "std::collections::HashSet") + )] + Vec, +); + +impl TokenizeInclude { + pub fn new(token_ids: bool, tokens: bool) -> Self { + Self( + [ + token_ids.then_some(TokenizeRepresentation::TokenIds), + tokens.then_some(TokenizeRepresentation::Tokens), + ] + .into_iter() + .flatten() + .collect(), + ) + } + + pub fn output(&self) -> TokenizeOutput { + match ( + self.0.contains(&TokenizeRepresentation::TokenIds), + self.0.contains(&TokenizeRepresentation::Tokens), + ) { + (false, false) => TokenizeOutput::Count, + (true, false) => TokenizeOutput::Ids, + (false, true) => TokenizeOutput::Tokens, + (true, true) => TokenizeOutput::Both, + } + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +fn deserialize_unique<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let values = Vec::::deserialize(deserializer)?; + if values + .iter() + .enumerate() + .any(|(index, value)| values[..index].contains(value)) + { + return Err(serde::de::Error::custom( + "include must not contain duplicate entries", + )); + } + Ok(values) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::{TokenizeInclude, TokenizeOutput}; + + #[rstest] + #[case::count(json!([]), TokenizeOutput::Count)] + #[case::ids(json!(["token_ids"]), TokenizeOutput::Ids)] + #[case::tokens(json!(["tokens"]), TokenizeOutput::Tokens)] + #[case::both(json!(["token_ids", "tokens"]), TokenizeOutput::Both)] + #[case::reversed(json!(["tokens", "token_ids"]), TokenizeOutput::Both)] + fn include_round_trips(#[case] body: Value, #[case] output: TokenizeOutput) { + let include: TokenizeInclude = serde_json::from_value(body.clone()).unwrap(); + assert_eq!(include.output(), output); + assert_eq!(serde_json::to_value(include).unwrap(), body); + } + + #[rstest] + #[case::duplicate_ids(json!(["token_ids", "token_ids"]))] + #[case::duplicate_tokens(json!(["tokens", "tokens"]))] + #[case::unknown(json!(["ids"]))] + #[case::wrong_case(json!(["Tokens"]))] + #[case::number(json!([1]))] + #[case::null_item(json!([null]))] + #[case::null(json!(null))] + #[case::string(json!("tokens"))] + #[case::object(json!({"tokens":true}))] + fn malformed_include_is_rejected(#[case] body: Value) { + assert!(serde_json::from_value::(body).is_err()); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/tokenize/mod.rs b/litellm-rust/crates/llms-types/src/formats/tokenize/mod.rs new file mode 100644 index 00000000000..7317750e151 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/tokenize/mod.rs @@ -0,0 +1,9 @@ +mod include; +mod output; +mod request; +mod response; + +pub use include::*; +pub use output::*; +pub use request::*; +pub use response::*; diff --git a/litellm-rust/crates/llms-types/src/formats/tokenize/output.rs b/litellm-rust/crates/llms-types/src/formats/tokenize/output.rs new file mode 100644 index 00000000000..853bb7d6c5d --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/tokenize/output.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; +use strum::VariantArray; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize, VariantArray)] +#[serde(rename_all = "snake_case")] +pub enum TokenizeOutput { + Count, + Ids, + Tokens, + Both, + #[default] + Available, +} + +impl TokenizeOutput { + pub fn is_available(&self) -> bool { + *self == Self::Available + } + + pub fn requires_ids(self) -> bool { + matches!(self, Self::Ids | Self::Both) + } + + pub fn requires_tokens(self) -> bool { + matches!(self, Self::Tokens | Self::Both) + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/tokenize/request.rs b/litellm-rust/crates/llms-types/src/formats/tokenize/request.rs new file mode 100644 index 00000000000..6f0d482fd77 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/tokenize/request.rs @@ -0,0 +1,164 @@ +use serde::{Deserialize, Deserializer}; + +use super::TokenizeInclude; + +#[macro_rules_attribute::apply(crate::wire_type)] +#[serde(deny_unknown_fields)] +pub struct TokenizeRequest { + #[serde(deserialize_with = "deserialize_model")] + #[cfg_attr(feature = "schema", schemars(length(min = 1)))] + pub model: String, + pub prompt: String, + #[serde( + default, + deserialize_with = "deserialize_include", + skip_serializing_if = "Option::is_none" + )] + #[cfg_attr(feature = "schema", schemars(with = "TokenizeInclude"))] + pub include: Option, + #[serde( + default, + deserialize_with = "deserialize_special_tokens", + skip_serializing_if = "Option::is_none" + )] + #[cfg_attr(feature = "schema", schemars(with = "bool"))] + pub add_special_tokens: Option, +} + +fn deserialize_include<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + TokenizeInclude::deserialize(deserializer).map(Some) +} + +fn deserialize_model<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let model = String::deserialize(deserializer)?; + if model.is_empty() { + return Err(serde::de::Error::custom("model must not be empty")); + } + Ok(model) +} + +fn deserialize_special_tokens<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + bool::deserialize(deserializer).map(Some) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::TokenizeRequest; + use crate::formats::tokenize::TokenizeOutput; + + #[rstest] + #[case::text("Hello")] + #[case::empty("")] + #[case::unicode(" café 世界\n")] + fn prompt_round_trips_without_normalization(#[case] prompt: &str) { + let body = json!({"model": "model-alias", "prompt": prompt}); + let request: TokenizeRequest = serde_json::from_value(body.clone()).unwrap(); + assert_eq!(request.include, None); + assert_eq!(request.add_special_tokens, None); + assert_eq!(serde_json::to_value(request).unwrap(), body); + } + + #[rstest] + #[case::count(json!([]), TokenizeOutput::Count)] + #[case::ids(json!(["token_ids"]), TokenizeOutput::Ids)] + #[case::tokens(json!(["tokens"]), TokenizeOutput::Tokens)] + #[case::both(json!(["token_ids", "tokens"]), TokenizeOutput::Both)] + fn explicit_include_round_trips(#[case] include: Value, #[case] output: TokenizeOutput) { + let body = json!({"model": "model-alias", "prompt": "Hello", "include": include}); + let request: TokenizeRequest = serde_json::from_value(body.clone()).unwrap(); + assert_eq!(request.include.as_ref().unwrap().output(), output); + assert_eq!(serde_json::to_value(request).unwrap(), body); + } + + #[rstest] + #[case::missing_model(json!({"prompt": "Hello"}))] + #[case::empty_model(json!({"model":"", "prompt":""}))] + #[case::null_model(json!({"model":null, "prompt":""}))] + #[case::numeric_model(json!({"model":1, "prompt":""}))] + #[case::missing_prompt(json!({"model": "model-alias"}))] + #[case::null_prompt(json!({"model": "model-alias", "prompt": null}))] + #[case::non_text_prompt(json!({"model": "model-alias", "prompt": [1, 2]}))] + #[case::non_object(json!([]))] + fn malformed_requests_are_rejected(#[case] body: Value) { + assert!(serde_json::from_value::(body).is_err()); + } + + #[rstest] + #[case::null_special("add_special_tokens", json!(null))] + #[case::string_special("add_special_tokens", json!("false"))] + #[case::numeric_special("add_special_tokens", json!(0))] + #[case::null_include("include", json!(null))] + #[case::output("output", json!("count"))] + #[case::messages("messages", json!([]))] + #[case::tools("tools", json!([]))] + #[case::images("images", json!([]))] + #[case::template("chat_template", json!(""))] + #[case::generation_prompt("add_generation_prompt", json!(false))] + fn invalid_options_are_rejected(#[case] field: &str, #[case] value: Value) { + let mut body = json!({"model":"m", "prompt":""}); + body[field] = value; + assert!(serde_json::from_value::(body).is_err()); + } + + #[rstest] + #[case::enabled(true)] + #[case::disabled(false)] + fn explicit_special_policy_round_trips(#[case] add_special_tokens: bool) { + let body = json!({"model":"m", "prompt":"", "include":["tokens"], "add_special_tokens":add_special_tokens}); + let request: TokenizeRequest = serde_json::from_value(body.clone()).unwrap(); + assert_eq!(request.add_special_tokens, Some(add_special_tokens)); + assert_eq!(serde_json::to_value(request).unwrap(), body); + } + + #[cfg(feature = "schema")] + #[rstest] + fn schema_matches_strict_request() { + let schema = schemars::schema_for!(TokenizeRequest).to_value(); + assert_eq!(schema["additionalProperties"], false); + assert_eq!(schema["properties"]["model"]["minLength"], 1); + assert_eq!( + schema["properties"]["add_special_tokens"]["type"], + "boolean" + ); + assert_eq!( + schema["properties"]["include"]["$ref"], + "#/$defs/TokenizeInclude" + ); + assert!(schema["properties"].get("output").is_none()); + assert!( + schema["required"] + .as_array() + .unwrap() + .contains(&json!("model")) + ); + assert!( + schema["required"] + .as_array() + .unwrap() + .contains(&json!("prompt")) + ); + assert!( + !schema["required"] + .as_array() + .unwrap() + .contains(&json!("include")) + ); + assert_eq!(schema["$defs"]["TokenizeInclude"]["uniqueItems"], true); + assert_eq!( + schema["$defs"]["TokenizeRepresentation"]["enum"], + json!(["token_ids", "tokens"]) + ); + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/tokenize/response.rs b/litellm-rust/crates/llms-types/src/formats/tokenize/response.rs new file mode 100644 index 00000000000..a4f0a03b7e7 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/tokenize/response.rs @@ -0,0 +1,37 @@ +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct TokenizeResponse { + pub model: String, + pub count: u64, + pub token_ids: Option>, + pub tokens: Option>, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::TokenizeResponse; + + #[rstest] + fn unavailable_token_ids_are_null_and_strings_remain_strings() { + let response = TokenizeResponse { + model: "model".into(), + count: 2, + token_ids: None, + tokens: Some(vec!["Hello".into(), " world".into()]), + }; + assert_eq!( + serde_json::to_value(response).unwrap(), + json!({"model": "model", "count": 2, "token_ids": null, "tokens": ["Hello", " world"]}) + ); + } + + #[rstest] + #[case::negative_count(json!({"model": "m", "count": -1}))] + #[case::string_ids(json!({"model": "m", "count": 1, "token_ids": ["hello"]}))] + #[case::numeric_strings(json!({"model": "m", "count": 1, "tokens": [123]}))] + fn token_representations_have_fixed_types(#[case] body: Value) { + assert!(serde_json::from_value::(body).is_err()); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs index 4c04c23a0bb..04a8239fa31 100644 --- a/litellm-rust/crates/llms/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -6,3 +6,4 @@ pub mod messages; pub mod ocr; pub mod responses; pub mod sse; +pub mod tokenize; diff --git a/litellm-rust/crates/llms/src/base_llm/tokenize/mod.rs b/litellm-rust/crates/llms/src/base_llm/tokenize/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/tokenize/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/tokenize/transformation.rs b/litellm-rust/crates/llms/src/base_llm/tokenize/transformation.rs new file mode 100644 index 00000000000..d2f8f255f43 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/tokenize/transformation.rs @@ -0,0 +1,185 @@ +use std::collections::BTreeMap; + +use litellm_auth::{InputSource, SecretValue, Sourced}; +use litellm_llms_types::formats::tokenize::{TokenizeOutput, TokenizeResponse}; +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; + +use crate::{Error, ErrorDetail}; + +use crate::base_llm::auth::{Headers, ValidatedEnvironment}; + +pub type TokenizeMap = BTreeMap>; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct TokenizeCapabilities { + pub token_ids: bool, + pub tokens: bool, +} + +impl TokenizeCapabilities { + pub fn supports(self, output: TokenizeOutput) -> bool { + match output { + TokenizeOutput::Count | TokenizeOutput::Available => true, + TokenizeOutput::Ids => self.token_ids, + TokenizeOutput::Tokens => self.tokens, + TokenizeOutput::Both => self.token_ids && self.tokens, + } + } +} + +pub struct TokenizeContext<'a> { + pub api_key: Option<&'a Sourced>, + pub api_base: Option<&'a Sourced>, + pub extra_headers: &'a TokenizeMap, + pub deployment_params: &'a TokenizeMap, + pub optional_params: &'a TokenizeMap, +} + +impl<'a> TokenizeContext<'a> { + pub fn new( + api_key: Option<&'a Sourced>, + api_base: Option<&'a Sourced>, + extra_headers: &'a TokenizeMap, + deployment_params: &'a TokenizeMap, + optional_params: &'a TokenizeMap, + ) -> Self { + Self { + api_key, + api_base, + extra_headers, + deployment_params, + optional_params, + } + } + + pub fn string_headers(&self) -> Result { + self.extra_headers + .iter() + .map(|(name, value)| { + let value = value.value().as_str().ok_or_else(|| { + Error::InvalidRequest(ErrorDetail::InvalidValue { + field: "extra_headers", + expected: "string header values", + actual: value.value().clone(), + }) + })?; + Ok((name.clone(), value.to_owned())) + }) + .collect() + } +} + +pub trait BaseTokenizeConfig: Sync { + fn get_supported_tokenize_params(&self) -> &'static [&'static str]; + + fn tokenize_capabilities(&self) -> TokenizeCapabilities { + TokenizeCapabilities::default() + } + + fn map_tokenize_params( + &self, + _model: &str, + params: &Map, + ) -> Result, Error> { + if let Some(parameter) = params.keys().find(|name| { + !self + .get_supported_tokenize_params() + .contains(&name.as_str()) + }) { + return Err(Error::InvalidRequest(ErrorDetail::InvalidChoice { + field: "parameter", + actual: parameter.clone(), + choices: self.get_supported_tokenize_params().to_vec(), + })); + } + Ok(params.clone()) + } + + fn validate_tokenize_input( + &self, + input: &TokenizeContext<'_>, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result<(), Error> { + validate_common_tokenize_input(input, self.get_supported_tokenize_params()) + } + + fn validate_environment( + &self, + input: &TokenizeContext<'_>, + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; + + fn get_complete_url( + &self, + api_base: Option<&str>, + model: &str, + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; + + fn transform_tokenize_request( + &self, + model: &str, + prompt: &str, + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; + + fn transform_tokenize_response( + &self, + model: &str, + response_json: Value, + ) -> Result; + + fn secret_names(&self) -> Vec<&'static str>; + + fn secret_param_names(&self) -> &'static [&'static str] { + &[] + } +} + +pub fn validate_common_tokenize_input( + input: &TokenizeContext<'_>, + supported_params: &[&'static str], +) -> Result<(), Error> { + if input + .deployment_params + .values() + .any(|value| value.source() == InputSource::Request) + { + return Err(Error::InvalidRequest( + "request parameters cannot be deployment parameters".into(), + )); + } + if let Some(parameter) = input + .optional_params + .keys() + .find(|name| !supported_params.contains(&name.as_str())) + { + return Err(Error::InvalidRequest(ErrorDetail::InvalidChoice { + field: "parameter", + actual: parameter.clone(), + choices: supported_params.to_vec(), + })); + } + Ok(()) +} + +pub fn decode_params(params: &Map) -> Result { + serde_json::from_value(Value::Object(params.clone())).map_err(|source| { + Error::InvalidRequest(ErrorDetail::invalid("tokenization parameters", source)) + }) +} + +pub fn validate_token_count(count: u64, length: usize) -> Result<(), Error> { + if count != length as u64 { + return Err(Error::InvalidResponse(ErrorDetail::InvalidValue { + field: "count", + expected: "the number of returned tokens", + actual: Value::from(count), + })); + } + Ok(()) +} diff --git a/litellm-rust/crates/llms/src/hosted_vllm/mod.rs b/litellm-rust/crates/llms/src/hosted_vllm/mod.rs new file mode 100644 index 00000000000..c378404939f --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/mod.rs @@ -0,0 +1 @@ +pub mod tokenize; diff --git a/litellm-rust/crates/llms/src/hosted_vllm/tokenize/AGENTS.md b/litellm-rust/crates/llms/src/hosted_vllm/tokenize/AGENTS.md new file mode 100644 index 00000000000..06ac1f2d20c --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/tokenize/AGENTS.md @@ -0,0 +1,9 @@ +# rules + +- Hosted vLLM tokenization uses `/tokenize` beside `/v1`, preserving any deployment path prefix +- Request token strings as well as IDs and preserve their separate normalized fields + +# references + +- https://docs.vllm.ai/en/v0.15.1/api/vllm/entrypoints/serve/tokenize/protocol/ +- https://docs.vllm.ai/en/v0.12.0/api/vllm/entrypoints/openai/api_server/ diff --git a/litellm-rust/crates/llms/src/hosted_vllm/tokenize/mod.rs b/litellm-rust/crates/llms/src/hosted_vllm/tokenize/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/tokenize/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/hosted_vllm/tokenize/transformation.rs b/litellm-rust/crates/llms/src/hosted_vllm/tokenize/transformation.rs new file mode 100644 index 00000000000..e85bdd3ab3d --- /dev/null +++ b/litellm-rust/crates/llms/src/hosted_vllm/tokenize/transformation.rs @@ -0,0 +1,388 @@ +use litellm_auth::{CredentialBoundary, CredentialPlacement, InputSource, SecretValue}; +use litellm_core_utils::settings::resolve_non_empty; +use litellm_llms_types::formats::tokenize::TokenizeResponse; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use url::Url; + +use crate::{ + Error, ErrorDetail, + base_llm::{ + auth::{AuthScheme, ValidatedEnvironment}, + tokenize::transformation::{ + BaseTokenizeConfig, TokenizeCapabilities, TokenizeContext, decode_params, + validate_common_tokenize_input, validate_token_count, + }, + }, +}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HostedVllmTokenizeParams { + add_special_tokens: Option, +} + +#[derive(Serialize)] +struct HostedVllmTokenizeRequest<'a> { + model: &'a str, + prompt: &'a str, + add_special_tokens: bool, + return_token_strs: bool, +} + +#[derive(Deserialize)] +struct HostedVllmTokenizeResponse { + count: u64, + tokens: Vec, + token_strs: Option>, +} + +pub struct HostedVllmTokenizeConfig; + +impl BaseTokenizeConfig for HostedVllmTokenizeConfig { + fn get_supported_tokenize_params(&self) -> &'static [&'static str] { + &["add_special_tokens"] + } + + fn tokenize_capabilities(&self) -> TokenizeCapabilities { + TokenizeCapabilities { + token_ids: true, + tokens: true, + } + } + + fn validate_environment( + &self, + input: &TokenizeContext<'_>, + _optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let headers = input.string_headers()?; + let api_key = input.api_key.map(|key| key.value().expose()); + if litellm_http::request::has_header(&headers, "authorization") { + return Ok(ValidatedEnvironment { + headers, + auth: AuthScheme::Forwarded, + }); + } + let key = resolve_non_empty(api_key, env_lookup, &["HOSTED_VLLM_API_KEY"]); + let auth = match key { + Some(key) => AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: SecretValue::new(key), + }, + None => AuthScheme::Forwarded, + }; + Ok(ValidatedEnvironment { headers, auth }) + } + + fn validate_tokenize_input( + &self, + input: &TokenizeContext<'_>, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result<(), Error> { + validate_common_tokenize_input(input, self.get_supported_tokenize_params())?; + input.string_headers()?; + let authorization = input + .extra_headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("authorization")) + .map(|(_, value)| value); + let mut sources: Vec = input + .api_key + .into_iter() + .map(|value| value.source()) + .chain(authorization.clone().map(|value| value.source())) + .collect(); + let key = resolve_non_empty( + input.api_key.map(|value| value.value().expose()), + env_lookup, + &["HOSTED_VLLM_API_KEY"], + ); + if authorization.clone().next().is_none() + && input + .api_key + .is_none_or(|value| value.value().expose().trim().is_empty()) + && key.is_some() + { + sources.push(InputSource::Environment); + } + if sources.is_empty() { + return Ok(()); + } + let boundary = CredentialBoundary::new(&sources) + .map_err(|error| Error::InvalidRequest(error.to_string().into()))?; + boundary + .validate_resources(input.api_base.into_iter().map(|value| value.source())) + .map_err(|error| Error::InvalidRequest(error.to_string().into())) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + _model: &str, + _optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let base = resolve_non_empty(api_base, env_lookup, &["HOSTED_VLLM_API_BASE"]).ok_or( + litellm_auth::Error::MissingApiBase { + provider: "hosted_vllm", + guidance: "set api_base or HOSTED_VLLM_API_BASE", + }, + )?; + tokenize_url(&base) + } + + fn transform_tokenize_request( + &self, + model: &str, + prompt: &str, + optional_params: &Map, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let params: HostedVllmTokenizeParams = decode_params(optional_params)?; + serde_json::to_value(HostedVllmTokenizeRequest { + model, + prompt, + add_special_tokens: params.add_special_tokens.unwrap_or(true), + return_token_strs: true, + }) + .map_err(|source| Error::InvalidRequest(ErrorDetail::invalid("tokenize request", source))) + } + + fn transform_tokenize_response( + &self, + model: &str, + response_json: Value, + ) -> Result { + let response: HostedVllmTokenizeResponse = + serde_json::from_value(response_json).map_err(|source| { + Error::InvalidResponse(ErrorDetail::invalid("tokenize response", source)) + })?; + validate_token_count(response.count, response.tokens.len())?; + if let Some(tokens) = &response.token_strs { + validate_token_count(response.count, tokens.len())?; + } + Ok(TokenizeResponse { + model: model.to_string(), + count: response.count, + token_ids: Some(response.tokens), + tokens: response.token_strs, + }) + } + + fn secret_names(&self) -> Vec<&'static str> { + vec!["HOSTED_VLLM_API_KEY", "HOSTED_VLLM_API_BASE"] + } +} + +fn tokenize_url(base: &str) -> Result { + let mut url = Url::parse(base.trim()) + .map_err(|source| Error::InvalidRequest(ErrorDetail::invalid("api_base", source)))?; + if !matches!(url.scheme(), "http" | "https") { + return Err(Error::Unsupported("api_base URL scheme")); + } + let path = url.path().trim_end_matches('/'); + let path = path.strip_suffix("/v1").unwrap_or(path); + let endpoint = if path.ends_with("/tokenize") { + path.to_string() + } else { + format!("{path}/tokenize") + }; + url.set_path(&endpoint); + Ok(url.into()) +} + +#[cfg(test)] +mod tests { + use litellm_auth::{InputSource, SecretValue, Sourced}; + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::base_llm::tokenize::transformation::TokenizeMap; + + fn no_env(_: &str) -> Option { + None + } + + #[rstest] + #[case::root("https://example.test", "https://example.test/tokenize")] + #[case::version("https://example.test/v1/", "https://example.test/tokenize")] + #[case::prefix( + "https://example.test/tenant/v1", + "https://example.test/tenant/tokenize" + )] + #[case::complete( + "https://example.test/tenant/tokenize/", + "https://example.test/tenant/tokenize" + )] + #[case::query( + "https://example.test/tenant/v1?tenant=a", + "https://example.test/tenant/tokenize?tenant=a" + )] + fn endpoint_preserves_the_deployment_prefix(#[case] base: &str, #[case] expected: &str) { + assert_eq!(tokenize_url(base).unwrap(), expected); + } + + #[rstest] + #[case::invalid("not a url")] + #[case::unsupported("file:///tokenize")] + fn invalid_endpoint_is_rejected(#[case] base: &str) { + assert!(tokenize_url(base).is_err()); + } + + #[rstest] + #[case::text("Hello")] + #[case::empty("")] + fn request_preserves_prompt_and_requests_both_representations(#[case] prompt: &str) { + let request = HostedVllmTokenizeConfig + .transform_tokenize_request("served-model", prompt, &Map::new(), &no_env) + .unwrap(); + assert_eq!( + request, + json!({"model": "served-model", "prompt": prompt, "add_special_tokens": true, "return_token_strs": true}) + ); + } + + #[rstest] + fn caller_can_omit_special_tokens() { + let params = Map::from_iter([("add_special_tokens".into(), json!(false))]); + let request = HostedVllmTokenizeConfig + .transform_tokenize_request("served-model", "Hello", ¶ms, &no_env) + .unwrap(); + assert_eq!(request["add_special_tokens"], false); + } + + #[rstest] + #[case::wrong_type(json!({"add_special_tokens": "false"}))] + #[case::unknown(json!({"unrecognized": true}))] + fn invalid_parameters_are_rejected(#[case] params: Value) { + assert!( + HostedVllmTokenizeConfig + .transform_tokenize_request("model", "Hello", params.as_object().unwrap(), &no_env) + .is_err() + ); + } + + #[rstest] + #[case::strings(Some(vec!["Hello".to_string(), " world".to_string()]))] + #[case::without_strings(None)] + fn response_keeps_ids_and_strings_separate(#[case] strings: Option>) { + let response = HostedVllmTokenizeConfig + .transform_tokenize_response( + "served-model", + json!({"count": 2, "tokens": [10, 20], "token_strs": strings, "max_model_len": 1000}), + ) + .unwrap(); + assert_eq!(response.model, "served-model"); + assert_eq!(response.count, 2); + assert_eq!(response.token_ids, Some(vec![10, 20])); + assert_eq!(response.tokens, strings); + } + + #[rstest] + #[case::missing_count(json!({"tokens": []}))] + #[case::negative_count(json!({"count": -1, "tokens": []}))] + #[case::negative_id(json!({"count": 1, "tokens": [-1]}))] + #[case::wrong_id_type(json!({"count": 1, "tokens": ["Hello"]}))] + #[case::missing_ids(json!({"count": 0}))] + #[case::wrong_count(json!({"count": 2, "tokens": [10]}))] + #[case::wrong_strings_length(json!({"count": 1, "tokens": [10], "token_strs": []}))] + fn malformed_responses_are_rejected(#[case] body: Value) { + assert!( + HostedVllmTokenizeConfig + .transform_tokenize_response("model", body) + .is_err() + ); + } + + #[rstest] + fn empty_token_arrays_remain_available() { + let response = HostedVllmTokenizeConfig + .transform_tokenize_response( + "model", + json!({"count": 0, "tokens": [], "token_strs": []}), + ) + .unwrap(); + assert_eq!(response.token_ids, Some(vec![])); + assert_eq!(response.tokens, Some(vec![])); + } + + #[rstest] + #[case::explicit(Some("explicit"), Some("environment"), Some("explicit"))] + #[case::environment(None, Some("environment"), Some("environment"))] + #[case::keyless(None, None, None)] + fn key_resolution_supports_authenticated_and_keyless_deployments( + #[case] explicit: Option<&str>, + #[case] environment: Option<&str>, + #[case] expected: Option<&str>, + ) { + let api_key = + explicit.map(|value| Sourced::new(SecretValue::new(value), InputSource::Request)); + let extra_headers = TokenizeMap::new(); + let deployment_params = TokenizeMap::new(); + let optional_params = TokenizeMap::new(); + let input = TokenizeContext::new( + api_key.as_ref(), + None, + &extra_headers, + &deployment_params, + &optional_params, + ); + let validated = HostedVllmTokenizeConfig + .validate_environment(&input, &Map::new(), &|name| { + (name == "HOSTED_VLLM_API_KEY") + .then_some(environment) + .flatten() + .map(str::to_string) + }) + .unwrap(); + match (validated.auth, expected) { + (AuthScheme::Credential { secret, .. }, Some(expected)) => { + assert_eq!(secret.expose(), expected) + } + (AuthScheme::Forwarded, None) => {} + _ => panic!("unexpected authentication"), + } + } + + #[rstest] + fn forwarded_authorization_is_preserved() { + let headers = vec![("AUTHORIZATION".into(), "Bearer forwarded".into())]; + let api_key = Sourced::new(SecretValue::new("configured"), InputSource::Request); + let extra_headers = TokenizeMap::from([( + "AUTHORIZATION".into(), + Sourced::new(json!("Bearer forwarded"), InputSource::Request), + )]); + let deployment_params = TokenizeMap::new(); + let optional_params = TokenizeMap::new(); + let input = TokenizeContext::new( + Some(&api_key), + None, + &extra_headers, + &deployment_params, + &optional_params, + ); + let validated = HostedVllmTokenizeConfig + .validate_environment(&input, &Map::new(), &no_env) + .unwrap(); + assert!(matches!(validated.auth, AuthScheme::Forwarded)); + assert_eq!(validated.headers, headers); + } + + #[rstest] + fn api_base_uses_environment_when_omitted() { + let url = HostedVllmTokenizeConfig + .get_complete_url(None, "model", &Map::new(), &|name| { + (name == "HOSTED_VLLM_API_BASE").then(|| "https://env.test/v1".into()) + }) + .unwrap(); + assert_eq!(url, "https://env.test/tokenize"); + assert!( + HostedVllmTokenizeConfig + .get_complete_url(None, "model", &Map::new(), &no_env) + .is_err() + ); + } +} diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs index 8a3a143c02b..61a903f25ab 100644 --- a/litellm-rust/crates/llms/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -8,6 +8,7 @@ pub mod cohere; pub mod deepseek; pub mod edenai; mod error; +pub mod hosted_vllm; pub mod minimax; pub mod mistral; pub mod openai;