diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 43966f2bee7..e0925843c71 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -91,6 +91,7 @@ impl Gateway { pub fn router(gateway: Arc) -> Router { Router::new() .route("/v1/messages", post(messages::create)) + .route("/v1/messages/count_tokens", post(messages::count_tokens)) .route("/ocr", post(ocr::create)) .route("/v1/ocr", post(ocr::create)) .route("/chat/completions", post(chat_completions::create)) diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index 58ecd40204d..2c9e8a94d3d 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -1,4 +1,5 @@ -//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it. +//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it, and +//! `POST /v1/messages/count_tokens`, answered by the deployment's own count-tokens API. use litellm_gateway_auth::AuthenticatedRequest; use std::sync::Arc; @@ -12,7 +13,10 @@ use axum::{ }; use litellm_host_http::Sse; use litellm_inference_messages::{MessagesCall, messages_body, route::Messages}; -use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders}; +use litellm_llms_types::{ + formats::messages::CountTokensResponse, + headers::{ProviderSpecificHeader, ProviderSpecificHeaders}, +}; use serde_json::{Map, Value}; use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request}; @@ -67,6 +71,34 @@ async fn handle( Ok(headers.apply(response)) } +pub async fn count_tokens( + State(gateway): State>, + identity: AuthenticatedRequest, + RequestId(request_id): RequestId, + headers: HeaderMap, + body: Result, +) -> impl IntoResponse { + let result = match body { + Ok(JsonObject(body)) => count(&gateway, &identity, &headers, body).await, + Err(error) => Err(error), + }; + result + .map(Json) + .map_err(|error| (error.status(), Json(error.body(request_id.as_deref())))) +} + +async fn count( + gateway: &Gateway, + identity: &AuthenticatedRequest, + headers: &HeaderMap, + body: Map, +) -> Result { + let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?; + request::authorize_model(identity, deployment, &body).await?; + let call = project(deployment, body, headers)?; + Ok(gateway.messages.count_tokens(call).await?) +} + fn project( deployment: &Deployment, body: Map, diff --git a/litellm-rust/crates/gateway-inference/tests/messages.rs b/litellm-rust/crates/gateway-inference/tests/messages.rs index 7d6d1f160d1..3ae6267afec 100644 --- a/litellm-rust/crates/gateway-inference/tests/messages.rs +++ b/litellm-rust/crates/gateway-inference/tests/messages.rs @@ -121,18 +121,20 @@ async fn a_stream_that_fails_after_opening_ends_with_an_sse_error_frame() { } #[rstest] +#[case::messages("/v1/messages")] +#[case::count_tokens("/v1/messages/count_tokens")] #[tokio::test] -async fn hosted_provider_failure_preserves_status_body_and_request_id() { +async fn hosted_provider_failure_preserves_status_body_and_request_id(#[case] route: &str) { let upstream = MockServer::start().await; let error = json!({"type": "error", "error": {"type": "rate_limit_error", "message": "retry later"}}); Mock::given(method("POST")) - .and(path("/v1/messages")) + .and(path(route)) .respond_with(ResponseTemplate::new(429).set_body_json(error.clone())) .expect(1) .mount(&upstream) .await; - let request = Request::post("/v1/messages") + let request = Request::post(route) .header("content-type", "application/json") .header("x-request-id", "host-http-request") .body(Body::from(json!({ @@ -150,8 +152,90 @@ async fn hosted_provider_failure_preserves_status_body_and_request_id() { assert_eq!(body["request_id"], "host-http-request"); } +#[rstest] +#[tokio::test] +async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model() { + let upstream = MockServer::start().await; + let messages = json!([{"role": "user", "content": "hi"}]); + let tools = json!([{"name": "lookup", "input_schema": {"type": "object"}}]); + let counted = json!({"input_tokens": 14, "context_management": {"original_input_tokens": 20}}); + Mock::given(method("POST")) + .and(path("/v1/messages/count_tokens")) + .and(header("x-api-key", "test-key")) + .and(header("anthropic-version", "test-version")) + .and(body_json(json!({ + "model": "test-model", "messages": messages, "system": "be brief", "tools": tools, + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(counted.clone())) + .expect(1) + .mount(&upstream) + .await; + let request = Request::post("/v1/messages/count_tokens") + .header("content-type", "application/json") + .header("anthropic-version", "test-version") + .body(Body::from( + json!({"model": "public/model", "messages": messages, "system": "be brief", "tools": tools}) + .to_string(), + )) + .unwrap(); + let response = support::app("anthropic/test-model", &upstream.uri()) + .oneshot(request) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!(support::json(response).await, counted); +} + +#[rstest] +#[tokio::test] +async fn count_tokens_for_a_provider_without_a_count_api_is_an_invalid_request() { + let upstream = MockServer::start().await; + let response = support::post( + support::app("deepseek/test-model", &upstream.uri()), + "/v1/messages/count_tokens", + json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["type"], "error"); + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert!( + body["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("deepseek")), + "{body}" + ); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn an_unknown_model_is_refused_alike_on_messages_and_count_tokens( + #[values("/v1/messages", "/v1/messages/count_tokens")] route: &str, +) { + let upstream = MockServer::start().await; + let response = support::post( + support::app("anthropic/test-model", &upstream.uri()), + route, + json!({"model": "other/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ) + .await; + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert!( + body["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("other/model")), + "{body}" + ); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + #[rstest] #[case::messages("/v1/messages")] +#[case::count_tokens("/v1/messages/count_tokens")] #[case::chat("/v1/chat/completions")] #[case::model_path("/engines/path-model/chat/completions")] #[case::ocr("/ocr")] diff --git a/litellm-rust/crates/gateway-inference/tests/routes.rs b/litellm-rust/crates/gateway-inference/tests/routes.rs index 24b8c5cc694..f019dcbe0da 100644 --- a/litellm-rust/crates/gateway-inference/tests/routes.rs +++ b/litellm-rust/crates/gateway-inference/tests/routes.rs @@ -235,6 +235,7 @@ async fn upload_audio_format_validation_matches_core( #[case::responses("/v1/responses", false)] #[case::deployment("/openai/deployments/public%2Fmodel/chat/completions", false)] #[case::messages("/v1/messages", true)] +#[case::count_tokens("/v1/messages/count_tokens", true)] #[case::ocr("/v1/ocr", false)] #[case::transcription("/v1/audio/transcriptions", false)] #[tokio::test] diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 9d0bf09197e..d8b729b6ec9 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -19,7 +19,9 @@ use litellm_llms::base_llm::{ transformation::BaseMessagesConfig, }, }; -use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; +use litellm_llms_types::formats::messages::{ + CountTokensResponse, MessagesRequest, MessagesResponse, +}; use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; use serde_json::Value; @@ -211,6 +213,92 @@ async fn provider_call( }) } +pub(super) async fn count_tokens( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + call: MessagesCall, + secrets: &(dyn Lookup + Send + Sync), +) -> Result { + let MessagesCall { + body, + api_key, + api_base, + litellm_params, + extra_headers, + provider_specific_header, + timeout, + shaping, + .. + } = call; + let connection_params = litellm_router_types::LitellmParams { + api_base: api_base.clone().or(litellm_params.api_base.clone()), + ..litellm_params + }; + let config = provider.config(); + let env_lookup = |key: &str| secrets.get(key); + let url = config + .get_count_tokens_url(api_base.as_deref(), &connection_params, &env_lookup) + .ok_or_else(|| { + Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedOperation { + operation: "token counting", + provider: provider.as_str(), + }) + })??; + + let sanitized = config.shape_request( + MessagesRequest { + model: config.count_tokens_model(&model).to_string(), + ..body + }, + shaping.settings.reasoning_auto_summary, + )?; + let request = + without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; + + let scoped = + get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); + let forwarded = string_headers(Some( + extra_headers.into_iter().flatten().chain(scoped).collect(), + ))?; + let validated = config.validate_environment( + forwarded, + api_key.as_deref(), + &request.model, + &request.params, + &connection_params, + &env_lookup, + )?; + let environment = ValidatedEnvironment { + headers: config.request_headers( + with_default_headers(validated.headers, config.default_headers()), + &request, + ), + auth: validated.auth, + }; + let authenticated = + resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?; + + let response = send( + &route.http, + authenticated, + &url, + &serde_json::to_value(&request).map_err(serialize_failure)?, + timeout.or_else(|| shaping.settings.request_timeout()), + ) + .await?; + if !response.status().is_success() { + return Err(provider_error(response).await); + } + let text = response.text().await.map_err(litellm_http::Error::from)?; + serde_json::from_str(&text).map_err(|err| { + Error::invalid_response(litellm_llms::ErrorDetail::invalid( + "count_tokens response JSON", + err, + )) + }) +} + fn without_additional_drop_params( request: MessagesRequest, paths: &[String], diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index bfefb9c63b0..1ac63b0af00 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -9,10 +9,11 @@ use litellm_auth::AuthServices; use litellm_host::interceptors::Interceptors; use litellm_inference::context::CallContext; use litellm_inference::provider::resolve_llm_provider; -use litellm_secrets::source::SecretSource; +use litellm_llms_types::formats::messages::CountTokensResponse; +use litellm_secrets::source::{SecretSource, Secrets}; use std::sync::Arc; -use crate::common_utils::messages_provider; +use crate::common_utils::{MessagesProvider, messages_provider}; pub use litellm_inference::Error; pub use types::{ @@ -75,24 +76,48 @@ impl MessagesRoute { context: CallContext<'_, impl Interceptors>, ) -> Result { litellm_inference::diagnostic::call(async { - let provider = resolve_llm_provider( - &call.body.model, - call.custom_llm_provider.as_deref(), - "messages", - )?; - let config = messages_provider(provider.provider, provider.model).ok_or_else(|| { - Error::InvalidProvider(<&str>::from(provider.provider).to_string()) - })?; - litellm_inference::diagnostic::provider( - provider.model, - <&'static str>::from(provider.provider), - ); - let model = provider.model.to_owned(); - let secrets = self.secrets.resolve(config.config().secret_names()).await?; - handler::execute(self, config, model, call, secrets, &context) + let (provider, model, secrets) = self.resolve_provider(&call).await?; + handler::execute(self, provider, model, call, secrets, &context) .boxed() .await }) .await } + + #[tracing::instrument(name = "litellm.route", skip_all, fields( + route = "messages_count_tokens", + model = %call.body.model, + provider, + resolved_model, + stream = false, + outcome + ))] + pub async fn count_tokens(&self, call: MessagesCall) -> Result { + litellm_inference::diagnostic::unary(async { + let (provider, model, secrets) = self.resolve_provider(&call).await?; + handler::count_tokens(self, provider, model, call, secrets.as_ref()) + .boxed() + .await + }) + .await + } + + async fn resolve_provider( + &self, + call: &MessagesCall, + ) -> Result<(MessagesProvider, String, Secrets), Error> { + let provider = resolve_llm_provider( + &call.body.model, + call.custom_llm_provider.as_deref(), + "messages", + )?; + let config = messages_provider(provider.provider, provider.model) + .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?; + litellm_inference::diagnostic::provider( + provider.model, + <&'static str>::from(provider.provider), + ); + let secrets = self.secrets.resolve(config.config().secret_names()).await?; + Ok((config, provider.model.to_owned(), secrets)) + } } diff --git a/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs b/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs new file mode 100644 index 00000000000..1928d048835 --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs @@ -0,0 +1,62 @@ +use rstest::rstest; + +use super::*; + +#[rstest] +#[tokio::test] +async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model( + call: MessagesCall, +) { + let upstream = upstream([json_response(json!({"input_tokens": 14}))]).await; + let messages = json!([{"role": "user", "content": "hi"}]); + + let counted = messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: body(json!({"model": "claude-sonnet-4-5@20250929", "messages": messages})), + custom_llm_provider: Some("vertex_ai".into()), + litellm_params: serde_json::from_value(json!({ + "model": "vertex_ai/claude-sonnet-4-5@20250929", + "vertex_project": "proj", + "vertex_location": "europe-west1", + })) + .unwrap(), + api_base: Some(upstream.uri()), + extra_headers: headers([ + ("Authorization", "Bearer caller-token"), + ("anthropic-version", "2023-06-01"), + ]), + ..call + }) + .await + .expect("count succeeds"); + + assert_eq!(counted.input_tokens, 14); + let request = only_request(&upstream).await; + assert_eq!( + request.url.path(), + "/v1/projects/proj/locations/europe-west1/publishers/anthropic/models/count-tokens:rawPredict" + ); + assert_eq!(request.header("authorization"), Some("Bearer caller-token")); + assert_eq!(request.header("anthropic-version"), None); + assert_eq!( + request.json(), + json!({"model": "claude-sonnet-4-5", "messages": messages}) + ); +} + +#[rstest] +#[tokio::test] +async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) { + let upstream = upstream([json_response(json!({"tokens": 14}))]).await; + + let error = messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .unwrap_err(); + + assert!(error.is_response(), "{error:?}"); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/main.rs b/litellm-rust/crates/inference-messages/tests/messages/main.rs index a01e571ec75..165ab6fb4fd 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/main.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/main.rs @@ -19,6 +19,7 @@ use wiremock::ResponseTemplate; mod support; use support::*; +mod count_tokens; mod host; mod request; mod response; diff --git a/litellm-rust/crates/llms-types/AGENTS.md b/litellm-rust/crates/llms-types/AGENTS.md index aa216d7bdab..0f76e783d3d 100644 --- a/litellm-rust/crates/llms-types/AGENTS.md +++ b/litellm-rust/crates/llms-types/AGENTS.md @@ -25,7 +25,7 @@ ## Providers - A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers//` -- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src///` +- Projections that only one adapter decodes (`InvokeChunkPayload`, the batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src///` - A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it - Provider types may use format types. Format types never depend on provider types diff --git a/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md index febc648fcd6..884c5ad8114 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md +++ b/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md @@ -19,6 +19,7 @@ # references - https://platform.claude.com/docs/en/api/http/messages/create +- https://platform.claude.com/docs/en/api/http/messages/count_tokens - https://platform.claude.com/docs/en/api/messages/create.md - https://platform.claude.com/docs/en/api/beta/messages/create.md - https://platform.claude.com/docs/en/build-with-claude/streaming.md diff --git a/litellm-rust/crates/llms-types/src/formats/messages/response.rs b/litellm-rust/crates/llms-types/src/formats/messages/response.rs index 9b0c5d31b3a..9bf702cd254 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/response.rs @@ -18,6 +18,13 @@ pub struct MessagesResponse { pub extra: Map, } +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct CountTokensResponse { + pub input_tokens: u64, + #[serde(flatten)] + pub extra: Map, +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md index f2984624b01..31c8824397c 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md @@ -1,6 +1,6 @@ # rules -- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers +- Shared across Anthropic's Messages, chat and batches adapters: API base, endpoint paths, header names, API version and default headers - `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one - Choosing which betas a request needs is policy in `llms/src/anthropic/` diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs index 941d1e75461..5ef719c7a17 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/constants.rs @@ -1,7 +1,6 @@ pub const API_BASE: &str = "https://api.anthropic.com"; pub const MESSAGES_PATH: &str = "/v1/messages"; pub const BATCHES_PATH: &str = "/v1/messages/batches"; -pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens"; pub const API_KEY_HEADER: &str = "x-api-key"; pub const BETA_HEADER: &str = "anthropic-beta"; diff --git a/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs b/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs index fccf7cdefe8..06dc30ea0be 100644 --- a/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs +++ b/litellm-rust/crates/llms-types/src/providers/vertex_ai/constants.rs @@ -1,6 +1,7 @@ pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com"; pub const GLOBAL_LOCATION: &str = "global"; pub const DEFAULT_LOCATION: &str = "us-central1"; +pub const COUNT_TOKENS_DEFAULT_LOCATION: &str = "us-east5"; pub const RAW_PREDICT: &str = "rawPredict"; pub const STREAM_RAW_PREDICT: &str = "streamRawPredict"; diff --git a/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md b/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md deleted file mode 100644 index 6d45f2ae14d..00000000000 --- a/litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md +++ /dev/null @@ -1,7 +0,0 @@ -# rules - -- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections - -# references - -- https://platform.claude.com/docs/en/api/http/messages/count_tokens diff --git a/litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs deleted file mode 100644 index 9b2ee44516c..00000000000 --- a/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs +++ /dev/null @@ -1,171 +0,0 @@ -use litellm_llms_types::{ - formats::messages::{Message, SystemPrompt}, - providers::anthropic::{ - API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER, - }, -}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX}; - -const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AnthropicCountTokensRequest { - pub model: String, - pub messages: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub system: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AnthropicCountTokensResponse { - pub input_tokens: u64, -} - -pub trait AnthropicCountTokensConfig { - fn endpoint(&self) -> String; - - fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>; - - fn transform_request( - &self, - model: &str, - messages: Vec, - tools: Option>, - system: Option, - ) -> Result; - - fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)>; -} - -pub struct AnthropicCountTokensTransformation; - -pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformation = - AnthropicCountTokensTransformation; - -impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { - fn endpoint(&self) -> String { - format!("{API_BASE}{COUNT_TOKENS_PATH}") - } - - fn transform_request( - &self, - model: &str, - messages: Vec, - tools: Option>, - system: Option, - ) -> Result { - self.validate_request(model, &messages)?; - - Ok(AnthropicCountTokensRequest { - model: model.to_string(), - messages, - tools, - system, - }) - } - - fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error> { - if model.is_empty() { - return Err(Error::MissingField("model")); - } - if messages.is_empty() { - return Err(Error::MissingField("messages")); - } - Ok(()) - } - - fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)> { - let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) { - ("authorization", format!("Bearer {api_key}")) - } else { - (API_KEY_HEADER, api_key.to_string()) - }; - vec![ - ("content-type", "application/json".to_string()), - auth, - (VERSION_HEADER, API_VERSION.to_string()), - (BETA_HEADER, TOKEN_COUNTING_BETA.to_string()), - ] - } -} - -#[cfg(test)] -mod tests { - use litellm_llms_types::formats::messages::MessageContent; - use serde_json::{Map, json}; - - use super::*; - - fn message() -> Message { - Message { - role: "user".into(), - content: MessageContent::Text("hello".into()), - extra: Map::new(), - } - } - - #[test] - fn maps_the_python_count_tokens_contract() { - let request = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION - .transform_request( - "claude-test", - vec![message()], - Some(vec![json!({"name": "lookup"})]), - Some(SystemPrompt::Text("system".into())), - ) - .unwrap(); - - assert_eq!( - serde_json::to_value(request).unwrap(), - json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hello"}], - "tools": [{"name": "lookup"}], - "system": "system" - }) - ); - assert_eq!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(), - "https://api.anthropic.com/v1/messages/count_tokens" - ); - } - - #[test] - fn rejects_the_invalid_requests_python_rejects() { - assert!(matches!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request( - "", - vec![message()], - None, - None - ), - Err(Error::MissingField("model")) - )); - assert!(matches!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request( - "claude-test", - vec![], - None, - None - ), - Err(Error::MissingField("messages")) - )); - } - - #[test] - fn uses_api_key_or_oauth_headers_without_combining_credentials() { - let api_key = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-api"); - assert!(api_key.contains(&("x-api-key", "sk-ant-api".into()))); - assert!(!api_key.iter().any(|(name, _)| *name == "authorization")); - - let oauth = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-oat-test"); - assert!(oauth.contains(&("authorization", "Bearer sk-ant-oat-test".into()))); - assert!(!oauth.iter().any(|(name, _)| *name == "x-api-key")); - assert!(oauth.contains(&("anthropic-beta", TOKEN_COUNTING_BETA.into()))); - } -} diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6657cbf262f..cf526a93f80 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -50,6 +50,18 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { Ok(complete_anthropic_url(api_base, env_lookup)) } + fn get_count_tokens_url( + &self, + api_base: Option<&str>, + _litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + Some(Ok(format!( + "{}/count_tokens", + complete_anthropic_url(api_base, env_lookup) + ))) + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, @@ -875,6 +887,34 @@ mod tests { ); } + #[rstest] + #[case::default_base(None, &[], "https://api.anthropic.com/v1/messages/count_tokens")] + #[case::env_base( + None, + &[(ANTHROPIC_API_BASE_ENV, "https://env.example.com")], + "https://env.example.com/v1/messages/count_tokens" + )] + #[case::custom_base(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages/count_tokens")] + #[case::complete_messages_endpoint( + Some("https://proxy.internal/v1/messages"), + &[], + "https://proxy.internal/v1/messages/count_tokens" + )] + fn count_tokens_url_is_under_the_messages_endpoint( + #[case] api_base: Option<&str>, + #[case] vars: Env, + #[case] expected: &str, + ) { + assert_eq!( + ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url( + api_base, + &LitellmParams::default(), + &env(vars) + ), + Some(Ok(expected.to_string())) + ); + } + fn betas(values: &[&str]) -> BetaSet { values.join(",").parse().unwrap() } diff --git a/litellm-rust/crates/llms/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs index 72c390cfe21..7d40e525772 100644 --- a/litellm-rust/crates/llms/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -3,7 +3,6 @@ pub mod common_utils; pub mod batches; pub mod chat; -pub mod count_tokens; pub mod messages; pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index 744ce89afdb..bfa007648cb 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -27,6 +27,19 @@ pub trait BaseMessagesConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; + fn get_count_tokens_url( + &self, + _api_base: Option<&str>, + _litellm_params: &LitellmParams, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + None + } + + fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str { + model + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index b3d1421083a..c2e593d5526 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -166,6 +166,11 @@ pub enum ErrorDetail { HostCall, #[error("{0} is not handled by the rust path")] Unhandled(&'static str), + #[error("{operation} is not supported for provider {provider}")] + UnsupportedOperation { + operation: &'static str, + provider: &'static str, + }, #[error("invalid {subject} on line {line}")] InvalidLine { subject: &'static str, diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md b/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md index d95da65ac9d..0a43ec9ef77 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md @@ -3,7 +3,9 @@ - Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token - Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it - Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it +- Counts tokens at the publisher's `count-tokens:rawPredict`, which rejects a versioned model, so the body's model loses its `@` suffix. The location is `vertex_count_tokens_location`, then the deployment's location, then `us-east5`, as Python's partner-model token counter picks it # references - https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude +- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 46e5a68dcb8..5e53c26b3b9 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -32,10 +32,12 @@ use crate::{ vertex_ai::common_utils::get_vertex_base_url, }; use litellm_llms_types::providers::vertex_ai::{ - ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT, + ANTHROPIC_VERSION, COUNT_TOKENS_DEFAULT_LOCATION, DEFAULT_LOCATION, RAW_PREDICT, + STREAM_RAW_PREDICT, }; const WEB_SEARCH_TOOL_PREFIX: &str = "web_search"; +const COUNT_TOKENS_LOCATION_PARAM: &str = "vertex_count_tokens_location"; const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"]; const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"]; @@ -64,14 +66,25 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { env_lookup: &dyn Fn(&str) -> Option, ) -> Result { let config = VertexConfig::from_params(&litellm_params.vertex); - let project = get_vertex_ai_project(&config, env_lookup) - .or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup)) - .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?; + let project = vertex_project(&config, env_lookup)?; let location = get_vertex_ai_location(&config, env_lookup) .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); complete_vertex_anthropic_url(api_base, &project, &location, model, stream) } + fn get_count_tokens_url( + &self, + api_base: Option<&str>, + litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + Some(count_tokens_url(api_base, litellm_params, env_lookup)) + } + + fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str { + model.split_once('@').map_or(model, |(name, _)| name) + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, @@ -211,6 +224,40 @@ pub fn complete_vertex_anthropic_url( }) } +fn vertex_project( + config: &VertexConfig, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + get_vertex_ai_project(config, env_lookup) + .or_else(|| get_vertex_ai_project_from_credentials(config, env_lookup)) + .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI"))) +} + +fn count_tokens_url( + api_base: Option<&str>, + litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + let config = VertexConfig::from_params(&litellm_params.vertex); + let project = vertex_project(&config, env_lookup)?; + let location = litellm_params + .extra + .get(COUNT_TOKENS_LOCATION_PARAM) + .and_then(Value::as_str) + .filter(|location| !location.is_empty()) + .map(str::to_string) + .or_else(|| get_vertex_ai_location(&config, env_lookup)) + .unwrap_or_else(|| COUNT_TOKENS_DEFAULT_LOCATION.to_string()); + let base = match api_base.map(str::trim).filter(|base| !base.is_empty()) { + Some(custom) => custom.to_string(), + None => get_vertex_base_url(&location)?, + }; + Ok(format!( + "{}/v1/projects/{project}/locations/{location}/publishers/anthropic/models/count-tokens:{RAW_PREDICT}", + base.trim_end_matches('/') + )) +} + /// Python's `sanitize_vertex_anthropic_output_params`: `effort` only reaches models whose /// capabilities accept it, a malformed `output_config` is dropped, and an emptied one is omitted. fn sanitize_output_config( @@ -458,6 +505,105 @@ mod tests { ); } + fn count_tokens_params(fields: Value) -> LitellmParams { + let (Value::Object(base), Value::Object(fields)) = ( + json!({"model": "vertex_ai/claude-sonnet-4-5", "vertex_project": "proj"}), + fields, + ) else { + panic!("litellm params are a json object"); + }; + serde_json::from_value(Value::Object(base.into_iter().chain(fields).collect())).unwrap() + } + + const COUNT_PATH: &str = "/publishers/anthropic/models/count-tokens:rawPredict"; + + #[rstest] + #[case::count_location_wins( + None, + json!({"vertex_count_tokens_location": "europe-west1", "vertex_location": "asia-southeast1"}), + &[], + "https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1" + )] + #[case::deployment_location( + None, + json!({"vertex_location": "asia-southeast1"}), + &[], + "https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1" + )] + #[case::empty_count_location_is_unset( + None, + json!({"vertex_count_tokens_location": "", "vertex_location": "asia-southeast1"}), + &[], + "https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1" + )] + #[case::environment_location( + None, + json!({}), + &[("VERTEXAI_LOCATION", "europe-west1")], + "https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1" + )] + #[case::count_default_location( + None, + json!({}), + &[], + "https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5" + )] + #[case::global( + None, + json!({"vertex_location": "global"}), + &[], + "https://aiplatform.googleapis.com/v1/projects/proj/locations/global" + )] + #[case::custom_base( + Some("https://proxy.example/prefix/"), + json!({"vertex_location": "global"}), + &[], + "https://proxy.example/prefix/v1/projects/proj/locations/global" + )] + fn count_tokens_url_follows_the_python_location_precedence( + #[case] api_base: Option<&str>, + #[case] fields: Value, + #[case] env: &[(&str, &str)], + #[case] prefix: &str, + ) { + let lookup = |name: &str| { + env.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }; + assert_eq!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url( + api_base, + &count_tokens_params(fields), + &lookup + ), + Some(Ok(format!("{prefix}{COUNT_PATH}"))) + ); + } + + #[rstest] + fn count_tokens_without_a_project_fails_before_sending() { + assert!(matches!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url( + None, + &LitellmParams::default(), + &no_env + ), + Some(Err(Error::Auth(_))) + )); + } + + #[rstest] + #[case::versioned("claude-sonnet-4-5@20250929", "claude-sonnet-4-5")] + #[case::default_version("claude-sonnet-4-5@default", "claude-sonnet-4-5")] + #[case::unversioned("claude-sonnet-4-5", "claude-sonnet-4-5")] + fn count_tokens_model_drops_the_version(#[case] model: &str, #[case] expected: &str) { + assert_eq!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.count_tokens_model(model), + expected + ); + } + #[rstest] fn without_a_project_anywhere_the_call_fails_before_sending() { let error = url(None, &LitellmParams::default(), false, &no_env).unwrap_err();