From fddadf3e29313ae302b971b01b7bd94cb9343bc4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:58:46 -0800 Subject: [PATCH] fix(rust-gateway): serve /v1/messages/count_tokens through the provider's count API The gateway answered 404 for POST /v1/messages/count_tokens, so the Anthropic SDK's messages.count_tokens failed while messages.create worked The route goes through the same deployment lookup, model access check and error envelope as /v1/messages. It keeps only the fields the count endpoints read, runs the same provider transform as /v1/messages, drops max_tokens from the wire, and sends the result to the provider's count API with the create path's header defaults and upstream error mapping. Anthropic counts at {api_base}/v1/messages/count_tokens, Azure AI under its Anthropic messages endpoint, and Vertex AI at the publisher count-tokens:rawPredict endpoint with the @version suffix stripped and Python's location order. Other providers get a 400 invalid_request_error that names the provider. The response carries litellm_count_source like the Python proxy's The unused Anthropic count-tokens adapter and its path constant are gone --- .../rust/crates/gateway-inference/src/lib.rs | 1 + .../crates/gateway-inference/src/messages.rs | 48 ++- .../gateway-inference/tests/messages.rs | 97 +++++- .../crates/gateway-inference/tests/routes.rs | 1 + .../crates/inference-messages/src/handler.rs | 124 ++++++- .../rust/crates/inference-messages/src/lib.rs | 21 ++ .../tests/messages/count_tokens.rs | 312 ++++++++++++++++++ .../inference-messages/tests/messages/main.rs | 1 + litellm/rust/crates/llms-types/AGENTS.md | 2 +- .../llms-types/src/formats/messages/AGENTS.md | 1 + .../src/formats/messages/request.rs | 14 + .../src/formats/messages/response.rs | 7 + .../src/providers/anthropic/AGENTS.md | 2 +- .../src/providers/anthropic/constants.rs | 1 - .../src/providers/vertex_ai/constants.rs | 1 + .../llms/src/anthropic/count_tokens/AGENTS.md | 7 - .../llms/src/anthropic/count_tokens/mod.rs | 1 - .../anthropic/count_tokens/transformation.rs | 171 ---------- .../src/anthropic/messages/transformation.rs | 40 +++ litellm/rust/crates/llms/src/anthropic/mod.rs | 1 - .../src/azure_ai/messages/transformation.rs | 37 +++ .../src/base_llm/messages/transformation.rs | 13 + litellm/rust/crates/llms/src/error.rs | 5 + .../llms/src/vertex_ai/messages/AGENTS.md | 2 + .../src/vertex_ai/messages/transformation.rs | 154 ++++++++- 25 files changed, 871 insertions(+), 193 deletions(-) create mode 100644 litellm/rust/crates/inference-messages/tests/messages/count_tokens.rs delete mode 100644 litellm/rust/crates/llms/src/anthropic/count_tokens/AGENTS.md delete mode 100644 litellm/rust/crates/llms/src/anthropic/count_tokens/mod.rs delete mode 100644 litellm/rust/crates/llms/src/anthropic/count_tokens/transformation.rs diff --git a/litellm/rust/crates/gateway-inference/src/lib.rs b/litellm/rust/crates/gateway-inference/src/lib.rs index 43966f2bee7..e0925843c71 100644 --- a/litellm/rust/crates/gateway-inference/src/lib.rs +++ b/litellm/rust/crates/gateway-inference/src/lib.rs @@ -91,6 +91,7 @@ impl Gateway { pub fn router(gateway: Arc) -> Router { Router::new() .route("/v1/messages", post(messages::create)) + .route("/v1/messages/count_tokens", post(messages::count_tokens)) .route("/ocr", post(ocr::create)) .route("/v1/ocr", post(ocr::create)) .route("/chat/completions", post(chat_completions::create)) diff --git a/litellm/rust/crates/gateway-inference/src/messages.rs b/litellm/rust/crates/gateway-inference/src/messages.rs index 58ecd40204d..a517de097ca 100644 --- a/litellm/rust/crates/gateway-inference/src/messages.rs +++ b/litellm/rust/crates/gateway-inference/src/messages.rs @@ -1,4 +1,5 @@ -//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it. +//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it, and +//! `POST /v1/messages/count_tokens`, answered by the deployment's own count-tokens API. use litellm_gateway_auth::AuthenticatedRequest; use std::sync::Arc; @@ -12,7 +13,11 @@ use axum::{ }; use litellm_host_http::Sse; use litellm_inference_messages::{MessagesCall, messages_body, route::Messages}; -use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders}; +use litellm_llms_types::{ + formats::messages::CountTokensResponse, + headers::{ProviderSpecificHeader, ProviderSpecificHeaders}, +}; +use serde::Serialize; use serde_json::{Map, Value}; use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request}; @@ -25,6 +30,14 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [ ]; const ANTHROPIC_API_HEADER_PROVIDERS: &str = "anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek"; +const PROVIDER_COUNT_SOURCE: &str = "provider"; + +#[derive(Serialize)] +struct SourcedCount { + #[serde(flatten)] + count: CountTokensResponse, + litellm_count_source: &'static str, +} pub async fn create( State(gateway): State>, @@ -67,6 +80,37 @@ async fn handle( Ok(headers.apply(response)) } +pub async fn count_tokens( + State(gateway): State>, + identity: AuthenticatedRequest, + RequestId(request_id): RequestId, + headers: HeaderMap, + body: Result, +) -> impl IntoResponse { + let result = match body { + Ok(JsonObject(body)) => count(&gateway, &identity, &headers, body).await, + Err(error) => Err(error), + }; + result + .map(Json) + .map_err(|error| (error.status(), Json(error.body(request_id.as_deref())))) +} + +async fn count( + gateway: &Gateway, + identity: &AuthenticatedRequest, + headers: &HeaderMap, + body: Map, +) -> Result { + let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?; + request::authorize_model(identity, deployment, &body).await?; + let call = project(deployment, body, headers)?; + Ok(SourcedCount { + count: gateway.messages.count_tokens(call).await?, + litellm_count_source: PROVIDER_COUNT_SOURCE, + }) +} + fn project( deployment: &Deployment, body: Map, diff --git a/litellm/rust/crates/gateway-inference/tests/messages.rs b/litellm/rust/crates/gateway-inference/tests/messages.rs index 7d6d1f160d1..3d0143aa741 100644 --- a/litellm/rust/crates/gateway-inference/tests/messages.rs +++ b/litellm/rust/crates/gateway-inference/tests/messages.rs @@ -121,18 +121,20 @@ async fn a_stream_that_fails_after_opening_ends_with_an_sse_error_frame() { } #[rstest] +#[case::messages("/v1/messages")] +#[case::count_tokens("/v1/messages/count_tokens")] #[tokio::test] -async fn hosted_provider_failure_preserves_status_body_and_request_id() { +async fn hosted_provider_failure_preserves_status_body_and_request_id(#[case] route: &str) { let upstream = MockServer::start().await; let error = json!({"type": "error", "error": {"type": "rate_limit_error", "message": "retry later"}}); Mock::given(method("POST")) - .and(path("/v1/messages")) + .and(path(route)) .respond_with(ResponseTemplate::new(429).set_body_json(error.clone())) .expect(1) .mount(&upstream) .await; - let request = Request::post("/v1/messages") + let request = Request::post(route) .header("content-type", "application/json") .header("x-request-id", "host-http-request") .body(Body::from(json!({ @@ -150,8 +152,97 @@ async fn hosted_provider_failure_preserves_status_body_and_request_id() { assert_eq!(body["request_id"], "host-http-request"); } +#[rstest] +#[tokio::test] +async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model() { + let upstream = MockServer::start().await; + let messages = json!([{"role": "user", "content": "hi"}]); + let tools = json!([{"name": "lookup", "input_schema": {"type": "object"}}]); + let counted = json!({"input_tokens": 14, "context_management": {"original_input_tokens": 20}}); + Mock::given(method("POST")) + .and(path("/v1/messages/count_tokens")) + .and(header("x-api-key", "test-key")) + .and(header("anthropic-version", "test-version")) + .and(body_json(json!({ + "model": "test-model", "messages": messages, "system": "be brief", "tools": tools, + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(counted)) + .expect(1) + .mount(&upstream) + .await; + let request = Request::post("/v1/messages/count_tokens") + .header("content-type", "application/json") + .header("anthropic-version", "test-version") + .body(Body::from( + json!({"model": "public/model", "messages": messages, "system": "be brief", "tools": tools}) + .to_string(), + )) + .unwrap(); + let response = support::app("anthropic/test-model", &upstream.uri()) + .oneshot(request) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!( + support::json(response).await, + json!({ + "input_tokens": 14, + "context_management": {"original_input_tokens": 20}, + "litellm_count_source": "provider", + }) + ); +} + +#[rstest] +#[tokio::test] +async fn count_tokens_for_a_provider_without_a_count_api_is_an_invalid_request() { + let upstream = MockServer::start().await; + let response = support::post( + support::app("deepseek/test-model", &upstream.uri()), + "/v1/messages/count_tokens", + json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["type"], "error"); + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert!( + body["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("deepseek")), + "{body}" + ); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn an_unknown_model_is_refused_alike_on_messages_and_count_tokens( + #[values("/v1/messages", "/v1/messages/count_tokens")] route: &str, +) { + let upstream = MockServer::start().await; + let response = support::post( + support::app("anthropic/test-model", &upstream.uri()), + route, + json!({"model": "other/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ) + .await; + assert_eq!(response.status(), 400); + let body = support::json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert!( + body["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("other/model")), + "{body}" + ); + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + #[rstest] #[case::messages("/v1/messages")] +#[case::count_tokens("/v1/messages/count_tokens")] #[case::chat("/v1/chat/completions")] #[case::model_path("/engines/path-model/chat/completions")] #[case::ocr("/ocr")] diff --git a/litellm/rust/crates/gateway-inference/tests/routes.rs b/litellm/rust/crates/gateway-inference/tests/routes.rs index 24b8c5cc694..f019dcbe0da 100644 --- a/litellm/rust/crates/gateway-inference/tests/routes.rs +++ b/litellm/rust/crates/gateway-inference/tests/routes.rs @@ -235,6 +235,7 @@ async fn upload_audio_format_validation_matches_core( #[case::responses("/v1/responses", false)] #[case::deployment("/openai/deployments/public%2Fmodel/chat/completions", false)] #[case::messages("/v1/messages", true)] +#[case::count_tokens("/v1/messages/count_tokens", true)] #[case::ocr("/v1/ocr", false)] #[case::transcription("/v1/audio/transcriptions", false)] #[tokio::test] diff --git a/litellm/rust/crates/inference-messages/src/handler.rs b/litellm/rust/crates/inference-messages/src/handler.rs index d54d4f5f8d1..53702bbb24b 100644 --- a/litellm/rust/crates/inference-messages/src/handler.rs +++ b/litellm/rust/crates/inference-messages/src/handler.rs @@ -22,8 +22,9 @@ use litellm_llms::base_llm::{ }, }; use litellm_llms_types::formats::messages::{ - MessagesOptionalParams, MessagesRequest, MessagesResponse, + CountTokensResponse, MessagesOptionalParams, MessagesRequest, MessagesResponse, }; +use litellm_llms_types::recognized::Recognized; use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; use serde_json::{Map, Value}; @@ -44,6 +45,8 @@ use litellm_inference::{ outbound::{raw_leaf_body, serialized_outbound_request}, }; +const UNCAPPED_MAX_TOKENS: u64 = u64::MAX; + struct ProviderCall { identity: ProviderIdentity, wire: WireRequest, @@ -389,6 +392,125 @@ async fn provider_call( }) } +pub(super) async fn count_tokens( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + call: MessagesCall, + secrets: &(dyn Lookup + Send + Sync), +) -> Result { + let MessagesCall { + body, + api_key, + api_base, + litellm_params, + extra_headers, + provider_specific_header, + timeout, + shaping, + .. + } = call; + let connection_params = litellm_router_types::LitellmParams { + api_base: api_base.clone().or(litellm_params.api_base.clone()), + ..litellm_params + }; + let config = provider.config(); + let env_lookup = |key: &str| secrets.get(key); + let url = config + .get_count_tokens_url(api_base.as_deref(), &connection_params, &env_lookup) + .ok_or_else(|| { + Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedOperation { + operation: "token counting", + provider: provider.as_str(), + }) + })??; + + let sanitized = config.shape_request( + MessagesRequest { + model: config.count_tokens_model(&model).to_string(), + params: body.params.countable(), + ..body + }, + shaping.settings.shape_options(), + )?; + let trimmed = + without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; + let scoped = + get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); + let forwarded = string_headers(Some(merge_messages_header_sources([ + extra_headers.unwrap_or_default(), + scoped, + ])))?; + let validated = config.validate_environment( + forwarded, + api_key.as_deref(), + &trimmed.model, + &trimmed.params, + &connection_params, + &env_lookup, + )?; + let transformed = config.transform_anthropic_messages_request( + with_a_max_tokens(trimmed), + &MessagesTransformContext { + drops_unsupported_params: shaping.settings.drops_unsupported_params, + ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) + }, + )?; + let environment = ValidatedEnvironment { + headers: config.headers_for_url( + config.request_headers( + with_python_default_headers(validated.headers, config.default_headers()), + &transformed, + ), + &url, + ), + auth: validated.auth, + }; + let authenticated = + resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?; + let wire_body = MessagesRequest { + params: MessagesOptionalParams { + max_tokens: None, + ..transformed.params + }, + ..transformed + }; + let timeout = timeout + .or_else(|| shaping.settings.request_timeout()) + .unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); + let response = send( + &route.http, + authenticated, + &url, + Bytes::from(serde_json::to_vec(&wire_body).map_err(serialize_failure)?), + timeout, + ) + .await?; + if !response.status().is_success() { + return Err(provider_failure(response, timeout, false).await?.into()); + } + let text = read_body(response, timeout).await?; + serde_json::from_str(&text).map_err(|err| { + Error::invalid_response(litellm_llms::ErrorDetail::invalid( + "count_tokens response JSON", + err, + )) + }) +} + +fn with_a_max_tokens(request: MessagesRequest) -> MessagesRequest { + MessagesRequest { + params: MessagesOptionalParams { + max_tokens: request + .params + .max_tokens + .or(Some(Recognized::Known(UNCAPPED_MAX_TOKENS))), + ..request.params + }, + ..request + } +} + fn with_requested_output_format( body: &MessagesRequest, requested: Option, diff --git a/litellm/rust/crates/inference-messages/src/lib.rs b/litellm/rust/crates/inference-messages/src/lib.rs index e65804ce108..3bcd8c8608b 100644 --- a/litellm/rust/crates/inference-messages/src/lib.rs +++ b/litellm/rust/crates/inference-messages/src/lib.rs @@ -16,6 +16,7 @@ use litellm_auth::AuthServices; use litellm_host::interceptors::Interceptors; use litellm_inference::context::CallContext; use litellm_inference::provider::resolve_llm_provider; +use litellm_llms_types::formats::messages::CountTokensResponse; use litellm_llms_types::recognized::Recognized; use litellm_secrets::source::SecretSource; use std::sync::Arc; @@ -189,6 +190,26 @@ impl MessagesRoute { .await } + #[tracing::instrument(name = "litellm.route", skip_all, fields( + route = "messages_count_tokens", + model = %call.body.model, + provider, + resolved_model, + stream = false, + outcome + ))] + pub async fn count_tokens(&self, call: MessagesCall) -> Result { + litellm_inference::diagnostic::unary(async { + let (config, model, provider) = Self::provider(&call)?; + litellm_inference::diagnostic::provider(&model, provider); + let secrets = self.secrets.resolve(config.config().secret_names()).await?; + handler::count_tokens(self, config, model, call, secrets.as_ref()) + .boxed() + .await + }) + .await + } + fn provider( call: &MessagesCall, ) -> Result<(common_utils::MessagesProvider, String, &'static str), Error> { diff --git a/litellm/rust/crates/inference-messages/tests/messages/count_tokens.rs b/litellm/rust/crates/inference-messages/tests/messages/count_tokens.rs new file mode 100644 index 00000000000..ec58c244bfd --- /dev/null +++ b/litellm/rust/crates/inference-messages/tests/messages/count_tokens.rs @@ -0,0 +1,312 @@ +use litellm_llms::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers}; +use rstest::rstest; + +use super::*; + +fn without_max_tokens(model: &str, fields: Value) -> MessagesRequest { + let messages = json!({"model": model, "messages": [{"role": "user", "content": "hi"}]}); + body(Value::Object( + object(messages).into_iter().chain(object(fields)).collect(), + )) +} + +async fn count_on_the_wire(call: MessagesCall) -> Value { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("count succeeds"); + only_request(&upstream).await.json() +} + +#[rstest] +#[tokio::test] +async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model( + call: MessagesCall, +) { + let upstream = upstream([json_response(json!({"input_tokens": 14}))]).await; + let messages = json!([{"role": "user", "content": "hi"}]); + + let counted = messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: body(json!({"model": "claude-sonnet-4-5@20250929", "messages": messages})), + custom_llm_provider: Some("vertex_ai".into()), + litellm_params: serde_json::from_value(json!({ + "model": "vertex_ai/claude-sonnet-4-5@20250929", + "vertex_project": "proj", + "vertex_location": "europe-west1", + })) + .unwrap(), + api_base: Some(upstream.uri()), + extra_headers: headers([ + ("Authorization", "Bearer caller-token"), + ("anthropic-version", "2023-06-01"), + ]), + ..call + }) + .await + .expect("count succeeds"); + + assert_eq!(counted.input_tokens, 14); + let request = only_request(&upstream).await; + assert_eq!( + request.url.path(), + "/v1/projects/proj/locations/europe-west1/publishers/anthropic/models/count-tokens:rawPredict" + ); + assert_eq!(request.header("authorization"), Some("Bearer caller-token")); + assert_eq!(request.header("anthropic-version"), None); + assert_eq!( + request.json(), + json!({"model": "claude-sonnet-4-5", "messages": messages}) + ); +} + +#[rstest] +#[tokio::test] +async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) { + let upstream = upstream([json_response(json!({"tokens": 14}))]).await; + + let error = messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .unwrap_err(); + + assert!(error.is_response(), "{error:?}"); +} + +#[rstest] +#[case::max_tokens(json!({"max_tokens": 1024}))] +#[case::stream(json!({"stream": true}))] +#[case::temperature(json!({"temperature": 0.5}))] +#[case::top_p(json!({"top_p": 0.5}))] +#[case::top_k(json!({"top_k": 5}))] +#[case::stop_sequences(json!({"stop_sequences": ["end"]}))] +#[case::metadata_the_messages_route_would_refuse(json!({"metadata": {"user_id": 7}}))] +#[case::speed(json!({"speed": "fast"}))] +#[case::service_tier(json!({"service_tier": "auto"}))] +#[case::container(json!({"container": "container_1"}))] +#[case::inference_geo(json!({"inference_geo": "us"}))] +#[case::mcp_servers(json!({"mcp_servers": [{"type": "url", "name": "docs", "url": "https://mcp.test"}]}))] +#[case::compaction(json!({"compaction": {"type": "summarize"}}))] +#[case::output_format(json!({"output_format": {"type": "json_schema"}}))] +#[case::field_no_provider_knows(json!({"litellm_metadata": {"tags": ["a"]}}))] +#[tokio::test] +async fn a_count_sends_only_what_the_count_endpoint_reads( + call: MessagesCall, + #[case] fields: Value, +) { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: without_max_tokens(MODEL, fields), + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_sampling_params: false, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + ..call + }) + .await + .expect("count succeeds"); + + let request = only_request(&upstream).await; + assert_eq!(request.header("anthropic-beta"), None); + assert_eq!( + request.json(), + json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]}) + ); +} + +#[rstest] +#[case::system(json!({"system": "be brief"}))] +#[case::tools(json!({"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]}))] +#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))] +#[case::thinking(json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}))] +#[case::output_config(json!({"output_config": {"format": {"type": "json_schema"}}}))] +#[case::context_management(json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}))] +#[tokio::test] +async fn a_count_keeps_what_the_count_endpoint_reads(call: MessagesCall, #[case] fields: Value) { + let expected = without_max_tokens(MODEL, fields.clone()); + + let sent = count_on_the_wire(MessagesCall { + body: without_max_tokens(MODEL, fields), + ..call + }) + .await; + + assert_eq!(sent, serde_json::to_value(expected).unwrap()); +} + +#[rstest] +#[tokio::test] +async fn a_count_asks_for_the_betas_of_the_fields_it_sends(call: MessagesCall) { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: without_max_tokens( + MODEL, + json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}), + ), + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("count succeeds"); + + assert_eq!( + only_request(&upstream).await.header("anthropic-beta"), + Some("context-management-2025-06-27") + ); +} + +#[rstest] +#[case::no_ceiling_keeps_the_whole_budget(json!({"reasoning_effort": "high"}), 4096)] +#[case::the_callers_ceiling_caps_the_budget( + json!({"reasoning_effort": "high", "max_tokens": 3000}), + 2999 +)] +#[tokio::test] +async fn a_count_translates_reasoning_like_the_messages_route( + call: MessagesCall, + #[case] fields: Value, + #[case] budget_tokens: u64, +) { + let sent = count_on_the_wire(MessagesCall { + body: without_max_tokens(MODEL, fields), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_reasoning: true, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + ..call + }) + .await; + + assert_eq!( + sent, + json!({ + "model": MODEL, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": budget_tokens}, + }) + ); +} + +#[rstest] +#[tokio::test] +async fn a_count_on_an_adaptive_model_sends_the_effort_the_messages_route_sends( + call: MessagesCall, +) { + let sent = count_on_the_wire(MessagesCall { + body: without_max_tokens(MODEL, json!({"reasoning_effort": "high"})), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + high: true, + ..SupportedEffortTiers::default() + }, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + ..call + }) + .await; + + assert_eq!(sent["thinking"]["type"], "adaptive"); + assert_eq!(sent["output_config"]["effort"], "high"); + assert_eq!(sent.get("reasoning_effort"), None); +} + +#[rstest] +#[tokio::test] +async fn a_vertex_count_lifts_the_system_role_and_strips_the_scope_vertex_refuses( + call: MessagesCall, +) { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: body(json!({ + "model": "claude-sonnet-4-5", + "messages": [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [{ + "type": "text", + "text": "hi", + "cache_control": {"type": "ephemeral", "scope": "global"}, + }]}, + ], + })), + custom_llm_provider: Some("vertex_ai".into()), + litellm_params: serde_json::from_value(json!({ + "model": "vertex_ai/claude-sonnet-4-5", + "vertex_project": "proj", + })) + .unwrap(), + api_base: Some(upstream.uri()), + extra_headers: headers([("Authorization", "Bearer caller-token")]), + ..call + }) + .await + .expect("count succeeds"); + + let sent = only_request(&upstream).await.json(); + assert_eq!( + sent["system"], + json!([{"type": "text", "text": "be brief"}]) + ); + assert_eq!( + sent["messages"], + json!([{"role": "user", "content": [{ + "type": "text", + "text": "hi", + "cache_control": {"type": "ephemeral"}, + }]}]) + ); +} + +#[rstest] +#[tokio::test] +async fn azure_ai_counts_at_its_anthropic_count_endpoint(call: MessagesCall) { + let upstream = upstream([json_response(json!({"input_tokens": 9}))]).await; + + let counted = messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + custom_llm_provider: Some("azure_ai".into()), + api_key: Some("azure-key".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("count succeeds"); + + assert_eq!(counted.input_tokens, 9); + let request = only_request(&upstream).await; + assert_eq!(request.url.path(), "/anthropic/v1/messages/count_tokens"); + assert_eq!(request.header("x-api-key"), Some("azure-key")); + assert_eq!( + request.json(), + json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]}) + ); +} diff --git a/litellm/rust/crates/inference-messages/tests/messages/main.rs b/litellm/rust/crates/inference-messages/tests/messages/main.rs index 1d515e87b90..8cb123dc9ba 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/main.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/main.rs @@ -19,6 +19,7 @@ use wiremock::ResponseTemplate; mod support; use support::*; +mod count_tokens; mod gateway; mod host; mod pre_request; diff --git a/litellm/rust/crates/llms-types/AGENTS.md b/litellm/rust/crates/llms-types/AGENTS.md index aa216d7bdab..0f76e783d3d 100644 --- a/litellm/rust/crates/llms-types/AGENTS.md +++ b/litellm/rust/crates/llms-types/AGENTS.md @@ -25,7 +25,7 @@ ## Providers - A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers//` -- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src///` +- Projections that only one adapter decodes (`InvokeChunkPayload`, the batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src///` - A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it - Provider types may use format types. Format types never depend on provider types diff --git a/litellm/rust/crates/llms-types/src/formats/messages/AGENTS.md b/litellm/rust/crates/llms-types/src/formats/messages/AGENTS.md index febc648fcd6..884c5ad8114 100644 --- a/litellm/rust/crates/llms-types/src/formats/messages/AGENTS.md +++ b/litellm/rust/crates/llms-types/src/formats/messages/AGENTS.md @@ -19,6 +19,7 @@ # references - https://platform.claude.com/docs/en/api/http/messages/create +- https://platform.claude.com/docs/en/api/http/messages/count_tokens - https://platform.claude.com/docs/en/api/messages/create.md - https://platform.claude.com/docs/en/api/beta/messages/create.md - https://platform.claude.com/docs/en/build-with-claude/streaming.md diff --git a/litellm/rust/crates/llms-types/src/formats/messages/request.rs b/litellm/rust/crates/llms-types/src/formats/messages/request.rs index 2e53b3adc5f..b00b3ca36e6 100644 --- a/litellm/rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm/rust/crates/llms-types/src/formats/messages/request.rs @@ -383,6 +383,20 @@ pub struct MessagesOptionalParams { } impl MessagesOptionalParams { + pub fn countable(self) -> Self { + Self { + max_tokens: self.max_tokens, + system: self.system, + tools: self.tools, + tool_choice: self.tool_choice, + thinking: self.thinking, + context_management: self.context_management, + output_config: self.output_config, + reasoning_effort: self.reasoning_effort, + ..Self::default() + } + } + pub fn streams(&self) -> bool { self.stream.as_ref().is_some_and(Recognized::is_truthy) } diff --git a/litellm/rust/crates/llms-types/src/formats/messages/response.rs b/litellm/rust/crates/llms-types/src/formats/messages/response.rs index 276a30a6cb4..b3e67fa1719 100644 --- a/litellm/rust/crates/llms-types/src/formats/messages/response.rs +++ b/litellm/rust/crates/llms-types/src/formats/messages/response.rs @@ -27,6 +27,13 @@ pub struct MessagesResponse { pub extra: Map, } +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct CountTokensResponse { + pub input_tokens: u64, + #[serde(flatten)] + pub extra: Map, +} + fn present_value<'de, D: serde::Deserializer<'de>>( deserializer: D, ) -> Result, D::Error> { diff --git a/litellm/rust/crates/llms-types/src/providers/anthropic/AGENTS.md b/litellm/rust/crates/llms-types/src/providers/anthropic/AGENTS.md index f2984624b01..31c8824397c 100644 --- a/litellm/rust/crates/llms-types/src/providers/anthropic/AGENTS.md +++ b/litellm/rust/crates/llms-types/src/providers/anthropic/AGENTS.md @@ -1,6 +1,6 @@ # rules -- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers +- Shared across Anthropic's Messages, chat and batches adapters: API base, endpoint paths, header names, API version and default headers - `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one - Choosing which betas a request needs is policy in `llms/src/anthropic/` diff --git a/litellm/rust/crates/llms-types/src/providers/anthropic/constants.rs b/litellm/rust/crates/llms-types/src/providers/anthropic/constants.rs index 941d1e75461..5ef719c7a17 100644 --- a/litellm/rust/crates/llms-types/src/providers/anthropic/constants.rs +++ b/litellm/rust/crates/llms-types/src/providers/anthropic/constants.rs @@ -1,7 +1,6 @@ pub const API_BASE: &str = "https://api.anthropic.com"; pub const MESSAGES_PATH: &str = "/v1/messages"; pub const BATCHES_PATH: &str = "/v1/messages/batches"; -pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens"; pub const API_KEY_HEADER: &str = "x-api-key"; pub const BETA_HEADER: &str = "anthropic-beta"; diff --git a/litellm/rust/crates/llms-types/src/providers/vertex_ai/constants.rs b/litellm/rust/crates/llms-types/src/providers/vertex_ai/constants.rs index fccf7cdefe8..06dc30ea0be 100644 --- a/litellm/rust/crates/llms-types/src/providers/vertex_ai/constants.rs +++ b/litellm/rust/crates/llms-types/src/providers/vertex_ai/constants.rs @@ -1,6 +1,7 @@ pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com"; pub const GLOBAL_LOCATION: &str = "global"; pub const DEFAULT_LOCATION: &str = "us-central1"; +pub const COUNT_TOKENS_DEFAULT_LOCATION: &str = "us-east5"; pub const RAW_PREDICT: &str = "rawPredict"; pub const STREAM_RAW_PREDICT: &str = "streamRawPredict"; diff --git a/litellm/rust/crates/llms/src/anthropic/count_tokens/AGENTS.md b/litellm/rust/crates/llms/src/anthropic/count_tokens/AGENTS.md deleted file mode 100644 index 6d45f2ae14d..00000000000 --- a/litellm/rust/crates/llms/src/anthropic/count_tokens/AGENTS.md +++ /dev/null @@ -1,7 +0,0 @@ -# rules - -- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections - -# references - -- https://platform.claude.com/docs/en/api/http/messages/count_tokens diff --git a/litellm/rust/crates/llms/src/anthropic/count_tokens/mod.rs b/litellm/rust/crates/llms/src/anthropic/count_tokens/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm/rust/crates/llms/src/anthropic/count_tokens/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm/rust/crates/llms/src/anthropic/count_tokens/transformation.rs b/litellm/rust/crates/llms/src/anthropic/count_tokens/transformation.rs deleted file mode 100644 index 9b2ee44516c..00000000000 --- a/litellm/rust/crates/llms/src/anthropic/count_tokens/transformation.rs +++ /dev/null @@ -1,171 +0,0 @@ -use litellm_llms_types::{ - formats::messages::{Message, SystemPrompt}, - providers::anthropic::{ - API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER, - }, -}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX}; - -const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AnthropicCountTokensRequest { - pub model: String, - pub messages: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub system: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AnthropicCountTokensResponse { - pub input_tokens: u64, -} - -pub trait AnthropicCountTokensConfig { - fn endpoint(&self) -> String; - - fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>; - - fn transform_request( - &self, - model: &str, - messages: Vec, - tools: Option>, - system: Option, - ) -> Result; - - fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)>; -} - -pub struct AnthropicCountTokensTransformation; - -pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformation = - AnthropicCountTokensTransformation; - -impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { - fn endpoint(&self) -> String { - format!("{API_BASE}{COUNT_TOKENS_PATH}") - } - - fn transform_request( - &self, - model: &str, - messages: Vec, - tools: Option>, - system: Option, - ) -> Result { - self.validate_request(model, &messages)?; - - Ok(AnthropicCountTokensRequest { - model: model.to_string(), - messages, - tools, - system, - }) - } - - fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error> { - if model.is_empty() { - return Err(Error::MissingField("model")); - } - if messages.is_empty() { - return Err(Error::MissingField("messages")); - } - Ok(()) - } - - fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)> { - let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) { - ("authorization", format!("Bearer {api_key}")) - } else { - (API_KEY_HEADER, api_key.to_string()) - }; - vec![ - ("content-type", "application/json".to_string()), - auth, - (VERSION_HEADER, API_VERSION.to_string()), - (BETA_HEADER, TOKEN_COUNTING_BETA.to_string()), - ] - } -} - -#[cfg(test)] -mod tests { - use litellm_llms_types::formats::messages::MessageContent; - use serde_json::{Map, json}; - - use super::*; - - fn message() -> Message { - Message { - role: "user".into(), - content: MessageContent::Text("hello".into()), - extra: Map::new(), - } - } - - #[test] - fn maps_the_python_count_tokens_contract() { - let request = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION - .transform_request( - "claude-test", - vec![message()], - Some(vec![json!({"name": "lookup"})]), - Some(SystemPrompt::Text("system".into())), - ) - .unwrap(); - - assert_eq!( - serde_json::to_value(request).unwrap(), - json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hello"}], - "tools": [{"name": "lookup"}], - "system": "system" - }) - ); - assert_eq!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(), - "https://api.anthropic.com/v1/messages/count_tokens" - ); - } - - #[test] - fn rejects_the_invalid_requests_python_rejects() { - assert!(matches!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request( - "", - vec![message()], - None, - None - ), - Err(Error::MissingField("model")) - )); - assert!(matches!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request( - "claude-test", - vec![], - None, - None - ), - Err(Error::MissingField("messages")) - )); - } - - #[test] - fn uses_api_key_or_oauth_headers_without_combining_credentials() { - let api_key = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-api"); - assert!(api_key.contains(&("x-api-key", "sk-ant-api".into()))); - assert!(!api_key.iter().any(|(name, _)| *name == "authorization")); - - let oauth = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-oat-test"); - assert!(oauth.contains(&("authorization", "Bearer sk-ant-oat-test".into()))); - assert!(!oauth.iter().any(|(name, _)| *name == "x-api-key")); - assert!(oauth.contains(&("anthropic-beta", TOKEN_COUNTING_BETA.into()))); - } -} diff --git a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs index bbb9b4e4c17..239daab8afa 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/transformation.rs @@ -67,6 +67,18 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { Ok(complete_anthropic_url(api_base, env_lookup)) } + fn get_count_tokens_url( + &self, + api_base: Option<&str>, + _litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + Some(Ok(format!( + "{}/count_tokens", + complete_anthropic_url(api_base, env_lookup) + ))) + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, @@ -1281,6 +1293,34 @@ mod tests { ); } + #[rstest] + #[case::default_base(None, &[], "https://api.anthropic.com/v1/messages/count_tokens")] + #[case::env_base( + None, + &[(ANTHROPIC_API_BASE_ENV, "https://env.example.com")], + "https://env.example.com/v1/messages/count_tokens" + )] + #[case::custom_base(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages/count_tokens")] + #[case::complete_messages_endpoint( + Some("https://proxy.internal/v1/messages"), + &[], + "https://proxy.internal/v1/messages/count_tokens" + )] + fn count_tokens_url_is_under_the_messages_endpoint( + #[case] api_base: Option<&str>, + #[case] vars: Env, + #[case] expected: &str, + ) { + assert_eq!( + ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url( + api_base, + &LitellmParams::default(), + &env(vars) + ), + Some(Ok(expected.to_string())) + ); + } + fn betas(values: &[&str]) -> BetaSet { values.join(",").parse().unwrap() } diff --git a/litellm/rust/crates/llms/src/anthropic/mod.rs b/litellm/rust/crates/llms/src/anthropic/mod.rs index 72c390cfe21..7d40e525772 100644 --- a/litellm/rust/crates/llms/src/anthropic/mod.rs +++ b/litellm/rust/crates/llms/src/anthropic/mod.rs @@ -3,7 +3,6 @@ pub mod common_utils; pub mod batches; pub mod chat; -pub mod count_tokens; pub mod messages; pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; diff --git a/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs index fb50d229bc7..9ac4d63a75c 100644 --- a/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -74,6 +74,18 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { complete_azure_anthropic_url(api_base, env_lookup) } + fn get_count_tokens_url( + &self, + api_base: Option<&str>, + _litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + Some( + complete_azure_anthropic_url(api_base, env_lookup) + .map(|messages_url| format!("{messages_url}/count_tokens")), + ) + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, @@ -218,6 +230,31 @@ mod tests { ); } + #[rstest] + #[case::resource_root("https://resource.services.ai.azure.com")] + #[case::anthropic_segment("https://resource.services.ai.azure.com/anthropic/")] + #[case::complete_messages_endpoint( + "https://resource.services.ai.azure.com/anthropic/v1/messages" + )] + fn count_url_sits_under_the_messages_endpoint(#[case] api_base: &str) { + let url = AZURE_ANTHROPIC_MESSAGES_CONFIG + .get_count_tokens_url(Some(api_base), &LitellmParams::default(), &|_| None) + .expect("azure ai has a count api") + .expect("url builds"); + assert_eq!( + url, + "https://resource.services.ai.azure.com/anthropic/v1/messages/count_tokens" + ); + } + + #[test] + fn count_url_without_a_base_is_an_error() { + let url = AZURE_ANTHROPIC_MESSAGES_CONFIG + .get_count_tokens_url(None, &LitellmParams::default(), &|_| None) + .expect("azure ai has a count api"); + assert!(url.is_err(), "{url:?}"); + } + #[test] fn url_falls_back_to_env_then_errors_when_absent() { let with_env = |key: &str| { diff --git a/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs index 13c021c2aec..aa34a2425b7 100644 --- a/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/base_llm/messages/transformation.rs @@ -41,6 +41,19 @@ pub trait BaseMessagesConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; + fn get_count_tokens_url( + &self, + _api_base: Option<&str>, + _litellm_params: &LitellmParams, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + None + } + + fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str { + model + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, diff --git a/litellm/rust/crates/llms/src/error.rs b/litellm/rust/crates/llms/src/error.rs index 77ee1f1036f..56416aa2a9d 100644 --- a/litellm/rust/crates/llms/src/error.rs +++ b/litellm/rust/crates/llms/src/error.rs @@ -207,6 +207,11 @@ pub enum ErrorDetail { HostCall, #[error("{0} is not handled by the rust path")] Unhandled(&'static str), + #[error("{operation} is not supported for provider {provider}")] + UnsupportedOperation { + operation: &'static str, + provider: &'static str, + }, #[error("invalid {subject} on line {line}")] InvalidLine { subject: &'static str, diff --git a/litellm/rust/crates/llms/src/vertex_ai/messages/AGENTS.md b/litellm/rust/crates/llms/src/vertex_ai/messages/AGENTS.md index d95da65ac9d..0a43ec9ef77 100644 --- a/litellm/rust/crates/llms/src/vertex_ai/messages/AGENTS.md +++ b/litellm/rust/crates/llms/src/vertex_ai/messages/AGENTS.md @@ -3,7 +3,9 @@ - Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token - Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it - Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it +- Counts tokens at the publisher's `count-tokens:rawPredict`, which rejects a versioned model, so the body's model loses its `@` suffix. The location is `vertex_count_tokens_location`, then the deployment's location, then `us-east5`, as Python's partner-model token counter picks it # references - https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude +- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens diff --git a/litellm/rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm/rust/crates/llms/src/vertex_ai/messages/transformation.rs index 674b9951d3e..09b4b064682 100644 --- a/litellm/rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm/rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -34,10 +34,12 @@ use crate::{ vertex_ai::common_utils::get_vertex_base_url, }; use litellm_llms_types::providers::vertex_ai::{ - ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT, + ANTHROPIC_VERSION, COUNT_TOKENS_DEFAULT_LOCATION, DEFAULT_LOCATION, RAW_PREDICT, + STREAM_RAW_PREDICT, }; const WEB_SEARCH_TOOL_PREFIX: &str = "web_search"; +const COUNT_TOKENS_LOCATION_PARAM: &str = "vertex_count_tokens_location"; const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"]; const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"]; @@ -82,14 +84,25 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { env_lookup: &dyn Fn(&str) -> Option, ) -> Result { let config = VertexConfig::from_params(&litellm_params.vertex); - let project = get_vertex_ai_project(&config, env_lookup) - .or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup)) - .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?; + let project = vertex_project(&config, env_lookup)?; let location = get_vertex_ai_location(&config, env_lookup) .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); complete_vertex_anthropic_url(api_base, &project, &location, model, stream) } + fn get_count_tokens_url( + &self, + api_base: Option<&str>, + litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + Some(count_tokens_url(api_base, litellm_params, env_lookup)) + } + + fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str { + model.split_once('@').map_or(model, |(name, _)| name) + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, @@ -229,6 +242,40 @@ pub fn complete_vertex_anthropic_url( }) } +fn vertex_project( + config: &VertexConfig, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + get_vertex_ai_project(config, env_lookup) + .or_else(|| get_vertex_ai_project_from_credentials(config, env_lookup)) + .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI"))) +} + +fn count_tokens_url( + api_base: Option<&str>, + litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + let config = VertexConfig::from_params(&litellm_params.vertex); + let project = vertex_project(&config, env_lookup)?; + let location = litellm_params + .extra + .get(COUNT_TOKENS_LOCATION_PARAM) + .and_then(Value::as_str) + .filter(|location| !location.is_empty()) + .map(str::to_string) + .or_else(|| get_vertex_ai_location(&config, env_lookup)) + .unwrap_or_else(|| COUNT_TOKENS_DEFAULT_LOCATION.to_string()); + let base = match api_base.map(str::trim).filter(|base| !base.is_empty()) { + Some(custom) => custom.to_string(), + None => get_vertex_base_url(&location)?, + }; + Ok(format!( + "{}/v1/projects/{project}/locations/{location}/publishers/anthropic/models/count-tokens:{RAW_PREDICT}", + base.trim_end_matches('/') + )) +} + /// Python's `sanitize_vertex_anthropic_output_params`: `effort` only reaches models whose /// capabilities accept it, a malformed `output_config` is dropped, and an emptied one is omitted. fn sanitize_output_config( @@ -476,6 +523,105 @@ mod tests { ); } + fn count_tokens_params(fields: Value) -> LitellmParams { + let (Value::Object(base), Value::Object(fields)) = ( + json!({"model": "vertex_ai/claude-sonnet-4-5", "vertex_project": "proj"}), + fields, + ) else { + panic!("litellm params are a json object"); + }; + serde_json::from_value(Value::Object(base.into_iter().chain(fields).collect())).unwrap() + } + + const COUNT_PATH: &str = "/publishers/anthropic/models/count-tokens:rawPredict"; + + #[rstest] + #[case::count_location_wins( + None, + json!({"vertex_count_tokens_location": "europe-west1", "vertex_location": "asia-southeast1"}), + &[], + "https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1" + )] + #[case::deployment_location( + None, + json!({"vertex_location": "asia-southeast1"}), + &[], + "https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1" + )] + #[case::empty_count_location_is_unset( + None, + json!({"vertex_count_tokens_location": "", "vertex_location": "asia-southeast1"}), + &[], + "https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1" + )] + #[case::environment_location( + None, + json!({}), + &[("VERTEXAI_LOCATION", "europe-west1")], + "https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1" + )] + #[case::count_default_location( + None, + json!({}), + &[], + "https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5" + )] + #[case::global( + None, + json!({"vertex_location": "global"}), + &[], + "https://aiplatform.googleapis.com/v1/projects/proj/locations/global" + )] + #[case::custom_base( + Some("https://proxy.example/prefix/"), + json!({"vertex_location": "global"}), + &[], + "https://proxy.example/prefix/v1/projects/proj/locations/global" + )] + fn count_tokens_url_follows_the_python_location_precedence( + #[case] api_base: Option<&str>, + #[case] fields: Value, + #[case] env: &[(&str, &str)], + #[case] prefix: &str, + ) { + let lookup = |name: &str| { + env.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }; + assert_eq!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url( + api_base, + &count_tokens_params(fields), + &lookup + ), + Some(Ok(format!("{prefix}{COUNT_PATH}"))) + ); + } + + #[rstest] + fn count_tokens_without_a_project_fails_before_sending() { + assert!(matches!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url( + None, + &LitellmParams::default(), + &no_env + ), + Some(Err(Error::Auth(_))) + )); + } + + #[rstest] + #[case::versioned("claude-sonnet-4-5@20250929", "claude-sonnet-4-5")] + #[case::default_version("claude-sonnet-4-5@default", "claude-sonnet-4-5")] + #[case::unversioned("claude-sonnet-4-5", "claude-sonnet-4-5")] + fn count_tokens_model_drops_the_version(#[case] model: &str, #[case] expected: &str) { + assert_eq!( + VERTEX_ANTHROPIC_MESSAGES_CONFIG.count_tokens_model(model), + expected + ); + } + #[rstest] fn without_a_project_anywhere_the_call_fails_before_sending() { let error = url(None, &LitellmParams::default(), false, &no_env).unwrap_err();