fix(rust-gateway): serve /v1/messages/count_tokens through the provider's count API

The gateway answered 404 for POST /v1/messages/count_tokens, so the
Anthropic SDK's messages.count_tokens failed while messages.create worked

The route now goes through the same deployment lookup, model access check,
and error envelope as /v1/messages, then forwards the caller's Anthropic
body with the deployment model to the provider's count-tokens API, using
the credentials and auth the Messages path already resolves. Anthropic
counts at {api_base}/v1/messages/count_tokens. Vertex AI counts at the
publisher count-tokens:rawPredict endpoint with the @version suffix
stripped and Python's location order (vertex_count_tokens_location, then
the deployment location, then us-east5). Other providers get a 400
invalid_request_error that names the provider

The unused Anthropic count-tokens adapter and its path constant are gone
This commit is contained in:
mateo-berri 2026-10-29 07:52:12 -07:00
parent 4212f1af27
commit ca1b5f2840
23 changed files with 538 additions and 210 deletions

View file

@ -91,6 +91,7 @@ impl Gateway {
pub fn router(gateway: Arc<Gateway>) -> Router {
Router::new()
.route("/v1/messages", post(messages::create))
.route("/v1/messages/count_tokens", post(messages::count_tokens))
.route("/ocr", post(ocr::create))
.route("/v1/ocr", post(ocr::create))
.route("/chat/completions", post(chat_completions::create))

View file

@ -1,4 +1,5 @@
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it.
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it, and
//! `POST /v1/messages/count_tokens`, answered by the deployment's own count-tokens API.
use litellm_gateway_auth::AuthenticatedRequest;
use std::sync::Arc;
@ -12,7 +13,10 @@ use axum::{
};
use litellm_host_http::Sse;
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
use litellm_llms_types::{
formats::messages::CountTokensResponse,
headers::{ProviderSpecificHeader, ProviderSpecificHeaders},
};
use serde_json::{Map, Value};
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
@ -67,6 +71,34 @@ async fn handle(
Ok(headers.apply(response))
}
pub async fn count_tokens(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
RequestId(request_id): RequestId,
headers: HeaderMap,
body: Result<JsonObject, Error>,
) -> impl IntoResponse {
let result = match body {
Ok(JsonObject(body)) => count(&gateway, &identity, &headers, body).await,
Err(error) => Err(error),
};
result
.map(Json)
.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
}
async fn count(
gateway: &Gateway,
identity: &AuthenticatedRequest,
headers: &HeaderMap,
body: Map<String, Value>,
) -> Result<CountTokensResponse, Error> {
let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?;
request::authorize_model(identity, deployment, &body).await?;
let call = project(deployment, body, headers)?;
Ok(gateway.messages.count_tokens(call).await?)
}
fn project(
deployment: &Deployment,
body: Map<String, Value>,

View file

@ -121,18 +121,20 @@ async fn a_stream_that_fails_after_opening_ends_with_an_sse_error_frame() {
}
#[rstest]
#[case::messages("/v1/messages")]
#[case::count_tokens("/v1/messages/count_tokens")]
#[tokio::test]
async fn hosted_provider_failure_preserves_status_body_and_request_id() {
async fn hosted_provider_failure_preserves_status_body_and_request_id(#[case] route: &str) {
let upstream = MockServer::start().await;
let error =
json!({"type": "error", "error": {"type": "rate_limit_error", "message": "retry later"}});
Mock::given(method("POST"))
.and(path("/v1/messages"))
.and(path(route))
.respond_with(ResponseTemplate::new(429).set_body_json(error.clone()))
.expect(1)
.mount(&upstream)
.await;
let request = Request::post("/v1/messages")
let request = Request::post(route)
.header("content-type", "application/json")
.header("x-request-id", "host-http-request")
.body(Body::from(json!({
@ -150,8 +152,90 @@ async fn hosted_provider_failure_preserves_status_body_and_request_id() {
assert_eq!(body["request_id"], "host-http-request");
}
#[rstest]
#[tokio::test]
async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model() {
let upstream = MockServer::start().await;
let messages = json!([{"role": "user", "content": "hi"}]);
let tools = json!([{"name": "lookup", "input_schema": {"type": "object"}}]);
let counted = json!({"input_tokens": 14, "context_management": {"original_input_tokens": 20}});
Mock::given(method("POST"))
.and(path("/v1/messages/count_tokens"))
.and(header("x-api-key", "test-key"))
.and(header("anthropic-version", "test-version"))
.and(body_json(json!({
"model": "test-model", "messages": messages, "system": "be brief", "tools": tools,
})))
.respond_with(ResponseTemplate::new(200).set_body_json(counted.clone()))
.expect(1)
.mount(&upstream)
.await;
let request = Request::post("/v1/messages/count_tokens")
.header("content-type", "application/json")
.header("anthropic-version", "test-version")
.body(Body::from(
json!({"model": "public/model", "messages": messages, "system": "be brief", "tools": tools})
.to_string(),
))
.unwrap();
let response = support::app("anthropic/test-model", &upstream.uri())
.oneshot(request)
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(support::json(response).await, counted);
}
#[rstest]
#[tokio::test]
async fn count_tokens_for_a_provider_without_a_count_api_is_an_invalid_request() {
let upstream = MockServer::start().await;
let response = support::post(
support::app("deepseek/test-model", &upstream.uri()),
"/v1/messages/count_tokens",
json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}),
)
.await;
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["type"], "error");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("deepseek")),
"{body}"
);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[tokio::test]
async fn an_unknown_model_is_refused_alike_on_messages_and_count_tokens(
#[values("/v1/messages", "/v1/messages/count_tokens")] route: &str,
) {
let upstream = MockServer::start().await;
let response = support::post(
support::app("anthropic/test-model", &upstream.uri()),
route,
json!({"model": "other/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
)
.await;
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("other/model")),
"{body}"
);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[case::messages("/v1/messages")]
#[case::count_tokens("/v1/messages/count_tokens")]
#[case::chat("/v1/chat/completions")]
#[case::model_path("/engines/path-model/chat/completions")]
#[case::ocr("/ocr")]

View file

@ -235,6 +235,7 @@ async fn upload_audio_format_validation_matches_core(
#[case::responses("/v1/responses", false)]
#[case::deployment("/openai/deployments/public%2Fmodel/chat/completions", false)]
#[case::messages("/v1/messages", true)]
#[case::count_tokens("/v1/messages/count_tokens", true)]
#[case::ocr("/v1/ocr", false)]
#[case::transcription("/v1/audio/transcriptions", false)]
#[tokio::test]

View file

@ -19,7 +19,9 @@ use litellm_llms::base_llm::{
transformation::BaseMessagesConfig,
},
};
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
use litellm_llms_types::formats::messages::{
CountTokensResponse, MessagesRequest, MessagesResponse,
};
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
use serde_json::Value;
@ -211,6 +213,92 @@ async fn provider_call(
})
}
pub(super) async fn count_tokens(
route: &MessagesRoute,
provider: MessagesProvider,
model: String,
call: MessagesCall,
secrets: &(dyn Lookup + Send + Sync),
) -> Result<CountTokensResponse, Error> {
let MessagesCall {
body,
api_key,
api_base,
litellm_params,
extra_headers,
provider_specific_header,
timeout,
shaping,
..
} = call;
let connection_params = litellm_router_types::LitellmParams {
api_base: api_base.clone().or(litellm_params.api_base.clone()),
..litellm_params
};
let config = provider.config();
let env_lookup = |key: &str| secrets.get(key);
let url = config
.get_count_tokens_url(api_base.as_deref(), &connection_params, &env_lookup)
.ok_or_else(|| {
Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedOperation {
operation: "token counting",
provider: provider.as_str(),
})
})??;
let sanitized = config.shape_request(
MessagesRequest {
model: config.count_tokens_model(&model).to_string(),
..body
},
shaping.settings.reasoning_auto_summary,
)?;
let request =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
&request.model,
&request.params,
&connection_params,
&env_lookup,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
&request,
),
auth: validated.auth,
};
let authenticated =
resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?;
let response = send(
&route.http,
authenticated,
&url,
&serde_json::to_value(&request).map_err(serialize_failure)?,
timeout.or_else(|| shaping.settings.request_timeout()),
)
.await?;
if !response.status().is_success() {
return Err(provider_error(response).await);
}
let text = response.text().await.map_err(litellm_http::Error::from)?;
serde_json::from_str(&text).map_err(|err| {
Error::invalid_response(litellm_llms::ErrorDetail::invalid(
"count_tokens response JSON",
err,
))
})
}
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],

View file

@ -9,10 +9,11 @@ use litellm_auth::AuthServices;
use litellm_host::interceptors::Interceptors;
use litellm_inference::context::CallContext;
use litellm_inference::provider::resolve_llm_provider;
use litellm_secrets::source::SecretSource;
use litellm_llms_types::formats::messages::CountTokensResponse;
use litellm_secrets::source::{SecretSource, Secrets};
use std::sync::Arc;
use crate::common_utils::messages_provider;
use crate::common_utils::{MessagesProvider, messages_provider};
pub use litellm_inference::Error;
pub use types::{
@ -75,24 +76,48 @@ impl MessagesRoute {
context: CallContext<'_, impl Interceptors<Error>>,
) -> Result<MessagesCallResponse, Error> {
litellm_inference::diagnostic::call(async {
let provider = resolve_llm_provider(
&call.body.model,
call.custom_llm_provider.as_deref(),
"messages",
)?;
let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
Error::InvalidProvider(<&str>::from(provider.provider).to_string())
})?;
litellm_inference::diagnostic::provider(
provider.model,
<&'static str>::from(provider.provider),
);
let model = provider.model.to_owned();
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
handler::execute(self, config, model, call, secrets, &context)
let (provider, model, secrets) = self.resolve_provider(&call).await?;
handler::execute(self, provider, model, call, secrets, &context)
.boxed()
.await
})
.await
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "messages_count_tokens",
model = %call.body.model,
provider,
resolved_model,
stream = false,
outcome
))]
pub async fn count_tokens(&self, call: MessagesCall) -> Result<CountTokensResponse, Error> {
litellm_inference::diagnostic::unary(async {
let (provider, model, secrets) = self.resolve_provider(&call).await?;
handler::count_tokens(self, provider, model, call, secrets.as_ref())
.boxed()
.await
})
.await
}
async fn resolve_provider(
&self,
call: &MessagesCall,
) -> Result<(MessagesProvider, String, Secrets), Error> {
let provider = resolve_llm_provider(
&call.body.model,
call.custom_llm_provider.as_deref(),
"messages",
)?;
let config = messages_provider(provider.provider, provider.model)
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
litellm_inference::diagnostic::provider(
provider.model,
<&'static str>::from(provider.provider),
);
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
Ok((config, provider.model.to_owned(), secrets))
}
}

View file

@ -0,0 +1,62 @@
use rstest::rstest;
use super::*;
#[rstest]
#[tokio::test]
async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model(
call: MessagesCall,
) {
let upstream = upstream([json_response(json!({"input_tokens": 14}))]).await;
let messages = json!([{"role": "user", "content": "hi"}]);
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: body(json!({"model": "claude-sonnet-4-5@20250929", "messages": messages})),
custom_llm_provider: Some("vertex_ai".into()),
litellm_params: serde_json::from_value(json!({
"model": "vertex_ai/claude-sonnet-4-5@20250929",
"vertex_project": "proj",
"vertex_location": "europe-west1",
}))
.unwrap(),
api_base: Some(upstream.uri()),
extra_headers: headers([
("Authorization", "Bearer caller-token"),
("anthropic-version", "2023-06-01"),
]),
..call
})
.await
.expect("count succeeds");
assert_eq!(counted.input_tokens, 14);
let request = only_request(&upstream).await;
assert_eq!(
request.url.path(),
"/v1/projects/proj/locations/europe-west1/publishers/anthropic/models/count-tokens:rawPredict"
);
assert_eq!(request.header("authorization"), Some("Bearer caller-token"));
assert_eq!(request.header("anthropic-version"), None);
assert_eq!(
request.json(),
json!({"model": "claude-sonnet-4-5", "messages": messages})
);
}
#[rstest]
#[tokio::test]
async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) {
let upstream = upstream([json_response(json!({"tokens": 14}))]).await;
let error = messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.unwrap_err();
assert!(error.is_response(), "{error:?}");
}

View file

@ -19,6 +19,7 @@ use wiremock::ResponseTemplate;
mod support;
use support::*;
mod count_tokens;
mod host;
mod request;
mod response;

View file

@ -25,7 +25,7 @@
## Providers
- A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers/<provider>/`
- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
- Projections that only one adapter decodes (`InvokeChunkPayload`, the batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
- A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it
- Provider types may use format types. Format types never depend on provider types

View file

@ -19,6 +19,7 @@
# references
- https://platform.claude.com/docs/en/api/http/messages/create
- https://platform.claude.com/docs/en/api/http/messages/count_tokens
- https://platform.claude.com/docs/en/api/messages/create.md
- https://platform.claude.com/docs/en/api/beta/messages/create.md
- https://platform.claude.com/docs/en/build-with-claude/streaming.md

View file

@ -18,6 +18,13 @@ pub struct MessagesResponse {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CountTokensResponse {
pub input_tokens: u64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -1,6 +1,6 @@
# rules
- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers
- Shared across Anthropic's Messages, chat and batches adapters: API base, endpoint paths, header names, API version and default headers
- `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one
- Choosing which betas a request needs is policy in `llms/src/anthropic/`

View file

@ -1,7 +1,6 @@
pub const API_BASE: &str = "https://api.anthropic.com";
pub const MESSAGES_PATH: &str = "/v1/messages";
pub const BATCHES_PATH: &str = "/v1/messages/batches";
pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens";
pub const API_KEY_HEADER: &str = "x-api-key";
pub const BETA_HEADER: &str = "anthropic-beta";

View file

@ -1,6 +1,7 @@
pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com";
pub const GLOBAL_LOCATION: &str = "global";
pub const DEFAULT_LOCATION: &str = "us-central1";
pub const COUNT_TOKENS_DEFAULT_LOCATION: &str = "us-east5";
pub const RAW_PREDICT: &str = "rawPredict";
pub const STREAM_RAW_PREDICT: &str = "streamRawPredict";

View file

@ -1,7 +0,0 @@
# rules
- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections
# references
- https://platform.claude.com/docs/en/api/http/messages/count_tokens

View file

@ -1 +0,0 @@
pub mod transformation;

View file

@ -1,171 +0,0 @@
use litellm_llms_types::{
formats::messages::{Message, SystemPrompt},
providers::anthropic::{
API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
},
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX};
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicCountTokensRequest {
pub model: String,
pub messages: Vec<Message>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub system: Option<SystemPrompt>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AnthropicCountTokensResponse {
pub input_tokens: u64,
}
pub trait AnthropicCountTokensConfig {
fn endpoint(&self) -> String;
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>;
fn transform_request(
&self,
model: &str,
messages: Vec<Message>,
tools: Option<Vec<Value>>,
system: Option<SystemPrompt>,
) -> Result<AnthropicCountTokensRequest, Error>;
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)>;
}
pub struct AnthropicCountTokensTransformation;
pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformation =
AnthropicCountTokensTransformation;
impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
fn endpoint(&self) -> String {
format!("{API_BASE}{COUNT_TOKENS_PATH}")
}
fn transform_request(
&self,
model: &str,
messages: Vec<Message>,
tools: Option<Vec<Value>>,
system: Option<SystemPrompt>,
) -> Result<AnthropicCountTokensRequest, Error> {
self.validate_request(model, &messages)?;
Ok(AnthropicCountTokensRequest {
model: model.to_string(),
messages,
tools,
system,
})
}
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error> {
if model.is_empty() {
return Err(Error::MissingField("model"));
}
if messages.is_empty() {
return Err(Error::MissingField("messages"));
}
Ok(())
}
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)> {
let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) {
("authorization", format!("Bearer {api_key}"))
} else {
(API_KEY_HEADER, api_key.to_string())
};
vec![
("content-type", "application/json".to_string()),
auth,
(VERSION_HEADER, API_VERSION.to_string()),
(BETA_HEADER, TOKEN_COUNTING_BETA.to_string()),
]
}
}
#[cfg(test)]
mod tests {
use litellm_llms_types::formats::messages::MessageContent;
use serde_json::{Map, json};
use super::*;
fn message() -> Message {
Message {
role: "user".into(),
content: MessageContent::Text("hello".into()),
extra: Map::new(),
}
}
#[test]
fn maps_the_python_count_tokens_contract() {
let request = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION
.transform_request(
"claude-test",
vec![message()],
Some(vec![json!({"name": "lookup"})]),
Some(SystemPrompt::Text("system".into())),
)
.unwrap();
assert_eq!(
serde_json::to_value(request).unwrap(),
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hello"}],
"tools": [{"name": "lookup"}],
"system": "system"
})
);
assert_eq!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(),
"https://api.anthropic.com/v1/messages/count_tokens"
);
}
#[test]
fn rejects_the_invalid_requests_python_rejects() {
assert!(matches!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
"",
vec![message()],
None,
None
),
Err(Error::MissingField("model"))
));
assert!(matches!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
"claude-test",
vec![],
None,
None
),
Err(Error::MissingField("messages"))
));
}
#[test]
fn uses_api_key_or_oauth_headers_without_combining_credentials() {
let api_key = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-api");
assert!(api_key.contains(&("x-api-key", "sk-ant-api".into())));
assert!(!api_key.iter().any(|(name, _)| *name == "authorization"));
let oauth = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-oat-test");
assert!(oauth.contains(&("authorization", "Bearer sk-ant-oat-test".into())));
assert!(!oauth.iter().any(|(name, _)| *name == "x-api-key"));
assert!(oauth.contains(&("anthropic-beta", TOKEN_COUNTING_BETA.into())));
}
}

View file

@ -50,6 +50,18 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
Ok(complete_anthropic_url(api_base, env_lookup))
}
fn get_count_tokens_url(
&self,
api_base: Option<&str>,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
Some(Ok(format!(
"{}/count_tokens",
complete_anthropic_url(api_base, env_lookup)
)))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
@ -875,6 +887,34 @@ mod tests {
);
}
#[rstest]
#[case::default_base(None, &[], "https://api.anthropic.com/v1/messages/count_tokens")]
#[case::env_base(
None,
&[(ANTHROPIC_API_BASE_ENV, "https://env.example.com")],
"https://env.example.com/v1/messages/count_tokens"
)]
#[case::custom_base(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages/count_tokens")]
#[case::complete_messages_endpoint(
Some("https://proxy.internal/v1/messages"),
&[],
"https://proxy.internal/v1/messages/count_tokens"
)]
fn count_tokens_url_is_under_the_messages_endpoint(
#[case] api_base: Option<&str>,
#[case] vars: Env,
#[case] expected: &str,
) {
assert_eq!(
ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
api_base,
&LitellmParams::default(),
&env(vars)
),
Some(Ok(expected.to_string()))
);
}
fn betas(values: &[&str]) -> BetaSet {
values.join(",").parse().unwrap()
}

View file

@ -3,7 +3,6 @@ pub mod common_utils;
pub mod batches;
pub mod chat;
pub mod count_tokens;
pub mod messages;
pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat";

View file

@ -27,6 +27,19 @@ pub trait BaseMessagesConfig: Sync {
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error>;
fn get_count_tokens_url(
&self,
_api_base: Option<&str>,
_litellm_params: &LitellmParams,
_env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
None
}
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
model
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,

View file

@ -166,6 +166,11 @@ pub enum ErrorDetail {
HostCall,
#[error("{0} is not handled by the rust path")]
Unhandled(&'static str),
#[error("{operation} is not supported for provider {provider}")]
UnsupportedOperation {
operation: &'static str,
provider: &'static str,
},
#[error("invalid {subject} on line {line}")]
InvalidLine {
subject: &'static str,

View file

@ -3,7 +3,9 @@
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
- Counts tokens at the publisher's `count-tokens:rawPredict`, which rejects a versioned model, so the body's model loses its `@` suffix. The location is `vertex_count_tokens_location`, then the deployment's location, then `us-east5`, as Python's partner-model token counter picks it
# references
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens

View file

@ -32,10 +32,12 @@ use crate::{
vertex_ai::common_utils::get_vertex_base_url,
};
use litellm_llms_types::providers::vertex_ai::{
ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT,
ANTHROPIC_VERSION, COUNT_TOKENS_DEFAULT_LOCATION, DEFAULT_LOCATION, RAW_PREDICT,
STREAM_RAW_PREDICT,
};
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
const COUNT_TOKENS_LOCATION_PARAM: &str = "vertex_count_tokens_location";
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
@ -64,14 +66,25 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let config = VertexConfig::from_params(&litellm_params.vertex);
let project = get_vertex_ai_project(&config, env_lookup)
.or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup))
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
let project = vertex_project(&config, env_lookup)?;
let location = get_vertex_ai_location(&config, env_lookup)
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
complete_vertex_anthropic_url(api_base, &project, &location, model, stream)
}
fn get_count_tokens_url(
&self,
api_base: Option<&str>,
litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
Some(count_tokens_url(api_base, litellm_params, env_lookup))
}
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
model.split_once('@').map_or(model, |(name, _)| name)
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
@ -211,6 +224,40 @@ pub fn complete_vertex_anthropic_url(
})
}
fn vertex_project(
config: &VertexConfig,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
get_vertex_ai_project(config, env_lookup)
.or_else(|| get_vertex_ai_project_from_credentials(config, env_lookup))
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))
}
fn count_tokens_url(
api_base: Option<&str>,
litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let config = VertexConfig::from_params(&litellm_params.vertex);
let project = vertex_project(&config, env_lookup)?;
let location = litellm_params
.extra
.get(COUNT_TOKENS_LOCATION_PARAM)
.and_then(Value::as_str)
.filter(|location| !location.is_empty())
.map(str::to_string)
.or_else(|| get_vertex_ai_location(&config, env_lookup))
.unwrap_or_else(|| COUNT_TOKENS_DEFAULT_LOCATION.to_string());
let base = match api_base.map(str::trim).filter(|base| !base.is_empty()) {
Some(custom) => custom.to_string(),
None => get_vertex_base_url(&location)?,
};
Ok(format!(
"{}/v1/projects/{project}/locations/{location}/publishers/anthropic/models/count-tokens:{RAW_PREDICT}",
base.trim_end_matches('/')
))
}
/// Python's `sanitize_vertex_anthropic_output_params`: `effort` only reaches models whose
/// capabilities accept it, a malformed `output_config` is dropped, and an emptied one is omitted.
fn sanitize_output_config(
@ -458,6 +505,105 @@ mod tests {
);
}
fn count_tokens_params(fields: Value) -> LitellmParams {
let (Value::Object(base), Value::Object(fields)) = (
json!({"model": "vertex_ai/claude-sonnet-4-5", "vertex_project": "proj"}),
fields,
) else {
panic!("litellm params are a json object");
};
serde_json::from_value(Value::Object(base.into_iter().chain(fields).collect())).unwrap()
}
const COUNT_PATH: &str = "/publishers/anthropic/models/count-tokens:rawPredict";
#[rstest]
#[case::count_location_wins(
None,
json!({"vertex_count_tokens_location": "europe-west1", "vertex_location": "asia-southeast1"}),
&[],
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
)]
#[case::deployment_location(
None,
json!({"vertex_location": "asia-southeast1"}),
&[],
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
)]
#[case::empty_count_location_is_unset(
None,
json!({"vertex_count_tokens_location": "", "vertex_location": "asia-southeast1"}),
&[],
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
)]
#[case::environment_location(
None,
json!({}),
&[("VERTEXAI_LOCATION", "europe-west1")],
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
)]
#[case::count_default_location(
None,
json!({}),
&[],
"https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5"
)]
#[case::global(
None,
json!({"vertex_location": "global"}),
&[],
"https://aiplatform.googleapis.com/v1/projects/proj/locations/global"
)]
#[case::custom_base(
Some("https://proxy.example/prefix/"),
json!({"vertex_location": "global"}),
&[],
"https://proxy.example/prefix/v1/projects/proj/locations/global"
)]
fn count_tokens_url_follows_the_python_location_precedence(
#[case] api_base: Option<&str>,
#[case] fields: Value,
#[case] env: &[(&str, &str)],
#[case] prefix: &str,
) {
let lookup = |name: &str| {
env.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
};
assert_eq!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
api_base,
&count_tokens_params(fields),
&lookup
),
Some(Ok(format!("{prefix}{COUNT_PATH}")))
);
}
#[rstest]
fn count_tokens_without_a_project_fails_before_sending() {
assert!(matches!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
None,
&LitellmParams::default(),
&no_env
),
Some(Err(Error::Auth(_)))
));
}
#[rstest]
#[case::versioned("claude-sonnet-4-5@20250929", "claude-sonnet-4-5")]
#[case::default_version("claude-sonnet-4-5@default", "claude-sonnet-4-5")]
#[case::unversioned("claude-sonnet-4-5", "claude-sonnet-4-5")]
fn count_tokens_model_drops_the_version(#[case] model: &str, #[case] expected: &str) {
assert_eq!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.count_tokens_model(model),
expected
);
}
#[rstest]
fn without_a_project_anywhere_the_call_fails_before_sending() {
let error = url(None, &LitellmParams::default(), false, &no_env).unwrap_err();