fix(rust-gateway): serve /v1/messages/count_tokens through the provider's count API

The gateway answered 404 for POST /v1/messages/count_tokens, so the
Anthropic SDK's messages.count_tokens failed while messages.create worked

The route goes through the same deployment lookup, model access check and
error envelope as /v1/messages. It keeps only the fields the count
endpoints read, runs the same provider transform as /v1/messages, drops
max_tokens from the wire, and sends the result to the provider's count
API with the create path's header defaults and upstream error mapping.
Anthropic counts at {api_base}/v1/messages/count_tokens, Azure AI under
its Anthropic messages endpoint, and Vertex AI at the publisher
count-tokens:rawPredict endpoint with the @version suffix stripped and
Python's location order. Other providers get a 400 invalid_request_error
that names the provider. The response carries litellm_count_source like
the Python proxy's

The unused Anthropic count-tokens adapter and its path constant are gone
This commit is contained in:
mateo-berri 2026-11-13 20:58:46 -08:00
parent 90122c07a2
commit fddadf3e29
25 changed files with 871 additions and 193 deletions

View file

@ -91,6 +91,7 @@ impl Gateway {
pub fn router(gateway: Arc<Gateway>) -> Router {
Router::new()
.route("/v1/messages", post(messages::create))
.route("/v1/messages/count_tokens", post(messages::count_tokens))
.route("/ocr", post(ocr::create))
.route("/v1/ocr", post(ocr::create))
.route("/chat/completions", post(chat_completions::create))

View file

@ -1,4 +1,5 @@
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it.
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it, and
//! `POST /v1/messages/count_tokens`, answered by the deployment's own count-tokens API.
use litellm_gateway_auth::AuthenticatedRequest;
use std::sync::Arc;
@ -12,7 +13,11 @@ use axum::{
};
use litellm_host_http::Sse;
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
use litellm_llms_types::{
formats::messages::CountTokensResponse,
headers::{ProviderSpecificHeader, ProviderSpecificHeaders},
};
use serde::Serialize;
use serde_json::{Map, Value};
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
@ -25,6 +30,14 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
const PROVIDER_COUNT_SOURCE: &str = "provider";
#[derive(Serialize)]
struct SourcedCount {
#[serde(flatten)]
count: CountTokensResponse,
litellm_count_source: &'static str,
}
pub async fn create(
State(gateway): State<Arc<Gateway>>,
@ -67,6 +80,37 @@ async fn handle(
Ok(headers.apply(response))
}
pub async fn count_tokens(
State(gateway): State<Arc<Gateway>>,
identity: AuthenticatedRequest,
RequestId(request_id): RequestId,
headers: HeaderMap,
body: Result<JsonObject, Error>,
) -> impl IntoResponse {
let result = match body {
Ok(JsonObject(body)) => count(&gateway, &identity, &headers, body).await,
Err(error) => Err(error),
};
result
.map(Json)
.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
}
async fn count(
gateway: &Gateway,
identity: &AuthenticatedRequest,
headers: &HeaderMap,
body: Map<String, Value>,
) -> Result<SourcedCount, Error> {
let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?;
request::authorize_model(identity, deployment, &body).await?;
let call = project(deployment, body, headers)?;
Ok(SourcedCount {
count: gateway.messages.count_tokens(call).await?,
litellm_count_source: PROVIDER_COUNT_SOURCE,
})
}
fn project(
deployment: &Deployment,
body: Map<String, Value>,

View file

@ -121,18 +121,20 @@ async fn a_stream_that_fails_after_opening_ends_with_an_sse_error_frame() {
}
#[rstest]
#[case::messages("/v1/messages")]
#[case::count_tokens("/v1/messages/count_tokens")]
#[tokio::test]
async fn hosted_provider_failure_preserves_status_body_and_request_id() {
async fn hosted_provider_failure_preserves_status_body_and_request_id(#[case] route: &str) {
let upstream = MockServer::start().await;
let error =
json!({"type": "error", "error": {"type": "rate_limit_error", "message": "retry later"}});
Mock::given(method("POST"))
.and(path("/v1/messages"))
.and(path(route))
.respond_with(ResponseTemplate::new(429).set_body_json(error.clone()))
.expect(1)
.mount(&upstream)
.await;
let request = Request::post("/v1/messages")
let request = Request::post(route)
.header("content-type", "application/json")
.header("x-request-id", "host-http-request")
.body(Body::from(json!({
@ -150,8 +152,97 @@ async fn hosted_provider_failure_preserves_status_body_and_request_id() {
assert_eq!(body["request_id"], "host-http-request");
}
#[rstest]
#[tokio::test]
async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model() {
let upstream = MockServer::start().await;
let messages = json!([{"role": "user", "content": "hi"}]);
let tools = json!([{"name": "lookup", "input_schema": {"type": "object"}}]);
let counted = json!({"input_tokens": 14, "context_management": {"original_input_tokens": 20}});
Mock::given(method("POST"))
.and(path("/v1/messages/count_tokens"))
.and(header("x-api-key", "test-key"))
.and(header("anthropic-version", "test-version"))
.and(body_json(json!({
"model": "test-model", "messages": messages, "system": "be brief", "tools": tools,
})))
.respond_with(ResponseTemplate::new(200).set_body_json(counted))
.expect(1)
.mount(&upstream)
.await;
let request = Request::post("/v1/messages/count_tokens")
.header("content-type", "application/json")
.header("anthropic-version", "test-version")
.body(Body::from(
json!({"model": "public/model", "messages": messages, "system": "be brief", "tools": tools})
.to_string(),
))
.unwrap();
let response = support::app("anthropic/test-model", &upstream.uri())
.oneshot(request)
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(
support::json(response).await,
json!({
"input_tokens": 14,
"context_management": {"original_input_tokens": 20},
"litellm_count_source": "provider",
})
);
}
#[rstest]
#[tokio::test]
async fn count_tokens_for_a_provider_without_a_count_api_is_an_invalid_request() {
let upstream = MockServer::start().await;
let response = support::post(
support::app("deepseek/test-model", &upstream.uri()),
"/v1/messages/count_tokens",
json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}),
)
.await;
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["type"], "error");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("deepseek")),
"{body}"
);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[tokio::test]
async fn an_unknown_model_is_refused_alike_on_messages_and_count_tokens(
#[values("/v1/messages", "/v1/messages/count_tokens")] route: &str,
) {
let upstream = MockServer::start().await;
let response = support::post(
support::app("anthropic/test-model", &upstream.uri()),
route,
json!({"model": "other/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
)
.await;
assert_eq!(response.status(), 400);
let body = support::json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("other/model")),
"{body}"
);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[case::messages("/v1/messages")]
#[case::count_tokens("/v1/messages/count_tokens")]
#[case::chat("/v1/chat/completions")]
#[case::model_path("/engines/path-model/chat/completions")]
#[case::ocr("/ocr")]

View file

@ -235,6 +235,7 @@ async fn upload_audio_format_validation_matches_core(
#[case::responses("/v1/responses", false)]
#[case::deployment("/openai/deployments/public%2Fmodel/chat/completions", false)]
#[case::messages("/v1/messages", true)]
#[case::count_tokens("/v1/messages/count_tokens", true)]
#[case::ocr("/v1/ocr", false)]
#[case::transcription("/v1/audio/transcriptions", false)]
#[tokio::test]

View file

@ -22,8 +22,9 @@ use litellm_llms::base_llm::{
},
};
use litellm_llms_types::formats::messages::{
MessagesOptionalParams, MessagesRequest, MessagesResponse,
CountTokensResponse, MessagesOptionalParams, MessagesRequest, MessagesResponse,
};
use litellm_llms_types::recognized::Recognized;
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
use serde_json::{Map, Value};
@ -44,6 +45,8 @@ use litellm_inference::{
outbound::{raw_leaf_body, serialized_outbound_request},
};
const UNCAPPED_MAX_TOKENS: u64 = u64::MAX;
struct ProviderCall {
identity: ProviderIdentity,
wire: WireRequest,
@ -389,6 +392,125 @@ async fn provider_call(
})
}
pub(super) async fn count_tokens(
route: &MessagesRoute,
provider: MessagesProvider,
model: String,
call: MessagesCall,
secrets: &(dyn Lookup + Send + Sync),
) -> Result<CountTokensResponse, Error> {
let MessagesCall {
body,
api_key,
api_base,
litellm_params,
extra_headers,
provider_specific_header,
timeout,
shaping,
..
} = call;
let connection_params = litellm_router_types::LitellmParams {
api_base: api_base.clone().or(litellm_params.api_base.clone()),
..litellm_params
};
let config = provider.config();
let env_lookup = |key: &str| secrets.get(key);
let url = config
.get_count_tokens_url(api_base.as_deref(), &connection_params, &env_lookup)
.ok_or_else(|| {
Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedOperation {
operation: "token counting",
provider: provider.as_str(),
})
})??;
let sanitized = config.shape_request(
MessagesRequest {
model: config.count_tokens_model(&model).to_string(),
params: body.params.countable(),
..body
},
shaping.settings.shape_options(),
)?;
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(merge_messages_header_sources([
extra_headers.unwrap_or_default(),
scoped,
])))?;
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
&trimmed.model,
&trimmed.params,
&connection_params,
&env_lookup,
)?;
let transformed = config.transform_anthropic_messages_request(
with_a_max_tokens(trimmed),
&MessagesTransformContext {
drops_unsupported_params: shaping.settings.drops_unsupported_params,
..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params)
},
)?;
let environment = ValidatedEnvironment {
headers: config.headers_for_url(
config.request_headers(
with_python_default_headers(validated.headers, config.default_headers()),
&transformed,
),
&url,
),
auth: validated.auth,
};
let authenticated =
resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?;
let wire_body = MessagesRequest {
params: MessagesOptionalParams {
max_tokens: None,
..transformed.params
},
..transformed
};
let timeout = timeout
.or_else(|| shaping.settings.request_timeout())
.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
let response = send(
&route.http,
authenticated,
&url,
Bytes::from(serde_json::to_vec(&wire_body).map_err(serialize_failure)?),
timeout,
)
.await?;
if !response.status().is_success() {
return Err(provider_failure(response, timeout, false).await?.into());
}
let text = read_body(response, timeout).await?;
serde_json::from_str(&text).map_err(|err| {
Error::invalid_response(litellm_llms::ErrorDetail::invalid(
"count_tokens response JSON",
err,
))
})
}
fn with_a_max_tokens(request: MessagesRequest) -> MessagesRequest {
MessagesRequest {
params: MessagesOptionalParams {
max_tokens: request
.params
.max_tokens
.or(Some(Recognized::Known(UNCAPPED_MAX_TOKENS))),
..request.params
},
..request
}
}
fn with_requested_output_format(
body: &MessagesRequest,
requested: Option<Value>,

View file

@ -16,6 +16,7 @@ use litellm_auth::AuthServices;
use litellm_host::interceptors::Interceptors;
use litellm_inference::context::CallContext;
use litellm_inference::provider::resolve_llm_provider;
use litellm_llms_types::formats::messages::CountTokensResponse;
use litellm_llms_types::recognized::Recognized;
use litellm_secrets::source::SecretSource;
use std::sync::Arc;
@ -189,6 +190,26 @@ impl MessagesRoute {
.await
}
#[tracing::instrument(name = "litellm.route", skip_all, fields(
route = "messages_count_tokens",
model = %call.body.model,
provider,
resolved_model,
stream = false,
outcome
))]
pub async fn count_tokens(&self, call: MessagesCall) -> Result<CountTokensResponse, Error> {
litellm_inference::diagnostic::unary(async {
let (config, model, provider) = Self::provider(&call)?;
litellm_inference::diagnostic::provider(&model, provider);
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
handler::count_tokens(self, config, model, call, secrets.as_ref())
.boxed()
.await
})
.await
}
fn provider(
call: &MessagesCall,
) -> Result<(common_utils::MessagesProvider, String, &'static str), Error> {

View file

@ -0,0 +1,312 @@
use litellm_llms::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers};
use rstest::rstest;
use super::*;
fn without_max_tokens(model: &str, fields: Value) -> MessagesRequest {
let messages = json!({"model": model, "messages": [{"role": "user", "content": "hi"}]});
body(Value::Object(
object(messages).into_iter().chain(object(fields)).collect(),
))
}
async fn count_on_the_wire(call: MessagesCall) -> Value {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect("count succeeds");
only_request(&upstream).await.json()
}
#[rstest]
#[tokio::test]
async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model(
call: MessagesCall,
) {
let upstream = upstream([json_response(json!({"input_tokens": 14}))]).await;
let messages = json!([{"role": "user", "content": "hi"}]);
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: body(json!({"model": "claude-sonnet-4-5@20250929", "messages": messages})),
custom_llm_provider: Some("vertex_ai".into()),
litellm_params: serde_json::from_value(json!({
"model": "vertex_ai/claude-sonnet-4-5@20250929",
"vertex_project": "proj",
"vertex_location": "europe-west1",
}))
.unwrap(),
api_base: Some(upstream.uri()),
extra_headers: headers([
("Authorization", "Bearer caller-token"),
("anthropic-version", "2023-06-01"),
]),
..call
})
.await
.expect("count succeeds");
assert_eq!(counted.input_tokens, 14);
let request = only_request(&upstream).await;
assert_eq!(
request.url.path(),
"/v1/projects/proj/locations/europe-west1/publishers/anthropic/models/count-tokens:rawPredict"
);
assert_eq!(request.header("authorization"), Some("Bearer caller-token"));
assert_eq!(request.header("anthropic-version"), None);
assert_eq!(
request.json(),
json!({"model": "claude-sonnet-4-5", "messages": messages})
);
}
#[rstest]
#[tokio::test]
async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) {
let upstream = upstream([json_response(json!({"tokens": 14}))]).await;
let error = messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.unwrap_err();
assert!(error.is_response(), "{error:?}");
}
#[rstest]
#[case::max_tokens(json!({"max_tokens": 1024}))]
#[case::stream(json!({"stream": true}))]
#[case::temperature(json!({"temperature": 0.5}))]
#[case::top_p(json!({"top_p": 0.5}))]
#[case::top_k(json!({"top_k": 5}))]
#[case::stop_sequences(json!({"stop_sequences": ["end"]}))]
#[case::metadata_the_messages_route_would_refuse(json!({"metadata": {"user_id": 7}}))]
#[case::speed(json!({"speed": "fast"}))]
#[case::service_tier(json!({"service_tier": "auto"}))]
#[case::container(json!({"container": "container_1"}))]
#[case::inference_geo(json!({"inference_geo": "us"}))]
#[case::mcp_servers(json!({"mcp_servers": [{"type": "url", "name": "docs", "url": "https://mcp.test"}]}))]
#[case::compaction(json!({"compaction": {"type": "summarize"}}))]
#[case::output_format(json!({"output_format": {"type": "json_schema"}}))]
#[case::field_no_provider_knows(json!({"litellm_metadata": {"tags": ["a"]}}))]
#[tokio::test]
async fn a_count_sends_only_what_the_count_endpoint_reads(
call: MessagesCall,
#[case] fields: Value,
) {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: without_max_tokens(MODEL, fields),
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_sampling_params: false,
..MessagesModelCapabilities::default()
},
..MessagesShaping::default()
},
..call
})
.await
.expect("count succeeds");
let request = only_request(&upstream).await;
assert_eq!(request.header("anthropic-beta"), None);
assert_eq!(
request.json(),
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
);
}
#[rstest]
#[case::system(json!({"system": "be brief"}))]
#[case::tools(json!({"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]}))]
#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))]
#[case::thinking(json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}))]
#[case::output_config(json!({"output_config": {"format": {"type": "json_schema"}}}))]
#[case::context_management(json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}))]
#[tokio::test]
async fn a_count_keeps_what_the_count_endpoint_reads(call: MessagesCall, #[case] fields: Value) {
let expected = without_max_tokens(MODEL, fields.clone());
let sent = count_on_the_wire(MessagesCall {
body: without_max_tokens(MODEL, fields),
..call
})
.await;
assert_eq!(sent, serde_json::to_value(expected).unwrap());
}
#[rstest]
#[tokio::test]
async fn a_count_asks_for_the_betas_of_the_fields_it_sends(call: MessagesCall) {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: without_max_tokens(
MODEL,
json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}),
),
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect("count succeeds");
assert_eq!(
only_request(&upstream).await.header("anthropic-beta"),
Some("context-management-2025-06-27")
);
}
#[rstest]
#[case::no_ceiling_keeps_the_whole_budget(json!({"reasoning_effort": "high"}), 4096)]
#[case::the_callers_ceiling_caps_the_budget(
json!({"reasoning_effort": "high", "max_tokens": 3000}),
2999
)]
#[tokio::test]
async fn a_count_translates_reasoning_like_the_messages_route(
call: MessagesCall,
#[case] fields: Value,
#[case] budget_tokens: u64,
) {
let sent = count_on_the_wire(MessagesCall {
body: without_max_tokens(MODEL, fields),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
..MessagesModelCapabilities::default()
},
..MessagesShaping::default()
},
..call
})
.await;
assert_eq!(
sent,
json!({
"model": MODEL,
"messages": [{"role": "user", "content": "hi"}],
"thinking": {"type": "enabled", "budget_tokens": budget_tokens},
})
);
}
#[rstest]
#[tokio::test]
async fn a_count_on_an_adaptive_model_sends_the_effort_the_messages_route_sends(
call: MessagesCall,
) {
let sent = count_on_the_wire(MessagesCall {
body: without_max_tokens(MODEL, json!({"reasoning_effort": "high"})),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: true,
supports_output_config: true,
effort_tiers: SupportedEffortTiers {
high: true,
..SupportedEffortTiers::default()
},
..MessagesModelCapabilities::default()
},
..MessagesShaping::default()
},
..call
})
.await;
assert_eq!(sent["thinking"]["type"], "adaptive");
assert_eq!(sent["output_config"]["effort"], "high");
assert_eq!(sent.get("reasoning_effort"), None);
}
#[rstest]
#[tokio::test]
async fn a_vertex_count_lifts_the_system_role_and_strips_the_scope_vertex_refuses(
call: MessagesCall,
) {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: body(json!({
"model": "claude-sonnet-4-5",
"messages": [
{"role": "system", "content": "be brief"},
{"role": "user", "content": [{
"type": "text",
"text": "hi",
"cache_control": {"type": "ephemeral", "scope": "global"},
}]},
],
})),
custom_llm_provider: Some("vertex_ai".into()),
litellm_params: serde_json::from_value(json!({
"model": "vertex_ai/claude-sonnet-4-5",
"vertex_project": "proj",
}))
.unwrap(),
api_base: Some(upstream.uri()),
extra_headers: headers([("Authorization", "Bearer caller-token")]),
..call
})
.await
.expect("count succeeds");
let sent = only_request(&upstream).await.json();
assert_eq!(
sent["system"],
json!([{"type": "text", "text": "be brief"}])
);
assert_eq!(
sent["messages"],
json!([{"role": "user", "content": [{
"type": "text",
"text": "hi",
"cache_control": {"type": "ephemeral"},
}]}])
);
}
#[rstest]
#[tokio::test]
async fn azure_ai_counts_at_its_anthropic_count_endpoint(call: MessagesCall) {
let upstream = upstream([json_response(json!({"input_tokens": 9}))]).await;
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
custom_llm_provider: Some("azure_ai".into()),
api_key: Some("azure-key".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect("count succeeds");
assert_eq!(counted.input_tokens, 9);
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/anthropic/v1/messages/count_tokens");
assert_eq!(request.header("x-api-key"), Some("azure-key"));
assert_eq!(
request.json(),
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
);
}

View file

@ -19,6 +19,7 @@ use wiremock::ResponseTemplate;
mod support;
use support::*;
mod count_tokens;
mod gateway;
mod host;
mod pre_request;

View file

@ -25,7 +25,7 @@
## Providers
- A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers/<provider>/`
- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
- Projections that only one adapter decodes (`InvokeChunkPayload`, the batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
- A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it
- Provider types may use format types. Format types never depend on provider types

View file

@ -19,6 +19,7 @@
# references
- https://platform.claude.com/docs/en/api/http/messages/create
- https://platform.claude.com/docs/en/api/http/messages/count_tokens
- https://platform.claude.com/docs/en/api/messages/create.md
- https://platform.claude.com/docs/en/api/beta/messages/create.md
- https://platform.claude.com/docs/en/build-with-claude/streaming.md

View file

@ -383,6 +383,20 @@ pub struct MessagesOptionalParams {
}
impl MessagesOptionalParams {
pub fn countable(self) -> Self {
Self {
max_tokens: self.max_tokens,
system: self.system,
tools: self.tools,
tool_choice: self.tool_choice,
thinking: self.thinking,
context_management: self.context_management,
output_config: self.output_config,
reasoning_effort: self.reasoning_effort,
..Self::default()
}
}
pub fn streams(&self) -> bool {
self.stream.as_ref().is_some_and(Recognized::is_truthy)
}

View file

@ -27,6 +27,13 @@ pub struct MessagesResponse {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CountTokensResponse {
pub input_tokens: u64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
fn present_value<'de, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<Option<Value>, D::Error> {

View file

@ -1,6 +1,6 @@
# rules
- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers
- Shared across Anthropic's Messages, chat and batches adapters: API base, endpoint paths, header names, API version and default headers
- `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one
- Choosing which betas a request needs is policy in `llms/src/anthropic/`

View file

@ -1,7 +1,6 @@
pub const API_BASE: &str = "https://api.anthropic.com";
pub const MESSAGES_PATH: &str = "/v1/messages";
pub const BATCHES_PATH: &str = "/v1/messages/batches";
pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens";
pub const API_KEY_HEADER: &str = "x-api-key";
pub const BETA_HEADER: &str = "anthropic-beta";

View file

@ -1,6 +1,7 @@
pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com";
pub const GLOBAL_LOCATION: &str = "global";
pub const DEFAULT_LOCATION: &str = "us-central1";
pub const COUNT_TOKENS_DEFAULT_LOCATION: &str = "us-east5";
pub const RAW_PREDICT: &str = "rawPredict";
pub const STREAM_RAW_PREDICT: &str = "streamRawPredict";

View file

@ -1,7 +0,0 @@
# rules
- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections
# references
- https://platform.claude.com/docs/en/api/http/messages/count_tokens

View file

@ -1 +0,0 @@
pub mod transformation;

View file

@ -1,171 +0,0 @@
use litellm_llms_types::{
formats::messages::{Message, SystemPrompt},
providers::anthropic::{
API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
},
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX};
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicCountTokensRequest {
pub model: String,
pub messages: Vec<Message>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub system: Option<SystemPrompt>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AnthropicCountTokensResponse {
pub input_tokens: u64,
}
pub trait AnthropicCountTokensConfig {
fn endpoint(&self) -> String;
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>;
fn transform_request(
&self,
model: &str,
messages: Vec<Message>,
tools: Option<Vec<Value>>,
system: Option<SystemPrompt>,
) -> Result<AnthropicCountTokensRequest, Error>;
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)>;
}
pub struct AnthropicCountTokensTransformation;
pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformation =
AnthropicCountTokensTransformation;
impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
fn endpoint(&self) -> String {
format!("{API_BASE}{COUNT_TOKENS_PATH}")
}
fn transform_request(
&self,
model: &str,
messages: Vec<Message>,
tools: Option<Vec<Value>>,
system: Option<SystemPrompt>,
) -> Result<AnthropicCountTokensRequest, Error> {
self.validate_request(model, &messages)?;
Ok(AnthropicCountTokensRequest {
model: model.to_string(),
messages,
tools,
system,
})
}
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error> {
if model.is_empty() {
return Err(Error::MissingField("model"));
}
if messages.is_empty() {
return Err(Error::MissingField("messages"));
}
Ok(())
}
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)> {
let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) {
("authorization", format!("Bearer {api_key}"))
} else {
(API_KEY_HEADER, api_key.to_string())
};
vec![
("content-type", "application/json".to_string()),
auth,
(VERSION_HEADER, API_VERSION.to_string()),
(BETA_HEADER, TOKEN_COUNTING_BETA.to_string()),
]
}
}
#[cfg(test)]
mod tests {
use litellm_llms_types::formats::messages::MessageContent;
use serde_json::{Map, json};
use super::*;
fn message() -> Message {
Message {
role: "user".into(),
content: MessageContent::Text("hello".into()),
extra: Map::new(),
}
}
#[test]
fn maps_the_python_count_tokens_contract() {
let request = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION
.transform_request(
"claude-test",
vec![message()],
Some(vec![json!({"name": "lookup"})]),
Some(SystemPrompt::Text("system".into())),
)
.unwrap();
assert_eq!(
serde_json::to_value(request).unwrap(),
json!({
"model": "claude-test",
"messages": [{"role": "user", "content": "hello"}],
"tools": [{"name": "lookup"}],
"system": "system"
})
);
assert_eq!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(),
"https://api.anthropic.com/v1/messages/count_tokens"
);
}
#[test]
fn rejects_the_invalid_requests_python_rejects() {
assert!(matches!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
"",
vec![message()],
None,
None
),
Err(Error::MissingField("model"))
));
assert!(matches!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
"claude-test",
vec![],
None,
None
),
Err(Error::MissingField("messages"))
));
}
#[test]
fn uses_api_key_or_oauth_headers_without_combining_credentials() {
let api_key = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-api");
assert!(api_key.contains(&("x-api-key", "sk-ant-api".into())));
assert!(!api_key.iter().any(|(name, _)| *name == "authorization"));
let oauth = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-oat-test");
assert!(oauth.contains(&("authorization", "Bearer sk-ant-oat-test".into())));
assert!(!oauth.iter().any(|(name, _)| *name == "x-api-key"));
assert!(oauth.contains(&("anthropic-beta", TOKEN_COUNTING_BETA.into())));
}
}

View file

@ -67,6 +67,18 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
Ok(complete_anthropic_url(api_base, env_lookup))
}
fn get_count_tokens_url(
&self,
api_base: Option<&str>,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
Some(Ok(format!(
"{}/count_tokens",
complete_anthropic_url(api_base, env_lookup)
)))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
@ -1281,6 +1293,34 @@ mod tests {
);
}
#[rstest]
#[case::default_base(None, &[], "https://api.anthropic.com/v1/messages/count_tokens")]
#[case::env_base(
None,
&[(ANTHROPIC_API_BASE_ENV, "https://env.example.com")],
"https://env.example.com/v1/messages/count_tokens"
)]
#[case::custom_base(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages/count_tokens")]
#[case::complete_messages_endpoint(
Some("https://proxy.internal/v1/messages"),
&[],
"https://proxy.internal/v1/messages/count_tokens"
)]
fn count_tokens_url_is_under_the_messages_endpoint(
#[case] api_base: Option<&str>,
#[case] vars: Env,
#[case] expected: &str,
) {
assert_eq!(
ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
api_base,
&LitellmParams::default(),
&env(vars)
),
Some(Ok(expected.to_string()))
);
}
fn betas(values: &[&str]) -> BetaSet {
values.join(",").parse().unwrap()
}

View file

@ -3,7 +3,6 @@ pub mod common_utils;
pub mod batches;
pub mod chat;
pub mod count_tokens;
pub mod messages;
pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat";

View file

@ -74,6 +74,18 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
complete_azure_anthropic_url(api_base, env_lookup)
}
fn get_count_tokens_url(
&self,
api_base: Option<&str>,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
Some(
complete_azure_anthropic_url(api_base, env_lookup)
.map(|messages_url| format!("{messages_url}/count_tokens")),
)
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
@ -218,6 +230,31 @@ mod tests {
);
}
#[rstest]
#[case::resource_root("https://resource.services.ai.azure.com")]
#[case::anthropic_segment("https://resource.services.ai.azure.com/anthropic/")]
#[case::complete_messages_endpoint(
"https://resource.services.ai.azure.com/anthropic/v1/messages"
)]
fn count_url_sits_under_the_messages_endpoint(#[case] api_base: &str) {
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
.get_count_tokens_url(Some(api_base), &LitellmParams::default(), &|_| None)
.expect("azure ai has a count api")
.expect("url builds");
assert_eq!(
url,
"https://resource.services.ai.azure.com/anthropic/v1/messages/count_tokens"
);
}
#[test]
fn count_url_without_a_base_is_an_error() {
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
.get_count_tokens_url(None, &LitellmParams::default(), &|_| None)
.expect("azure ai has a count api");
assert!(url.is_err(), "{url:?}");
}
#[test]
fn url_falls_back_to_env_then_errors_when_absent() {
let with_env = |key: &str| {

View file

@ -41,6 +41,19 @@ pub trait BaseMessagesConfig: Sync {
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error>;
fn get_count_tokens_url(
&self,
_api_base: Option<&str>,
_litellm_params: &LitellmParams,
_env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
None
}
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
model
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,

View file

@ -207,6 +207,11 @@ pub enum ErrorDetail {
HostCall,
#[error("{0} is not handled by the rust path")]
Unhandled(&'static str),
#[error("{operation} is not supported for provider {provider}")]
UnsupportedOperation {
operation: &'static str,
provider: &'static str,
},
#[error("invalid {subject} on line {line}")]
InvalidLine {
subject: &'static str,

View file

@ -3,7 +3,9 @@
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
- Counts tokens at the publisher's `count-tokens:rawPredict`, which rejects a versioned model, so the body's model loses its `@` suffix. The location is `vertex_count_tokens_location`, then the deployment's location, then `us-east5`, as Python's partner-model token counter picks it
# references
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens

View file

@ -34,10 +34,12 @@ use crate::{
vertex_ai::common_utils::get_vertex_base_url,
};
use litellm_llms_types::providers::vertex_ai::{
ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT,
ANTHROPIC_VERSION, COUNT_TOKENS_DEFAULT_LOCATION, DEFAULT_LOCATION, RAW_PREDICT,
STREAM_RAW_PREDICT,
};
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
const COUNT_TOKENS_LOCATION_PARAM: &str = "vertex_count_tokens_location";
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
@ -82,14 +84,25 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let config = VertexConfig::from_params(&litellm_params.vertex);
let project = get_vertex_ai_project(&config, env_lookup)
.or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup))
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
let project = vertex_project(&config, env_lookup)?;
let location = get_vertex_ai_location(&config, env_lookup)
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
complete_vertex_anthropic_url(api_base, &project, &location, model, stream)
}
fn get_count_tokens_url(
&self,
api_base: Option<&str>,
litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
Some(count_tokens_url(api_base, litellm_params, env_lookup))
}
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
model.split_once('@').map_or(model, |(name, _)| name)
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
@ -229,6 +242,40 @@ pub fn complete_vertex_anthropic_url(
})
}
fn vertex_project(
config: &VertexConfig,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
get_vertex_ai_project(config, env_lookup)
.or_else(|| get_vertex_ai_project_from_credentials(config, env_lookup))
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))
}
fn count_tokens_url(
api_base: Option<&str>,
litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let config = VertexConfig::from_params(&litellm_params.vertex);
let project = vertex_project(&config, env_lookup)?;
let location = litellm_params
.extra
.get(COUNT_TOKENS_LOCATION_PARAM)
.and_then(Value::as_str)
.filter(|location| !location.is_empty())
.map(str::to_string)
.or_else(|| get_vertex_ai_location(&config, env_lookup))
.unwrap_or_else(|| COUNT_TOKENS_DEFAULT_LOCATION.to_string());
let base = match api_base.map(str::trim).filter(|base| !base.is_empty()) {
Some(custom) => custom.to_string(),
None => get_vertex_base_url(&location)?,
};
Ok(format!(
"{}/v1/projects/{project}/locations/{location}/publishers/anthropic/models/count-tokens:{RAW_PREDICT}",
base.trim_end_matches('/')
))
}
/// Python's `sanitize_vertex_anthropic_output_params`: `effort` only reaches models whose
/// capabilities accept it, a malformed `output_config` is dropped, and an emptied one is omitted.
fn sanitize_output_config(
@ -476,6 +523,105 @@ mod tests {
);
}
fn count_tokens_params(fields: Value) -> LitellmParams {
let (Value::Object(base), Value::Object(fields)) = (
json!({"model": "vertex_ai/claude-sonnet-4-5", "vertex_project": "proj"}),
fields,
) else {
panic!("litellm params are a json object");
};
serde_json::from_value(Value::Object(base.into_iter().chain(fields).collect())).unwrap()
}
const COUNT_PATH: &str = "/publishers/anthropic/models/count-tokens:rawPredict";
#[rstest]
#[case::count_location_wins(
None,
json!({"vertex_count_tokens_location": "europe-west1", "vertex_location": "asia-southeast1"}),
&[],
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
)]
#[case::deployment_location(
None,
json!({"vertex_location": "asia-southeast1"}),
&[],
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
)]
#[case::empty_count_location_is_unset(
None,
json!({"vertex_count_tokens_location": "", "vertex_location": "asia-southeast1"}),
&[],
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
)]
#[case::environment_location(
None,
json!({}),
&[("VERTEXAI_LOCATION", "europe-west1")],
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
)]
#[case::count_default_location(
None,
json!({}),
&[],
"https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5"
)]
#[case::global(
None,
json!({"vertex_location": "global"}),
&[],
"https://aiplatform.googleapis.com/v1/projects/proj/locations/global"
)]
#[case::custom_base(
Some("https://proxy.example/prefix/"),
json!({"vertex_location": "global"}),
&[],
"https://proxy.example/prefix/v1/projects/proj/locations/global"
)]
fn count_tokens_url_follows_the_python_location_precedence(
#[case] api_base: Option<&str>,
#[case] fields: Value,
#[case] env: &[(&str, &str)],
#[case] prefix: &str,
) {
let lookup = |name: &str| {
env.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
};
assert_eq!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
api_base,
&count_tokens_params(fields),
&lookup
),
Some(Ok(format!("{prefix}{COUNT_PATH}")))
);
}
#[rstest]
fn count_tokens_without_a_project_fails_before_sending() {
assert!(matches!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
None,
&LitellmParams::default(),
&no_env
),
Some(Err(Error::Auth(_)))
));
}
#[rstest]
#[case::versioned("claude-sonnet-4-5@20250929", "claude-sonnet-4-5")]
#[case::default_version("claude-sonnet-4-5@default", "claude-sonnet-4-5")]
#[case::unversioned("claude-sonnet-4-5", "claude-sonnet-4-5")]
fn count_tokens_model_drops_the_version(#[case] model: &str, #[case] expected: &str) {
assert_eq!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.count_tokens_model(model),
expected
);
}
#[rstest]
fn without_a_project_anywhere_the_call_fails_before_sending() {
let error = url(None, &LitellmParams::default(), false, &no_env).unwrap_err();