mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(rust-gateway): serve /v1/messages/count_tokens through the provider's count API
The gateway answered 404 for POST /v1/messages/count_tokens, so the
Anthropic SDK's messages.count_tokens failed while messages.create worked
The route now goes through the same deployment lookup, model access check,
and error envelope as /v1/messages, then forwards the caller's Anthropic
body with the deployment model to the provider's count-tokens API, using
the credentials and auth the Messages path already resolves. Anthropic
counts at {api_base}/v1/messages/count_tokens. Vertex AI counts at the
publisher count-tokens:rawPredict endpoint with the @version suffix
stripped and Python's location order (vertex_count_tokens_location, then
the deployment location, then us-east5). Other providers get a 400
invalid_request_error that names the provider
The unused Anthropic count-tokens adapter and its path constant are gone
This commit is contained in:
parent
4212f1af27
commit
ca1b5f2840
23 changed files with 538 additions and 210 deletions
|
|
@ -91,6 +91,7 @@ impl Gateway {
|
|||
pub fn router(gateway: Arc<Gateway>) -> Router {
|
||||
Router::new()
|
||||
.route("/v1/messages", post(messages::create))
|
||||
.route("/v1/messages/count_tokens", post(messages::count_tokens))
|
||||
.route("/ocr", post(ocr::create))
|
||||
.route("/v1/ocr", post(ocr::create))
|
||||
.route("/chat/completions", post(chat_completions::create))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it.
|
||||
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it, and
|
||||
//! `POST /v1/messages/count_tokens`, answered by the deployment's own count-tokens API.
|
||||
|
||||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -12,7 +13,10 @@ use axum::{
|
|||
};
|
||||
use litellm_host_http::Sse;
|
||||
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
|
||||
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::CountTokensResponse,
|
||||
headers::{ProviderSpecificHeader, ProviderSpecificHeaders},
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
|
||||
|
|
@ -67,6 +71,34 @@ async fn handle(
|
|||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
pub async fn count_tokens(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
RequestId(request_id): RequestId,
|
||||
headers: HeaderMap,
|
||||
body: Result<JsonObject, Error>,
|
||||
) -> impl IntoResponse {
|
||||
let result = match body {
|
||||
Ok(JsonObject(body)) => count(&gateway, &identity, &headers, body).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
result
|
||||
.map(Json)
|
||||
.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
|
||||
}
|
||||
|
||||
async fn count(
|
||||
gateway: &Gateway,
|
||||
identity: &AuthenticatedRequest,
|
||||
headers: &HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<CountTokensResponse, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
let call = project(deployment, body, headers)?;
|
||||
Ok(gateway.messages.count_tokens(call).await?)
|
||||
}
|
||||
|
||||
fn project(
|
||||
deployment: &Deployment,
|
||||
body: Map<String, Value>,
|
||||
|
|
|
|||
|
|
@ -121,18 +121,20 @@ async fn a_stream_that_fails_after_opening_ends_with_an_sse_error_frame() {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[case::count_tokens("/v1/messages/count_tokens")]
|
||||
#[tokio::test]
|
||||
async fn hosted_provider_failure_preserves_status_body_and_request_id() {
|
||||
async fn hosted_provider_failure_preserves_status_body_and_request_id(#[case] route: &str) {
|
||||
let upstream = MockServer::start().await;
|
||||
let error =
|
||||
json!({"type": "error", "error": {"type": "rate_limit_error", "message": "retry later"}});
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages"))
|
||||
.and(path(route))
|
||||
.respond_with(ResponseTemplate::new(429).set_body_json(error.clone()))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let request = Request::post("/v1/messages")
|
||||
let request = Request::post(route)
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", "host-http-request")
|
||||
.body(Body::from(json!({
|
||||
|
|
@ -150,8 +152,90 @@ async fn hosted_provider_failure_preserves_status_body_and_request_id() {
|
|||
assert_eq!(body["request_id"], "host-http-request");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model() {
|
||||
let upstream = MockServer::start().await;
|
||||
let messages = json!([{"role": "user", "content": "hi"}]);
|
||||
let tools = json!([{"name": "lookup", "input_schema": {"type": "object"}}]);
|
||||
let counted = json!({"input_tokens": 14, "context_management": {"original_input_tokens": 20}});
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages/count_tokens"))
|
||||
.and(header("x-api-key", "test-key"))
|
||||
.and(header("anthropic-version", "test-version"))
|
||||
.and(body_json(json!({
|
||||
"model": "test-model", "messages": messages, "system": "be brief", "tools": tools,
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(counted.clone()))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let request = Request::post("/v1/messages/count_tokens")
|
||||
.header("content-type", "application/json")
|
||||
.header("anthropic-version", "test-version")
|
||||
.body(Body::from(
|
||||
json!({"model": "public/model", "messages": messages, "system": "be brief", "tools": tools})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
let response = support::app("anthropic/test-model", &upstream.uri())
|
||||
.oneshot(request)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(support::json(response).await, counted);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn count_tokens_for_a_provider_without_a_count_api_is_an_invalid_request() {
|
||||
let upstream = MockServer::start().await;
|
||||
let response = support::post(
|
||||
support::app("deepseek/test-model", &upstream.uri()),
|
||||
"/v1/messages/count_tokens",
|
||||
json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["type"], "error");
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|message| message.contains("deepseek")),
|
||||
"{body}"
|
||||
);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_unknown_model_is_refused_alike_on_messages_and_count_tokens(
|
||||
#[values("/v1/messages", "/v1/messages/count_tokens")] route: &str,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
let response = support::post(
|
||||
support::app("anthropic/test-model", &upstream.uri()),
|
||||
route,
|
||||
json!({"model": "other/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|message| message.contains("other/model")),
|
||||
"{body}"
|
||||
);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[case::count_tokens("/v1/messages/count_tokens")]
|
||||
#[case::chat("/v1/chat/completions")]
|
||||
#[case::model_path("/engines/path-model/chat/completions")]
|
||||
#[case::ocr("/ocr")]
|
||||
|
|
|
|||
|
|
@ -235,6 +235,7 @@ async fn upload_audio_format_validation_matches_core(
|
|||
#[case::responses("/v1/responses", false)]
|
||||
#[case::deployment("/openai/deployments/public%2Fmodel/chat/completions", false)]
|
||||
#[case::messages("/v1/messages", true)]
|
||||
#[case::count_tokens("/v1/messages/count_tokens", true)]
|
||||
#[case::ocr("/v1/ocr", false)]
|
||||
#[case::transcription("/v1/audio/transcriptions", false)]
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ use litellm_llms::base_llm::{
|
|||
transformation::BaseMessagesConfig,
|
||||
},
|
||||
};
|
||||
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
|
||||
use litellm_llms_types::formats::messages::{
|
||||
CountTokensResponse, MessagesRequest, MessagesResponse,
|
||||
};
|
||||
use litellm_secrets::source::Secrets;
|
||||
use litellm_tracing::ByteChunk;
|
||||
use serde_json::Value;
|
||||
|
|
@ -211,6 +213,92 @@ async fn provider_call(
|
|||
})
|
||||
}
|
||||
|
||||
pub(super) async fn count_tokens(
|
||||
route: &MessagesRoute,
|
||||
provider: MessagesProvider,
|
||||
model: String,
|
||||
call: MessagesCall,
|
||||
secrets: &(dyn Lookup + Send + Sync),
|
||||
) -> Result<CountTokensResponse, Error> {
|
||||
let MessagesCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
litellm_params,
|
||||
extra_headers,
|
||||
provider_specific_header,
|
||||
timeout,
|
||||
shaping,
|
||||
..
|
||||
} = call;
|
||||
let connection_params = litellm_router_types::LitellmParams {
|
||||
api_base: api_base.clone().or(litellm_params.api_base.clone()),
|
||||
..litellm_params
|
||||
};
|
||||
let config = provider.config();
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
let url = config
|
||||
.get_count_tokens_url(api_base.as_deref(), &connection_params, &env_lookup)
|
||||
.ok_or_else(|| {
|
||||
Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedOperation {
|
||||
operation: "token counting",
|
||||
provider: provider.as_str(),
|
||||
})
|
||||
})??;
|
||||
|
||||
let sanitized = config.shape_request(
|
||||
MessagesRequest {
|
||||
model: config.count_tokens_model(&model).to_string(),
|
||||
..body
|
||||
},
|
||||
shaping.settings.reasoning_auto_summary,
|
||||
)?;
|
||||
let request =
|
||||
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
|
||||
|
||||
let scoped =
|
||||
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
|
||||
let forwarded = string_headers(Some(
|
||||
extra_headers.into_iter().flatten().chain(scoped).collect(),
|
||||
))?;
|
||||
let validated = config.validate_environment(
|
||||
forwarded,
|
||||
api_key.as_deref(),
|
||||
&request.model,
|
||||
&request.params,
|
||||
&connection_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.request_headers(
|
||||
with_default_headers(validated.headers, config.default_headers()),
|
||||
&request,
|
||||
),
|
||||
auth: validated.auth,
|
||||
};
|
||||
let authenticated =
|
||||
resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?;
|
||||
|
||||
let response = send(
|
||||
&route.http,
|
||||
authenticated,
|
||||
&url,
|
||||
&serde_json::to_value(&request).map_err(serialize_failure)?,
|
||||
timeout.or_else(|| shaping.settings.request_timeout()),
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(provider_error(response).await);
|
||||
}
|
||||
let text = response.text().await.map_err(litellm_http::Error::from)?;
|
||||
serde_json::from_str(&text).map_err(|err| {
|
||||
Error::invalid_response(litellm_llms::ErrorDetail::invalid(
|
||||
"count_tokens response JSON",
|
||||
err,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn without_additional_drop_params(
|
||||
request: MessagesRequest,
|
||||
paths: &[String],
|
||||
|
|
|
|||
|
|
@ -9,10 +9,11 @@ use litellm_auth::AuthServices;
|
|||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_inference::context::CallContext;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use litellm_llms_types::formats::messages::CountTokensResponse;
|
||||
use litellm_secrets::source::{SecretSource, Secrets};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::common_utils::messages_provider;
|
||||
use crate::common_utils::{MessagesProvider, messages_provider};
|
||||
|
||||
pub use litellm_inference::Error;
|
||||
pub use types::{
|
||||
|
|
@ -75,24 +76,48 @@ impl MessagesRoute {
|
|||
context: CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<MessagesCallResponse, Error> {
|
||||
litellm_inference::diagnostic::call(async {
|
||||
let provider = resolve_llm_provider(
|
||||
&call.body.model,
|
||||
call.custom_llm_provider.as_deref(),
|
||||
"messages",
|
||||
)?;
|
||||
let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
|
||||
Error::InvalidProvider(<&str>::from(provider.provider).to_string())
|
||||
})?;
|
||||
litellm_inference::diagnostic::provider(
|
||||
provider.model,
|
||||
<&'static str>::from(provider.provider),
|
||||
);
|
||||
let model = provider.model.to_owned();
|
||||
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
|
||||
handler::execute(self, config, model, call, secrets, &context)
|
||||
let (provider, model, secrets) = self.resolve_provider(&call).await?;
|
||||
handler::execute(self, provider, model, call, secrets, &context)
|
||||
.boxed()
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "messages_count_tokens",
|
||||
model = %call.body.model,
|
||||
provider,
|
||||
resolved_model,
|
||||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
pub async fn count_tokens(&self, call: MessagesCall) -> Result<CountTokensResponse, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let (provider, model, secrets) = self.resolve_provider(&call).await?;
|
||||
handler::count_tokens(self, provider, model, call, secrets.as_ref())
|
||||
.boxed()
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn resolve_provider(
|
||||
&self,
|
||||
call: &MessagesCall,
|
||||
) -> Result<(MessagesProvider, String, Secrets), Error> {
|
||||
let provider = resolve_llm_provider(
|
||||
&call.body.model,
|
||||
call.custom_llm_provider.as_deref(),
|
||||
"messages",
|
||||
)?;
|
||||
let config = messages_provider(provider.provider, provider.model)
|
||||
.ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
|
||||
litellm_inference::diagnostic::provider(
|
||||
provider.model,
|
||||
<&'static str>::from(provider.provider),
|
||||
);
|
||||
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
|
||||
Ok((config, provider.model.to_owned(), secrets))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 14}))]).await;
|
||||
let messages = json!([{"role": "user", "content": "hi"}]);
|
||||
|
||||
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: body(json!({"model": "claude-sonnet-4-5@20250929", "messages": messages})),
|
||||
custom_llm_provider: Some("vertex_ai".into()),
|
||||
litellm_params: serde_json::from_value(json!({
|
||||
"model": "vertex_ai/claude-sonnet-4-5@20250929",
|
||||
"vertex_project": "proj",
|
||||
"vertex_location": "europe-west1",
|
||||
}))
|
||||
.unwrap(),
|
||||
api_base: Some(upstream.uri()),
|
||||
extra_headers: headers([
|
||||
("Authorization", "Bearer caller-token"),
|
||||
("anthropic-version", "2023-06-01"),
|
||||
]),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
assert_eq!(counted.input_tokens, 14);
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(
|
||||
request.url.path(),
|
||||
"/v1/projects/proj/locations/europe-west1/publishers/anthropic/models/count-tokens:rawPredict"
|
||||
);
|
||||
assert_eq!(request.header("authorization"), Some("Bearer caller-token"));
|
||||
assert_eq!(request.header("anthropic-version"), None);
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"model": "claude-sonnet-4-5", "messages": messages})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) {
|
||||
let upstream = upstream([json_response(json!({"tokens": 14}))]).await;
|
||||
|
||||
let error = messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.is_response(), "{error:?}");
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ use wiremock::ResponseTemplate;
|
|||
mod support;
|
||||
use support::*;
|
||||
|
||||
mod count_tokens;
|
||||
mod host;
|
||||
mod request;
|
||||
mod response;
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@
|
|||
## Providers
|
||||
|
||||
- A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers/<provider>/`
|
||||
- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
|
||||
- Projections that only one adapter decodes (`InvokeChunkPayload`, the batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
|
||||
- A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it
|
||||
- Provider types may use format types. Format types never depend on provider types
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
# references
|
||||
|
||||
- https://platform.claude.com/docs/en/api/http/messages/create
|
||||
- https://platform.claude.com/docs/en/api/http/messages/count_tokens
|
||||
- https://platform.claude.com/docs/en/api/messages/create.md
|
||||
- https://platform.claude.com/docs/en/api/beta/messages/create.md
|
||||
- https://platform.claude.com/docs/en/build-with-claude/streaming.md
|
||||
|
|
|
|||
|
|
@ -18,6 +18,13 @@ pub struct MessagesResponse {
|
|||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct CountTokensResponse {
|
||||
pub input_tokens: u64,
|
||||
#[serde(flatten)]
|
||||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# rules
|
||||
|
||||
- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers
|
||||
- Shared across Anthropic's Messages, chat and batches adapters: API base, endpoint paths, header names, API version and default headers
|
||||
- `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one
|
||||
- Choosing which betas a request needs is policy in `llms/src/anthropic/`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
pub const API_BASE: &str = "https://api.anthropic.com";
|
||||
pub const MESSAGES_PATH: &str = "/v1/messages";
|
||||
pub const BATCHES_PATH: &str = "/v1/messages/batches";
|
||||
pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens";
|
||||
|
||||
pub const API_KEY_HEADER: &str = "x-api-key";
|
||||
pub const BETA_HEADER: &str = "anthropic-beta";
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com";
|
||||
pub const GLOBAL_LOCATION: &str = "global";
|
||||
pub const DEFAULT_LOCATION: &str = "us-central1";
|
||||
pub const COUNT_TOKENS_DEFAULT_LOCATION: &str = "us-east5";
|
||||
|
||||
pub const RAW_PREDICT: &str = "rawPredict";
|
||||
pub const STREAM_RAW_PREDICT: &str = "streamRawPredict";
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
# rules
|
||||
|
||||
- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections
|
||||
|
||||
# references
|
||||
|
||||
- https://platform.claude.com/docs/en/api/http/messages/count_tokens
|
||||
|
|
@ -1 +0,0 @@
|
|||
pub mod transformation;
|
||||
|
|
@ -1,171 +0,0 @@
|
|||
use litellm_llms_types::{
|
||||
formats::messages::{Message, SystemPrompt},
|
||||
providers::anthropic::{
|
||||
API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
|
||||
},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX};
|
||||
|
||||
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct AnthropicCountTokensRequest {
|
||||
pub model: String,
|
||||
pub messages: Vec<Message>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<Value>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system: Option<SystemPrompt>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AnthropicCountTokensResponse {
|
||||
pub input_tokens: u64,
|
||||
}
|
||||
|
||||
pub trait AnthropicCountTokensConfig {
|
||||
fn endpoint(&self) -> String;
|
||||
|
||||
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>;
|
||||
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: Vec<Message>,
|
||||
tools: Option<Vec<Value>>,
|
||||
system: Option<SystemPrompt>,
|
||||
) -> Result<AnthropicCountTokensRequest, Error>;
|
||||
|
||||
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)>;
|
||||
}
|
||||
|
||||
pub struct AnthropicCountTokensTransformation;
|
||||
|
||||
pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformation =
|
||||
AnthropicCountTokensTransformation;
|
||||
|
||||
impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
|
||||
fn endpoint(&self) -> String {
|
||||
format!("{API_BASE}{COUNT_TOKENS_PATH}")
|
||||
}
|
||||
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: Vec<Message>,
|
||||
tools: Option<Vec<Value>>,
|
||||
system: Option<SystemPrompt>,
|
||||
) -> Result<AnthropicCountTokensRequest, Error> {
|
||||
self.validate_request(model, &messages)?;
|
||||
|
||||
Ok(AnthropicCountTokensRequest {
|
||||
model: model.to_string(),
|
||||
messages,
|
||||
tools,
|
||||
system,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error> {
|
||||
if model.is_empty() {
|
||||
return Err(Error::MissingField("model"));
|
||||
}
|
||||
if messages.is_empty() {
|
||||
return Err(Error::MissingField("messages"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)> {
|
||||
let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) {
|
||||
("authorization", format!("Bearer {api_key}"))
|
||||
} else {
|
||||
(API_KEY_HEADER, api_key.to_string())
|
||||
};
|
||||
vec![
|
||||
("content-type", "application/json".to_string()),
|
||||
auth,
|
||||
(VERSION_HEADER, API_VERSION.to_string()),
|
||||
(BETA_HEADER, TOKEN_COUNTING_BETA.to_string()),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_llms_types::formats::messages::MessageContent;
|
||||
use serde_json::{Map, json};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn message() -> Message {
|
||||
Message {
|
||||
role: "user".into(),
|
||||
content: MessageContent::Text("hello".into()),
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_the_python_count_tokens_contract() {
|
||||
let request = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION
|
||||
.transform_request(
|
||||
"claude-test",
|
||||
vec![message()],
|
||||
Some(vec![json!({"name": "lookup"})]),
|
||||
Some(SystemPrompt::Text("system".into())),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(request).unwrap(),
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"tools": [{"name": "lookup"}],
|
||||
"system": "system"
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(),
|
||||
"https://api.anthropic.com/v1/messages/count_tokens"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_the_invalid_requests_python_rejects() {
|
||||
assert!(matches!(
|
||||
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
|
||||
"",
|
||||
vec![message()],
|
||||
None,
|
||||
None
|
||||
),
|
||||
Err(Error::MissingField("model"))
|
||||
));
|
||||
assert!(matches!(
|
||||
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
|
||||
"claude-test",
|
||||
vec![],
|
||||
None,
|
||||
None
|
||||
),
|
||||
Err(Error::MissingField("messages"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_api_key_or_oauth_headers_without_combining_credentials() {
|
||||
let api_key = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-api");
|
||||
assert!(api_key.contains(&("x-api-key", "sk-ant-api".into())));
|
||||
assert!(!api_key.iter().any(|(name, _)| *name == "authorization"));
|
||||
|
||||
let oauth = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-oat-test");
|
||||
assert!(oauth.contains(&("authorization", "Bearer sk-ant-oat-test".into())));
|
||||
assert!(!oauth.iter().any(|(name, _)| *name == "x-api-key"));
|
||||
assert!(oauth.contains(&("anthropic-beta", TOKEN_COUNTING_BETA.into())));
|
||||
}
|
||||
}
|
||||
|
|
@ -50,6 +50,18 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
|
|||
Ok(complete_anthropic_url(api_base, env_lookup))
|
||||
}
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
Some(Ok(format!(
|
||||
"{}/count_tokens",
|
||||
complete_anthropic_url(api_base, env_lookup)
|
||||
)))
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
@ -875,6 +887,34 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default_base(None, &[], "https://api.anthropic.com/v1/messages/count_tokens")]
|
||||
#[case::env_base(
|
||||
None,
|
||||
&[(ANTHROPIC_API_BASE_ENV, "https://env.example.com")],
|
||||
"https://env.example.com/v1/messages/count_tokens"
|
||||
)]
|
||||
#[case::custom_base(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages/count_tokens")]
|
||||
#[case::complete_messages_endpoint(
|
||||
Some("https://proxy.internal/v1/messages"),
|
||||
&[],
|
||||
"https://proxy.internal/v1/messages/count_tokens"
|
||||
)]
|
||||
fn count_tokens_url_is_under_the_messages_endpoint(
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] vars: Env,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
|
||||
api_base,
|
||||
&LitellmParams::default(),
|
||||
&env(vars)
|
||||
),
|
||||
Some(Ok(expected.to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
fn betas(values: &[&str]) -> BetaSet {
|
||||
values.join(",").parse().unwrap()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ pub mod common_utils;
|
|||
|
||||
pub mod batches;
|
||||
pub mod chat;
|
||||
pub mod count_tokens;
|
||||
pub mod messages;
|
||||
|
||||
pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat";
|
||||
|
|
|
|||
|
|
@ -27,6 +27,19 @@ pub trait BaseMessagesConfig: Sync {
|
|||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
_api_base: Option<&str>,
|
||||
_litellm_params: &LitellmParams,
|
||||
_env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
None
|
||||
}
|
||||
|
||||
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
|
||||
model
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
|
|||
|
|
@ -166,6 +166,11 @@ pub enum ErrorDetail {
|
|||
HostCall,
|
||||
#[error("{0} is not handled by the rust path")]
|
||||
Unhandled(&'static str),
|
||||
#[error("{operation} is not supported for provider {provider}")]
|
||||
UnsupportedOperation {
|
||||
operation: &'static str,
|
||||
provider: &'static str,
|
||||
},
|
||||
#[error("invalid {subject} on line {line}")]
|
||||
InvalidLine {
|
||||
subject: &'static str,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
|
||||
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
|
||||
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
|
||||
- Counts tokens at the publisher's `count-tokens:rawPredict`, which rejects a versioned model, so the body's model loses its `@` suffix. The location is `vertex_count_tokens_location`, then the deployment's location, then `us-east5`, as Python's partner-model token counter picks it
|
||||
|
||||
# references
|
||||
|
||||
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude
|
||||
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
|
||||
|
|
|
|||
|
|
@ -32,10 +32,12 @@ use crate::{
|
|||
vertex_ai::common_utils::get_vertex_base_url,
|
||||
};
|
||||
use litellm_llms_types::providers::vertex_ai::{
|
||||
ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT,
|
||||
ANTHROPIC_VERSION, COUNT_TOKENS_DEFAULT_LOCATION, DEFAULT_LOCATION, RAW_PREDICT,
|
||||
STREAM_RAW_PREDICT,
|
||||
};
|
||||
|
||||
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
|
||||
const COUNT_TOKENS_LOCATION_PARAM: &str = "vertex_count_tokens_location";
|
||||
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
|
||||
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
|
||||
|
||||
|
|
@ -64,14 +66,25 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
let config = VertexConfig::from_params(&litellm_params.vertex);
|
||||
let project = get_vertex_ai_project(&config, env_lookup)
|
||||
.or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup))
|
||||
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
|
||||
let project = vertex_project(&config, env_lookup)?;
|
||||
let location = get_vertex_ai_location(&config, env_lookup)
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
complete_vertex_anthropic_url(api_base, &project, &location, model, stream)
|
||||
}
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
Some(count_tokens_url(api_base, litellm_params, env_lookup))
|
||||
}
|
||||
|
||||
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
|
||||
model.split_once('@').map_or(model, |(name, _)| name)
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
@ -211,6 +224,40 @@ pub fn complete_vertex_anthropic_url(
|
|||
})
|
||||
}
|
||||
|
||||
fn vertex_project(
|
||||
config: &VertexConfig,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
get_vertex_ai_project(config, env_lookup)
|
||||
.or_else(|| get_vertex_ai_project_from_credentials(config, env_lookup))
|
||||
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))
|
||||
}
|
||||
|
||||
fn count_tokens_url(
|
||||
api_base: Option<&str>,
|
||||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
let config = VertexConfig::from_params(&litellm_params.vertex);
|
||||
let project = vertex_project(&config, env_lookup)?;
|
||||
let location = litellm_params
|
||||
.extra
|
||||
.get(COUNT_TOKENS_LOCATION_PARAM)
|
||||
.and_then(Value::as_str)
|
||||
.filter(|location| !location.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_vertex_ai_location(&config, env_lookup))
|
||||
.unwrap_or_else(|| COUNT_TOKENS_DEFAULT_LOCATION.to_string());
|
||||
let base = match api_base.map(str::trim).filter(|base| !base.is_empty()) {
|
||||
Some(custom) => custom.to_string(),
|
||||
None => get_vertex_base_url(&location)?,
|
||||
};
|
||||
Ok(format!(
|
||||
"{}/v1/projects/{project}/locations/{location}/publishers/anthropic/models/count-tokens:{RAW_PREDICT}",
|
||||
base.trim_end_matches('/')
|
||||
))
|
||||
}
|
||||
|
||||
/// Python's `sanitize_vertex_anthropic_output_params`: `effort` only reaches models whose
|
||||
/// capabilities accept it, a malformed `output_config` is dropped, and an emptied one is omitted.
|
||||
fn sanitize_output_config(
|
||||
|
|
@ -458,6 +505,105 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
fn count_tokens_params(fields: Value) -> LitellmParams {
|
||||
let (Value::Object(base), Value::Object(fields)) = (
|
||||
json!({"model": "vertex_ai/claude-sonnet-4-5", "vertex_project": "proj"}),
|
||||
fields,
|
||||
) else {
|
||||
panic!("litellm params are a json object");
|
||||
};
|
||||
serde_json::from_value(Value::Object(base.into_iter().chain(fields).collect())).unwrap()
|
||||
}
|
||||
|
||||
const COUNT_PATH: &str = "/publishers/anthropic/models/count-tokens:rawPredict";
|
||||
|
||||
#[rstest]
|
||||
#[case::count_location_wins(
|
||||
None,
|
||||
json!({"vertex_count_tokens_location": "europe-west1", "vertex_location": "asia-southeast1"}),
|
||||
&[],
|
||||
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
|
||||
)]
|
||||
#[case::deployment_location(
|
||||
None,
|
||||
json!({"vertex_location": "asia-southeast1"}),
|
||||
&[],
|
||||
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
|
||||
)]
|
||||
#[case::empty_count_location_is_unset(
|
||||
None,
|
||||
json!({"vertex_count_tokens_location": "", "vertex_location": "asia-southeast1"}),
|
||||
&[],
|
||||
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
|
||||
)]
|
||||
#[case::environment_location(
|
||||
None,
|
||||
json!({}),
|
||||
&[("VERTEXAI_LOCATION", "europe-west1")],
|
||||
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
|
||||
)]
|
||||
#[case::count_default_location(
|
||||
None,
|
||||
json!({}),
|
||||
&[],
|
||||
"https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5"
|
||||
)]
|
||||
#[case::global(
|
||||
None,
|
||||
json!({"vertex_location": "global"}),
|
||||
&[],
|
||||
"https://aiplatform.googleapis.com/v1/projects/proj/locations/global"
|
||||
)]
|
||||
#[case::custom_base(
|
||||
Some("https://proxy.example/prefix/"),
|
||||
json!({"vertex_location": "global"}),
|
||||
&[],
|
||||
"https://proxy.example/prefix/v1/projects/proj/locations/global"
|
||||
)]
|
||||
fn count_tokens_url_follows_the_python_location_precedence(
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] fields: Value,
|
||||
#[case] env: &[(&str, &str)],
|
||||
#[case] prefix: &str,
|
||||
) {
|
||||
let lookup = |name: &str| {
|
||||
env.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
};
|
||||
assert_eq!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
|
||||
api_base,
|
||||
&count_tokens_params(fields),
|
||||
&lookup
|
||||
),
|
||||
Some(Ok(format!("{prefix}{COUNT_PATH}")))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn count_tokens_without_a_project_fails_before_sending() {
|
||||
assert!(matches!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
|
||||
None,
|
||||
&LitellmParams::default(),
|
||||
&no_env
|
||||
),
|
||||
Some(Err(Error::Auth(_)))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::versioned("claude-sonnet-4-5@20250929", "claude-sonnet-4-5")]
|
||||
#[case::default_version("claude-sonnet-4-5@default", "claude-sonnet-4-5")]
|
||||
#[case::unversioned("claude-sonnet-4-5", "claude-sonnet-4-5")]
|
||||
fn count_tokens_model_drops_the_version(#[case] model: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.count_tokens_model(model),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn without_a_project_anywhere_the_call_fails_before_sending() {
|
||||
let error = url(None, &LitellmParams::default(), false, &no_env).unwrap_err();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue