mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(rust-gateway): serve /v1/messages/count_tokens through the provider's count API
The gateway answered 404 for POST /v1/messages/count_tokens, so the
Anthropic SDK's messages.count_tokens failed while messages.create worked
The route goes through the same deployment lookup, model access check and
error envelope as /v1/messages. It keeps only the fields the count
endpoints read, runs the same provider transform as /v1/messages, drops
max_tokens from the wire, and sends the result to the provider's count
API with the create path's header defaults and upstream error mapping.
Anthropic counts at {api_base}/v1/messages/count_tokens, Azure AI under
its Anthropic messages endpoint, and Vertex AI at the publisher
count-tokens:rawPredict endpoint with the @version suffix stripped and
Python's location order. Other providers get a 400 invalid_request_error
that names the provider. The response carries litellm_count_source like
the Python proxy's
The unused Anthropic count-tokens adapter and its path constant are gone
This commit is contained in:
parent
90122c07a2
commit
fddadf3e29
25 changed files with 871 additions and 193 deletions
|
|
@ -91,6 +91,7 @@ impl Gateway {
|
|||
pub fn router(gateway: Arc<Gateway>) -> Router {
|
||||
Router::new()
|
||||
.route("/v1/messages", post(messages::create))
|
||||
.route("/v1/messages/count_tokens", post(messages::count_tokens))
|
||||
.route("/ocr", post(ocr::create))
|
||||
.route("/v1/ocr", post(ocr::create))
|
||||
.route("/chat/completions", post(chat_completions::create))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it.
|
||||
//! `POST /v1/messages`, as the Python proxy's `anthropic_response` serves it, and
|
||||
//! `POST /v1/messages/count_tokens`, answered by the deployment's own count-tokens API.
|
||||
|
||||
use litellm_gateway_auth::AuthenticatedRequest;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -12,7 +13,11 @@ use axum::{
|
|||
};
|
||||
use litellm_host_http::Sse;
|
||||
use litellm_inference_messages::{MessagesCall, messages_body, route::Messages};
|
||||
use litellm_llms_types::headers::{ProviderSpecificHeader, ProviderSpecificHeaders};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::CountTokensResponse,
|
||||
headers::{ProviderSpecificHeader, ProviderSpecificHeaders},
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
|
||||
|
|
@ -25,6 +30,14 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
|
|||
];
|
||||
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
|
||||
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
|
||||
const PROVIDER_COUNT_SOURCE: &str = "provider";
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SourcedCount {
|
||||
#[serde(flatten)]
|
||||
count: CountTokensResponse,
|
||||
litellm_count_source: &'static str,
|
||||
}
|
||||
|
||||
pub async fn create(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
|
|
@ -67,6 +80,37 @@ async fn handle(
|
|||
Ok(headers.apply(response))
|
||||
}
|
||||
|
||||
pub async fn count_tokens(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
identity: AuthenticatedRequest,
|
||||
RequestId(request_id): RequestId,
|
||||
headers: HeaderMap,
|
||||
body: Result<JsonObject, Error>,
|
||||
) -> impl IntoResponse {
|
||||
let result = match body {
|
||||
Ok(JsonObject(body)) => count(&gateway, &identity, &headers, body).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
result
|
||||
.map(Json)
|
||||
.map_err(|error| (error.status(), Json(error.body(request_id.as_deref()))))
|
||||
}
|
||||
|
||||
async fn count(
|
||||
gateway: &Gateway,
|
||||
identity: &AuthenticatedRequest,
|
||||
headers: &HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<SourcedCount, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
let call = project(deployment, body, headers)?;
|
||||
Ok(SourcedCount {
|
||||
count: gateway.messages.count_tokens(call).await?,
|
||||
litellm_count_source: PROVIDER_COUNT_SOURCE,
|
||||
})
|
||||
}
|
||||
|
||||
fn project(
|
||||
deployment: &Deployment,
|
||||
body: Map<String, Value>,
|
||||
|
|
|
|||
|
|
@ -121,18 +121,20 @@ async fn a_stream_that_fails_after_opening_ends_with_an_sse_error_frame() {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[case::count_tokens("/v1/messages/count_tokens")]
|
||||
#[tokio::test]
|
||||
async fn hosted_provider_failure_preserves_status_body_and_request_id() {
|
||||
async fn hosted_provider_failure_preserves_status_body_and_request_id(#[case] route: &str) {
|
||||
let upstream = MockServer::start().await;
|
||||
let error =
|
||||
json!({"type": "error", "error": {"type": "rate_limit_error", "message": "retry later"}});
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages"))
|
||||
.and(path(route))
|
||||
.respond_with(ResponseTemplate::new(429).set_body_json(error.clone()))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let request = Request::post("/v1/messages")
|
||||
let request = Request::post(route)
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", "host-http-request")
|
||||
.body(Body::from(json!({
|
||||
|
|
@ -150,8 +152,97 @@ async fn hosted_provider_failure_preserves_status_body_and_request_id() {
|
|||
assert_eq!(body["request_id"], "host-http-request");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model() {
|
||||
let upstream = MockServer::start().await;
|
||||
let messages = json!([{"role": "user", "content": "hi"}]);
|
||||
let tools = json!([{"name": "lookup", "input_schema": {"type": "object"}}]);
|
||||
let counted = json!({"input_tokens": 14, "context_management": {"original_input_tokens": 20}});
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/messages/count_tokens"))
|
||||
.and(header("x-api-key", "test-key"))
|
||||
.and(header("anthropic-version", "test-version"))
|
||||
.and(body_json(json!({
|
||||
"model": "test-model", "messages": messages, "system": "be brief", "tools": tools,
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(counted))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let request = Request::post("/v1/messages/count_tokens")
|
||||
.header("content-type", "application/json")
|
||||
.header("anthropic-version", "test-version")
|
||||
.body(Body::from(
|
||||
json!({"model": "public/model", "messages": messages, "system": "be brief", "tools": tools})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
let response = support::app("anthropic/test-model", &upstream.uri())
|
||||
.oneshot(request)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(
|
||||
support::json(response).await,
|
||||
json!({
|
||||
"input_tokens": 14,
|
||||
"context_management": {"original_input_tokens": 20},
|
||||
"litellm_count_source": "provider",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn count_tokens_for_a_provider_without_a_count_api_is_an_invalid_request() {
|
||||
let upstream = MockServer::start().await;
|
||||
let response = support::post(
|
||||
support::app("deepseek/test-model", &upstream.uri()),
|
||||
"/v1/messages/count_tokens",
|
||||
json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["type"], "error");
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|message| message.contains("deepseek")),
|
||||
"{body}"
|
||||
);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_unknown_model_is_refused_alike_on_messages_and_count_tokens(
|
||||
#[values("/v1/messages", "/v1/messages/count_tokens")] route: &str,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
let response = support::post(
|
||||
support::app("anthropic/test-model", &upstream.uri()),
|
||||
route,
|
||||
json!({"model": "other/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 400);
|
||||
let body = support::json(response).await;
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.is_some_and(|message| message.contains("other/model")),
|
||||
"{body}"
|
||||
);
|
||||
assert!(upstream.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::messages("/v1/messages")]
|
||||
#[case::count_tokens("/v1/messages/count_tokens")]
|
||||
#[case::chat("/v1/chat/completions")]
|
||||
#[case::model_path("/engines/path-model/chat/completions")]
|
||||
#[case::ocr("/ocr")]
|
||||
|
|
|
|||
|
|
@ -235,6 +235,7 @@ async fn upload_audio_format_validation_matches_core(
|
|||
#[case::responses("/v1/responses", false)]
|
||||
#[case::deployment("/openai/deployments/public%2Fmodel/chat/completions", false)]
|
||||
#[case::messages("/v1/messages", true)]
|
||||
#[case::count_tokens("/v1/messages/count_tokens", true)]
|
||||
#[case::ocr("/v1/ocr", false)]
|
||||
#[case::transcription("/v1/audio/transcriptions", false)]
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -22,8 +22,9 @@ use litellm_llms::base_llm::{
|
|||
},
|
||||
};
|
||||
use litellm_llms_types::formats::messages::{
|
||||
MessagesOptionalParams, MessagesRequest, MessagesResponse,
|
||||
CountTokensResponse, MessagesOptionalParams, MessagesRequest, MessagesResponse,
|
||||
};
|
||||
use litellm_llms_types::recognized::Recognized;
|
||||
use litellm_secrets::source::Secrets;
|
||||
use litellm_tracing::ByteChunk;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
@ -44,6 +45,8 @@ use litellm_inference::{
|
|||
outbound::{raw_leaf_body, serialized_outbound_request},
|
||||
};
|
||||
|
||||
const UNCAPPED_MAX_TOKENS: u64 = u64::MAX;
|
||||
|
||||
struct ProviderCall {
|
||||
identity: ProviderIdentity,
|
||||
wire: WireRequest,
|
||||
|
|
@ -389,6 +392,125 @@ async fn provider_call(
|
|||
})
|
||||
}
|
||||
|
||||
pub(super) async fn count_tokens(
|
||||
route: &MessagesRoute,
|
||||
provider: MessagesProvider,
|
||||
model: String,
|
||||
call: MessagesCall,
|
||||
secrets: &(dyn Lookup + Send + Sync),
|
||||
) -> Result<CountTokensResponse, Error> {
|
||||
let MessagesCall {
|
||||
body,
|
||||
api_key,
|
||||
api_base,
|
||||
litellm_params,
|
||||
extra_headers,
|
||||
provider_specific_header,
|
||||
timeout,
|
||||
shaping,
|
||||
..
|
||||
} = call;
|
||||
let connection_params = litellm_router_types::LitellmParams {
|
||||
api_base: api_base.clone().or(litellm_params.api_base.clone()),
|
||||
..litellm_params
|
||||
};
|
||||
let config = provider.config();
|
||||
let env_lookup = |key: &str| secrets.get(key);
|
||||
let url = config
|
||||
.get_count_tokens_url(api_base.as_deref(), &connection_params, &env_lookup)
|
||||
.ok_or_else(|| {
|
||||
Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedOperation {
|
||||
operation: "token counting",
|
||||
provider: provider.as_str(),
|
||||
})
|
||||
})??;
|
||||
|
||||
let sanitized = config.shape_request(
|
||||
MessagesRequest {
|
||||
model: config.count_tokens_model(&model).to_string(),
|
||||
params: body.params.countable(),
|
||||
..body
|
||||
},
|
||||
shaping.settings.shape_options(),
|
||||
)?;
|
||||
let trimmed =
|
||||
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
|
||||
let scoped =
|
||||
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
|
||||
let forwarded = string_headers(Some(merge_messages_header_sources([
|
||||
extra_headers.unwrap_or_default(),
|
||||
scoped,
|
||||
])))?;
|
||||
let validated = config.validate_environment(
|
||||
forwarded,
|
||||
api_key.as_deref(),
|
||||
&trimmed.model,
|
||||
&trimmed.params,
|
||||
&connection_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let transformed = config.transform_anthropic_messages_request(
|
||||
with_a_max_tokens(trimmed),
|
||||
&MessagesTransformContext {
|
||||
drops_unsupported_params: shaping.settings.drops_unsupported_params,
|
||||
..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params)
|
||||
},
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.headers_for_url(
|
||||
config.request_headers(
|
||||
with_python_default_headers(validated.headers, config.default_headers()),
|
||||
&transformed,
|
||||
),
|
||||
&url,
|
||||
),
|
||||
auth: validated.auth,
|
||||
};
|
||||
let authenticated =
|
||||
resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?;
|
||||
let wire_body = MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
max_tokens: None,
|
||||
..transformed.params
|
||||
},
|
||||
..transformed
|
||||
};
|
||||
let timeout = timeout
|
||||
.or_else(|| shaping.settings.request_timeout())
|
||||
.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
|
||||
let response = send(
|
||||
&route.http,
|
||||
authenticated,
|
||||
&url,
|
||||
Bytes::from(serde_json::to_vec(&wire_body).map_err(serialize_failure)?),
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(provider_failure(response, timeout, false).await?.into());
|
||||
}
|
||||
let text = read_body(response, timeout).await?;
|
||||
serde_json::from_str(&text).map_err(|err| {
|
||||
Error::invalid_response(litellm_llms::ErrorDetail::invalid(
|
||||
"count_tokens response JSON",
|
||||
err,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn with_a_max_tokens(request: MessagesRequest) -> MessagesRequest {
|
||||
MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
max_tokens: request
|
||||
.params
|
||||
.max_tokens
|
||||
.or(Some(Recognized::Known(UNCAPPED_MAX_TOKENS))),
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
}
|
||||
}
|
||||
|
||||
fn with_requested_output_format(
|
||||
body: &MessagesRequest,
|
||||
requested: Option<Value>,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ use litellm_auth::AuthServices;
|
|||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_inference::context::CallContext;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_llms_types::formats::messages::CountTokensResponse;
|
||||
use litellm_llms_types::recognized::Recognized;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use std::sync::Arc;
|
||||
|
|
@ -189,6 +190,26 @@ impl MessagesRoute {
|
|||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "litellm.route", skip_all, fields(
|
||||
route = "messages_count_tokens",
|
||||
model = %call.body.model,
|
||||
provider,
|
||||
resolved_model,
|
||||
stream = false,
|
||||
outcome
|
||||
))]
|
||||
pub async fn count_tokens(&self, call: MessagesCall) -> Result<CountTokensResponse, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let (config, model, provider) = Self::provider(&call)?;
|
||||
litellm_inference::diagnostic::provider(&model, provider);
|
||||
let secrets = self.secrets.resolve(config.config().secret_names()).await?;
|
||||
handler::count_tokens(self, config, model, call, secrets.as_ref())
|
||||
.boxed()
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn provider(
|
||||
call: &MessagesCall,
|
||||
) -> Result<(common_utils::MessagesProvider, String, &'static str), Error> {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,312 @@
|
|||
use litellm_llms::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn without_max_tokens(model: &str, fields: Value) -> MessagesRequest {
|
||||
let messages = json!({"model": model, "messages": [{"role": "user", "content": "hi"}]});
|
||||
body(Value::Object(
|
||||
object(messages).into_iter().chain(object(fields)).collect(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn count_on_the_wire(call: MessagesCall) -> Value {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
only_request(&upstream).await.json()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 14}))]).await;
|
||||
let messages = json!([{"role": "user", "content": "hi"}]);
|
||||
|
||||
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: body(json!({"model": "claude-sonnet-4-5@20250929", "messages": messages})),
|
||||
custom_llm_provider: Some("vertex_ai".into()),
|
||||
litellm_params: serde_json::from_value(json!({
|
||||
"model": "vertex_ai/claude-sonnet-4-5@20250929",
|
||||
"vertex_project": "proj",
|
||||
"vertex_location": "europe-west1",
|
||||
}))
|
||||
.unwrap(),
|
||||
api_base: Some(upstream.uri()),
|
||||
extra_headers: headers([
|
||||
("Authorization", "Bearer caller-token"),
|
||||
("anthropic-version", "2023-06-01"),
|
||||
]),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
assert_eq!(counted.input_tokens, 14);
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(
|
||||
request.url.path(),
|
||||
"/v1/projects/proj/locations/europe-west1/publishers/anthropic/models/count-tokens:rawPredict"
|
||||
);
|
||||
assert_eq!(request.header("authorization"), Some("Bearer caller-token"));
|
||||
assert_eq!(request.header("anthropic-version"), None);
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"model": "claude-sonnet-4-5", "messages": messages})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) {
|
||||
let upstream = upstream([json_response(json!({"tokens": 14}))]).await;
|
||||
|
||||
let error = messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.is_response(), "{error:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::max_tokens(json!({"max_tokens": 1024}))]
|
||||
#[case::stream(json!({"stream": true}))]
|
||||
#[case::temperature(json!({"temperature": 0.5}))]
|
||||
#[case::top_p(json!({"top_p": 0.5}))]
|
||||
#[case::top_k(json!({"top_k": 5}))]
|
||||
#[case::stop_sequences(json!({"stop_sequences": ["end"]}))]
|
||||
#[case::metadata_the_messages_route_would_refuse(json!({"metadata": {"user_id": 7}}))]
|
||||
#[case::speed(json!({"speed": "fast"}))]
|
||||
#[case::service_tier(json!({"service_tier": "auto"}))]
|
||||
#[case::container(json!({"container": "container_1"}))]
|
||||
#[case::inference_geo(json!({"inference_geo": "us"}))]
|
||||
#[case::mcp_servers(json!({"mcp_servers": [{"type": "url", "name": "docs", "url": "https://mcp.test"}]}))]
|
||||
#[case::compaction(json!({"compaction": {"type": "summarize"}}))]
|
||||
#[case::output_format(json!({"output_format": {"type": "json_schema"}}))]
|
||||
#[case::field_no_provider_knows(json!({"litellm_metadata": {"tags": ["a"]}}))]
|
||||
#[tokio::test]
|
||||
async fn a_count_sends_only_what_the_count_endpoint_reads(
|
||||
call: MessagesCall,
|
||||
#[case] fields: Value,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: without_max_tokens(MODEL, fields),
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
shaping: MessagesShaping {
|
||||
capabilities: MessagesModelCapabilities {
|
||||
supports_sampling_params: false,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.header("anthropic-beta"), None);
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::system(json!({"system": "be brief"}))]
|
||||
#[case::tools(json!({"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]}))]
|
||||
#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))]
|
||||
#[case::thinking(json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}))]
|
||||
#[case::output_config(json!({"output_config": {"format": {"type": "json_schema"}}}))]
|
||||
#[case::context_management(json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}))]
|
||||
#[tokio::test]
|
||||
async fn a_count_keeps_what_the_count_endpoint_reads(call: MessagesCall, #[case] fields: Value) {
|
||||
let expected = without_max_tokens(MODEL, fields.clone());
|
||||
|
||||
let sent = count_on_the_wire(MessagesCall {
|
||||
body: without_max_tokens(MODEL, fields),
|
||||
..call
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(sent, serde_json::to_value(expected).unwrap());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_count_asks_for_the_betas_of_the_fields_it_sends(call: MessagesCall) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: without_max_tokens(
|
||||
MODEL,
|
||||
json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}),
|
||||
),
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.header("anthropic-beta"),
|
||||
Some("context-management-2025-06-27")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_ceiling_keeps_the_whole_budget(json!({"reasoning_effort": "high"}), 4096)]
|
||||
#[case::the_callers_ceiling_caps_the_budget(
|
||||
json!({"reasoning_effort": "high", "max_tokens": 3000}),
|
||||
2999
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn a_count_translates_reasoning_like_the_messages_route(
|
||||
call: MessagesCall,
|
||||
#[case] fields: Value,
|
||||
#[case] budget_tokens: u64,
|
||||
) {
|
||||
let sent = count_on_the_wire(MessagesCall {
|
||||
body: without_max_tokens(MODEL, fields),
|
||||
shaping: MessagesShaping {
|
||||
capabilities: MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..call
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
sent,
|
||||
json!({
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": budget_tokens},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_count_on_an_adaptive_model_sends_the_effort_the_messages_route_sends(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let sent = count_on_the_wire(MessagesCall {
|
||||
body: without_max_tokens(MODEL, json!({"reasoning_effort": "high"})),
|
||||
shaping: MessagesShaping {
|
||||
capabilities: MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_adaptive_thinking: true,
|
||||
supports_output_config: true,
|
||||
effort_tiers: SupportedEffortTiers {
|
||||
high: true,
|
||||
..SupportedEffortTiers::default()
|
||||
},
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..call
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(sent["thinking"]["type"], "adaptive");
|
||||
assert_eq!(sent["output_config"]["effort"], "high");
|
||||
assert_eq!(sent.get("reasoning_effort"), None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_vertex_count_lifts_the_system_role_and_strips_the_scope_vertex_refuses(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: body(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"messages": [
|
||||
{"role": "system", "content": "be brief"},
|
||||
{"role": "user", "content": [{
|
||||
"type": "text",
|
||||
"text": "hi",
|
||||
"cache_control": {"type": "ephemeral", "scope": "global"},
|
||||
}]},
|
||||
],
|
||||
})),
|
||||
custom_llm_provider: Some("vertex_ai".into()),
|
||||
litellm_params: serde_json::from_value(json!({
|
||||
"model": "vertex_ai/claude-sonnet-4-5",
|
||||
"vertex_project": "proj",
|
||||
}))
|
||||
.unwrap(),
|
||||
api_base: Some(upstream.uri()),
|
||||
extra_headers: headers([("Authorization", "Bearer caller-token")]),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await.json();
|
||||
assert_eq!(
|
||||
sent["system"],
|
||||
json!([{"type": "text", "text": "be brief"}])
|
||||
);
|
||||
assert_eq!(
|
||||
sent["messages"],
|
||||
json!([{"role": "user", "content": [{
|
||||
"type": "text",
|
||||
"text": "hi",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}]}])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn azure_ai_counts_at_its_anthropic_count_endpoint(call: MessagesCall) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 9}))]).await;
|
||||
|
||||
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
custom_llm_provider: Some("azure_ai".into()),
|
||||
api_key: Some("azure-key".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
assert_eq!(counted.input_tokens, 9);
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.url.path(), "/anthropic/v1/messages/count_tokens");
|
||||
assert_eq!(request.header("x-api-key"), Some("azure-key"));
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
|
||||
);
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ use wiremock::ResponseTemplate;
|
|||
mod support;
|
||||
use support::*;
|
||||
|
||||
mod count_tokens;
|
||||
mod gateway;
|
||||
mod host;
|
||||
mod pre_request;
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@
|
|||
## Providers
|
||||
|
||||
- A provider with more than one endpoint format puts shared endpoint paths, default headers and wire types used by several of its formats in `providers/<provider>/`
|
||||
- Projections that only one adapter decodes (`InvokeChunkPayload`, the count-tokens and batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
|
||||
- Projections that only one adapter decodes (`InvokeChunkPayload`, the batches adapter structs, `ReplayedWebSearchResult`) stay in `llms/src/<provider>/<format>/`
|
||||
- A typed provider extension of a format (MiniMax media blocks) belongs here once a consumer must construct or interpret it
|
||||
- Provider types may use format types. Format types never depend on provider types
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
# references
|
||||
|
||||
- https://platform.claude.com/docs/en/api/http/messages/create
|
||||
- https://platform.claude.com/docs/en/api/http/messages/count_tokens
|
||||
- https://platform.claude.com/docs/en/api/messages/create.md
|
||||
- https://platform.claude.com/docs/en/api/beta/messages/create.md
|
||||
- https://platform.claude.com/docs/en/build-with-claude/streaming.md
|
||||
|
|
|
|||
|
|
@ -383,6 +383,20 @@ pub struct MessagesOptionalParams {
|
|||
}
|
||||
|
||||
impl MessagesOptionalParams {
|
||||
pub fn countable(self) -> Self {
|
||||
Self {
|
||||
max_tokens: self.max_tokens,
|
||||
system: self.system,
|
||||
tools: self.tools,
|
||||
tool_choice: self.tool_choice,
|
||||
thinking: self.thinking,
|
||||
context_management: self.context_management,
|
||||
output_config: self.output_config,
|
||||
reasoning_effort: self.reasoning_effort,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn streams(&self) -> bool {
|
||||
self.stream.as_ref().is_some_and(Recognized::is_truthy)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,13 @@ pub struct MessagesResponse {
|
|||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct CountTokensResponse {
|
||||
pub input_tokens: u64,
|
||||
#[serde(flatten)]
|
||||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
fn present_value<'de, D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Value>, D::Error> {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# rules
|
||||
|
||||
- Shared across Anthropic's Messages, chat, count-tokens and batches adapters: API base, endpoint paths, header names, API version and default headers
|
||||
- Shared across Anthropic's Messages, chat and batches adapters: API base, endpoint paths, header names, API version and default headers
|
||||
- `beta.rs` holds `AnthropicBeta`, `BetaSet` and `BetaProvider`, the `anthropic-beta` values, their wire spelling and which hosts accept each one
|
||||
- Choosing which betas a request needs is policy in `llms/src/anthropic/`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
pub const API_BASE: &str = "https://api.anthropic.com";
|
||||
pub const MESSAGES_PATH: &str = "/v1/messages";
|
||||
pub const BATCHES_PATH: &str = "/v1/messages/batches";
|
||||
pub const COUNT_TOKENS_PATH: &str = "/v1/messages/count_tokens";
|
||||
|
||||
pub const API_KEY_HEADER: &str = "x-api-key";
|
||||
pub const BETA_HEADER: &str = "anthropic-beta";
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
pub const GLOBAL_API_BASE: &str = "https://aiplatform.googleapis.com";
|
||||
pub const GLOBAL_LOCATION: &str = "global";
|
||||
pub const DEFAULT_LOCATION: &str = "us-central1";
|
||||
pub const COUNT_TOKENS_DEFAULT_LOCATION: &str = "us-east5";
|
||||
|
||||
pub const RAW_PREDICT: &str = "rawPredict";
|
||||
pub const STREAM_RAW_PREDICT: &str = "streamRawPredict";
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
# rules
|
||||
|
||||
- Builds the count-tokens request against the public Anthropic host. The request and response structs are adapter-only projections
|
||||
|
||||
# references
|
||||
|
||||
- https://platform.claude.com/docs/en/api/http/messages/count_tokens
|
||||
|
|
@ -1 +0,0 @@
|
|||
pub mod transformation;
|
||||
|
|
@ -1,171 +0,0 @@
|
|||
use litellm_llms_types::{
|
||||
formats::messages::{Message, SystemPrompt},
|
||||
providers::anthropic::{
|
||||
API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
|
||||
},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX};
|
||||
|
||||
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct AnthropicCountTokensRequest {
|
||||
pub model: String,
|
||||
pub messages: Vec<Message>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<Value>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system: Option<SystemPrompt>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AnthropicCountTokensResponse {
|
||||
pub input_tokens: u64,
|
||||
}
|
||||
|
||||
pub trait AnthropicCountTokensConfig {
|
||||
fn endpoint(&self) -> String;
|
||||
|
||||
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>;
|
||||
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: Vec<Message>,
|
||||
tools: Option<Vec<Value>>,
|
||||
system: Option<SystemPrompt>,
|
||||
) -> Result<AnthropicCountTokensRequest, Error>;
|
||||
|
||||
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)>;
|
||||
}
|
||||
|
||||
pub struct AnthropicCountTokensTransformation;
|
||||
|
||||
pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformation =
|
||||
AnthropicCountTokensTransformation;
|
||||
|
||||
impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
|
||||
fn endpoint(&self) -> String {
|
||||
format!("{API_BASE}{COUNT_TOKENS_PATH}")
|
||||
}
|
||||
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: Vec<Message>,
|
||||
tools: Option<Vec<Value>>,
|
||||
system: Option<SystemPrompt>,
|
||||
) -> Result<AnthropicCountTokensRequest, Error> {
|
||||
self.validate_request(model, &messages)?;
|
||||
|
||||
Ok(AnthropicCountTokensRequest {
|
||||
model: model.to_string(),
|
||||
messages,
|
||||
tools,
|
||||
system,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error> {
|
||||
if model.is_empty() {
|
||||
return Err(Error::MissingField("model"));
|
||||
}
|
||||
if messages.is_empty() {
|
||||
return Err(Error::MissingField("messages"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn required_headers(&self, api_key: &str) -> Vec<(&'static str, String)> {
|
||||
let auth = if api_key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) {
|
||||
("authorization", format!("Bearer {api_key}"))
|
||||
} else {
|
||||
(API_KEY_HEADER, api_key.to_string())
|
||||
};
|
||||
vec![
|
||||
("content-type", "application/json".to_string()),
|
||||
auth,
|
||||
(VERSION_HEADER, API_VERSION.to_string()),
|
||||
(BETA_HEADER, TOKEN_COUNTING_BETA.to_string()),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_llms_types::formats::messages::MessageContent;
|
||||
use serde_json::{Map, json};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn message() -> Message {
|
||||
Message {
|
||||
role: "user".into(),
|
||||
content: MessageContent::Text("hello".into()),
|
||||
extra: Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_the_python_count_tokens_contract() {
|
||||
let request = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION
|
||||
.transform_request(
|
||||
"claude-test",
|
||||
vec![message()],
|
||||
Some(vec![json!({"name": "lookup"})]),
|
||||
Some(SystemPrompt::Text("system".into())),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(request).unwrap(),
|
||||
json!({
|
||||
"model": "claude-test",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"tools": [{"name": "lookup"}],
|
||||
"system": "system"
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(),
|
||||
"https://api.anthropic.com/v1/messages/count_tokens"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_the_invalid_requests_python_rejects() {
|
||||
assert!(matches!(
|
||||
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
|
||||
"",
|
||||
vec![message()],
|
||||
None,
|
||||
None
|
||||
),
|
||||
Err(Error::MissingField("model"))
|
||||
));
|
||||
assert!(matches!(
|
||||
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.transform_request(
|
||||
"claude-test",
|
||||
vec![],
|
||||
None,
|
||||
None
|
||||
),
|
||||
Err(Error::MissingField("messages"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_api_key_or_oauth_headers_without_combining_credentials() {
|
||||
let api_key = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-api");
|
||||
assert!(api_key.contains(&("x-api-key", "sk-ant-api".into())));
|
||||
assert!(!api_key.iter().any(|(name, _)| *name == "authorization"));
|
||||
|
||||
let oauth = ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.required_headers("sk-ant-oat-test");
|
||||
assert!(oauth.contains(&("authorization", "Bearer sk-ant-oat-test".into())));
|
||||
assert!(!oauth.iter().any(|(name, _)| *name == "x-api-key"));
|
||||
assert!(oauth.contains(&("anthropic-beta", TOKEN_COUNTING_BETA.into())));
|
||||
}
|
||||
}
|
||||
|
|
@ -67,6 +67,18 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
|
|||
Ok(complete_anthropic_url(api_base, env_lookup))
|
||||
}
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
Some(Ok(format!(
|
||||
"{}/count_tokens",
|
||||
complete_anthropic_url(api_base, env_lookup)
|
||||
)))
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
@ -1281,6 +1293,34 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default_base(None, &[], "https://api.anthropic.com/v1/messages/count_tokens")]
|
||||
#[case::env_base(
|
||||
None,
|
||||
&[(ANTHROPIC_API_BASE_ENV, "https://env.example.com")],
|
||||
"https://env.example.com/v1/messages/count_tokens"
|
||||
)]
|
||||
#[case::custom_base(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages/count_tokens")]
|
||||
#[case::complete_messages_endpoint(
|
||||
Some("https://proxy.internal/v1/messages"),
|
||||
&[],
|
||||
"https://proxy.internal/v1/messages/count_tokens"
|
||||
)]
|
||||
fn count_tokens_url_is_under_the_messages_endpoint(
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] vars: Env,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
|
||||
api_base,
|
||||
&LitellmParams::default(),
|
||||
&env(vars)
|
||||
),
|
||||
Some(Ok(expected.to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
fn betas(values: &[&str]) -> BetaSet {
|
||||
values.join(",").parse().unwrap()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ pub mod common_utils;
|
|||
|
||||
pub mod batches;
|
||||
pub mod chat;
|
||||
pub mod count_tokens;
|
||||
pub mod messages;
|
||||
|
||||
pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat";
|
||||
|
|
|
|||
|
|
@ -74,6 +74,18 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
complete_azure_anthropic_url(api_base, env_lookup)
|
||||
}
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
Some(
|
||||
complete_azure_anthropic_url(api_base, env_lookup)
|
||||
.map(|messages_url| format!("{messages_url}/count_tokens")),
|
||||
)
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
@ -218,6 +230,31 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::resource_root("https://resource.services.ai.azure.com")]
|
||||
#[case::anthropic_segment("https://resource.services.ai.azure.com/anthropic/")]
|
||||
#[case::complete_messages_endpoint(
|
||||
"https://resource.services.ai.azure.com/anthropic/v1/messages"
|
||||
)]
|
||||
fn count_url_sits_under_the_messages_endpoint(#[case] api_base: &str) {
|
||||
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
|
||||
.get_count_tokens_url(Some(api_base), &LitellmParams::default(), &|_| None)
|
||||
.expect("azure ai has a count api")
|
||||
.expect("url builds");
|
||||
assert_eq!(
|
||||
url,
|
||||
"https://resource.services.ai.azure.com/anthropic/v1/messages/count_tokens"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_url_without_a_base_is_an_error() {
|
||||
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
|
||||
.get_count_tokens_url(None, &LitellmParams::default(), &|_| None)
|
||||
.expect("azure ai has a count api");
|
||||
assert!(url.is_err(), "{url:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_falls_back_to_env_then_errors_when_absent() {
|
||||
let with_env = |key: &str| {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,19 @@ pub trait BaseMessagesConfig: Sync {
|
|||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
_api_base: Option<&str>,
|
||||
_litellm_params: &LitellmParams,
|
||||
_env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
None
|
||||
}
|
||||
|
||||
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
|
||||
model
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
|
|||
|
|
@ -207,6 +207,11 @@ pub enum ErrorDetail {
|
|||
HostCall,
|
||||
#[error("{0} is not handled by the rust path")]
|
||||
Unhandled(&'static str),
|
||||
#[error("{operation} is not supported for provider {provider}")]
|
||||
UnsupportedOperation {
|
||||
operation: &'static str,
|
||||
provider: &'static str,
|
||||
},
|
||||
#[error("invalid {subject} on line {line}")]
|
||||
InvalidLine {
|
||||
subject: &'static str,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
|
||||
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
|
||||
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
|
||||
- Counts tokens at the publisher's `count-tokens:rawPredict`, which rejects a versioned model, so the body's model loses its `@` suffix. The location is `vertex_count_tokens_location`, then the deployment's location, then `us-east5`, as Python's partner-model token counter picks it
|
||||
|
||||
# references
|
||||
|
||||
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude
|
||||
- https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
|
||||
|
|
|
|||
|
|
@ -34,10 +34,12 @@ use crate::{
|
|||
vertex_ai::common_utils::get_vertex_base_url,
|
||||
};
|
||||
use litellm_llms_types::providers::vertex_ai::{
|
||||
ANTHROPIC_VERSION, DEFAULT_LOCATION, RAW_PREDICT, STREAM_RAW_PREDICT,
|
||||
ANTHROPIC_VERSION, COUNT_TOKENS_DEFAULT_LOCATION, DEFAULT_LOCATION, RAW_PREDICT,
|
||||
STREAM_RAW_PREDICT,
|
||||
};
|
||||
|
||||
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
|
||||
const COUNT_TOKENS_LOCATION_PARAM: &str = "vertex_count_tokens_location";
|
||||
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
|
||||
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
|
||||
|
||||
|
|
@ -82,14 +84,25 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
let config = VertexConfig::from_params(&litellm_params.vertex);
|
||||
let project = get_vertex_ai_project(&config, env_lookup)
|
||||
.or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup))
|
||||
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
|
||||
let project = vertex_project(&config, env_lookup)?;
|
||||
let location = get_vertex_ai_location(&config, env_lookup)
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
complete_vertex_anthropic_url(api_base, &project, &location, model, stream)
|
||||
}
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
Some(count_tokens_url(api_base, litellm_params, env_lookup))
|
||||
}
|
||||
|
||||
fn count_tokens_model<'a>(&self, model: &'a str) -> &'a str {
|
||||
model.split_once('@').map_or(model, |(name, _)| name)
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
@ -229,6 +242,40 @@ pub fn complete_vertex_anthropic_url(
|
|||
})
|
||||
}
|
||||
|
||||
fn vertex_project(
|
||||
config: &VertexConfig,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
get_vertex_ai_project(config, env_lookup)
|
||||
.or_else(|| get_vertex_ai_project_from_credentials(config, env_lookup))
|
||||
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))
|
||||
}
|
||||
|
||||
fn count_tokens_url(
|
||||
api_base: Option<&str>,
|
||||
litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
let config = VertexConfig::from_params(&litellm_params.vertex);
|
||||
let project = vertex_project(&config, env_lookup)?;
|
||||
let location = litellm_params
|
||||
.extra
|
||||
.get(COUNT_TOKENS_LOCATION_PARAM)
|
||||
.and_then(Value::as_str)
|
||||
.filter(|location| !location.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| get_vertex_ai_location(&config, env_lookup))
|
||||
.unwrap_or_else(|| COUNT_TOKENS_DEFAULT_LOCATION.to_string());
|
||||
let base = match api_base.map(str::trim).filter(|base| !base.is_empty()) {
|
||||
Some(custom) => custom.to_string(),
|
||||
None => get_vertex_base_url(&location)?,
|
||||
};
|
||||
Ok(format!(
|
||||
"{}/v1/projects/{project}/locations/{location}/publishers/anthropic/models/count-tokens:{RAW_PREDICT}",
|
||||
base.trim_end_matches('/')
|
||||
))
|
||||
}
|
||||
|
||||
/// Python's `sanitize_vertex_anthropic_output_params`: `effort` only reaches models whose
|
||||
/// capabilities accept it, a malformed `output_config` is dropped, and an emptied one is omitted.
|
||||
fn sanitize_output_config(
|
||||
|
|
@ -476,6 +523,105 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
fn count_tokens_params(fields: Value) -> LitellmParams {
|
||||
let (Value::Object(base), Value::Object(fields)) = (
|
||||
json!({"model": "vertex_ai/claude-sonnet-4-5", "vertex_project": "proj"}),
|
||||
fields,
|
||||
) else {
|
||||
panic!("litellm params are a json object");
|
||||
};
|
||||
serde_json::from_value(Value::Object(base.into_iter().chain(fields).collect())).unwrap()
|
||||
}
|
||||
|
||||
const COUNT_PATH: &str = "/publishers/anthropic/models/count-tokens:rawPredict";
|
||||
|
||||
#[rstest]
|
||||
#[case::count_location_wins(
|
||||
None,
|
||||
json!({"vertex_count_tokens_location": "europe-west1", "vertex_location": "asia-southeast1"}),
|
||||
&[],
|
||||
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
|
||||
)]
|
||||
#[case::deployment_location(
|
||||
None,
|
||||
json!({"vertex_location": "asia-southeast1"}),
|
||||
&[],
|
||||
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
|
||||
)]
|
||||
#[case::empty_count_location_is_unset(
|
||||
None,
|
||||
json!({"vertex_count_tokens_location": "", "vertex_location": "asia-southeast1"}),
|
||||
&[],
|
||||
"https://asia-southeast1-aiplatform.googleapis.com/v1/projects/proj/locations/asia-southeast1"
|
||||
)]
|
||||
#[case::environment_location(
|
||||
None,
|
||||
json!({}),
|
||||
&[("VERTEXAI_LOCATION", "europe-west1")],
|
||||
"https://europe-west1-aiplatform.googleapis.com/v1/projects/proj/locations/europe-west1"
|
||||
)]
|
||||
#[case::count_default_location(
|
||||
None,
|
||||
json!({}),
|
||||
&[],
|
||||
"https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5"
|
||||
)]
|
||||
#[case::global(
|
||||
None,
|
||||
json!({"vertex_location": "global"}),
|
||||
&[],
|
||||
"https://aiplatform.googleapis.com/v1/projects/proj/locations/global"
|
||||
)]
|
||||
#[case::custom_base(
|
||||
Some("https://proxy.example/prefix/"),
|
||||
json!({"vertex_location": "global"}),
|
||||
&[],
|
||||
"https://proxy.example/prefix/v1/projects/proj/locations/global"
|
||||
)]
|
||||
fn count_tokens_url_follows_the_python_location_precedence(
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] fields: Value,
|
||||
#[case] env: &[(&str, &str)],
|
||||
#[case] prefix: &str,
|
||||
) {
|
||||
let lookup = |name: &str| {
|
||||
env.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
};
|
||||
assert_eq!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
|
||||
api_base,
|
||||
&count_tokens_params(fields),
|
||||
&lookup
|
||||
),
|
||||
Some(Ok(format!("{prefix}{COUNT_PATH}")))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn count_tokens_without_a_project_fails_before_sending() {
|
||||
assert!(matches!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.get_count_tokens_url(
|
||||
None,
|
||||
&LitellmParams::default(),
|
||||
&no_env
|
||||
),
|
||||
Some(Err(Error::Auth(_)))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::versioned("claude-sonnet-4-5@20250929", "claude-sonnet-4-5")]
|
||||
#[case::default_version("claude-sonnet-4-5@default", "claude-sonnet-4-5")]
|
||||
#[case::unversioned("claude-sonnet-4-5", "claude-sonnet-4-5")]
|
||||
fn count_tokens_model_drops_the_version(#[case] model: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.count_tokens_model(model),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn without_a_project_anywhere_the_call_fails_before_sending() {
|
||||
let error = url(None, &LitellmParams::default(), false, &no_env).unwrap_err();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue