fix(rust): shape count_tokens like /v1/messages and count on Azure AI
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Narrow the request to what the count endpoints read before shaping, so a
field the provider refuses on count stops answering 400. Run the same
provider transform as /v1/messages so the count matches what inference
sends, then drop max_tokens from the wire. Add the Azure AI count URL and
the litellm_count_source field the Python proxy returns.
This commit is contained in:
mateo-berri 2026-10-30 02:09:04 -07:00
parent 2c5ee64fcc
commit 94c049a60d
6 changed files with 356 additions and 12 deletions

View file

@ -17,6 +17,7 @@ use litellm_llms_types::{
formats::messages::CountTokensResponse,
headers::{ProviderSpecificHeader, ProviderSpecificHeaders},
};
use serde::Serialize;
use serde_json::{Map, Value};
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
@ -29,6 +30,14 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
const PROVIDER_COUNT_SOURCE: &str = "provider";
#[derive(Serialize)]
struct SourcedCount {
#[serde(flatten)]
count: CountTokensResponse,
litellm_count_source: &'static str,
}
pub async fn create(
State(gateway): State<Arc<Gateway>>,
@ -92,11 +101,14 @@ async fn count(
identity: &AuthenticatedRequest,
headers: &HeaderMap,
body: Map<String, Value>,
) -> Result<CountTokensResponse, Error> {
) -> Result<SourcedCount, Error> {
let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?;
request::authorize_model(identity, deployment, &body).await?;
let call = project(deployment, body, headers)?;
Ok(gateway.messages.count_tokens(call).await?)
Ok(SourcedCount {
count: gateway.messages.count_tokens(call).await?,
litellm_count_source: PROVIDER_COUNT_SOURCE,
})
}
fn project(

View file

@ -166,7 +166,7 @@ async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model(
.and(body_json(json!({
"model": "test-model", "messages": messages, "system": "be brief", "tools": tools,
})))
.respond_with(ResponseTemplate::new(200).set_body_json(counted.clone()))
.respond_with(ResponseTemplate::new(200).set_body_json(counted))
.expect(1)
.mount(&upstream)
.await;
@ -183,7 +183,14 @@ async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model(
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(support::json(response).await, counted);
assert_eq!(
support::json(response).await,
json!({
"input_tokens": 14,
"context_management": {"original_input_tokens": 20},
"litellm_count_source": "provider",
})
);
}
#[rstest]

View file

@ -20,7 +20,7 @@ use litellm_llms::base_llm::{
},
};
use litellm_llms_types::formats::messages::{
CountTokensResponse, MessagesRequest, MessagesResponse,
CountTokensResponse, MessagesOptionalParams, MessagesRequest, MessagesResponse,
};
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
@ -35,6 +35,8 @@ use super::{
};
use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
const UNCAPPED_MAX_TOKENS: u64 = u64::MAX;
struct ProviderCall {
identity: ProviderIdentity,
wire: WireRequest,
@ -249,13 +251,13 @@ pub(super) async fn count_tokens(
let sanitized = config.shape_request(
MessagesRequest {
model: config.count_tokens_model(&model).to_string(),
params: body.params.countable(),
..body
},
shaping.settings.reasoning_auto_summary,
)?;
let request =
let trimmed =
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
let forwarded = string_headers(Some(
@ -264,26 +266,36 @@ pub(super) async fn count_tokens(
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
&request.model,
&request.params,
&trimmed.model,
&trimmed.params,
&connection_params,
&env_lookup,
)?;
let transformed = config.transform_anthropic_messages_request(
with_a_max_tokens(trimmed),
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
&request,
&transformed,
),
auth: validated.auth,
};
let authenticated =
resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?;
let wire_body = MessagesRequest {
params: MessagesOptionalParams {
max_tokens: None,
..transformed.params
},
..transformed
};
let response = send(
&route.http,
authenticated,
&url,
&serde_json::to_value(&request).map_err(serialize_failure)?,
&serde_json::to_value(&wire_body).map_err(serialize_failure)?,
timeout.or_else(|| shaping.settings.request_timeout()),
)
.await?;
@ -299,6 +311,16 @@ pub(super) async fn count_tokens(
})
}
fn with_a_max_tokens(request: MessagesRequest) -> MessagesRequest {
MessagesRequest {
params: MessagesOptionalParams {
max_tokens: request.params.max_tokens.or(Some(UNCAPPED_MAX_TOKENS)),
..request.params
},
..request
}
}
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],

View file

@ -1,7 +1,28 @@
use litellm_llms::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers};
use rstest::rstest;
use super::*;
fn without_max_tokens(model: &str, fields: Value) -> MessagesRequest {
let messages = json!({"model": model, "messages": [{"role": "user", "content": "hi"}]});
body(Value::Object(
object(messages).into_iter().chain(object(fields)).collect(),
))
}
async fn count_on_the_wire(call: MessagesCall) -> Value {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect("count succeeds");
only_request(&upstream).await.json()
}
#[rstest]
#[tokio::test]
async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model(
@ -60,3 +81,232 @@ async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall)
assert!(error.is_response(), "{error:?}");
}
#[rstest]
#[case::max_tokens(json!({"max_tokens": 1024}))]
#[case::stream(json!({"stream": true}))]
#[case::temperature(json!({"temperature": 0.5}))]
#[case::top_p(json!({"top_p": 0.5}))]
#[case::top_k(json!({"top_k": 5}))]
#[case::stop_sequences(json!({"stop_sequences": ["end"]}))]
#[case::metadata_the_messages_route_would_refuse(json!({"metadata": {"user_id": 7}}))]
#[case::speed(json!({"speed": "fast"}))]
#[case::service_tier(json!({"service_tier": "auto"}))]
#[case::container(json!({"container": "container_1"}))]
#[case::inference_geo(json!({"inference_geo": "us"}))]
#[case::mcp_servers(json!({"mcp_servers": [{"type": "url", "name": "docs", "url": "https://mcp.test"}]}))]
#[case::compaction(json!({"compaction": {"type": "summarize"}}))]
#[case::output_format(json!({"output_format": {"type": "json_schema"}}))]
#[case::field_no_provider_knows(json!({"litellm_metadata": {"tags": ["a"]}}))]
#[tokio::test]
async fn a_count_sends_only_what_the_count_endpoint_reads(
call: MessagesCall,
#[case] fields: Value,
) {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: without_max_tokens(MODEL, fields),
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_sampling_params: false,
..MessagesModelCapabilities::default()
},
..MessagesShaping::default()
},
..call
})
.await
.expect("count succeeds");
let request = only_request(&upstream).await;
assert_eq!(request.header("anthropic-beta"), None);
assert_eq!(
request.json(),
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
);
}
#[rstest]
#[case::system(json!({"system": "be brief"}))]
#[case::tools(json!({"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]}))]
#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))]
#[case::thinking(json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}))]
#[case::output_config(json!({"output_config": {"format": {"type": "json_schema"}}}))]
#[case::context_management(json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}))]
#[tokio::test]
async fn a_count_keeps_what_the_count_endpoint_reads(call: MessagesCall, #[case] fields: Value) {
let expected = without_max_tokens(MODEL, fields.clone());
let sent = count_on_the_wire(MessagesCall {
body: without_max_tokens(MODEL, fields),
..call
})
.await;
assert_eq!(sent, serde_json::to_value(expected).unwrap());
}
#[rstest]
#[tokio::test]
async fn a_count_asks_for_the_betas_of_the_fields_it_sends(call: MessagesCall) {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: without_max_tokens(
MODEL,
json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}),
),
api_key: Some("sk-ant".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect("count succeeds");
assert_eq!(
only_request(&upstream).await.header("anthropic-beta"),
Some("context-management-2025-06-27")
);
}
#[rstest]
#[case::no_ceiling_keeps_the_whole_budget(json!({"reasoning_effort": "high"}), 4096)]
#[case::the_callers_ceiling_caps_the_budget(
json!({"reasoning_effort": "high", "max_tokens": 3000}),
2999
)]
#[tokio::test]
async fn a_count_translates_reasoning_like_the_messages_route(
call: MessagesCall,
#[case] fields: Value,
#[case] budget_tokens: u64,
) {
let sent = count_on_the_wire(MessagesCall {
body: without_max_tokens(MODEL, fields),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
..MessagesModelCapabilities::default()
},
..MessagesShaping::default()
},
..call
})
.await;
assert_eq!(
sent,
json!({
"model": MODEL,
"messages": [{"role": "user", "content": "hi"}],
"thinking": {"type": "enabled", "budget_tokens": budget_tokens},
})
);
}
#[rstest]
#[tokio::test]
async fn a_count_on_an_adaptive_model_sends_the_effort_the_messages_route_sends(
call: MessagesCall,
) {
let sent = count_on_the_wire(MessagesCall {
body: without_max_tokens(MODEL, json!({"reasoning_effort": "high"})),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: true,
supports_output_config: true,
effort_tiers: SupportedEffortTiers {
high: true,
..SupportedEffortTiers::default()
},
..MessagesModelCapabilities::default()
},
..MessagesShaping::default()
},
..call
})
.await;
assert_eq!(sent["thinking"]["type"], "adaptive");
assert_eq!(sent["output_config"]["effort"], "high");
assert_eq!(sent.get("reasoning_effort"), None);
}
#[rstest]
#[tokio::test]
async fn a_vertex_count_lifts_the_system_role_and_strips_the_scope_vertex_refuses(
call: MessagesCall,
) {
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
body: body(json!({
"model": "claude-sonnet-4-5",
"messages": [
{"role": "system", "content": "be brief"},
{"role": "user", "content": [{
"type": "text",
"text": "hi",
"cache_control": {"type": "ephemeral", "scope": "global"},
}]},
],
})),
custom_llm_provider: Some("vertex_ai".into()),
litellm_params: serde_json::from_value(json!({
"model": "vertex_ai/claude-sonnet-4-5",
"vertex_project": "proj",
}))
.unwrap(),
api_base: Some(upstream.uri()),
extra_headers: headers([("Authorization", "Bearer caller-token")]),
..call
})
.await
.expect("count succeeds");
let sent = only_request(&upstream).await.json();
assert_eq!(
sent["system"],
json!([{"type": "text", "text": "be brief"}])
);
assert_eq!(
sent["messages"],
json!([{"role": "user", "content": [{
"type": "text",
"text": "hi",
"cache_control": {"type": "ephemeral"},
}]}])
);
}
#[rstest]
#[tokio::test]
async fn azure_ai_counts_at_its_anthropic_count_endpoint(call: MessagesCall) {
let upstream = upstream([json_response(json!({"input_tokens": 9}))]).await;
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
.count_tokens(MessagesCall {
custom_llm_provider: Some("azure_ai".into()),
api_key: Some("azure-key".into()),
api_base: Some(upstream.uri()),
..call
})
.await
.expect("count succeeds");
assert_eq!(counted.input_tokens, 9);
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/anthropic/v1/messages/count_tokens");
assert_eq!(request.header("x-api-key"), Some("azure-key"));
assert_eq!(
request.json(),
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
);
}

View file

@ -369,6 +369,22 @@ pub struct MessagesOptionalParams {
pub extra: Map<String, Value>,
}
impl MessagesOptionalParams {
pub fn countable(self) -> Self {
Self {
max_tokens: self.max_tokens,
system: self.system,
tools: self.tools,
tool_choice: self.tool_choice,
thinking: self.thinking,
context_management: self.context_management,
output_config: self.output_config,
reasoning_effort: self.reasoning_effort,
..Self::default()
}
}
}
impl Message {
pub fn blocks(&self) -> &[ContentBlock] {
match &self.content {

View file

@ -55,6 +55,18 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
complete_azure_anthropic_url(api_base, env_lookup)
}
fn get_count_tokens_url(
&self,
api_base: Option<&str>,
_litellm_params: &LitellmParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<Result<String, Error>> {
Some(
complete_azure_anthropic_url(api_base, env_lookup)
.map(|messages_url| format!("{messages_url}/count_tokens")),
)
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
@ -199,6 +211,31 @@ mod tests {
);
}
#[rstest]
#[case::resource_root("https://resource.services.ai.azure.com")]
#[case::anthropic_segment("https://resource.services.ai.azure.com/anthropic/")]
#[case::complete_messages_endpoint(
"https://resource.services.ai.azure.com/anthropic/v1/messages"
)]
fn count_url_sits_under_the_messages_endpoint(#[case] api_base: &str) {
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
.get_count_tokens_url(Some(api_base), &LitellmParams::default(), &|_| None)
.expect("azure ai has a count api")
.expect("url builds");
assert_eq!(
url,
"https://resource.services.ai.azure.com/anthropic/v1/messages/count_tokens"
);
}
#[test]
fn count_url_without_a_base_is_an_error() {
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
.get_count_tokens_url(None, &LitellmParams::default(), &|_| None)
.expect("azure ai has a count api");
assert!(url.is_err(), "{url:?}");
}
#[test]
fn url_falls_back_to_env_then_errors_when_absent() {
let with_env = |key: &str| {