diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index 2c9e8a94d3d..a517de097ca 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -17,6 +17,7 @@ use litellm_llms_types::{ formats::messages::CountTokensResponse, headers::{ProviderSpecificHeader, ProviderSpecificHeaders}, }; +use serde::Serialize; use serde_json::{Map, Value}; use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request}; @@ -29,6 +30,14 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [ ]; const ANTHROPIC_API_HEADER_PROVIDERS: &str = "anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek"; +const PROVIDER_COUNT_SOURCE: &str = "provider"; + +#[derive(Serialize)] +struct SourcedCount { + #[serde(flatten)] + count: CountTokensResponse, + litellm_count_source: &'static str, +} pub async fn create( State(gateway): State>, @@ -92,11 +101,14 @@ async fn count( identity: &AuthenticatedRequest, headers: &HeaderMap, body: Map, -) -> Result { +) -> Result { let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?; request::authorize_model(identity, deployment, &body).await?; let call = project(deployment, body, headers)?; - Ok(gateway.messages.count_tokens(call).await?) + Ok(SourcedCount { + count: gateway.messages.count_tokens(call).await?, + litellm_count_source: PROVIDER_COUNT_SOURCE, + }) } fn project( diff --git a/litellm-rust/crates/gateway-inference/tests/messages.rs b/litellm-rust/crates/gateway-inference/tests/messages.rs index 3ae6267afec..3d0143aa741 100644 --- a/litellm-rust/crates/gateway-inference/tests/messages.rs +++ b/litellm-rust/crates/gateway-inference/tests/messages.rs @@ -166,7 +166,7 @@ async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model( .and(body_json(json!({ "model": "test-model", "messages": messages, "system": "be brief", "tools": tools, }))) - .respond_with(ResponseTemplate::new(200).set_body_json(counted.clone())) + .respond_with(ResponseTemplate::new(200).set_body_json(counted)) .expect(1) .mount(&upstream) .await; @@ -183,7 +183,14 @@ async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model( .await .unwrap(); assert_eq!(response.status(), 200); - assert_eq!(support::json(response).await, counted); + assert_eq!( + support::json(response).await, + json!({ + "input_tokens": 14, + "context_management": {"original_input_tokens": 20}, + "litellm_count_source": "provider", + }) + ); } #[rstest] diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index d8b729b6ec9..500962af4db 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -20,7 +20,7 @@ use litellm_llms::base_llm::{ }, }; use litellm_llms_types::formats::messages::{ - CountTokensResponse, MessagesRequest, MessagesResponse, + CountTokensResponse, MessagesOptionalParams, MessagesRequest, MessagesResponse, }; use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; @@ -35,6 +35,8 @@ use super::{ }; use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request}; +const UNCAPPED_MAX_TOKENS: u64 = u64::MAX; + struct ProviderCall { identity: ProviderIdentity, wire: WireRequest, @@ -249,13 +251,13 @@ pub(super) async fn count_tokens( let sanitized = config.shape_request( MessagesRequest { model: config.count_tokens_model(&model).to_string(), + params: body.params.countable(), ..body }, shaping.settings.reasoning_auto_summary, )?; - let request = + let trimmed = without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; - let scoped = get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); let forwarded = string_headers(Some( @@ -264,26 +266,36 @@ pub(super) async fn count_tokens( let validated = config.validate_environment( forwarded, api_key.as_deref(), - &request.model, - &request.params, + &trimmed.model, + &trimmed.params, &connection_params, &env_lookup, )?; + let transformed = config.transform_anthropic_messages_request( + with_a_max_tokens(trimmed), + &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params), + )?; let environment = ValidatedEnvironment { headers: config.request_headers( with_default_headers(validated.headers, config.default_headers()), - &request, + &transformed, ), auth: validated.auth, }; let authenticated = resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?; - + let wire_body = MessagesRequest { + params: MessagesOptionalParams { + max_tokens: None, + ..transformed.params + }, + ..transformed + }; let response = send( &route.http, authenticated, &url, - &serde_json::to_value(&request).map_err(serialize_failure)?, + &serde_json::to_value(&wire_body).map_err(serialize_failure)?, timeout.or_else(|| shaping.settings.request_timeout()), ) .await?; @@ -299,6 +311,16 @@ pub(super) async fn count_tokens( }) } +fn with_a_max_tokens(request: MessagesRequest) -> MessagesRequest { + MessagesRequest { + params: MessagesOptionalParams { + max_tokens: request.params.max_tokens.or(Some(UNCAPPED_MAX_TOKENS)), + ..request.params + }, + ..request + } +} + fn without_additional_drop_params( request: MessagesRequest, paths: &[String], diff --git a/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs b/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs index 1928d048835..ec58c244bfd 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/count_tokens.rs @@ -1,7 +1,28 @@ +use litellm_llms::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers}; use rstest::rstest; use super::*; +fn without_max_tokens(model: &str, fields: Value) -> MessagesRequest { + let messages = json!({"model": model, "messages": [{"role": "user", "content": "hi"}]}); + body(Value::Object( + object(messages).into_iter().chain(object(fields)).collect(), + )) +} + +async fn count_on_the_wire(call: MessagesCall) -> Value { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("count succeeds"); + only_request(&upstream).await.json() +} + #[rstest] #[tokio::test] async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model( @@ -60,3 +81,232 @@ async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall) assert!(error.is_response(), "{error:?}"); } + +#[rstest] +#[case::max_tokens(json!({"max_tokens": 1024}))] +#[case::stream(json!({"stream": true}))] +#[case::temperature(json!({"temperature": 0.5}))] +#[case::top_p(json!({"top_p": 0.5}))] +#[case::top_k(json!({"top_k": 5}))] +#[case::stop_sequences(json!({"stop_sequences": ["end"]}))] +#[case::metadata_the_messages_route_would_refuse(json!({"metadata": {"user_id": 7}}))] +#[case::speed(json!({"speed": "fast"}))] +#[case::service_tier(json!({"service_tier": "auto"}))] +#[case::container(json!({"container": "container_1"}))] +#[case::inference_geo(json!({"inference_geo": "us"}))] +#[case::mcp_servers(json!({"mcp_servers": [{"type": "url", "name": "docs", "url": "https://mcp.test"}]}))] +#[case::compaction(json!({"compaction": {"type": "summarize"}}))] +#[case::output_format(json!({"output_format": {"type": "json_schema"}}))] +#[case::field_no_provider_knows(json!({"litellm_metadata": {"tags": ["a"]}}))] +#[tokio::test] +async fn a_count_sends_only_what_the_count_endpoint_reads( + call: MessagesCall, + #[case] fields: Value, +) { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: without_max_tokens(MODEL, fields), + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_sampling_params: false, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + ..call + }) + .await + .expect("count succeeds"); + + let request = only_request(&upstream).await; + assert_eq!(request.header("anthropic-beta"), None); + assert_eq!( + request.json(), + json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]}) + ); +} + +#[rstest] +#[case::system(json!({"system": "be brief"}))] +#[case::tools(json!({"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]}))] +#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))] +#[case::thinking(json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}))] +#[case::output_config(json!({"output_config": {"format": {"type": "json_schema"}}}))] +#[case::context_management(json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}))] +#[tokio::test] +async fn a_count_keeps_what_the_count_endpoint_reads(call: MessagesCall, #[case] fields: Value) { + let expected = without_max_tokens(MODEL, fields.clone()); + + let sent = count_on_the_wire(MessagesCall { + body: without_max_tokens(MODEL, fields), + ..call + }) + .await; + + assert_eq!(sent, serde_json::to_value(expected).unwrap()); +} + +#[rstest] +#[tokio::test] +async fn a_count_asks_for_the_betas_of_the_fields_it_sends(call: MessagesCall) { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: without_max_tokens( + MODEL, + json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}), + ), + api_key: Some("sk-ant".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("count succeeds"); + + assert_eq!( + only_request(&upstream).await.header("anthropic-beta"), + Some("context-management-2025-06-27") + ); +} + +#[rstest] +#[case::no_ceiling_keeps_the_whole_budget(json!({"reasoning_effort": "high"}), 4096)] +#[case::the_callers_ceiling_caps_the_budget( + json!({"reasoning_effort": "high", "max_tokens": 3000}), + 2999 +)] +#[tokio::test] +async fn a_count_translates_reasoning_like_the_messages_route( + call: MessagesCall, + #[case] fields: Value, + #[case] budget_tokens: u64, +) { + let sent = count_on_the_wire(MessagesCall { + body: without_max_tokens(MODEL, fields), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_reasoning: true, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + ..call + }) + .await; + + assert_eq!( + sent, + json!({ + "model": MODEL, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": budget_tokens}, + }) + ); +} + +#[rstest] +#[tokio::test] +async fn a_count_on_an_adaptive_model_sends_the_effort_the_messages_route_sends( + call: MessagesCall, +) { + let sent = count_on_the_wire(MessagesCall { + body: without_max_tokens(MODEL, json!({"reasoning_effort": "high"})), + shaping: MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + high: true, + ..SupportedEffortTiers::default() + }, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + ..call + }) + .await; + + assert_eq!(sent["thinking"]["type"], "adaptive"); + assert_eq!(sent["output_config"]["effort"], "high"); + assert_eq!(sent.get("reasoning_effort"), None); +} + +#[rstest] +#[tokio::test] +async fn a_vertex_count_lifts_the_system_role_and_strips_the_scope_vertex_refuses( + call: MessagesCall, +) { + let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await; + + messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + body: body(json!({ + "model": "claude-sonnet-4-5", + "messages": [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [{ + "type": "text", + "text": "hi", + "cache_control": {"type": "ephemeral", "scope": "global"}, + }]}, + ], + })), + custom_llm_provider: Some("vertex_ai".into()), + litellm_params: serde_json::from_value(json!({ + "model": "vertex_ai/claude-sonnet-4-5", + "vertex_project": "proj", + })) + .unwrap(), + api_base: Some(upstream.uri()), + extra_headers: headers([("Authorization", "Bearer caller-token")]), + ..call + }) + .await + .expect("count succeeds"); + + let sent = only_request(&upstream).await.json(); + assert_eq!( + sent["system"], + json!([{"type": "text", "text": "be brief"}]) + ); + assert_eq!( + sent["messages"], + json!([{"role": "user", "content": [{ + "type": "text", + "text": "hi", + "cache_control": {"type": "ephemeral"}, + }]}]) + ); +} + +#[rstest] +#[tokio::test] +async fn azure_ai_counts_at_its_anthropic_count_endpoint(call: MessagesCall) { + let upstream = upstream([json_response(json!({"input_tokens": 9}))]).await; + + let counted = messages_route(Arc::new(RecordingSecrets::empty())) + .count_tokens(MessagesCall { + custom_llm_provider: Some("azure_ai".into()), + api_key: Some("azure-key".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("count succeeds"); + + assert_eq!(counted.input_tokens, 9); + let request = only_request(&upstream).await; + assert_eq!(request.url.path(), "/anthropic/v1/messages/count_tokens"); + assert_eq!(request.header("x-api-key"), Some("azure-key")); + assert_eq!( + request.json(), + json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]}) + ); +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index 9dbfa068025..b0270ff1551 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -369,6 +369,22 @@ pub struct MessagesOptionalParams { pub extra: Map, } +impl MessagesOptionalParams { + pub fn countable(self) -> Self { + Self { + max_tokens: self.max_tokens, + system: self.system, + tools: self.tools, + tool_choice: self.tool_choice, + thinking: self.thinking, + context_management: self.context_management, + output_config: self.output_config, + reasoning_effort: self.reasoning_effort, + ..Self::default() + } + } +} + impl Message { pub fn blocks(&self) -> &[ContentBlock] { match &self.content { diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index bf934fb4d4c..9d83aa11342 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -55,6 +55,18 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { complete_azure_anthropic_url(api_base, env_lookup) } + fn get_count_tokens_url( + &self, + api_base: Option<&str>, + _litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Option> { + Some( + complete_azure_anthropic_url(api_base, env_lookup) + .map(|messages_url| format!("{messages_url}/count_tokens")), + ) + } + fn transform_anthropic_messages_request( &self, request: MessagesRequest, @@ -199,6 +211,31 @@ mod tests { ); } + #[rstest] + #[case::resource_root("https://resource.services.ai.azure.com")] + #[case::anthropic_segment("https://resource.services.ai.azure.com/anthropic/")] + #[case::complete_messages_endpoint( + "https://resource.services.ai.azure.com/anthropic/v1/messages" + )] + fn count_url_sits_under_the_messages_endpoint(#[case] api_base: &str) { + let url = AZURE_ANTHROPIC_MESSAGES_CONFIG + .get_count_tokens_url(Some(api_base), &LitellmParams::default(), &|_| None) + .expect("azure ai has a count api") + .expect("url builds"); + assert_eq!( + url, + "https://resource.services.ai.azure.com/anthropic/v1/messages/count_tokens" + ); + } + + #[test] + fn count_url_without_a_base_is_an_error() { + let url = AZURE_ANTHROPIC_MESSAGES_CONFIG + .get_count_tokens_url(None, &LitellmParams::default(), &|_| None) + .expect("azure ai has a count api"); + assert!(url.is_err(), "{url:?}"); + } + #[test] fn url_falls_back_to_env_then_errors_when_absent() { let with_env = |key: &str| {