mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(rust): shape count_tokens like /v1/messages and count on Azure AI
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Narrow the request to what the count endpoints read before shaping, so a field the provider refuses on count stops answering 400. Run the same provider transform as /v1/messages so the count matches what inference sends, then drop max_tokens from the wire. Add the Azure AI count URL and the litellm_count_source field the Python proxy returns.
This commit is contained in:
parent
2c5ee64fcc
commit
94c049a60d
6 changed files with 356 additions and 12 deletions
|
|
@ -17,6 +17,7 @@ use litellm_llms_types::{
|
|||
formats::messages::CountTokensResponse,
|
||||
headers::{ProviderSpecificHeader, ProviderSpecificHeaders},
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
|
||||
|
|
@ -29,6 +30,14 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
|
|||
];
|
||||
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
|
||||
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
|
||||
const PROVIDER_COUNT_SOURCE: &str = "provider";
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SourcedCount {
|
||||
#[serde(flatten)]
|
||||
count: CountTokensResponse,
|
||||
litellm_count_source: &'static str,
|
||||
}
|
||||
|
||||
pub async fn create(
|
||||
State(gateway): State<Arc<Gateway>>,
|
||||
|
|
@ -92,11 +101,14 @@ async fn count(
|
|||
identity: &AuthenticatedRequest,
|
||||
headers: &HeaderMap,
|
||||
body: Map<String, Value>,
|
||||
) -> Result<CountTokensResponse, Error> {
|
||||
) -> Result<SourcedCount, Error> {
|
||||
let deployment = request::resolve_deployment(gateway, "/v1/messages/count_tokens", &body)?;
|
||||
request::authorize_model(identity, deployment, &body).await?;
|
||||
let call = project(deployment, body, headers)?;
|
||||
Ok(gateway.messages.count_tokens(call).await?)
|
||||
Ok(SourcedCount {
|
||||
count: gateway.messages.count_tokens(call).await?,
|
||||
litellm_count_source: PROVIDER_COUNT_SOURCE,
|
||||
})
|
||||
}
|
||||
|
||||
fn project(
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model(
|
|||
.and(body_json(json!({
|
||||
"model": "test-model", "messages": messages, "system": "be brief", "tools": tools,
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(counted.clone()))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(counted))
|
||||
.expect(1)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
|
|
@ -183,7 +183,14 @@ async fn count_tokens_answers_with_the_providers_count_for_the_deployment_model(
|
|||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(support::json(response).await, counted);
|
||||
assert_eq!(
|
||||
support::json(response).await,
|
||||
json!({
|
||||
"input_tokens": 14,
|
||||
"context_management": {"original_input_tokens": 20},
|
||||
"litellm_count_source": "provider",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ use litellm_llms::base_llm::{
|
|||
},
|
||||
};
|
||||
use litellm_llms_types::formats::messages::{
|
||||
CountTokensResponse, MessagesRequest, MessagesResponse,
|
||||
CountTokensResponse, MessagesOptionalParams, MessagesRequest, MessagesResponse,
|
||||
};
|
||||
use litellm_secrets::source::Secrets;
|
||||
use litellm_tracing::ByteChunk;
|
||||
|
|
@ -35,6 +35,8 @@ use super::{
|
|||
};
|
||||
use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
|
||||
|
||||
const UNCAPPED_MAX_TOKENS: u64 = u64::MAX;
|
||||
|
||||
struct ProviderCall {
|
||||
identity: ProviderIdentity,
|
||||
wire: WireRequest,
|
||||
|
|
@ -249,13 +251,13 @@ pub(super) async fn count_tokens(
|
|||
let sanitized = config.shape_request(
|
||||
MessagesRequest {
|
||||
model: config.count_tokens_model(&model).to_string(),
|
||||
params: body.params.countable(),
|
||||
..body
|
||||
},
|
||||
shaping.settings.reasoning_auto_summary,
|
||||
)?;
|
||||
let request =
|
||||
let trimmed =
|
||||
without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
|
||||
|
||||
let scoped =
|
||||
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
|
||||
let forwarded = string_headers(Some(
|
||||
|
|
@ -264,26 +266,36 @@ pub(super) async fn count_tokens(
|
|||
let validated = config.validate_environment(
|
||||
forwarded,
|
||||
api_key.as_deref(),
|
||||
&request.model,
|
||||
&request.params,
|
||||
&trimmed.model,
|
||||
&trimmed.params,
|
||||
&connection_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let transformed = config.transform_anthropic_messages_request(
|
||||
with_a_max_tokens(trimmed),
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.request_headers(
|
||||
with_default_headers(validated.headers, config.default_headers()),
|
||||
&request,
|
||||
&transformed,
|
||||
),
|
||||
auth: validated.auth,
|
||||
};
|
||||
let authenticated =
|
||||
resolve_auth(&route.auth, environment, &|key| std::env::var(key).ok()).await?;
|
||||
|
||||
let wire_body = MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
max_tokens: None,
|
||||
..transformed.params
|
||||
},
|
||||
..transformed
|
||||
};
|
||||
let response = send(
|
||||
&route.http,
|
||||
authenticated,
|
||||
&url,
|
||||
&serde_json::to_value(&request).map_err(serialize_failure)?,
|
||||
&serde_json::to_value(&wire_body).map_err(serialize_failure)?,
|
||||
timeout.or_else(|| shaping.settings.request_timeout()),
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -299,6 +311,16 @@ pub(super) async fn count_tokens(
|
|||
})
|
||||
}
|
||||
|
||||
fn with_a_max_tokens(request: MessagesRequest) -> MessagesRequest {
|
||||
MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
max_tokens: request.params.max_tokens.or(Some(UNCAPPED_MAX_TOKENS)),
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
}
|
||||
}
|
||||
|
||||
fn without_additional_drop_params(
|
||||
request: MessagesRequest,
|
||||
paths: &[String],
|
||||
|
|
|
|||
|
|
@ -1,7 +1,28 @@
|
|||
use litellm_llms::base_llm::messages::context::{MessagesModelCapabilities, SupportedEffortTiers};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn without_max_tokens(model: &str, fields: Value) -> MessagesRequest {
|
||||
let messages = json!({"model": model, "messages": [{"role": "user", "content": "hi"}]});
|
||||
body(Value::Object(
|
||||
object(messages).into_iter().chain(object(fields)).collect(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn count_on_the_wire(call: MessagesCall) -> Value {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
only_request(&upstream).await.json()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn vertex_ai_counts_at_the_publisher_count_endpoint_with_the_unversioned_model(
|
||||
|
|
@ -60,3 +81,232 @@ async fn a_count_without_input_tokens_is_an_invalid_response(call: MessagesCall)
|
|||
|
||||
assert!(error.is_response(), "{error:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::max_tokens(json!({"max_tokens": 1024}))]
|
||||
#[case::stream(json!({"stream": true}))]
|
||||
#[case::temperature(json!({"temperature": 0.5}))]
|
||||
#[case::top_p(json!({"top_p": 0.5}))]
|
||||
#[case::top_k(json!({"top_k": 5}))]
|
||||
#[case::stop_sequences(json!({"stop_sequences": ["end"]}))]
|
||||
#[case::metadata_the_messages_route_would_refuse(json!({"metadata": {"user_id": 7}}))]
|
||||
#[case::speed(json!({"speed": "fast"}))]
|
||||
#[case::service_tier(json!({"service_tier": "auto"}))]
|
||||
#[case::container(json!({"container": "container_1"}))]
|
||||
#[case::inference_geo(json!({"inference_geo": "us"}))]
|
||||
#[case::mcp_servers(json!({"mcp_servers": [{"type": "url", "name": "docs", "url": "https://mcp.test"}]}))]
|
||||
#[case::compaction(json!({"compaction": {"type": "summarize"}}))]
|
||||
#[case::output_format(json!({"output_format": {"type": "json_schema"}}))]
|
||||
#[case::field_no_provider_knows(json!({"litellm_metadata": {"tags": ["a"]}}))]
|
||||
#[tokio::test]
|
||||
async fn a_count_sends_only_what_the_count_endpoint_reads(
|
||||
call: MessagesCall,
|
||||
#[case] fields: Value,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: without_max_tokens(MODEL, fields),
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
shaping: MessagesShaping {
|
||||
capabilities: MessagesModelCapabilities {
|
||||
supports_sampling_params: false,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.header("anthropic-beta"), None);
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::system(json!({"system": "be brief"}))]
|
||||
#[case::tools(json!({"tools": [{"name": "lookup", "input_schema": {"type": "object"}}]}))]
|
||||
#[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))]
|
||||
#[case::thinking(json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}))]
|
||||
#[case::output_config(json!({"output_config": {"format": {"type": "json_schema"}}}))]
|
||||
#[case::context_management(json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}))]
|
||||
#[tokio::test]
|
||||
async fn a_count_keeps_what_the_count_endpoint_reads(call: MessagesCall, #[case] fields: Value) {
|
||||
let expected = without_max_tokens(MODEL, fields.clone());
|
||||
|
||||
let sent = count_on_the_wire(MessagesCall {
|
||||
body: without_max_tokens(MODEL, fields),
|
||||
..call
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(sent, serde_json::to_value(expected).unwrap());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_count_asks_for_the_betas_of_the_fields_it_sends(call: MessagesCall) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: without_max_tokens(
|
||||
MODEL,
|
||||
json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}),
|
||||
),
|
||||
api_key: Some("sk-ant".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
assert_eq!(
|
||||
only_request(&upstream).await.header("anthropic-beta"),
|
||||
Some("context-management-2025-06-27")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_ceiling_keeps_the_whole_budget(json!({"reasoning_effort": "high"}), 4096)]
|
||||
#[case::the_callers_ceiling_caps_the_budget(
|
||||
json!({"reasoning_effort": "high", "max_tokens": 3000}),
|
||||
2999
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn a_count_translates_reasoning_like_the_messages_route(
|
||||
call: MessagesCall,
|
||||
#[case] fields: Value,
|
||||
#[case] budget_tokens: u64,
|
||||
) {
|
||||
let sent = count_on_the_wire(MessagesCall {
|
||||
body: without_max_tokens(MODEL, fields),
|
||||
shaping: MessagesShaping {
|
||||
capabilities: MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..call
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
sent,
|
||||
json!({
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": budget_tokens},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_count_on_an_adaptive_model_sends_the_effort_the_messages_route_sends(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let sent = count_on_the_wire(MessagesCall {
|
||||
body: without_max_tokens(MODEL, json!({"reasoning_effort": "high"})),
|
||||
shaping: MessagesShaping {
|
||||
capabilities: MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_adaptive_thinking: true,
|
||||
supports_output_config: true,
|
||||
effort_tiers: SupportedEffortTiers {
|
||||
high: true,
|
||||
..SupportedEffortTiers::default()
|
||||
},
|
||||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..call
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(sent["thinking"]["type"], "adaptive");
|
||||
assert_eq!(sent["output_config"]["effort"], "high");
|
||||
assert_eq!(sent.get("reasoning_effort"), None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_vertex_count_lifts_the_system_role_and_strips_the_scope_vertex_refuses(
|
||||
call: MessagesCall,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 3}))]).await;
|
||||
|
||||
messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
body: body(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"messages": [
|
||||
{"role": "system", "content": "be brief"},
|
||||
{"role": "user", "content": [{
|
||||
"type": "text",
|
||||
"text": "hi",
|
||||
"cache_control": {"type": "ephemeral", "scope": "global"},
|
||||
}]},
|
||||
],
|
||||
})),
|
||||
custom_llm_provider: Some("vertex_ai".into()),
|
||||
litellm_params: serde_json::from_value(json!({
|
||||
"model": "vertex_ai/claude-sonnet-4-5",
|
||||
"vertex_project": "proj",
|
||||
}))
|
||||
.unwrap(),
|
||||
api_base: Some(upstream.uri()),
|
||||
extra_headers: headers([("Authorization", "Bearer caller-token")]),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
let sent = only_request(&upstream).await.json();
|
||||
assert_eq!(
|
||||
sent["system"],
|
||||
json!([{"type": "text", "text": "be brief"}])
|
||||
);
|
||||
assert_eq!(
|
||||
sent["messages"],
|
||||
json!([{"role": "user", "content": [{
|
||||
"type": "text",
|
||||
"text": "hi",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}]}])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn azure_ai_counts_at_its_anthropic_count_endpoint(call: MessagesCall) {
|
||||
let upstream = upstream([json_response(json!({"input_tokens": 9}))]).await;
|
||||
|
||||
let counted = messages_route(Arc::new(RecordingSecrets::empty()))
|
||||
.count_tokens(MessagesCall {
|
||||
custom_llm_provider: Some("azure_ai".into()),
|
||||
api_key: Some("azure-key".into()),
|
||||
api_base: Some(upstream.uri()),
|
||||
..call
|
||||
})
|
||||
.await
|
||||
.expect("count succeeds");
|
||||
|
||||
assert_eq!(counted.input_tokens, 9);
|
||||
let request = only_request(&upstream).await;
|
||||
assert_eq!(request.url.path(), "/anthropic/v1/messages/count_tokens");
|
||||
assert_eq!(request.header("x-api-key"), Some("azure-key"));
|
||||
assert_eq!(
|
||||
request.json(),
|
||||
json!({"model": MODEL, "messages": [{"role": "user", "content": "hi"}]})
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -369,6 +369,22 @@ pub struct MessagesOptionalParams {
|
|||
pub extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl MessagesOptionalParams {
|
||||
pub fn countable(self) -> Self {
|
||||
Self {
|
||||
max_tokens: self.max_tokens,
|
||||
system: self.system,
|
||||
tools: self.tools,
|
||||
tool_choice: self.tool_choice,
|
||||
thinking: self.thinking,
|
||||
context_management: self.context_management,
|
||||
output_config: self.output_config,
|
||||
reasoning_effort: self.reasoning_effort,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Message {
|
||||
pub fn blocks(&self) -> &[ContentBlock] {
|
||||
match &self.content {
|
||||
|
|
|
|||
|
|
@ -55,6 +55,18 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
|
|||
complete_azure_anthropic_url(api_base, env_lookup)
|
||||
}
|
||||
|
||||
fn get_count_tokens_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
_litellm_params: &LitellmParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<Result<String, Error>> {
|
||||
Some(
|
||||
complete_azure_anthropic_url(api_base, env_lookup)
|
||||
.map(|messages_url| format!("{messages_url}/count_tokens")),
|
||||
)
|
||||
}
|
||||
|
||||
fn transform_anthropic_messages_request(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
|
|
@ -199,6 +211,31 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::resource_root("https://resource.services.ai.azure.com")]
|
||||
#[case::anthropic_segment("https://resource.services.ai.azure.com/anthropic/")]
|
||||
#[case::complete_messages_endpoint(
|
||||
"https://resource.services.ai.azure.com/anthropic/v1/messages"
|
||||
)]
|
||||
fn count_url_sits_under_the_messages_endpoint(#[case] api_base: &str) {
|
||||
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
|
||||
.get_count_tokens_url(Some(api_base), &LitellmParams::default(), &|_| None)
|
||||
.expect("azure ai has a count api")
|
||||
.expect("url builds");
|
||||
assert_eq!(
|
||||
url,
|
||||
"https://resource.services.ai.azure.com/anthropic/v1/messages/count_tokens"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_url_without_a_base_is_an_error() {
|
||||
let url = AZURE_ANTHROPIC_MESSAGES_CONFIG
|
||||
.get_count_tokens_url(None, &LitellmParams::default(), &|_| None)
|
||||
.expect("azure ai has a count api");
|
||||
assert!(url.is_err(), "{url:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_falls_back_to_env_then_errors_when_absent() {
|
||||
let with_env = |key: &str| {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue