From e06685c3e487db6c97bab1d26a52d3b0fc72581a Mon Sep 17 00:00:00 2001 From: James Date: Wed, 23 Sep 2026 14:16:39 -0700 Subject: [PATCH 1/3] fix(rust): count tool call arguments and tool_call_id like the python token counter --- .../crates/token-counter/src/counter.rs | 26 ++++++++++++++++++- .../crates/token-counter/src/types.rs | 25 ++++++++++++++++++ .../token-counter/tests/token_counter.rs | 16 +++++++++--- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/litellm-rust/crates/token-counter/src/counter.rs b/litellm-rust/crates/token-counter/src/counter.rs index ce08e225be4..9906b8ab16f 100644 --- a/litellm-rust/crates/token-counter/src/counter.rs +++ b/litellm-rust/crates/token-counter/src/counter.rs @@ -124,7 +124,31 @@ impl TokenCounter { .sum::>()?, None => 0, }; - Ok(TOKENS_PER_MESSAGE + role_tokens + name_tokens + content_tokens) + // Python counts only each tool call's `arguments` string + // (`_count_function_call_tokens`); names ride with the tool + // definitions and `tool_choice`. + let tool_call_tokens = match &message.tool_calls { + Some(calls) => calls + .iter() + .map(|call| self.count_text(call.function.arguments.as_deref().unwrap_or(""))) + .sum::>()?, + None => 0, + }; + let tool_call_id_tokens = match &message.tool_call_id { + Some(id) => self.count_text(id)?, + None => 0, + }; + let legacy_call_tokens = match &message.function_call { + Some(call) => self.count_text(call.arguments.as_deref().unwrap_or(""))?, + None => 0, + }; + Ok(TOKENS_PER_MESSAGE + + role_tokens + + name_tokens + + content_tokens + + tool_call_tokens + + tool_call_id_tokens + + legacy_call_tokens) } fn count_content_item(&self, item: &ContentItem) -> Result { diff --git a/litellm-rust/crates/token-counter/src/types.rs b/litellm-rust/crates/token-counter/src/types.rs index d25554beaac..16ab1d86f90 100644 --- a/litellm-rust/crates/token-counter/src/types.rs +++ b/litellm-rust/crates/token-counter/src/types.rs @@ -137,6 +137,28 @@ impl<'de> Visitor<'de> for TextValueVisitor { } } +/// The parts of an assistant tool call Python counts: only the `arguments` +/// string contributes (`_count_function_call_tokens` in +/// `litellm_core_utils/token_counter.py`). Absent arguments count as the empty +/// string; a non-string `arguments` declines so Python handles the fallback +/// instead of miscounting. +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub(crate) struct ToolCall { + pub(crate) function: ToolCallFunction, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub(crate) struct ToolCallFunction { + #[serde(default)] + pub(crate) arguments: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub(crate) struct LegacyFunctionCall { + #[serde(default)] + pub(crate) arguments: Option, +} + /// Python counts every string-valued key of a message, so any key beyond these /// makes the shape unsupported rather than silently uncounted. #[derive(Clone, Debug, Deserialize, PartialEq)] @@ -145,6 +167,9 @@ pub(crate) struct Message { pub(crate) role: Option, pub(crate) name: Option, pub(crate) content: Option, + pub(crate) tool_call_id: Option, + pub(crate) tool_calls: Option>, + pub(crate) function_call: Option, } #[derive(Clone, Debug, Deserialize, PartialEq)] diff --git a/litellm-rust/crates/token-counter/tests/token_counter.rs b/litellm-rust/crates/token-counter/tests/token_counter.rs index 542bd4a1fc4..2161652f0b1 100644 --- a/litellm-rust/crates/token-counter/tests/token_counter.rs +++ b/litellm-rust/crates/token-counter/tests/token_counter.rs @@ -70,6 +70,16 @@ mod json { const RERANK: &str = r#"{"model":"claude-sonnet-4-5","query":"best harbour", "documents":["doc one",{"text":"doc two","title":"T","n":3,"ok":true,"none":null,"tags":["a","b"]}]}"#; + const TOOL_CALLS_AND_TOOL_RESULT: &str = r#"{"model":"claude-sonnet-4-5","messages":[ + {"role":"assistant","tool_calls":[{"id":"1","type":"function","function":{"name":"f","arguments":"{\"city\": \"Tokyo\"}"}}]}, + {"role":"tool","tool_call_id":"1","content":"Sunny, 22C"}]}"#; + + const LEGACY_FUNCTION_CALL: &str = r#"{"model":"claude-sonnet-4-5","messages":[ + {"role":"assistant","function_call":{"name":"f","arguments":"{\"city\": \"Tokyo\"}"}}]}"#; + + const ASSISTANT_CONTENT_WITH_TOOL_CALL: &str = r#"{"model":"claude-sonnet-4-5","messages":[ + {"role":"assistant","content":"calling","tool_calls":[{"id":"1","type":"function","function":{"name":"get_weather","arguments":"{\"city\": \"Paris, France\"}"}}]}]}"#; + fn assert_count_request_matches_python_token_counter( load: JsonLoader, body: &str, @@ -154,6 +164,9 @@ mod json { #[case::responses_input_items(super::RESPONSES_INPUT, 62)] #[case::embeddings_token_ids(super::EMBEDDINGS_TOKEN_IDS, 5)] #[case::rerank_query_and_documents(super::RERANK, 41)] + #[case::tool_calls_and_tool_result(super::TOOL_CALLS_AND_TOOL_RESULT, 24)] + #[case::legacy_function_call(super::LEGACY_FUNCTION_CALL, 14)] + #[case::assistant_content_with_tool_call(super::ASSISTANT_CONTENT_WITH_TOOL_CALL, 16)] fn count_request_matches_python_token_counter( #[case] body: &str, #[case] expected: usize, @@ -235,9 +248,6 @@ mod json { #[rstest] #[case::not_json(b"not json" as &[u8])] #[case::messages_not_a_list(br#"{"model":"m","messages":"hi"}"#)] -#[case::message_with_tool_calls( - br#"{"model":"m","messages":[{"role":"assistant","tool_calls":[{"id":"1","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"# -)] #[case::dict_content( br#"{"model":"m","messages":[{"role":"user","content":{"type":"text","text":"x"}}]}"# )] From be79444f9cc7331c249e2ddddcc625de38885fa4 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 23 Sep 2026 16:52:31 -0700 Subject: [PATCH 2/3] fix(rust): count explicit null tool call arguments as the string None like python --- litellm-rust/crates/token-counter/src/types.rs | 17 +++++++++++++++-- .../crates/token-counter/tests/token_counter.rs | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/token-counter/src/types.rs b/litellm-rust/crates/token-counter/src/types.rs index 16ab1d86f90..7f01219d4c0 100644 --- a/litellm-rust/crates/token-counter/src/types.rs +++ b/litellm-rust/crates/token-counter/src/types.rs @@ -147,15 +147,28 @@ pub(crate) struct ToolCall { pub(crate) function: ToolCallFunction, } +/// Python runs `str(function.get("arguments", ""))`: a missing key counts as +/// empty, but an explicit null counts as the string "None". `serde(default)` +/// keeps a missing key at `None`, and the deserializer turns an explicit null +/// into "None" so the counter sees what Python counts. +fn explicit_null_arguments_as_none<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + match Option::::deserialize(deserializer)? { + Some(arguments) => Ok(Some(arguments)), + None => Ok(Some(String::from("None"))), + } +} + #[derive(Clone, Debug, Deserialize, PartialEq)] pub(crate) struct ToolCallFunction { - #[serde(default)] + #[serde(default, deserialize_with = "explicit_null_arguments_as_none")] pub(crate) arguments: Option, } #[derive(Clone, Debug, Deserialize, PartialEq)] pub(crate) struct LegacyFunctionCall { - #[serde(default)] + #[serde(default, deserialize_with = "explicit_null_arguments_as_none")] pub(crate) arguments: Option, } diff --git a/litellm-rust/crates/token-counter/tests/token_counter.rs b/litellm-rust/crates/token-counter/tests/token_counter.rs index 2161652f0b1..10dd5abbbb0 100644 --- a/litellm-rust/crates/token-counter/tests/token_counter.rs +++ b/litellm-rust/crates/token-counter/tests/token_counter.rs @@ -80,6 +80,17 @@ mod json { const ASSISTANT_CONTENT_WITH_TOOL_CALL: &str = r#"{"model":"claude-sonnet-4-5","messages":[ {"role":"assistant","content":"calling","tool_calls":[{"id":"1","type":"function","function":{"name":"get_weather","arguments":"{\"city\": \"Paris, France\"}"}}]}]}"#; + const NULL_TOOL_CALL_ARGUMENTS: &str = r#"{"model":"claude-sonnet-4-5","messages":[ + {"role":"assistant","tool_calls":[{"id":"1","type":"function","function":{"name":"f","arguments":null}}]}, + {"role":"tool","tool_call_id":"1","content":"x"}]}"#; + + const MISSING_TOOL_CALL_ARGUMENTS: &str = r#"{"model":"claude-sonnet-4-5","messages":[ + {"role":"assistant","tool_calls":[{"id":"1","type":"function","function":{"name":"f"}}]}, + {"role":"tool","tool_call_id":"1","content":"x"}]}"#; + + const LEGACY_NULL_FUNCTION_CALL_ARGUMENTS: &str = r#"{"model":"claude-sonnet-4-5","messages":[ + {"role":"assistant","function_call":{"name":"f","arguments":null}}]}"#; + fn assert_count_request_matches_python_token_counter( load: JsonLoader, body: &str, @@ -167,6 +178,9 @@ mod json { #[case::tool_calls_and_tool_result(super::TOOL_CALLS_AND_TOOL_RESULT, 24)] #[case::legacy_function_call(super::LEGACY_FUNCTION_CALL, 14)] #[case::assistant_content_with_tool_call(super::ASSISTANT_CONTENT_WITH_TOOL_CALL, 16)] + #[case::null_tool_call_arguments_count_as_none(super::NULL_TOOL_CALL_ARGUMENTS, 14)] + #[case::missing_tool_call_arguments_count_as_empty(super::MISSING_TOOL_CALL_ARGUMENTS, 13)] + #[case::legacy_null_function_call_arguments_count_as_none(super::LEGACY_NULL_FUNCTION_CALL_ARGUMENTS, 8)] fn count_request_matches_python_token_counter( #[case] body: &str, #[case] expected: usize, From bc7cce592057d785c5dfc7467b8bdb84ac43660f Mon Sep 17 00:00:00 2001 From: James Date: Fri, 25 Sep 2026 11:57:33 -0700 Subject: [PATCH 3/3] docs(rust): describe the null arguments mapping and decline coverage Fix the tool call doc comment that still claimed a non-string arguments declines, and spell out the null-to-None mapping it now shares with the deserializer. Note the deliberate absence of deny_unknown_fields, since python reads only function.arguments and the extra id, type and name keys are ignored rather than declined. Restore decline coverage for malformed tool call shapes: tool_calls that is not a list, a tool call with no function key, and a non-string non-null arguments such as a number or an object, all surfaced as the crate parse error so the host falls back to the python counter --- litellm-rust/crates/token-counter/src/types.rs | 12 +++++++++--- .../crates/token-counter/tests/token_counter.rs | 12 ++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/litellm-rust/crates/token-counter/src/types.rs b/litellm-rust/crates/token-counter/src/types.rs index 7f01219d4c0..73c21039fcf 100644 --- a/litellm-rust/crates/token-counter/src/types.rs +++ b/litellm-rust/crates/token-counter/src/types.rs @@ -139,9 +139,15 @@ impl<'de> Visitor<'de> for TextValueVisitor { /// The parts of an assistant tool call Python counts: only the `arguments` /// string contributes (`_count_function_call_tokens` in -/// `litellm_core_utils/token_counter.py`). Absent arguments count as the empty -/// string; a non-string `arguments` declines so Python handles the fallback -/// instead of miscounting. +/// `litellm_core_utils/token_counter.py`). An absent arguments key counts as the +/// empty string, while an explicit null counts as the string "None" the way +/// Python's `str(None)` does; any other non-string `arguments` declines so +/// Python handles the fallback instead of miscounting. +/// +/// `deny_unknown_fields` is deliberately absent from this shape, and from +/// `ToolCallFunction` and `LegacyFunctionCall`: Python reads only +/// `function.arguments` off a tool call, so the extra `id`, `type` and `name` +/// keys the OpenAI shape carries are ignored rather than declined. #[derive(Clone, Debug, Deserialize, PartialEq)] pub(crate) struct ToolCall { pub(crate) function: ToolCallFunction, diff --git a/litellm-rust/crates/token-counter/tests/token_counter.rs b/litellm-rust/crates/token-counter/tests/token_counter.rs index 10dd5abbbb0..77dc618db38 100644 --- a/litellm-rust/crates/token-counter/tests/token_counter.rs +++ b/litellm-rust/crates/token-counter/tests/token_counter.rs @@ -262,6 +262,18 @@ mod json { #[rstest] #[case::not_json(b"not json" as &[u8])] #[case::messages_not_a_list(br#"{"model":"m","messages":"hi"}"#)] +#[case::tool_calls_not_a_list( + br#"{"model":"m","messages":[{"role":"assistant","tool_calls":"nope"}]}"# +)] +#[case::tool_call_without_function( + br#"{"model":"m","messages":[{"role":"assistant","tool_calls":[{"id":"1","type":"function"}]}]}"# +)] +#[case::tool_call_arguments_is_a_number( + br#"{"model":"m","messages":[{"role":"assistant","tool_calls":[{"id":"1","type":"function","function":{"name":"f","arguments":42}}]}]}"# +)] +#[case::tool_call_arguments_is_an_object( + br#"{"model":"m","messages":[{"role":"assistant","tool_calls":[{"id":"1","type":"function","function":{"name":"f","arguments":{"city":"Tokyo"}}}]}]}"# +)] #[case::dict_content( br#"{"model":"m","messages":[{"role":"user","content":{"type":"text","text":"x"}}]}"# )]