From 516c87028e21715e608adcd130be32aca90cdb36 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:23:36 -0700 Subject: [PATCH] fix(gateway): answer chat completions with id, object and the usage and provider fields The gateway's 200 body had no id, object, usage.completion_tokens_details or message.provider_specific_fields, so clients that read them got nothing where the Python proxy returns all four. The route now mints the chatcmpl id before the cache store, so a cache hit repeats it, and each provider fills the details it knows --- litellm-rust/Cargo.lock | 2 + .../crates/core-utils/src/core_helpers.rs | 33 +++++++++- .../crates/gateway-inference/Cargo.toml | 1 + .../crates/gateway-inference/tests/routes.rs | 63 ++++++++++++++++++ litellm-rust/crates/inference-chat/Cargo.toml | 1 + .../crates/inference-chat/src/handler.rs | 14 +++- litellm-rust/crates/inference-chat/src/lib.rs | 34 +++++----- .../crates/inference-chat/src/route.rs | 4 +- .../inference-chat/tests/chat_completions.rs | 14 ++-- .../crates/inference-chat/tests/request.rs | 2 +- litellm-rust/crates/llms-types/AGENTS.md | 2 +- .../src/formats/chat_completions/AGENTS.md | 2 +- .../src/formats/chat_completions/response.rs | 65 +++++++++++++++++-- .../llms/src/anthropic/chat/transformation.rs | 60 ++++++++++++++++- .../bedrock/chat/converse_transformation.rs | 22 +++++++ .../src/openai_like/chat/transformation.rs | 52 +++++++++++++++ .../chat_completions/test_requests.py | 15 +++++ 17 files changed, 346 insertions(+), 40 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index e959f77917b..429d5c96076 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3887,6 +3887,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "tower", + "uuid", "wiremock", ] @@ -4100,6 +4101,7 @@ dependencies = [ "serde_json", "tokio", "tracing", + "uuid", "wiremock", ] diff --git a/litellm-rust/crates/core-utils/src/core_helpers.rs b/litellm-rust/crates/core-utils/src/core_helpers.rs index 58ee88d301c..83fa685fadc 100644 --- a/litellm-rust/crates/core-utils/src/core_helpers.rs +++ b/litellm-rust/crates/core-utils/src/core_helpers.rs @@ -2,7 +2,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; -use litellm_llms_types::formats::chat_completions::{ChatCompletionsUsage, PromptTokensDetails}; +use litellm_llms_types::formats::chat_completions::{ + ChatCompletionsUsage, CompletionTokensDetails, PromptTokensDetails, +}; /// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the /// reasons the providers on this route can emit. Python warns and falls back to @@ -30,13 +32,16 @@ pub fn finish_reason_for(provider_reason: &str) -> &'static str { /// Python folds cache tokens into `prompt_tokens` and reports the split under /// `prompt_tokens_details`; mirror that so cost tracking agrees on both paths. +/// Python caps the reported reasoning count at the output, and the rest is text. pub fn usage_from_parts( input_tokens: u64, output_tokens: u64, cache_read_tokens: u64, cache_creation_tokens: u64, + reported_reasoning_tokens: u64, ) -> ChatCompletionsUsage { let prompt_tokens = input_tokens + cache_read_tokens + cache_creation_tokens; + let reasoning_tokens = reported_reasoning_tokens.min(output_tokens); ChatCompletionsUsage { prompt_tokens, completion_tokens: output_tokens, @@ -46,6 +51,11 @@ pub fn usage_from_parts( cache_creation_tokens, text_tokens: input_tokens, }, + completion_tokens_details: Some(CompletionTokensDetails { + reasoning_tokens: Some(reasoning_tokens), + text_tokens: Some(output_tokens - reasoning_tokens), + ..CompletionTokensDetails::default() + }), } } @@ -93,7 +103,7 @@ mod tests { #[test] fn folds_cache_tokens_into_prompt_tokens() { - let usage = usage_from_parts(10, 4, 7, 3); + let usage = usage_from_parts(10, 4, 7, 3, 0); assert_eq!(usage.prompt_tokens, 20); assert_eq!(usage.completion_tokens, 4); assert_eq!(usage.total_tokens, 24); @@ -104,9 +114,26 @@ mod tests { #[test] fn reports_raw_input_tokens_when_no_cache_is_involved() { - let usage = usage_from_parts(12, 5, 0, 0); + let usage = usage_from_parts(12, 5, 0, 0, 0); assert_eq!(usage.prompt_tokens, 12); assert_eq!(usage.total_tokens, 17); assert_eq!(usage.prompt_tokens_details.text_tokens, 12); } + + #[rstest] + #[case::no_reasoning(0, 0, 10)] + #[case::splits_the_output(3, 3, 7)] + #[case::all_reasoning(10, 10, 0)] + #[case::over_report_is_capped_at_the_output(20, 10, 0)] + fn splits_completion_tokens_into_reasoning_and_text( + #[case] reported: u64, + #[case] reasoning: u64, + #[case] text: u64, + ) { + let details = usage_from_parts(1, 10, 0, 0, reported) + .completion_tokens_details + .expect("details are always reported"); + assert_eq!(details.reasoning_tokens, Some(reasoning)); + assert_eq!(details.text_tokens, Some(text)); + } } diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 034c3a9826e..dbc1a4b2433 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -37,4 +37,5 @@ tokio = { workspace = true, features = ["io-util"] } rstest.workspace = true tokio-tungstenite.workspace = true tower = { version = "0.5.3", features = ["util"] } +uuid.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/gateway-inference/tests/routes.rs b/litellm-rust/crates/gateway-inference/tests/routes.rs index 24b8c5cc694..5a8f9552086 100644 --- a/litellm-rust/crates/gateway-inference/tests/routes.rs +++ b/litellm-rust/crates/gateway-inference/tests/routes.rs @@ -67,6 +67,69 @@ async fn chat_aliases_call_core_and_use_the_body_model_before_the_path( ); } +#[rstest] +#[case::anthropic( + "anthropic/test-model", + json!({ + "id": "msg_upstream", "model": "test-model", "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 10, "output_tokens_details": {"thinking_tokens": 4}} + }), + json!({"reasoning_tokens": 4, "text_tokens": 6}), + json!({"citations": null, "thinking_blocks": null}) +)] +#[case::openai_like( + "openai_like/test-model", + json!({ + "id": "chatcmpl-upstream", "model": "test-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 3, "completion_tokens": 10, "total_tokens": 13, + "completion_tokens_details": {"reasoning_tokens": 4, "audio_tokens": 0} + } + }), + json!({"reasoning_tokens": 4, "audio_tokens": 0}), + Value::Null +)] +#[tokio::test] +async fn chat_answers_with_a_minted_id_and_the_full_openai_shape( + #[case] model: &str, + #[case] provider_body: Value, + #[case] completion_tokens_details: Value, + #[case] provider_specific_fields: Value, +) { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(provider_body)) + .mount(&upstream) + .await; + let app = support::app(model, &upstream.uri()); + let request = json!({ + "model": "public/model", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + }); + + let first = + support::json(support::post(app.clone(), "/v1/chat/completions", request.clone()).await) + .await; + let second = support::json(support::post(app, "/v1/chat/completions", request).await).await; + + let id = first["id"].as_str().expect("the body carries an id"); + let uuid = id.strip_prefix("chatcmpl-").expect("a chatcmpl id"); + assert!(uuid::Uuid::parse_str(uuid).is_ok(), "{id} ends in a uuid"); + assert_ne!(first["id"], second["id"]); + assert_eq!(first["object"], "chat.completion"); + assert_eq!( + first["usage"]["completion_tokens_details"], + completion_tokens_details + ); + assert_eq!( + first["choices"][0]["message"]["provider_specific_fields"], + provider_specific_fields + ); +} + #[rstest] #[case::streaming(json!({"messages": [{"role": "user", "content": "hi"}], "stream": true}), 501)] #[case::missing_messages(json!({}), 400)] diff --git a/litellm-rust/crates/inference-chat/Cargo.toml b/litellm-rust/crates/inference-chat/Cargo.toml index 8a779d44976..9d0dabf97a3 100644 --- a/litellm-rust/crates/inference-chat/Cargo.toml +++ b/litellm-rust/crates/inference-chat/Cargo.toml @@ -19,6 +19,7 @@ litellm-secrets.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } futures-util.workspace = true tracing.workspace = true +uuid.workspace = true [dev-dependencies] litellm-cache-memory.workspace = true diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 1c724a47956..7d6134d0939 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -9,9 +9,10 @@ use litellm_llms::base_llm::{ chat::transformation::BaseConfig, chat::transformation::ProviderChatResponseData, }; -use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; +use litellm_llms_types::formats::chat_completions::{ChatCompletion, ChatCompletionsResponse}; use litellm_secrets::source::Secrets; use serde_json::Value; +use uuid::Uuid; use super::Error; use crate::{ @@ -19,6 +20,14 @@ use crate::{ types::ResolvedChatCompletionsRequest, }; +fn minted(body: ChatCompletionsResponse) -> ChatCompletion { + ChatCompletion { + id: format!("chatcmpl-{}", Uuid::new_v4()), + object: "chat.completion".to_owned(), + body, + } +} + #[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately pub(super) async fn execute( http: &Client, @@ -31,7 +40,7 @@ pub(super) async fn execute( cache_options: Option, interceptors: &impl Interceptors, observers: Option<&ObservationSender>, -) -> Result { +) -> Result { let validated = config.validate_environment( string_headers(call.extra_headers)?, call.api_key.as_deref(), @@ -127,6 +136,7 @@ pub(super) async fn execute( })?; config .transform_response(provider.model, ProviderChatResponseData { body }) + .map(minted) .map_err(Error::from) .map_err(Error::after_send) }, diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index 3be83840901..61b58330ccc 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -5,7 +5,7 @@ pub use litellm_inference::Error; mod common_utils; pub mod constants; pub(crate) mod handler; -use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; +use litellm_llms_types::formats::chat_completions::ChatCompletion; use litellm_auth::AuthServices; use litellm_inference::provider::resolve_llm_provider; @@ -47,7 +47,7 @@ impl ChatCompletionsRoute { request: ChatCompletionsRequest<'_>, interceptors: &impl litellm_host::interceptors::Interceptors, options: impl Into, - ) -> Result { + ) -> Result { let litellm_inference::CallOptions { cache: cache_options, observers, @@ -78,7 +78,7 @@ impl ChatCompletionsRoute { cache_options: Option, interceptors: &impl litellm_host::interceptors::Interceptors, observers: Option<&ObservationSender>, - ) -> Result { + ) -> Result { litellm_inference::diagnostic::unary(async { let model = call.model.clone(); let custom_llm_provider = call.custom_llm_provider.clone(); @@ -93,21 +93,19 @@ impl ChatCompletionsRoute { <&'static str>::from(provider.provider), ); let secrets = self.secrets.resolve(&config.secret_names()).await?; - let execute: futures_util::future::BoxFuture< - '_, - Result, - > = Box::pin(handler::execute( - &self.http, - &self.auth, - config, - provider, - call, - secrets, - self.cache.clone(), - cache_options, - interceptors, - observers, - )); + let execute: futures_util::future::BoxFuture<'_, Result> = + Box::pin(handler::execute( + &self.http, + &self.auth, + config, + provider, + call, + secrets, + self.cache.clone(), + cache_options, + interceptors, + observers, + )); execute.await }) .await diff --git a/litellm-rust/crates/inference-chat/src/route.rs b/litellm-rust/crates/inference-chat/src/route.rs index 4a7bc0e169f..8ae6140bf40 100644 --- a/litellm-rust/crates/inference-chat/src/route.rs +++ b/litellm-rust/crates/inference-chat/src/route.rs @@ -4,14 +4,14 @@ use litellm_host::{ call::{CallOutput, HostedMachine, hosted_call}, protocol::Protocol, }; -use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; +use litellm_llms_types::formats::chat_completions::ChatCompletion; use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall}; pub struct ChatCompletions; impl Protocol for ChatCompletions { - type Response = ChatCompletionsResponse; + type Response = ChatCompletion; type Error = Error; type Request = ChatCompletionsCall; type HostCall = Infallible; diff --git a/litellm-rust/crates/inference-chat/tests/chat_completions.rs b/litellm-rust/crates/inference-chat/tests/chat_completions.rs index 6ed513ba32a..726c904e89a 100644 --- a/litellm-rust/crates/inference-chat/tests/chat_completions.rs +++ b/litellm-rust/crates/inference-chat/tests/chat_completions.rs @@ -6,7 +6,7 @@ use litellm_host::{ use std::time::Duration; use litellm_inference_chat::{Error, types::ChatCompletionsRequest}; -use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; +use litellm_llms_types::formats::chat_completions::ChatCompletion; use rstest::{fixture, rstest}; use serde_json::{Map, Value, json}; use wiremock::ResponseTemplate; @@ -16,7 +16,7 @@ use support::*; const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5-20260101","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#; -async fn complete(request: ChatCompletionsRequest<'_>) -> Result { +async fn complete(request: ChatCompletionsRequest<'_>) -> Result { chat_completions_route().execute(request, &(), None).await } @@ -83,10 +83,10 @@ async fn anthropic_round_trip_translates_the_conversation_and_normalizes_the_res ); assert_eq!(body["max_tokens"], 16); assert_eq!( - response.choices[0].message.content.as_deref(), + response.body.choices[0].message.content.as_deref(), Some("hello") ); - assert_eq!(response.usage.total_tokens, 15); + assert_eq!(response.body.usage.total_tokens, 15); } #[rstest] @@ -152,10 +152,10 @@ async fn bedrock_round_trip_is_signed_and_normalized(request: ChatCompletionsReq json!([{"role": "user", "content": [{"text": "hi"}]}]) ); assert_eq!( - response.choices[0].message.content.as_deref(), + response.body.choices[0].message.content.as_deref(), Some("hello") ); - assert_eq!(response.usage.total_tokens, 15); + assert_eq!(response.body.usage.total_tokens, 15); } #[rstest] @@ -308,7 +308,7 @@ async fn direct_and_hosted_calls_share_hooks_and_lifecycle( .unwrap() }; assert_eq!( - response.choices[0].message.content.as_deref(), + response.body.choices[0].message.content.as_deref(), Some("hello") ); assert_eq!( diff --git a/litellm-rust/crates/inference-chat/tests/request.rs b/litellm-rust/crates/inference-chat/tests/request.rs index 121f7347c24..e8576445ce1 100644 --- a/litellm-rust/crates/inference-chat/tests/request.rs +++ b/litellm-rust/crates/inference-chat/tests/request.rs @@ -33,7 +33,7 @@ fn request<'a>( async fn complete( request: ChatCompletionsRequest<'_>, -) -> Result { +) -> Result { chat_completions_route().execute(request, &(), None).await } diff --git a/litellm-rust/crates/llms-types/AGENTS.md b/litellm-rust/crates/llms-types/AGENTS.md index aa216d7bdab..8d8bf1241d0 100644 --- a/litellm-rust/crates/llms-types/AGENTS.md +++ b/litellm-rust/crates/llms-types/AGENTS.md @@ -20,7 +20,7 @@ - Name types after the format, never the originator (`formats::messages::MessagesRequest`, not `AnthropicMessagesRequest`) - A field defined by the format stays in the format type even when provider support varies - Keep one canonical definition and import path. Move consumers with the type instead of adding re-exports -- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the response handed to the host, so do not swap it for a fuller upstream schema +- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the normalized provider response, and `ChatCompletion` wraps it with the id the route mints for the host, so do not swap either for a fuller upstream schema ## Providers diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md index 34c9b02c012..8fc818d0867 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md @@ -7,7 +7,7 @@ - accept a subset of parameters, which is recorded per provider in `supported_openai_param_mappings`, not by trimming the type - add top-level request or response fields, preserved through the existing extra-field maps instead of new typed fields - return slightly different usage or finish reasons, normalized in the provider transformation -- `ChatCompletionsResponse` is LiteLLM's response contract to the host, a typed projection rather than the full upstream schema +- `ChatCompletion` is LiteLLM's response contract to the host, the id the route mints plus `ChatCompletionsResponse`, a typed projection rather than the full upstream schema # references diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs index 2ce07e3c465..a75b9f0328c 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs @@ -48,6 +48,21 @@ pub struct PromptTokensDetails { pub text_tokens: u64, } +#[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] +pub struct CompletionTokensDetails { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audio_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accepted_prediction_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rejected_prediction_tokens: Option, +} + #[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ChatCompletionsUsage { @@ -55,6 +70,8 @@ pub struct ChatCompletionsUsage { pub completion_tokens: u64, pub total_tokens: u64, pub prompt_tokens_details: PromptTokensDetails, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_tokens_details: Option, } #[macro_rules_attribute::apply(crate::wire_type)] @@ -65,6 +82,8 @@ pub struct ChatCompletionsChoiceMessage { // while Converse assigns the joined string unconditionally. Each config // mirrors its own, so keep this optional and serialize it even when None. pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, } #[macro_rules_attribute::apply(crate::wire_type)] @@ -74,11 +93,12 @@ pub struct ChatCompletionsChoice { pub finish_reason: String, } -/// The normalized response handed back to the host. +/// The provider's response, normalized by its transformation. /// -/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the -/// `ModelResponse` it already created, and echoing the provider's own id here -/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests. +/// There is deliberately no `id`: the route mints the `chatcmpl-…` id on +/// `ChatCompletion`, and echoing the provider's own id here would compete with +/// it. Pinned by `response_carries_no_id` in the Anthropic chat transformation +/// tests. #[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionsResponse { pub created: u64, @@ -86,3 +106,40 @@ pub struct ChatCompletionsResponse { pub choices: Vec, pub usage: ChatCompletionsUsage, } + +/// What the route hands back to the host. The id is stored in the cached entry, +/// so a cache hit repeats it, and an entry cached without one decodes to a miss. +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct ChatCompletion { + pub id: String, + pub object: String, + #[serde(flatten)] + pub body: ChatCompletionsResponse, +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::ChatCompletion; + + #[test] + fn an_entry_cached_without_an_id_is_not_a_completion() { + let entry = json!({ + "created": 1, + "model": "claude", + "choices": [], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_creation_tokens": 0, + "text_tokens": 1, + }, + }, + }); + assert!(serde_json::from_value::(entry).is_err()); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index eb796a900f5..c4f87358b75 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -74,6 +74,8 @@ struct ResponseUsageProjection { cache_read_input_tokens: Option, #[serde(default)] cache_creation_input_tokens: Option, + #[serde(default)] + output_tokens_details: Option, } pub struct AnthropicConfig; @@ -153,6 +155,10 @@ impl BaseConfig for AnthropicConfig { message: ChatCompletionsChoiceMessage { role: "assistant".to_string(), content: (!text.is_empty()).then_some(text), + provider_specific_fields: Some(Map::from_iter([ + ("citations".to_string(), Value::Null), + ("thinking_blocks".to_string(), Value::Null), + ])), }, finish_reason: finish_reason_for(body.stop_reason.as_deref().unwrap_or("")) .to_string(), @@ -162,6 +168,12 @@ impl BaseConfig for AnthropicConfig { body.usage.output_tokens, body.usage.cache_read_input_tokens.unwrap_or(0), body.usage.cache_creation_input_tokens.unwrap_or(0), + body.usage + .output_tokens_details + .as_ref() + .and_then(|details| details.get("thinking_tokens")) + .and_then(Value::as_u64) + .unwrap_or(0), ), }) } @@ -630,8 +642,54 @@ mod tests { assert_eq!(response.choices[0].message.content, None); } + #[rstest] + #[case::reported(json!({"thinking_tokens": 4}), 4, 6)] + #[case::over_reported_is_capped_at_the_output(json!({"thinking_tokens": 40}), 10, 0)] + #[case::not_reported(json!({}), 0, 10)] + #[case::not_a_count(json!({"thinking_tokens": "many"}), 0, 10)] + #[case::not_an_object(json!("none"), 0, 10)] + fn usage_splits_reported_thinking_tokens_from_the_text( + #[case] output_tokens_details: Value, + #[case] reasoning: u64, + #[case] text: u64, + ) { + let response = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1, + "output_tokens": 10, + "output_tokens_details": output_tokens_details, + } + })) + .expect("response transforms"); + let details = response + .usage + .completion_tokens_details + .expect("details are always reported"); + assert_eq!(details.reasoning_tokens, Some(reasoning)); + assert_eq!(details.text_tokens, Some(text)); + } + #[test] - fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() { + fn message_reports_null_citations_and_thinking_blocks_like_python() { + let response = transform_response(json!({ + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + })) + .expect("response transforms"); + let value = serde_json::to_value(response).expect("serializable"); + assert_eq!( + value["choices"][0]["message"]["provider_specific_fields"], + json!({"citations": null, "thinking_blocks": null}) + ); + } + + #[test] + fn response_carries_no_id_so_the_route_mints_the_chatcmpl_id() { let response = transform_response(json!({ "id": "msg_should_not_leak", "model": "claude-sonnet-4-5", diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index 976c9161dd2..beba73b5c1f 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -176,6 +176,7 @@ impl BaseConfig for AmazonConverseConfig { response.usage.output_tokens, response.usage.cache_read_input_tokens, response.usage.cache_write_input_tokens, + 0, ); // Converse reports `totalTokens` and Python passes it straight through, // where Anthropic has no such field and Python adds the two counts @@ -201,6 +202,7 @@ impl BaseConfig for AmazonConverseConfig { // Anthropic. A caller calling `.strip()` on it would break // on this path alone. content: Some(text), + provider_specific_fields: None, }, finish_reason: finish_reason_for( ConverseStopReason::parse(response.stop_reason) @@ -788,6 +790,26 @@ mod tests { assert_eq!(response.usage.completion_tokens, 4); } + #[test] + fn reports_all_output_as_text_and_no_provider_fields_for_a_text_response() { + let response = transform_response(json!({ + "output": {"message": {"content": [{"text": "x"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 4, "totalTokens": 14} + })) + .expect("response transforms"); + let value = serde_json::to_value(response).expect("serializable"); + assert_eq!( + value["usage"]["completion_tokens_details"], + json!({"reasoning_tokens": 0, "text_tokens": 4}) + ); + assert!( + value["choices"][0]["message"] + .get("provider_specific_fields") + .is_none() + ); + } + #[test] fn falls_back_to_the_computed_total_when_converse_omits_it() { // Python raises a KeyError on a body with no `totalTokens`. Reporting a zero diff --git a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs index e998f8785cb..7ec65d12fcf 100644 --- a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs @@ -177,6 +177,10 @@ impl BaseConfig for OpenAILikeChatConfig { .and_then(Value::as_u64) .unwrap_or(0), }, + completion_tokens_details: usage + .and_then(|usage| usage.get("completion_tokens_details")) + .cloned() + .and_then(|details| serde_json::from_value(details).ok()), }, }) } @@ -273,6 +277,7 @@ fn normalize_choice(position: usize, choice: &Value) -> Result None: + result: Final = await litellm.acompletion( + MESSAGES_MODEL, list(MESSAGES), base_url=server.base_url, api_key="test-key", max_tokens=32 + ) + + assert isinstance(result, ModelResponse) + assert_served_natively(server) + assert result.id.startswith("chatcmpl-") + assert result.object == "chat.completion" + assert result.usage.completion_tokens_details.reasoning_tokens == 0 + assert result.usage.completion_tokens_details.text_tokens == result.usage.completion_tokens + assert result.choices[0].message.provider_specific_fields == {"citations": None, "thinking_blocks": None}