fix(gateway): answer chat completions with id, object and the usage and provider fields

The gateway's 200 body had no id, object, usage.completion_tokens_details or
message.provider_specific_fields, so clients that read them got nothing where the
Python proxy returns all four. The route now mints the chatcmpl id before the cache
store, so a cache hit repeats it, and each provider fills the details it knows
This commit is contained in:
mateo-berri 2026-10-30 10:23:36 -07:00
parent d18efdfe13
commit 516c87028e
17 changed files with 346 additions and 40 deletions

View file

@ -3887,6 +3887,7 @@ dependencies = [
"tokio",
"tokio-tungstenite",
"tower",
"uuid",
"wiremock",
]
@ -4100,6 +4101,7 @@ dependencies = [
"serde_json",
"tokio",
"tracing",
"uuid",
"wiremock",
]

View file

@ -2,7 +2,9 @@
use std::time::{SystemTime, UNIX_EPOCH};
use litellm_llms_types::formats::chat_completions::{ChatCompletionsUsage, PromptTokensDetails};
use litellm_llms_types::formats::chat_completions::{
ChatCompletionsUsage, CompletionTokensDetails, PromptTokensDetails,
};
/// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the
/// reasons the providers on this route can emit. Python warns and falls back to
@ -30,13 +32,16 @@ pub fn finish_reason_for(provider_reason: &str) -> &'static str {
/// Python folds cache tokens into `prompt_tokens` and reports the split under
/// `prompt_tokens_details`; mirror that so cost tracking agrees on both paths.
/// Python caps the reported reasoning count at the output, and the rest is text.
pub fn usage_from_parts(
input_tokens: u64,
output_tokens: u64,
cache_read_tokens: u64,
cache_creation_tokens: u64,
reported_reasoning_tokens: u64,
) -> ChatCompletionsUsage {
let prompt_tokens = input_tokens + cache_read_tokens + cache_creation_tokens;
let reasoning_tokens = reported_reasoning_tokens.min(output_tokens);
ChatCompletionsUsage {
prompt_tokens,
completion_tokens: output_tokens,
@ -46,6 +51,11 @@ pub fn usage_from_parts(
cache_creation_tokens,
text_tokens: input_tokens,
},
completion_tokens_details: Some(CompletionTokensDetails {
reasoning_tokens: Some(reasoning_tokens),
text_tokens: Some(output_tokens - reasoning_tokens),
..CompletionTokensDetails::default()
}),
}
}
@ -93,7 +103,7 @@ mod tests {
#[test]
fn folds_cache_tokens_into_prompt_tokens() {
let usage = usage_from_parts(10, 4, 7, 3);
let usage = usage_from_parts(10, 4, 7, 3, 0);
assert_eq!(usage.prompt_tokens, 20);
assert_eq!(usage.completion_tokens, 4);
assert_eq!(usage.total_tokens, 24);
@ -104,9 +114,26 @@ mod tests {
#[test]
fn reports_raw_input_tokens_when_no_cache_is_involved() {
let usage = usage_from_parts(12, 5, 0, 0);
let usage = usage_from_parts(12, 5, 0, 0, 0);
assert_eq!(usage.prompt_tokens, 12);
assert_eq!(usage.total_tokens, 17);
assert_eq!(usage.prompt_tokens_details.text_tokens, 12);
}
#[rstest]
#[case::no_reasoning(0, 0, 10)]
#[case::splits_the_output(3, 3, 7)]
#[case::all_reasoning(10, 10, 0)]
#[case::over_report_is_capped_at_the_output(20, 10, 0)]
fn splits_completion_tokens_into_reasoning_and_text(
#[case] reported: u64,
#[case] reasoning: u64,
#[case] text: u64,
) {
let details = usage_from_parts(1, 10, 0, 0, reported)
.completion_tokens_details
.expect("details are always reported");
assert_eq!(details.reasoning_tokens, Some(reasoning));
assert_eq!(details.text_tokens, Some(text));
}
}

View file

@ -37,4 +37,5 @@ tokio = { workspace = true, features = ["io-util"] }
rstest.workspace = true
tokio-tungstenite.workspace = true
tower = { version = "0.5.3", features = ["util"] }
uuid.workspace = true
wiremock.workspace = true

View file

@ -67,6 +67,69 @@ async fn chat_aliases_call_core_and_use_the_body_model_before_the_path(
);
}
#[rstest]
#[case::anthropic(
"anthropic/test-model",
json!({
"id": "msg_upstream", "model": "test-model", "content": [{"type": "text", "text": "hello"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 10, "output_tokens_details": {"thinking_tokens": 4}}
}),
json!({"reasoning_tokens": 4, "text_tokens": 6}),
json!({"citations": null, "thinking_blocks": null})
)]
#[case::openai_like(
"openai_like/test-model",
json!({
"id": "chatcmpl-upstream", "model": "test-model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}],
"usage": {
"prompt_tokens": 3, "completion_tokens": 10, "total_tokens": 13,
"completion_tokens_details": {"reasoning_tokens": 4, "audio_tokens": 0}
}
}),
json!({"reasoning_tokens": 4, "audio_tokens": 0}),
Value::Null
)]
#[tokio::test]
async fn chat_answers_with_a_minted_id_and_the_full_openai_shape(
#[case] model: &str,
#[case] provider_body: Value,
#[case] completion_tokens_details: Value,
#[case] provider_specific_fields: Value,
) {
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200).set_body_json(provider_body))
.mount(&upstream)
.await;
let app = support::app(model, &upstream.uri());
let request = json!({
"model": "public/model",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 16,
});
let first =
support::json(support::post(app.clone(), "/v1/chat/completions", request.clone()).await)
.await;
let second = support::json(support::post(app, "/v1/chat/completions", request).await).await;
let id = first["id"].as_str().expect("the body carries an id");
let uuid = id.strip_prefix("chatcmpl-").expect("a chatcmpl id");
assert!(uuid::Uuid::parse_str(uuid).is_ok(), "{id} ends in a uuid");
assert_ne!(first["id"], second["id"]);
assert_eq!(first["object"], "chat.completion");
assert_eq!(
first["usage"]["completion_tokens_details"],
completion_tokens_details
);
assert_eq!(
first["choices"][0]["message"]["provider_specific_fields"],
provider_specific_fields
);
}
#[rstest]
#[case::streaming(json!({"messages": [{"role": "user", "content": "hi"}], "stream": true}), 501)]
#[case::missing_messages(json!({}), 400)]

View file

@ -19,6 +19,7 @@ litellm-secrets.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
futures-util.workspace = true
tracing.workspace = true
uuid.workspace = true
[dev-dependencies]
litellm-cache-memory.workspace = true

View file

@ -9,9 +9,10 @@ use litellm_llms::base_llm::{
chat::transformation::BaseConfig,
chat::transformation::ProviderChatResponseData,
};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use litellm_llms_types::formats::chat_completions::{ChatCompletion, ChatCompletionsResponse};
use litellm_secrets::source::Secrets;
use serde_json::Value;
use uuid::Uuid;
use super::Error;
use crate::{
@ -19,6 +20,14 @@ use crate::{
types::ResolvedChatCompletionsRequest,
};
fn minted(body: ChatCompletionsResponse) -> ChatCompletion {
ChatCompletion {
id: format!("chatcmpl-{}", Uuid::new_v4()),
object: "chat.completion".to_owned(),
body,
}
}
#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
pub(super) async fn execute(
http: &Client,
@ -31,7 +40,7 @@ pub(super) async fn execute(
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ChatCompletionsResponse, Error> {
) -> Result<ChatCompletion, Error> {
let validated = config.validate_environment(
string_headers(call.extra_headers)?,
call.api_key.as_deref(),
@ -127,6 +136,7 @@ pub(super) async fn execute(
})?;
config
.transform_response(provider.model, ProviderChatResponseData { body })
.map(minted)
.map_err(Error::from)
.map_err(Error::after_send)
},

View file

@ -5,7 +5,7 @@ pub use litellm_inference::Error;
mod common_utils;
pub mod constants;
pub(crate) mod handler;
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use litellm_llms_types::formats::chat_completions::ChatCompletion;
use litellm_auth::AuthServices;
use litellm_inference::provider::resolve_llm_provider;
@ -47,7 +47,7 @@ impl ChatCompletionsRoute {
request: ChatCompletionsRequest<'_>,
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
options: impl Into<litellm_inference::CallOptions>,
) -> Result<ChatCompletionsResponse, Error> {
) -> Result<ChatCompletion, Error> {
let litellm_inference::CallOptions {
cache: cache_options,
observers,
@ -78,7 +78,7 @@ impl ChatCompletionsRoute {
cache_options: Option<litellm_cache_response::CachePolicy>,
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<ChatCompletionsResponse, Error> {
) -> Result<ChatCompletion, Error> {
litellm_inference::diagnostic::unary(async {
let model = call.model.clone();
let custom_llm_provider = call.custom_llm_provider.clone();
@ -93,21 +93,19 @@ impl ChatCompletionsRoute {
<&'static str>::from(provider.provider),
);
let secrets = self.secrets.resolve(&config.secret_names()).await?;
let execute: futures_util::future::BoxFuture<
'_,
Result<ChatCompletionsResponse, Error>,
> = Box::pin(handler::execute(
&self.http,
&self.auth,
config,
provider,
call,
secrets,
self.cache.clone(),
cache_options,
interceptors,
observers,
));
let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletion, Error>> =
Box::pin(handler::execute(
&self.http,
&self.auth,
config,
provider,
call,
secrets,
self.cache.clone(),
cache_options,
interceptors,
observers,
));
execute.await
})
.await

View file

@ -4,14 +4,14 @@ use litellm_host::{
call::{CallOutput, HostedMachine, hosted_call},
protocol::Protocol,
};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use litellm_llms_types::formats::chat_completions::ChatCompletion;
use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
pub struct ChatCompletions;
impl Protocol for ChatCompletions {
type Response = ChatCompletionsResponse;
type Response = ChatCompletion;
type Error = Error;
type Request = ChatCompletionsCall;
type HostCall = Infallible;

View file

@ -6,7 +6,7 @@ use litellm_host::{
use std::time::Duration;
use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use litellm_llms_types::formats::chat_completions::ChatCompletion;
use rstest::{fixture, rstest};
use serde_json::{Map, Value, json};
use wiremock::ResponseTemplate;
@ -16,7 +16,7 @@ use support::*;
const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5-20260101","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
async fn complete(request: ChatCompletionsRequest<'_>) -> Result<ChatCompletionsResponse, Error> {
async fn complete(request: ChatCompletionsRequest<'_>) -> Result<ChatCompletion, Error> {
chat_completions_route().execute(request, &(), None).await
}
@ -83,10 +83,10 @@ async fn anthropic_round_trip_translates_the_conversation_and_normalizes_the_res
);
assert_eq!(body["max_tokens"], 16);
assert_eq!(
response.choices[0].message.content.as_deref(),
response.body.choices[0].message.content.as_deref(),
Some("hello")
);
assert_eq!(response.usage.total_tokens, 15);
assert_eq!(response.body.usage.total_tokens, 15);
}
#[rstest]
@ -152,10 +152,10 @@ async fn bedrock_round_trip_is_signed_and_normalized(request: ChatCompletionsReq
json!([{"role": "user", "content": [{"text": "hi"}]}])
);
assert_eq!(
response.choices[0].message.content.as_deref(),
response.body.choices[0].message.content.as_deref(),
Some("hello")
);
assert_eq!(response.usage.total_tokens, 15);
assert_eq!(response.body.usage.total_tokens, 15);
}
#[rstest]
@ -308,7 +308,7 @@ async fn direct_and_hosted_calls_share_hooks_and_lifecycle(
.unwrap()
};
assert_eq!(
response.choices[0].message.content.as_deref(),
response.body.choices[0].message.content.as_deref(),
Some("hello")
);
assert_eq!(

View file

@ -33,7 +33,7 @@ fn request<'a>(
async fn complete(
request: ChatCompletionsRequest<'_>,
) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletion, Error> {
chat_completions_route().execute(request, &(), None).await
}

View file

@ -20,7 +20,7 @@
- Name types after the format, never the originator (`formats::messages::MessagesRequest`, not `AnthropicMessagesRequest`)
- A field defined by the format stays in the format type even when provider support varies
- Keep one canonical definition and import path. Move consumers with the type instead of adding re-exports
- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the response handed to the host, so do not swap it for a fuller upstream schema
- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the normalized provider response, and `ChatCompletion` wraps it with the id the route mints for the host, so do not swap either for a fuller upstream schema
## Providers

View file

@ -7,7 +7,7 @@
- accept a subset of parameters, which is recorded per provider in `supported_openai_param_mappings`, not by trimming the type
- add top-level request or response fields, preserved through the existing extra-field maps instead of new typed fields
- return slightly different usage or finish reasons, normalized in the provider transformation
- `ChatCompletionsResponse` is LiteLLM's response contract to the host, a typed projection rather than the full upstream schema
- `ChatCompletion` is LiteLLM's response contract to the host, the id the route mints plus `ChatCompletionsResponse`, a typed projection rather than the full upstream schema
# references

View file

@ -48,6 +48,21 @@ pub struct PromptTokensDetails {
pub text_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CompletionTokensDetails {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub audio_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub accepted_prediction_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rejected_prediction_tokens: Option<u64>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionsUsage {
@ -55,6 +70,8 @@ pub struct ChatCompletionsUsage {
pub completion_tokens: u64,
pub total_tokens: u64,
pub prompt_tokens_details: PromptTokensDetails,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completion_tokens_details: Option<CompletionTokensDetails>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
@ -65,6 +82,8 @@ pub struct ChatCompletionsChoiceMessage {
// while Converse assigns the joined string unconditionally. Each config
// mirrors its own, so keep this optional and serialize it even when None.
pub content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
@ -74,11 +93,12 @@ pub struct ChatCompletionsChoice {
pub finish_reason: String,
}
/// The normalized response handed back to the host.
/// The provider's response, normalized by its transformation.
///
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
/// `ModelResponse` it already created, and echoing the provider's own id here
/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests.
/// There is deliberately no `id`: the route mints the `chatcmpl-…` id on
/// `ChatCompletion`, and echoing the provider's own id here would compete with
/// it. Pinned by `response_carries_no_id` in the Anthropic chat transformation
/// tests.
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsResponse {
pub created: u64,
@ -86,3 +106,40 @@ pub struct ChatCompletionsResponse {
pub choices: Vec<ChatCompletionsChoice>,
pub usage: ChatCompletionsUsage,
}
/// What the route hands back to the host. The id is stored in the cached entry,
/// so a cache hit repeats it, and an entry cached without one decodes to a miss.
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletion {
pub id: String,
pub object: String,
#[serde(flatten)]
pub body: ChatCompletionsResponse,
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::ChatCompletion;
#[test]
fn an_entry_cached_without_an_id_is_not_a_completion() {
let entry = json!({
"created": 1,
"model": "claude",
"choices": [],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
"prompt_tokens_details": {
"cached_tokens": 0,
"cache_creation_tokens": 0,
"text_tokens": 1,
},
},
});
assert!(serde_json::from_value::<ChatCompletion>(entry).is_err());
}
}

View file

@ -74,6 +74,8 @@ struct ResponseUsageProjection {
cache_read_input_tokens: Option<u64>,
#[serde(default)]
cache_creation_input_tokens: Option<u64>,
#[serde(default)]
output_tokens_details: Option<Value>,
}
pub struct AnthropicConfig;
@ -153,6 +155,10 @@ impl BaseConfig for AnthropicConfig {
message: ChatCompletionsChoiceMessage {
role: "assistant".to_string(),
content: (!text.is_empty()).then_some(text),
provider_specific_fields: Some(Map::from_iter([
("citations".to_string(), Value::Null),
("thinking_blocks".to_string(), Value::Null),
])),
},
finish_reason: finish_reason_for(body.stop_reason.as_deref().unwrap_or(""))
.to_string(),
@ -162,6 +168,12 @@ impl BaseConfig for AnthropicConfig {
body.usage.output_tokens,
body.usage.cache_read_input_tokens.unwrap_or(0),
body.usage.cache_creation_input_tokens.unwrap_or(0),
body.usage
.output_tokens_details
.as_ref()
.and_then(|details| details.get("thinking_tokens"))
.and_then(Value::as_u64)
.unwrap_or(0),
),
})
}
@ -630,8 +642,54 @@ mod tests {
assert_eq!(response.choices[0].message.content, None);
}
#[rstest]
#[case::reported(json!({"thinking_tokens": 4}), 4, 6)]
#[case::over_reported_is_capped_at_the_output(json!({"thinking_tokens": 40}), 10, 0)]
#[case::not_reported(json!({}), 0, 10)]
#[case::not_a_count(json!({"thinking_tokens": "many"}), 0, 10)]
#[case::not_an_object(json!("none"), 0, 10)]
fn usage_splits_reported_thinking_tokens_from_the_text(
#[case] output_tokens_details: Value,
#[case] reasoning: u64,
#[case] text: u64,
) {
let response = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {
"input_tokens": 1,
"output_tokens": 10,
"output_tokens_details": output_tokens_details,
}
}))
.expect("response transforms");
let details = response
.usage
.completion_tokens_details
.expect("details are always reported");
assert_eq!(details.reasoning_tokens, Some(reasoning));
assert_eq!(details.text_tokens, Some(text));
}
#[test]
fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() {
fn message_reports_null_citations_and_thinking_blocks_like_python() {
let response = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.expect("response transforms");
let value = serde_json::to_value(response).expect("serializable");
assert_eq!(
value["choices"][0]["message"]["provider_specific_fields"],
json!({"citations": null, "thinking_blocks": null})
);
}
#[test]
fn response_carries_no_id_so_the_route_mints_the_chatcmpl_id() {
let response = transform_response(json!({
"id": "msg_should_not_leak",
"model": "claude-sonnet-4-5",

View file

@ -176,6 +176,7 @@ impl BaseConfig for AmazonConverseConfig {
response.usage.output_tokens,
response.usage.cache_read_input_tokens,
response.usage.cache_write_input_tokens,
0,
);
// Converse reports `totalTokens` and Python passes it straight through,
// where Anthropic has no such field and Python adds the two counts
@ -201,6 +202,7 @@ impl BaseConfig for AmazonConverseConfig {
// Anthropic. A caller calling `.strip()` on it would break
// on this path alone.
content: Some(text),
provider_specific_fields: None,
},
finish_reason: finish_reason_for(
ConverseStopReason::parse(response.stop_reason)
@ -788,6 +790,26 @@ mod tests {
assert_eq!(response.usage.completion_tokens, 4);
}
#[test]
fn reports_all_output_as_text_and_no_provider_fields_for_a_text_response() {
let response = transform_response(json!({
"output": {"message": {"content": [{"text": "x"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 10, "outputTokens": 4, "totalTokens": 14}
}))
.expect("response transforms");
let value = serde_json::to_value(response).expect("serializable");
assert_eq!(
value["usage"]["completion_tokens_details"],
json!({"reasoning_tokens": 0, "text_tokens": 4})
);
assert!(
value["choices"][0]["message"]
.get("provider_specific_fields")
.is_none()
);
}
#[test]
fn falls_back_to_the_computed_total_when_converse_omits_it() {
// Python raises a KeyError on a body with no `totalTokens`. Reporting a zero

View file

@ -177,6 +177,10 @@ impl BaseConfig for OpenAILikeChatConfig {
.and_then(Value::as_u64)
.unwrap_or(0),
},
completion_tokens_details: usage
.and_then(|usage| usage.get("completion_tokens_details"))
.cloned()
.and_then(|details| serde_json::from_value(details).ok()),
},
})
}
@ -273,6 +277,7 @@ fn normalize_choice(position: usize, choice: &Value) -> Result<ChatCompletionsCh
.unwrap_or("assistant")
.to_string(),
content: content.and_then(Value::as_str).map(str::to_string),
provider_specific_fields: None,
},
finish_reason: choice
.get("finish_reason")
@ -526,6 +531,53 @@ mod tests {
assert_eq!(response.usage.total_tokens, 8);
}
#[rstest]
#[case::passes_through_what_the_upstream_reports(
json!({"reasoning_tokens": 2, "audio_tokens": 0, "accepted_prediction_tokens": 1, "rejected_prediction_tokens": 3}),
json!({"reasoning_tokens": 2, "audio_tokens": 0, "accepted_prediction_tokens": 1, "rejected_prediction_tokens": 3})
)]
#[case::null_upstream_field_is_left_out(
json!({"reasoning_tokens": 2, "audio_tokens": null}),
json!({"reasoning_tokens": 2})
)]
#[case::null_details_are_left_out(Value::Null, Value::Null)]
#[case::malformed_details_are_left_out(json!({"reasoning_tokens": "two"}), Value::Null)]
fn completion_tokens_details_mirror_the_upstream(
#[case] details: Value,
#[case] expected: Value,
) {
let response = transform_response(json!({
"model": "m",
"choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 5,
"total_tokens": 6,
"completion_tokens_details": details,
},
}))
.expect("response normalizes");
let value = serde_json::to_value(response).expect("serializable");
assert_eq!(value["usage"]["completion_tokens_details"], expected);
}
#[rstest]
fn an_upstream_without_details_reports_none() {
let response = transform_response(json!({
"model": "m",
"choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 5, "total_tokens": 6},
}))
.expect("response normalizes");
let value = serde_json::to_value(response).expect("serializable");
assert!(value["usage"].get("completion_tokens_details").is_none());
assert!(
value["choices"][0]["message"]
.get("provider_specific_fields")
.is_none()
);
}
#[rstest]
fn null_token_fields_in_usage_become_zero() {
// `_sanitize_usage_obj`: providers that return null token values break

View file

@ -38,3 +38,18 @@ async def test_positional_parameters_reach_the_wire(server: RecordingServer, asy
body: Final = _OBJECT.validate_python(server.requests[0].body)
assert body["temperature"] == temperature
assert body["max_tokens"] == supplied["max_tokens"]
@pytest.mark.asyncio
async def test_a_native_response_has_the_id_and_detail_fields_the_python_path_reports(server: RecordingServer) -> None:
result: Final = await litellm.acompletion(
MESSAGES_MODEL, list(MESSAGES), base_url=server.base_url, api_key="test-key", max_tokens=32
)
assert isinstance(result, ModelResponse)
assert_served_natively(server)
assert result.id.startswith("chatcmpl-")
assert result.object == "chat.completion"
assert result.usage.completion_tokens_details.reasoning_tokens == 0
assert result.usage.completion_tokens_details.text_tokens == result.usage.completion_tokens
assert result.choices[0].message.provider_specific_fields == {"citations": None, "thinking_blocks": None}