mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(gateway): answer chat completions with id, object and the usage and provider fields
The gateway's 200 body had no id, object, usage.completion_tokens_details or message.provider_specific_fields, so clients that read them got nothing where the Python proxy returns all four. The route now mints the chatcmpl id before the cache store, so a cache hit repeats it, and each provider fills the details it knows
This commit is contained in:
parent
d18efdfe13
commit
516c87028e
17 changed files with 346 additions and 40 deletions
2
litellm-rust/Cargo.lock
generated
2
litellm-rust/Cargo.lock
generated
|
|
@ -3887,6 +3887,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tower",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
|
|
@ -4100,6 +4101,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_llms_types::formats::chat_completions::{ChatCompletionsUsage, PromptTokensDetails};
|
||||
use litellm_llms_types::formats::chat_completions::{
|
||||
ChatCompletionsUsage, CompletionTokensDetails, PromptTokensDetails,
|
||||
};
|
||||
|
||||
/// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the
|
||||
/// reasons the providers on this route can emit. Python warns and falls back to
|
||||
|
|
@ -30,13 +32,16 @@ pub fn finish_reason_for(provider_reason: &str) -> &'static str {
|
|||
|
||||
/// Python folds cache tokens into `prompt_tokens` and reports the split under
|
||||
/// `prompt_tokens_details`; mirror that so cost tracking agrees on both paths.
|
||||
/// Python caps the reported reasoning count at the output, and the rest is text.
|
||||
pub fn usage_from_parts(
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
cache_read_tokens: u64,
|
||||
cache_creation_tokens: u64,
|
||||
reported_reasoning_tokens: u64,
|
||||
) -> ChatCompletionsUsage {
|
||||
let prompt_tokens = input_tokens + cache_read_tokens + cache_creation_tokens;
|
||||
let reasoning_tokens = reported_reasoning_tokens.min(output_tokens);
|
||||
ChatCompletionsUsage {
|
||||
prompt_tokens,
|
||||
completion_tokens: output_tokens,
|
||||
|
|
@ -46,6 +51,11 @@ pub fn usage_from_parts(
|
|||
cache_creation_tokens,
|
||||
text_tokens: input_tokens,
|
||||
},
|
||||
completion_tokens_details: Some(CompletionTokensDetails {
|
||||
reasoning_tokens: Some(reasoning_tokens),
|
||||
text_tokens: Some(output_tokens - reasoning_tokens),
|
||||
..CompletionTokensDetails::default()
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -93,7 +103,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn folds_cache_tokens_into_prompt_tokens() {
|
||||
let usage = usage_from_parts(10, 4, 7, 3);
|
||||
let usage = usage_from_parts(10, 4, 7, 3, 0);
|
||||
assert_eq!(usage.prompt_tokens, 20);
|
||||
assert_eq!(usage.completion_tokens, 4);
|
||||
assert_eq!(usage.total_tokens, 24);
|
||||
|
|
@ -104,9 +114,26 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn reports_raw_input_tokens_when_no_cache_is_involved() {
|
||||
let usage = usage_from_parts(12, 5, 0, 0);
|
||||
let usage = usage_from_parts(12, 5, 0, 0, 0);
|
||||
assert_eq!(usage.prompt_tokens, 12);
|
||||
assert_eq!(usage.total_tokens, 17);
|
||||
assert_eq!(usage.prompt_tokens_details.text_tokens, 12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_reasoning(0, 0, 10)]
|
||||
#[case::splits_the_output(3, 3, 7)]
|
||||
#[case::all_reasoning(10, 10, 0)]
|
||||
#[case::over_report_is_capped_at_the_output(20, 10, 0)]
|
||||
fn splits_completion_tokens_into_reasoning_and_text(
|
||||
#[case] reported: u64,
|
||||
#[case] reasoning: u64,
|
||||
#[case] text: u64,
|
||||
) {
|
||||
let details = usage_from_parts(1, 10, 0, 0, reported)
|
||||
.completion_tokens_details
|
||||
.expect("details are always reported");
|
||||
assert_eq!(details.reasoning_tokens, Some(reasoning));
|
||||
assert_eq!(details.text_tokens, Some(text));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,4 +37,5 @@ tokio = { workspace = true, features = ["io-util"] }
|
|||
rstest.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
uuid.workspace = true
|
||||
wiremock.workspace = true
|
||||
|
|
|
|||
|
|
@ -67,6 +67,69 @@ async fn chat_aliases_call_core_and_use_the_body_model_before_the_path(
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::anthropic(
|
||||
"anthropic/test-model",
|
||||
json!({
|
||||
"id": "msg_upstream", "model": "test-model", "content": [{"type": "text", "text": "hello"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 10, "output_tokens_details": {"thinking_tokens": 4}}
|
||||
}),
|
||||
json!({"reasoning_tokens": 4, "text_tokens": 6}),
|
||||
json!({"citations": null, "thinking_blocks": null})
|
||||
)]
|
||||
#[case::openai_like(
|
||||
"openai_like/test-model",
|
||||
json!({
|
||||
"id": "chatcmpl-upstream", "model": "test-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 3, "completion_tokens": 10, "total_tokens": 13,
|
||||
"completion_tokens_details": {"reasoning_tokens": 4, "audio_tokens": 0}
|
||||
}
|
||||
}),
|
||||
json!({"reasoning_tokens": 4, "audio_tokens": 0}),
|
||||
Value::Null
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn chat_answers_with_a_minted_id_and_the_full_openai_shape(
|
||||
#[case] model: &str,
|
||||
#[case] provider_body: Value,
|
||||
#[case] completion_tokens_details: Value,
|
||||
#[case] provider_specific_fields: Value,
|
||||
) {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(provider_body))
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let app = support::app(model, &upstream.uri());
|
||||
let request = json!({
|
||||
"model": "public/model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 16,
|
||||
});
|
||||
|
||||
let first =
|
||||
support::json(support::post(app.clone(), "/v1/chat/completions", request.clone()).await)
|
||||
.await;
|
||||
let second = support::json(support::post(app, "/v1/chat/completions", request).await).await;
|
||||
|
||||
let id = first["id"].as_str().expect("the body carries an id");
|
||||
let uuid = id.strip_prefix("chatcmpl-").expect("a chatcmpl id");
|
||||
assert!(uuid::Uuid::parse_str(uuid).is_ok(), "{id} ends in a uuid");
|
||||
assert_ne!(first["id"], second["id"]);
|
||||
assert_eq!(first["object"], "chat.completion");
|
||||
assert_eq!(
|
||||
first["usage"]["completion_tokens_details"],
|
||||
completion_tokens_details
|
||||
);
|
||||
assert_eq!(
|
||||
first["choices"][0]["message"]["provider_specific_fields"],
|
||||
provider_specific_fields
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::streaming(json!({"messages": [{"role": "user", "content": "hi"}], "stream": true}), 501)]
|
||||
#[case::missing_messages(json!({}), 400)]
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ litellm-secrets.workspace = true
|
|||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
futures-util.workspace = true
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-cache-memory.workspace = true
|
||||
|
|
|
|||
|
|
@ -9,9 +9,10 @@ use litellm_llms::base_llm::{
|
|||
chat::transformation::BaseConfig,
|
||||
chat::transformation::ProviderChatResponseData,
|
||||
};
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
use litellm_llms_types::formats::chat_completions::{ChatCompletion, ChatCompletionsResponse};
|
||||
use litellm_secrets::source::Secrets;
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::Error;
|
||||
use crate::{
|
||||
|
|
@ -19,6 +20,14 @@ use crate::{
|
|||
types::ResolvedChatCompletionsRequest,
|
||||
};
|
||||
|
||||
fn minted(body: ChatCompletionsResponse) -> ChatCompletion {
|
||||
ChatCompletion {
|
||||
id: format!("chatcmpl-{}", Uuid::new_v4()),
|
||||
object: "chat.completion".to_owned(),
|
||||
body,
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
|
||||
pub(super) async fn execute(
|
||||
http: &Client,
|
||||
|
|
@ -31,7 +40,7 @@ pub(super) async fn execute(
|
|||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ChatCompletionsResponse, Error> {
|
||||
) -> Result<ChatCompletion, Error> {
|
||||
let validated = config.validate_environment(
|
||||
string_headers(call.extra_headers)?,
|
||||
call.api_key.as_deref(),
|
||||
|
|
@ -127,6 +136,7 @@ pub(super) async fn execute(
|
|||
})?;
|
||||
config
|
||||
.transform_response(provider.model, ProviderChatResponseData { body })
|
||||
.map(minted)
|
||||
.map_err(Error::from)
|
||||
.map_err(Error::after_send)
|
||||
},
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ pub use litellm_inference::Error;
|
|||
mod common_utils;
|
||||
pub mod constants;
|
||||
pub(crate) mod handler;
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletion;
|
||||
|
||||
use litellm_auth::AuthServices;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
|
|
@ -47,7 +47,7 @@ impl ChatCompletionsRoute {
|
|||
request: ChatCompletionsRequest<'_>,
|
||||
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
|
||||
options: impl Into<litellm_inference::CallOptions>,
|
||||
) -> Result<ChatCompletionsResponse, Error> {
|
||||
) -> Result<ChatCompletion, Error> {
|
||||
let litellm_inference::CallOptions {
|
||||
cache: cache_options,
|
||||
observers,
|
||||
|
|
@ -78,7 +78,7 @@ impl ChatCompletionsRoute {
|
|||
cache_options: Option<litellm_cache_response::CachePolicy>,
|
||||
interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<ChatCompletionsResponse, Error> {
|
||||
) -> Result<ChatCompletion, Error> {
|
||||
litellm_inference::diagnostic::unary(async {
|
||||
let model = call.model.clone();
|
||||
let custom_llm_provider = call.custom_llm_provider.clone();
|
||||
|
|
@ -93,21 +93,19 @@ impl ChatCompletionsRoute {
|
|||
<&'static str>::from(provider.provider),
|
||||
);
|
||||
let secrets = self.secrets.resolve(&config.secret_names()).await?;
|
||||
let execute: futures_util::future::BoxFuture<
|
||||
'_,
|
||||
Result<ChatCompletionsResponse, Error>,
|
||||
> = Box::pin(handler::execute(
|
||||
&self.http,
|
||||
&self.auth,
|
||||
config,
|
||||
provider,
|
||||
call,
|
||||
secrets,
|
||||
self.cache.clone(),
|
||||
cache_options,
|
||||
interceptors,
|
||||
observers,
|
||||
));
|
||||
let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletion, Error>> =
|
||||
Box::pin(handler::execute(
|
||||
&self.http,
|
||||
&self.auth,
|
||||
config,
|
||||
provider,
|
||||
call,
|
||||
secrets,
|
||||
self.cache.clone(),
|
||||
cache_options,
|
||||
interceptors,
|
||||
observers,
|
||||
));
|
||||
execute.await
|
||||
})
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -4,14 +4,14 @@ use litellm_host::{
|
|||
call::{CallOutput, HostedMachine, hosted_call},
|
||||
protocol::Protocol,
|
||||
};
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletion;
|
||||
|
||||
use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
|
||||
|
||||
pub struct ChatCompletions;
|
||||
|
||||
impl Protocol for ChatCompletions {
|
||||
type Response = ChatCompletionsResponse;
|
||||
type Response = ChatCompletion;
|
||||
type Error = Error;
|
||||
type Request = ChatCompletionsCall;
|
||||
type HostCall = Infallible;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use litellm_host::{
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
|
||||
use litellm_llms_types::formats::chat_completions::ChatCompletion;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Map, Value, json};
|
||||
use wiremock::ResponseTemplate;
|
||||
|
|
@ -16,7 +16,7 @@ use support::*;
|
|||
|
||||
const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5-20260101","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
|
||||
|
||||
async fn complete(request: ChatCompletionsRequest<'_>) -> Result<ChatCompletionsResponse, Error> {
|
||||
async fn complete(request: ChatCompletionsRequest<'_>) -> Result<ChatCompletion, Error> {
|
||||
chat_completions_route().execute(request, &(), None).await
|
||||
}
|
||||
|
||||
|
|
@ -83,10 +83,10 @@ async fn anthropic_round_trip_translates_the_conversation_and_normalizes_the_res
|
|||
);
|
||||
assert_eq!(body["max_tokens"], 16);
|
||||
assert_eq!(
|
||||
response.choices[0].message.content.as_deref(),
|
||||
response.body.choices[0].message.content.as_deref(),
|
||||
Some("hello")
|
||||
);
|
||||
assert_eq!(response.usage.total_tokens, 15);
|
||||
assert_eq!(response.body.usage.total_tokens, 15);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -152,10 +152,10 @@ async fn bedrock_round_trip_is_signed_and_normalized(request: ChatCompletionsReq
|
|||
json!([{"role": "user", "content": [{"text": "hi"}]}])
|
||||
);
|
||||
assert_eq!(
|
||||
response.choices[0].message.content.as_deref(),
|
||||
response.body.choices[0].message.content.as_deref(),
|
||||
Some("hello")
|
||||
);
|
||||
assert_eq!(response.usage.total_tokens, 15);
|
||||
assert_eq!(response.body.usage.total_tokens, 15);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -308,7 +308,7 @@ async fn direct_and_hosted_calls_share_hooks_and_lifecycle(
|
|||
.unwrap()
|
||||
};
|
||||
assert_eq!(
|
||||
response.choices[0].message.content.as_deref(),
|
||||
response.body.choices[0].message.content.as_deref(),
|
||||
Some("hello")
|
||||
);
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ fn request<'a>(
|
|||
|
||||
async fn complete(
|
||||
request: ChatCompletionsRequest<'_>,
|
||||
) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
|
||||
) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletion, Error> {
|
||||
chat_completions_route().execute(request, &(), None).await
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
- Name types after the format, never the originator (`formats::messages::MessagesRequest`, not `AnthropicMessagesRequest`)
|
||||
- A field defined by the format stays in the format type even when provider support varies
|
||||
- Keep one canonical definition and import path. Move consumers with the type instead of adding re-exports
|
||||
- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the response handed to the host, so do not swap it for a fuller upstream schema
|
||||
- LiteLLM's normalized contracts live here too. `ChatCompletionsResponse` is the normalized provider response, and `ChatCompletion` wraps it with the id the route mints for the host, so do not swap either for a fuller upstream schema
|
||||
|
||||
## Providers
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
- accept a subset of parameters, which is recorded per provider in `supported_openai_param_mappings`, not by trimming the type
|
||||
- add top-level request or response fields, preserved through the existing extra-field maps instead of new typed fields
|
||||
- return slightly different usage or finish reasons, normalized in the provider transformation
|
||||
- `ChatCompletionsResponse` is LiteLLM's response contract to the host, a typed projection rather than the full upstream schema
|
||||
- `ChatCompletion` is LiteLLM's response contract to the host, the id the route mints plus `ChatCompletionsResponse`, a typed projection rather than the full upstream schema
|
||||
|
||||
# references
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,21 @@ pub struct PromptTokensDetails {
|
|||
pub text_tokens: u64,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
#[derive(Default)]
|
||||
pub struct CompletionTokensDetails {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub audio_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub accepted_prediction_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub rejected_prediction_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
#[derive(Default)]
|
||||
pub struct ChatCompletionsUsage {
|
||||
|
|
@ -55,6 +70,8 @@ pub struct ChatCompletionsUsage {
|
|||
pub completion_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub prompt_tokens_details: PromptTokensDetails,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
|
|
@ -65,6 +82,8 @@ pub struct ChatCompletionsChoiceMessage {
|
|||
// while Converse assigns the joined string unconditionally. Each config
|
||||
// mirrors its own, so keep this optional and serialize it even when None.
|
||||
pub content: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_specific_fields: Option<Map<String, Value>>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
|
|
@ -74,11 +93,12 @@ pub struct ChatCompletionsChoice {
|
|||
pub finish_reason: String,
|
||||
}
|
||||
|
||||
/// The normalized response handed back to the host.
|
||||
/// The provider's response, normalized by its transformation.
|
||||
///
|
||||
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
|
||||
/// `ModelResponse` it already created, and echoing the provider's own id here
|
||||
/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests.
|
||||
/// There is deliberately no `id`: the route mints the `chatcmpl-…` id on
|
||||
/// `ChatCompletion`, and echoing the provider's own id here would compete with
|
||||
/// it. Pinned by `response_carries_no_id` in the Anthropic chat transformation
|
||||
/// tests.
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct ChatCompletionsResponse {
|
||||
pub created: u64,
|
||||
|
|
@ -86,3 +106,40 @@ pub struct ChatCompletionsResponse {
|
|||
pub choices: Vec<ChatCompletionsChoice>,
|
||||
pub usage: ChatCompletionsUsage,
|
||||
}
|
||||
|
||||
/// What the route hands back to the host. The id is stored in the cached entry,
|
||||
/// so a cache hit repeats it, and an entry cached without one decodes to a miss.
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct ChatCompletion {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
#[serde(flatten)]
|
||||
pub body: ChatCompletionsResponse,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::ChatCompletion;
|
||||
|
||||
#[test]
|
||||
fn an_entry_cached_without_an_id_is_not_a_completion() {
|
||||
let entry = json!({
|
||||
"created": 1,
|
||||
"model": "claude",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
"text_tokens": 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
assert!(serde_json::from_value::<ChatCompletion>(entry).is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ struct ResponseUsageProjection {
|
|||
cache_read_input_tokens: Option<u64>,
|
||||
#[serde(default)]
|
||||
cache_creation_input_tokens: Option<u64>,
|
||||
#[serde(default)]
|
||||
output_tokens_details: Option<Value>,
|
||||
}
|
||||
|
||||
pub struct AnthropicConfig;
|
||||
|
|
@ -153,6 +155,10 @@ impl BaseConfig for AnthropicConfig {
|
|||
message: ChatCompletionsChoiceMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: (!text.is_empty()).then_some(text),
|
||||
provider_specific_fields: Some(Map::from_iter([
|
||||
("citations".to_string(), Value::Null),
|
||||
("thinking_blocks".to_string(), Value::Null),
|
||||
])),
|
||||
},
|
||||
finish_reason: finish_reason_for(body.stop_reason.as_deref().unwrap_or(""))
|
||||
.to_string(),
|
||||
|
|
@ -162,6 +168,12 @@ impl BaseConfig for AnthropicConfig {
|
|||
body.usage.output_tokens,
|
||||
body.usage.cache_read_input_tokens.unwrap_or(0),
|
||||
body.usage.cache_creation_input_tokens.unwrap_or(0),
|
||||
body.usage
|
||||
.output_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|details| details.get("thinking_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
|
@ -630,8 +642,54 @@ mod tests {
|
|||
assert_eq!(response.choices[0].message.content, None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::reported(json!({"thinking_tokens": 4}), 4, 6)]
|
||||
#[case::over_reported_is_capped_at_the_output(json!({"thinking_tokens": 40}), 10, 0)]
|
||||
#[case::not_reported(json!({}), 0, 10)]
|
||||
#[case::not_a_count(json!({"thinking_tokens": "many"}), 0, 10)]
|
||||
#[case::not_an_object(json!("none"), 0, 10)]
|
||||
fn usage_splits_reported_thinking_tokens_from_the_text(
|
||||
#[case] output_tokens_details: Value,
|
||||
#[case] reasoning: u64,
|
||||
#[case] text: u64,
|
||||
) {
|
||||
let response = transform_response(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 10,
|
||||
"output_tokens_details": output_tokens_details,
|
||||
}
|
||||
}))
|
||||
.expect("response transforms");
|
||||
let details = response
|
||||
.usage
|
||||
.completion_tokens_details
|
||||
.expect("details are always reported");
|
||||
assert_eq!(details.reasoning_tokens, Some(reasoning));
|
||||
assert_eq!(details.text_tokens, Some(text));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() {
|
||||
fn message_reports_null_citations_and_thinking_blocks_like_python() {
|
||||
let response = transform_response(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1}
|
||||
}))
|
||||
.expect("response transforms");
|
||||
let value = serde_json::to_value(response).expect("serializable");
|
||||
assert_eq!(
|
||||
value["choices"][0]["message"]["provider_specific_fields"],
|
||||
json!({"citations": null, "thinking_blocks": null})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_carries_no_id_so_the_route_mints_the_chatcmpl_id() {
|
||||
let response = transform_response(json!({
|
||||
"id": "msg_should_not_leak",
|
||||
"model": "claude-sonnet-4-5",
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ impl BaseConfig for AmazonConverseConfig {
|
|||
response.usage.output_tokens,
|
||||
response.usage.cache_read_input_tokens,
|
||||
response.usage.cache_write_input_tokens,
|
||||
0,
|
||||
);
|
||||
// Converse reports `totalTokens` and Python passes it straight through,
|
||||
// where Anthropic has no such field and Python adds the two counts
|
||||
|
|
@ -201,6 +202,7 @@ impl BaseConfig for AmazonConverseConfig {
|
|||
// Anthropic. A caller calling `.strip()` on it would break
|
||||
// on this path alone.
|
||||
content: Some(text),
|
||||
provider_specific_fields: None,
|
||||
},
|
||||
finish_reason: finish_reason_for(
|
||||
ConverseStopReason::parse(response.stop_reason)
|
||||
|
|
@ -788,6 +790,26 @@ mod tests {
|
|||
assert_eq!(response.usage.completion_tokens, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_all_output_as_text_and_no_provider_fields_for_a_text_response() {
|
||||
let response = transform_response(json!({
|
||||
"output": {"message": {"content": [{"text": "x"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 10, "outputTokens": 4, "totalTokens": 14}
|
||||
}))
|
||||
.expect("response transforms");
|
||||
let value = serde_json::to_value(response).expect("serializable");
|
||||
assert_eq!(
|
||||
value["usage"]["completion_tokens_details"],
|
||||
json!({"reasoning_tokens": 0, "text_tokens": 4})
|
||||
);
|
||||
assert!(
|
||||
value["choices"][0]["message"]
|
||||
.get("provider_specific_fields")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_the_computed_total_when_converse_omits_it() {
|
||||
// Python raises a KeyError on a body with no `totalTokens`. Reporting a zero
|
||||
|
|
|
|||
|
|
@ -177,6 +177,10 @@ impl BaseConfig for OpenAILikeChatConfig {
|
|||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0),
|
||||
},
|
||||
completion_tokens_details: usage
|
||||
.and_then(|usage| usage.get("completion_tokens_details"))
|
||||
.cloned()
|
||||
.and_then(|details| serde_json::from_value(details).ok()),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -273,6 +277,7 @@ fn normalize_choice(position: usize, choice: &Value) -> Result<ChatCompletionsCh
|
|||
.unwrap_or("assistant")
|
||||
.to_string(),
|
||||
content: content.and_then(Value::as_str).map(str::to_string),
|
||||
provider_specific_fields: None,
|
||||
},
|
||||
finish_reason: choice
|
||||
.get("finish_reason")
|
||||
|
|
@ -526,6 +531,53 @@ mod tests {
|
|||
assert_eq!(response.usage.total_tokens, 8);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::passes_through_what_the_upstream_reports(
|
||||
json!({"reasoning_tokens": 2, "audio_tokens": 0, "accepted_prediction_tokens": 1, "rejected_prediction_tokens": 3}),
|
||||
json!({"reasoning_tokens": 2, "audio_tokens": 0, "accepted_prediction_tokens": 1, "rejected_prediction_tokens": 3})
|
||||
)]
|
||||
#[case::null_upstream_field_is_left_out(
|
||||
json!({"reasoning_tokens": 2, "audio_tokens": null}),
|
||||
json!({"reasoning_tokens": 2})
|
||||
)]
|
||||
#[case::null_details_are_left_out(Value::Null, Value::Null)]
|
||||
#[case::malformed_details_are_left_out(json!({"reasoning_tokens": "two"}), Value::Null)]
|
||||
fn completion_tokens_details_mirror_the_upstream(
|
||||
#[case] details: Value,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
let response = transform_response(json!({
|
||||
"model": "m",
|
||||
"choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 6,
|
||||
"completion_tokens_details": details,
|
||||
},
|
||||
}))
|
||||
.expect("response normalizes");
|
||||
let value = serde_json::to_value(response).expect("serializable");
|
||||
assert_eq!(value["usage"]["completion_tokens_details"], expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn an_upstream_without_details_reports_none() {
|
||||
let response = transform_response(json!({
|
||||
"model": "m",
|
||||
"choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 5, "total_tokens": 6},
|
||||
}))
|
||||
.expect("response normalizes");
|
||||
let value = serde_json::to_value(response).expect("serializable");
|
||||
assert!(value["usage"].get("completion_tokens_details").is_none());
|
||||
assert!(
|
||||
value["choices"][0]["message"]
|
||||
.get("provider_specific_fields")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn null_token_fields_in_usage_become_zero() {
|
||||
// `_sanitize_usage_obj`: providers that return null token values break
|
||||
|
|
|
|||
|
|
@ -38,3 +38,18 @@ async def test_positional_parameters_reach_the_wire(server: RecordingServer, asy
|
|||
body: Final = _OBJECT.validate_python(server.requests[0].body)
|
||||
assert body["temperature"] == temperature
|
||||
assert body["max_tokens"] == supplied["max_tokens"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_native_response_has_the_id_and_detail_fields_the_python_path_reports(server: RecordingServer) -> None:
|
||||
result: Final = await litellm.acompletion(
|
||||
MESSAGES_MODEL, list(MESSAGES), base_url=server.base_url, api_key="test-key", max_tokens=32
|
||||
)
|
||||
|
||||
assert isinstance(result, ModelResponse)
|
||||
assert_served_natively(server)
|
||||
assert result.id.startswith("chatcmpl-")
|
||||
assert result.object == "chat.completion"
|
||||
assert result.usage.completion_tokens_details.reasoning_tokens == 0
|
||||
assert result.usage.completion_tokens_details.text_tokens == result.usage.completion_tokens
|
||||
assert result.choices[0].message.provider_specific_fields == {"citations": None, "thinking_blocks": None}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue