mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
fix(spend_tracking): populate cache_creation_input_tokens for Responses API logs
On the /v1/responses path the response usage is not chat-Usage-shaped, so additional_usage_values could not derive cache tokens from response_obj.usage and the Admin UI Logs cache-creation token row stayed empty. Fall back to the normalized standard_logging usage_object's prompt_tokens_details for both the cache-read and cache-creation counts. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
2f502a1bfc
commit
5e1b08b355
2 changed files with 101 additions and 6 deletions
|
|
@ -374,15 +374,17 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
|
|||
if isinstance(v, BaseModel):
|
||||
v = v.model_dump()
|
||||
additional_usage_values.update({k: v})
|
||||
if "cache_read_input_tokens" not in additional_usage_values:
|
||||
prompt_tokens_details = additional_usage_values.get("prompt_tokens_details")
|
||||
if isinstance(prompt_tokens_details, dict):
|
||||
prompt_tokens_details = additional_usage_values.get("prompt_tokens_details")
|
||||
if not isinstance(prompt_tokens_details, dict):
|
||||
usage_object = clean_metadata.get("usage_object")
|
||||
if isinstance(usage_object, dict):
|
||||
prompt_tokens_details = usage_object.get("prompt_tokens_details")
|
||||
if isinstance(prompt_tokens_details, dict):
|
||||
if "cache_read_input_tokens" not in additional_usage_values:
|
||||
cached_tokens = prompt_tokens_details.get("cached_tokens")
|
||||
if isinstance(cached_tokens, int) and cached_tokens > 0:
|
||||
additional_usage_values["cache_read_input_tokens"] = cached_tokens
|
||||
if "cache_creation_input_tokens" not in additional_usage_values:
|
||||
prompt_tokens_details = additional_usage_values.get("prompt_tokens_details")
|
||||
if isinstance(prompt_tokens_details, dict):
|
||||
if "cache_creation_input_tokens" not in additional_usage_values:
|
||||
cache_write_tokens = prompt_tokens_details.get("cache_write_tokens") or prompt_tokens_details.get(
|
||||
"cache_creation_tokens"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -153,6 +153,99 @@ def test_get_logging_payload_does_not_map_missing_or_zero_cache_write_tokens(pro
|
|||
assert "cache_creation_input_tokens" not in additional_usage_values
|
||||
|
||||
|
||||
def _make_standard_logging_payload_with_usage_object(usage_object: dict) -> StandardLoggingPayload:
|
||||
return StandardLoggingPayload(
|
||||
id="test-id-responses",
|
||||
call_type="responses",
|
||||
stream=False,
|
||||
response_cost=0.02,
|
||||
status="success",
|
||||
total_tokens=1010,
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=10,
|
||||
startTime=1234567890.0,
|
||||
endTime=1234567891.0,
|
||||
completionStartTime=None,
|
||||
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-5.6", model_map_value=None),
|
||||
model="gpt-5.6",
|
||||
model_id="model-123",
|
||||
model_group="openai",
|
||||
custom_llm_provider="openai",
|
||||
api_base="https://api.openai.com",
|
||||
metadata=StandardLoggingMetadata(
|
||||
user_api_key_hash="test_hash",
|
||||
user_api_key_alias=None,
|
||||
user_api_key_team_id=None,
|
||||
user_api_key_org_id=None,
|
||||
user_api_key_user_id=None,
|
||||
user_api_key_team_alias=None,
|
||||
spend_logs_metadata=None,
|
||||
requester_ip_address=None,
|
||||
requester_metadata=None,
|
||||
user_api_key_end_user_id=None,
|
||||
usage_object=usage_object,
|
||||
),
|
||||
cache_hit=False,
|
||||
cache_key=None,
|
||||
saved_cache_cost=0.0,
|
||||
request_tags=[],
|
||||
end_user=None,
|
||||
requester_ip_address=None,
|
||||
messages=[],
|
||||
response={},
|
||||
error_str=None,
|
||||
model_parameters={},
|
||||
hidden_params=StandardLoggingHiddenParams(
|
||||
model_id="model-123",
|
||||
cache_key=None,
|
||||
api_base="https://api.openai.com",
|
||||
response_cost="0.02",
|
||||
litellm_overhead_time_ms=None,
|
||||
additional_headers=None,
|
||||
batch_models=None,
|
||||
litellm_model_name=None,
|
||||
usage_object=None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_get_logging_payload_maps_responses_api_cache_write_tokens_from_usage_object():
|
||||
"""Responses API (/v1/responses) usage is not chat-Usage-shaped, so
|
||||
additional_usage_values can't derive cache tokens from response_obj.usage.
|
||||
The Admin UI Logs "Cache Creation Tokens" row reads
|
||||
additional_usage_values.cache_creation_input_tokens, so it must be filled
|
||||
from the normalized standard_logging usage_object (LIT-4633)."""
|
||||
standard_logging_payload = _make_standard_logging_payload_with_usage_object(
|
||||
usage_object={
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 10,
|
||||
"total_tokens": 1010,
|
||||
"prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 800, "cache_creation_tokens": 800},
|
||||
}
|
||||
)
|
||||
payload = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "gpt-5.6",
|
||||
"call_type": "responses",
|
||||
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
|
||||
"standard_logging_object": standard_logging_payload,
|
||||
},
|
||||
response_obj={
|
||||
"id": "resp-test",
|
||||
"usage": {
|
||||
"input_tokens": 1000,
|
||||
"output_tokens": 10,
|
||||
"total_tokens": 1010,
|
||||
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 800},
|
||||
},
|
||||
},
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
additional_usage_values = json.loads(payload["metadata"])["additional_usage_values"]
|
||||
assert additional_usage_values["cache_creation_input_tokens"] == 800
|
||||
|
||||
|
||||
def test_sanitize_request_body_for_spend_logs_payload_basic():
|
||||
request_body = {
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue