scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

Merge pull request #18033 from BerriAI/litellm_ui_logs_overhead

Browse source 
[Feature] Add LiteLLM Overhead to Logs
This commit is contained in:
yuneng-jiang 2025-12-16 14:51:50 -08:00 • committed by GitHub
commit f316caa088
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 234 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/proxy/_types.py
View file

@ -2671,6 +2671,9 @@ class SpendLogsMetadata(TypedDict):
cold_storage_object_key: Optional[
str
] # S3/GCS object key for cold storage retrieval
litellm_overhead_time_ms: Optional[
float
] # LiteLLM overhead time in milliseconds
class SpendLogsPayload(TypedDict):

10
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -55,6 +55,7 @@ def _get_spend_logs_metadata(
usage_object: Optional[dict] = None,
model_map_information: Optional[StandardLoggingModelInformation] = None,
cold_storage_object_key: Optional[str] = None,
litellm_overhead_time_ms: Optional[float] = None,
) -> SpendLogsMetadata:
if metadata is None:
return SpendLogsMetadata(
@ -78,6 +79,7 @@ def _get_spend_logs_metadata(
usage_object=None,
guardrail_information=None,
cold_storage_object_key=cold_storage_object_key,
litellm_overhead_time_ms=None,
)
verbose_proxy_logger.debug(
"getting payload for SpendLogs, available keys in metadata: "
@ -102,6 +104,7 @@ def _get_spend_logs_metadata(
clean_metadata["usage_object"] = usage_object
clean_metadata["model_map_information"] = model_map_information
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms
return clean_metadata
@ -298,6 +301,12 @@ def get_logging_payload( # noqa: PLR0915
_model_id = metadata.get("model_info", {}).get("id", "")
_model_group = metadata.get("model_group", "")
# Extract overhead from hidden_params if available
litellm_overhead_time_ms = None
if standard_logging_payload is not None:
hidden_params = standard_logging_payload.get("hidden_params", {})
litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms")
# clean up litellm metadata
clean_metadata = _get_spend_logs_metadata(
metadata,
@ -343,6 +352,7 @@ def get_logging_payload( # noqa: PLR0915
if standard_logging_payload is not None
else None
),
litellm_overhead_time_ms=litellm_overhead_time_ms,
)
special_usage_fields = ["completion_tokens", "prompt_tokens", "total_tokens"]

1
tests/logging_callback_tests/test_gcs_pub_sub.py
View file

@ -39,6 +39,7 @@ ignored_keys = [
"metadata.model_map_information",
"metadata.usage_object",
"metadata.cold_storage_object_key",
"metadata.litellm_overhead_time_ms",
]

1
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -201,6 +201,7 @@ ignored_keys = [
"metadata.usage_object",
"metadata.cold_storage_object_key",
"metadata.additional_usage_values.prompt_tokens_details.cache_creation_tokens",
"metadata.litellm_overhead_time_ms",
]
MODEL_LIST = [

220
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -24,7 +24,12 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
_sanitize_request_body_for_spend_logs_payload,
get_logging_payload,
)
from litellm.types.utils import StandardLoggingPayload
from litellm.types.utils import (
StandardLoggingHiddenParams,
StandardLoggingMetadata,
StandardLoggingModelInformation,
StandardLoggingPayload,
)
def test_sanitize_request_body_for_spend_logs_payload_basic():
@ -632,3 +637,216 @@ def test_get_logging_payload_includes_agent_id_from_kwargs():
assert payload["agent_id"] == test_agent_id, f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_includes_overhead_in_spend_logs_metadata():
"""
Test that get_logging_payload extracts litellm_overhead_time_ms from hidden_params
and stores it in spend_logs_metadata within the metadata JSON.
"""
test_overhead_ms = 123.45
# Create StandardLoggingPayload with hidden_params containing overhead
standard_logging_payload = StandardLoggingPayload(
id="test-id-123",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=test_overhead_ms,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
)
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": standard_logging_payload,
}
response_obj = {
"id": "test-response-123",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
# Parse the metadata JSON string
metadata_json = payload.get("metadata")
assert metadata_json is not None, "metadata should not be None"
metadata = json.loads(metadata_json)
# Verify overhead is stored directly in metadata
assert (
metadata.get("litellm_overhead_time_ms") == test_overhead_ms
), f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_handles_missing_overhead_gracefully():
"""
Test that get_logging_payload handles missing overhead gracefully
(backward compatibility - when overhead is not present, it should not break).
"""
# Create StandardLoggingPayload WITHOUT overhead in hidden_params
standard_logging_payload = StandardLoggingPayload(
id="test-id-456",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None, # No overhead
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
)
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": standard_logging_payload,
}
response_obj = {
"id": "test-response-456",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
# Should not raise an exception
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
# Parse the metadata JSON string
metadata_json = payload.get("metadata")
assert metadata_json is not None, "metadata should not be None"
metadata = json.loads(metadata_json)
# When overhead is None, litellm_overhead_time_ms should be None or not present
assert (
metadata.get("litellm_overhead_time_ms") is None
), "litellm_overhead_time_ms should be None when overhead is not provided"