From 75635258db7f2e0a478a2e9276aade7517bb1eb6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 20:37:02 -0800 Subject: [PATCH 1/4] Base commit --- litellm/proxy/spend_tracking/spend_management_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 774b971de3a..21e6212488a 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1680,7 +1680,7 @@ async def ui_view_spend_logs( # noqa: PLR0915 ), ): """ - View spend logs with pagination support. + View spend logs with pagination support Available at both `/spend/logs/v2` (public API) and `/spend/logs/ui` (internal UI). Returns paginated response with data, total, page, page_size, and total_pages. From 66306f21b1d51f8e26e4af2423be938169aa0b61 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 20:56:55 -0800 Subject: [PATCH 2/4] Add litellm overhead to spend logs metadata --- litellm/proxy/_types.py | 3 + .../spend_management_endpoints.py | 2 +- .../spend_tracking/spend_tracking_utils.py | 10 + .../test_spend_tracking_utils.py | 220 +++++++++++++++++- 4 files changed, 233 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fcc4097e452..b9b35f24554 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2664,6 +2664,9 @@ class SpendLogsMetadata(TypedDict): cold_storage_object_key: Optional[ str ] # S3/GCS object key for cold storage retrieval + litellm_overhead_time_ms: Optional[ + float + ] # LiteLLM overhead time in milliseconds class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 21e6212488a..774b971de3a 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1680,7 +1680,7 @@ async def ui_view_spend_logs( # noqa: PLR0915 ), ): """ - View spend logs with pagination support + View spend logs with pagination support. Available at both `/spend/logs/v2` (public API) and `/spend/logs/ui` (internal UI). Returns paginated response with data, total, page, page_size, and total_pages. diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 090d870ba72..687af8a4514 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -55,6 +55,7 @@ def _get_spend_logs_metadata( usage_object: Optional[dict] = None, model_map_information: Optional[StandardLoggingModelInformation] = None, cold_storage_object_key: Optional[str] = None, + litellm_overhead_time_ms: Optional[float] = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -78,6 +79,7 @@ def _get_spend_logs_metadata( usage_object=None, guardrail_information=None, cold_storage_object_key=cold_storage_object_key, + litellm_overhead_time_ms=None, ) verbose_proxy_logger.debug( "getting payload for SpendLogs, available keys in metadata: " @@ -102,6 +104,7 @@ def _get_spend_logs_metadata( clean_metadata["usage_object"] = usage_object clean_metadata["model_map_information"] = model_map_information clean_metadata["cold_storage_object_key"] = cold_storage_object_key + clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms return clean_metadata @@ -298,6 +301,12 @@ def get_logging_payload( # noqa: PLR0915 _model_id = metadata.get("model_info", {}).get("id", "") _model_group = metadata.get("model_group", "") + # Extract overhead from hidden_params if available + litellm_overhead_time_ms = None + if standard_logging_payload is not None: + hidden_params = standard_logging_payload.get("hidden_params", {}) + litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") + # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( metadata, @@ -343,6 +352,7 @@ def get_logging_payload( # noqa: PLR0915 if standard_logging_payload is not None else None ), + litellm_overhead_time_ms=litellm_overhead_time_ms, ) special_usage_fields = ["completion_tokens", "prompt_tokens", "total_tokens"] diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5adf0bb1a3d..69b7e504184 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -24,7 +24,12 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_request_body_for_spend_logs_payload, get_logging_payload, ) -from litellm.types.utils import StandardLoggingPayload +from litellm.types.utils import ( + StandardLoggingHiddenParams, + StandardLoggingMetadata, + StandardLoggingModelInformation, + StandardLoggingPayload, +) def test_sanitize_request_body_for_spend_logs_payload_basic(): @@ -632,3 +637,216 @@ def test_get_logging_payload_includes_agent_id_from_kwargs(): assert payload["agent_id"] == test_agent_id, f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'" + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_includes_overhead_in_spend_logs_metadata(): + """ + Test that get_logging_payload extracts litellm_overhead_time_ms from hidden_params + and stores it in spend_logs_metadata within the metadata JSON. + """ + test_overhead_ms = 123.45 + + # Create StandardLoggingPayload with hidden_params containing overhead + standard_logging_payload = StandardLoggingPayload( + id="test-id-123", + call_type="completion", + stream=False, + response_cost=0.001, + status="success", + total_tokens=100, + prompt_tokens=50, + completion_tokens=50, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=None, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai", + custom_llm_provider="openai", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_alias=None, + user_api_key_team_id=None, + user_api_key_org_id=None, + user_api_key_user_id=None, + user_api_key_team_alias=None, + spend_logs_metadata=None, + requester_ip_address=None, + requester_metadata=None, + user_api_key_end_user_id=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address=None, + messages=[], + response={}, + error_str=None, + model_parameters={}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.001", + litellm_overhead_time_ms=test_overhead_ms, + additional_headers=None, + batch_models=None, + litellm_model_name=None, + usage_object=None, + ), + ) + + kwargs = { + "model": "gpt-3.5-turbo", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test-key", + } + }, + "standard_logging_object": standard_logging_payload, + } + + response_obj = { + "id": "test-response-123", + "choices": [{"message": {"content": "Hello!"}}], + "usage": { + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + }, + } + + start_time = datetime.datetime.now(timezone.utc) + end_time = datetime.datetime.now(timezone.utc) + + payload = get_logging_payload( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + + # Parse the metadata JSON string + metadata_json = payload.get("metadata") + assert metadata_json is not None, "metadata should not be None" + + metadata = json.loads(metadata_json) + + # Verify overhead is stored directly in metadata + assert ( + metadata.get("litellm_overhead_time_ms") == test_overhead_ms + ), f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'" + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_handles_missing_overhead_gracefully(): + """ + Test that get_logging_payload handles missing overhead gracefully + (backward compatibility - when overhead is not present, it should not break). + """ + # Create StandardLoggingPayload WITHOUT overhead in hidden_params + standard_logging_payload = StandardLoggingPayload( + id="test-id-456", + call_type="completion", + stream=False, + response_cost=0.001, + status="success", + total_tokens=100, + prompt_tokens=50, + completion_tokens=50, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=None, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai", + custom_llm_provider="openai", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_alias=None, + user_api_key_team_id=None, + user_api_key_org_id=None, + user_api_key_user_id=None, + user_api_key_team_alias=None, + spend_logs_metadata=None, + requester_ip_address=None, + requester_metadata=None, + user_api_key_end_user_id=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address=None, + messages=[], + response={}, + error_str=None, + model_parameters={}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.001", + litellm_overhead_time_ms=None, # No overhead + additional_headers=None, + batch_models=None, + litellm_model_name=None, + usage_object=None, + ), + ) + + kwargs = { + "model": "gpt-3.5-turbo", + "litellm_params": { + "metadata": { + "user_api_key": "sk-test-key", + } + }, + "standard_logging_object": standard_logging_payload, + } + + response_obj = { + "id": "test-response-456", + "choices": [{"message": {"content": "Hello!"}}], + "usage": { + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + }, + } + + start_time = datetime.datetime.now(timezone.utc) + end_time = datetime.datetime.now(timezone.utc) + + # Should not raise an exception + payload = get_logging_payload( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + + # Parse the metadata JSON string + metadata_json = payload.get("metadata") + assert metadata_json is not None, "metadata should not be None" + + metadata = json.loads(metadata_json) + + # When overhead is None, litellm_overhead_time_ms should be None or not present + assert ( + metadata.get("litellm_overhead_time_ms") is None + ), "litellm_overhead_time_ms should be None when overhead is not provided" + From 0e5b56eddf0e2aa059744c5a5c19defffedabf24 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 12:51:44 -0800 Subject: [PATCH 3/4] Fixing tests --- .../proxy/spend_tracking/test_spend_management_endpoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index b64706e5ac2..e08f2ad98dd 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -201,6 +201,7 @@ ignored_keys = [ "metadata.usage_object", "metadata.cold_storage_object_key", "metadata.additional_usage_values.prompt_tokens_details.cache_creation_tokens", + "metadata.litellm_overhead_time_ms", ] MODEL_LIST = [ From c2f79681b699a96b64b612eab171613c194377c4 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 13:44:53 -0800 Subject: [PATCH 4/4] Fixing test 2 --- tests/logging_callback_tests/test_gcs_pub_sub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 4172659e659..d45110b3277 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -39,6 +39,7 @@ ignored_keys = [ "metadata.model_map_information", "metadata.usage_object", "metadata.cold_storage_object_key", + "metadata.litellm_overhead_time_ms", ]