From 54e95af071b0c0f665607cb8c2602ad5cbf00894 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:04:07 +0800 Subject: [PATCH 01/14] fix: set overhead duration metric for all route types update_response_metadata was only called in litellm.utils.completion() for /v1/chat/completions. Routes like /v1/messages (Anthropic) and /v1/responses (Responses API) skipped this call. Call update_response_metadata from base_process_llm_request after the LLM response is received, but only when litellm_overhead_time_ms is not already set, so chat completions keep their SDK-level timing. Fixes #30566 --- litellm/proxy/common_request_processing.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 97f7d51970c..428f8688aa1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1366,6 +1366,8 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, ) + self.data["start_time"] = datetime.now() + # Defer async logging when post-call guardrails are configured so the # StandardLoggingPayload is built after guardrails write to metadata. # Cache the result to avoid scanning litellm.callbacks twice. @@ -1427,6 +1429,23 @@ class ProxyBaseLLMRequestProcessing: response = responses[1] + # GH#30566: overhead for non-chat-completions routes + _hidden_params = getattr(response, "_hidden_params", {}) or {} + if not _hidden_params.get("litellm_overhead_time_ms"): + end_time = datetime.now() + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, + ) + + update_response_metadata( + result=response, + logging_obj=self.data.get("litellm_logging_obj"), + model=self.data.get("model"), + kwargs=self.data, + start_time=self.data.get("start_time", end_time), + end_time=end_time, + ) + _exception_raised = False try: hidden_params = getattr(response, "_hidden_params", {}) or {} From 742e0feb124e7a3d58dc9d5b462d641ee2fdbaa1 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Thu, 18 Jun 2026 18:44:02 +0800 Subject: [PATCH 02/14] fix: remove start_time from self.data to avoid snapshot test pollution start_time was not needed for overhead computation on non-chat routes because llm_api_duration_ms is also not available. Use end_time for both start and end; the guard prevents affecting chat completions. Fixes #30566 --- litellm/proxy/common_request_processing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 428f8688aa1..919e9dc0633 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1366,8 +1366,6 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, ) - self.data["start_time"] = datetime.now() - # Defer async logging when post-call guardrails are configured so the # StandardLoggingPayload is built after guardrails write to metadata. # Cache the result to avoid scanning litellm.callbacks twice. @@ -1442,7 +1440,7 @@ class ProxyBaseLLMRequestProcessing: logging_obj=self.data.get("litellm_logging_obj"), model=self.data.get("model"), kwargs=self.data, - start_time=self.data.get("start_time", end_time), + start_time=end_time, end_time=end_time, ) From d10ec508078a610433b9f48c5ed194ca6375b6b2 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Thu, 18 Jun 2026 18:48:58 +0800 Subject: [PATCH 03/14] test: verify overhead computed for routes without pre-existing value GH#30566 --- .../test_response_metadata.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py index 203c6d3da0d..d481743d29a 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -8,6 +8,8 @@ through _hidden_params to the x-litellm-callback-duration-ms response header. import datetime from unittest.mock import MagicMock +import pytest + import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod from litellm.litellm_core_utils.litellm_logging import Logging @@ -91,6 +93,32 @@ class TestCallbackDurationMs: # overhead should also be set assert hidden.get("litellm_overhead_time_ms") is not None + def test_overhead_computed_for_routes_without_pre_existing_value(self): + """GH#30566: overhead is set even when _hidden_params + does not already contain litellm_overhead_time_ms. + This simulates non-chat-completions routes that skip + the SDK-level update_response_metadata call.""" + result = ModelResponse() + logging_obj = self._make_logging_obj(llm_api_duration_ms=900.0) + logging_obj._response_cost_calculator = MagicMock(return_value=0.001) + logging_obj.litellm_call_id = "test-gh30566" + + start = datetime.datetime(2025, 1, 1, 0, 0, 0) + end = datetime.datetime(2025, 1, 1, 0, 0, 1) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-4", + kwargs={}, + start_time=start, + end_time=end, + ) + + hidden = result._hidden_params + assert hidden.get("litellm_overhead_time_ms") == pytest.approx(100.0, rel=0.01) + assert hidden.get("_response_ms") == pytest.approx(1000.0, rel=0.01) + class TestCallbackDurationInCustomHeaders: """Test that callback_duration_ms flows into get_custom_headers.""" From 9f03795b88266ed9dacb983affc02ec871e54338 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Thu, 18 Jun 2026 19:05:44 +0800 Subject: [PATCH 04/14] fix: guard against None logging_obj to prevent AttributeError --- litellm/proxy/common_request_processing.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 919e9dc0633..80262a146c5 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1431,18 +1431,20 @@ class ProxyBaseLLMRequestProcessing: _hidden_params = getattr(response, "_hidden_params", {}) or {} if not _hidden_params.get("litellm_overhead_time_ms"): end_time = datetime.now() - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - update_response_metadata, - ) + logging_obj = self.data.get("litellm_logging_obj") + if logging_obj is not None: + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, + ) - update_response_metadata( - result=response, - logging_obj=self.data.get("litellm_logging_obj"), - model=self.data.get("model"), - kwargs=self.data, - start_time=end_time, - end_time=end_time, - ) + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=self.data.get("model"), + kwargs=self.data, + start_time=end_time, + end_time=end_time, + ) _exception_raised = False try: From 78aa0b2ea7812ba343c88e5041ffb491e01d543e Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Sun, 21 Jun 2026 13:37:55 +0800 Subject: [PATCH 05/14] fix: use logging_obj.start_time for overhead calculation Previously the call passed start_time=end_time, producing _response_ms=0 and negative litellm_overhead_time_ms. Use logging_obj.start_time with end_time fallback instead. --- litellm/proxy/common_request_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 80262a146c5..f08962000c5 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1437,12 +1437,13 @@ class ProxyBaseLLMRequestProcessing: update_response_metadata, ) + start_time = logging_obj.start_time or end_time update_response_metadata( result=response, logging_obj=logging_obj, model=self.data.get("model"), kwargs=self.data, - start_time=end_time, + start_time=start_time, end_time=end_time, ) From df130fea02213e0cfd90c178d328caaca3dffb2c Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Sun, 21 Jun 2026 14:11:21 +0800 Subject: [PATCH 06/14] fix: avoid shadowing logging_obj in overhead duration code path Renamed local logging_obj to _logging_obj and start_time to _start_time to prevent overwriting the function-level logging_obj variable, which caused proxy-infra test failures when self.data had no litellm_logging_obj. --- litellm/proxy/common_request_processing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f08962000c5..4349ec130c4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1431,19 +1431,19 @@ class ProxyBaseLLMRequestProcessing: _hidden_params = getattr(response, "_hidden_params", {}) or {} if not _hidden_params.get("litellm_overhead_time_ms"): end_time = datetime.now() - logging_obj = self.data.get("litellm_logging_obj") - if logging_obj is not None: + _logging_obj = self.data.get("litellm_logging_obj") + if _logging_obj is not None: from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( update_response_metadata, ) - start_time = logging_obj.start_time or end_time + _start_time = _logging_obj.start_time or end_time update_response_metadata( result=response, - logging_obj=logging_obj, + logging_obj=_logging_obj, model=self.data.get("model"), kwargs=self.data, - start_time=start_time, + start_time=_start_time, end_time=end_time, ) From e40bda538f7c7f4c8ab423b0a7f2a728e136e585 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:11:42 +0800 Subject: [PATCH 07/14] fix: use timezone-aware datetime.now for overhead calculation --- litellm/proxy/common_request_processing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 4349ec130c4..4002cb861c3 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4,7 +4,7 @@ import logging import math import time import traceback -from datetime import datetime +from datetime import datetime, timezone from typing import ( TYPE_CHECKING, Any, @@ -1430,7 +1430,7 @@ class ProxyBaseLLMRequestProcessing: # GH#30566: overhead for non-chat-completions routes _hidden_params = getattr(response, "_hidden_params", {}) or {} if not _hidden_params.get("litellm_overhead_time_ms"): - end_time = datetime.now() + end_time = datetime.now(tz=timezone.utc) _logging_obj = self.data.get("litellm_logging_obj") if _logging_obj is not None: from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( From 92bbd51250a2f33f36b676b1b839deaca265279c Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:22:38 +0800 Subject: [PATCH 08/14] fix: revert timezone-aware datetime, restore test file datetime.now(tz=timezone.utc) is aware but logging_obj.start_time is naive, causing "can't subtract offset-naive and offset-aware datetimes" TypeError. Revert to naive datetime.now(). Also restore test file lost in previous API push. --- litellm/proxy/common_request_processing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 4002cb861c3..4349ec130c4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4,7 +4,7 @@ import logging import math import time import traceback -from datetime import datetime, timezone +from datetime import datetime from typing import ( TYPE_CHECKING, Any, @@ -1430,7 +1430,7 @@ class ProxyBaseLLMRequestProcessing: # GH#30566: overhead for non-chat-completions routes _hidden_params = getattr(response, "_hidden_params", {}) or {} if not _hidden_params.get("litellm_overhead_time_ms"): - end_time = datetime.now(tz=timezone.utc) + end_time = datetime.now() _logging_obj = self.data.get("litellm_logging_obj") if _logging_obj is not None: from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( From 084432f9ede0856e91dde4b3ad8753018cd0adf1 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:07:12 +0800 Subject: [PATCH 09/14] test: add guard condition test for overhead not overwritten --- .../test_response_metadata.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py index d481743d29a..ca620dd7e7e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -119,6 +119,24 @@ class TestCallbackDurationMs: assert hidden.get("litellm_overhead_time_ms") == pytest.approx(100.0, rel=0.01) assert hidden.get("_response_ms") == pytest.approx(1000.0, rel=0.01) + def test_overhead_guard_skips_when_already_present(self): + """GH#30566: the guard in base_process_llm_request prevents + calling update_response_metadata when litellm_overhead_time_ms + is already set by the SDK layer (e.g. /v1/chat/completions). + This test verifies the guard condition directly.""" + # Chat-completions path: overhead already populated + result = ModelResponse() + result._hidden_params = {"litellm_overhead_time_ms": 50.0} + hidden_params = getattr(result, "_hidden_params", {}) or {} + should_skip = bool(hidden_params.get("litellm_overhead_time_ms")) + assert should_skip is True + + # Non-chat path (/v1/messages, /v1/responses): no overhead yet + result2 = ModelResponse() + hidden_params2 = getattr(result2, "_hidden_params", {}) or {} + should_skip2 = bool(hidden_params2.get("litellm_overhead_time_ms")) + assert should_skip2 is False + class TestCallbackDurationInCustomHeaders: """Test that callback_duration_ms flows into get_custom_headers.""" From b69a4a47ac8558792b228e0e9de7047d60173a6e Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:01:33 +0800 Subject: [PATCH 10/14] fix: skip overhead metric for chat completions route types Chat completions already set litellm_overhead_time_ms in the SDK layer. Skip our overhead calculation for acompletion/completion routes to avoid interfering with object responses. --- litellm/proxy/common_request_processing.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 4349ec130c4..cb93c98cfae 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1427,9 +1427,14 @@ class ProxyBaseLLMRequestProcessing: response = responses[1] - # GH#30566: overhead for non-chat-completions routes + # GH#30566: overhead for non-chat-completions routes. + # Skip chat routes (acompletion/completion) because the SDK already + # sets litellm_overhead_time_ms in litellm.utils.completion(). _hidden_params = getattr(response, "_hidden_params", {}) or {} - if not _hidden_params.get("litellm_overhead_time_ms"): + if ( + not _hidden_params.get("litellm_overhead_time_ms") + and route_type not in ("acompletion", "completion") + ): end_time = datetime.now() _logging_obj = self.data.get("litellm_logging_obj") if _logging_obj is not None: From 78a8c2d6d89ee6043d8515096c49f93253fb063a Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:04:55 +0800 Subject: [PATCH 11/14] chore: ruff format --- litellm/proxy/common_request_processing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index cb93c98cfae..8d3d4b29705 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1431,10 +1431,7 @@ class ProxyBaseLLMRequestProcessing: # Skip chat routes (acompletion/completion) because the SDK already # sets litellm_overhead_time_ms in litellm.utils.completion(). _hidden_params = getattr(response, "_hidden_params", {}) or {} - if ( - not _hidden_params.get("litellm_overhead_time_ms") - and route_type not in ("acompletion", "completion") - ): + if not _hidden_params.get("litellm_overhead_time_ms") and route_type not in ("acompletion", "completion"): end_time = datetime.now() _logging_obj = self.data.get("litellm_logging_obj") if _logging_obj is not None: From 321e98147f43b723db84cade53e80e07163e1c45 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:23:31 +0800 Subject: [PATCH 12/14] fix: set overhead directly instead of calling update_response_metadata Only set litellm_overhead_time_ms in _hidden_params without going through update_response_metadata which also touches response_cost and calls the cost calculator. --- litellm/proxy/common_request_processing.py | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 8d3d4b29705..f9c9a6c4aa2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1427,27 +1427,27 @@ class ProxyBaseLLMRequestProcessing: response = responses[1] - # GH#30566: overhead for non-chat-completions routes. - # Skip chat routes (acompletion/completion) because the SDK already - # sets litellm_overhead_time_ms in litellm.utils.completion(). - _hidden_params = getattr(response, "_hidden_params", {}) or {} - if not _hidden_params.get("litellm_overhead_time_ms") and route_type not in ("acompletion", "completion"): + # GH#30566: set overhead duration for non-chat-completions + # routes (/v1/messages, /v1/responses). Chat completions + # already have litellm_overhead_time_ms from the SDK. + # Only set the overhead field directly (don't call + # update_response_metadata which also touches cost). + _overhead_hidden_params = getattr(response, "_hidden_params", {}) or {} + if not _overhead_hidden_params.get("litellm_overhead_time_ms") and route_type not in ( + "acompletion", + "completion", + ): end_time = datetime.now() _logging_obj = self.data.get("litellm_logging_obj") - if _logging_obj is not None: - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - update_response_metadata, - ) - - _start_time = _logging_obj.start_time or end_time - update_response_metadata( - result=response, - logging_obj=_logging_obj, - model=self.data.get("model"), - kwargs=self.data, - start_time=_start_time, - end_time=end_time, + if _logging_obj is not None and _logging_obj.start_time is not None: + overhead_ms = (end_time - _logging_obj.start_time).total_seconds() * 1000 - ( + _logging_obj.model_call_details.get("llm_api_duration_ms", 0) ) + if not isinstance(_overhead_hidden_params, dict): + _overhead_hidden_params = {} + _overhead_hidden_params["litellm_overhead_time_ms"] = overhead_ms + if hasattr(response, "_hidden_params"): + response._hidden_params = _overhead_hidden_params _exception_raised = False try: From faf9133ea377feb98bf7c1a5c741762543255438 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:54:07 +0800 Subject: [PATCH 13/14] fix: write overhead to dict-typed responses via key assignment TypedDict responses (e.g. AnthropicMessagesResponse) are plain dicts at runtime, so hasattr(response, "_hidden_params") returns False. Also assign via response["_hidden_params"] for dict types. --- litellm/proxy/common_request_processing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f9c9a6c4aa2..f4d0cb52af6 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1448,6 +1448,8 @@ class ProxyBaseLLMRequestProcessing: _overhead_hidden_params["litellm_overhead_time_ms"] = overhead_ms if hasattr(response, "_hidden_params"): response._hidden_params = _overhead_hidden_params + elif isinstance(response, dict): + response["_hidden_params"] = _overhead_hidden_params _exception_raised = False try: From 75b049d3cc225c1f1871f046ce2e418c8f40d009 Mon Sep 17 00:00:00 2001 From: factnn <166481866+factnn@users.noreply.github.com> Date: Sat, 4 Jul 2026 16:07:30 +0800 Subject: [PATCH 14/14] chore: re-trigger CI