From b4259fb2e21bddc31049595cc8a01c38e5880bb9 Mon Sep 17 00:00:00 2001 From: shivam Date: Fri, 27 Mar 2026 17:04:44 -0700 Subject: [PATCH] =?UTF-8?q?refactor:=20extract=20hidden=5Fparams=5Ftiming?= =?UTF-8?q?=20to=20break=20proxy=E2=86=94response=5Fmetadata=20import=20cy?= =?UTF-8?q?cle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made-with: Cursor --- litellm/litellm_core_utils/litellm_logging.py | 6 +- .../hidden_params_timing.py | 132 ++++++++++++++++++ .../llm_response_utils/response_metadata.py | 132 ++---------------- litellm/proxy/common_request_processing.py | 13 +- 4 files changed, 153 insertions(+), 130 deletions(-) create mode 100644 litellm/litellm_core_utils/llm_response_utils/hidden_params_timing.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 75b75f9421a..e15b62f661c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1702,7 +1702,7 @@ class Logging(LiteLLMLoggingBaseClass): """ if logging_result is None: return - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + from litellm.litellm_core_utils.llm_response_utils.hidden_params_timing import ( get_response_hidden_params, hidden_params_to_plain_dict, ) @@ -1734,7 +1734,7 @@ class Logging(LiteLLMLoggingBaseClass): start_time, end_time, ): - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + from litellm.litellm_core_utils.llm_response_utils.hidden_params_timing import ( get_response_hidden_params, hidden_params_to_plain_dict, ) @@ -5322,7 +5322,7 @@ def _extract_response_obj_and_hidden_params( hidden_params = getattr(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + from litellm.litellm_core_utils.llm_response_utils.hidden_params_timing import ( get_response_hidden_params, ) diff --git a/litellm/litellm_core_utils/llm_response_utils/hidden_params_timing.py b/litellm/litellm_core_utils/llm_response_utils/hidden_params_timing.py new file mode 100644 index 00000000000..85a61309218 --- /dev/null +++ b/litellm/litellm_core_utils/llm_response_utils/hidden_params_timing.py @@ -0,0 +1,132 @@ +""" +Hidden-params helpers and proxy timing merge. + +Kept separate from ``response_metadata`` so the proxy can import timing merge logic +without pulling in ``response_metadata`` → ``logging_utils`` import chains that +CodeQL flags as cyclic with ``litellm.proxy.common_request_processing``. +""" + +import datetime +from typing import Any, Optional, Union + +from litellm.constants import LITELLM_DETAILED_TIMING +from litellm.types.utils import HiddenParams + + +def get_response_hidden_params(response: Any) -> Union[HiddenParams, dict]: + """ + Read LiteLLM internal fields from ModelResponse/Streaming responses or from + dict-shaped provider responses (e.g. Anthropic ``/v1/messages`` JSON bodies). + + Dict responses store timing/cost under the ``_hidden_params`` key; the proxy + strips that key before returning JSON to clients. + """ + if response is None: + return {} + hp = getattr(response, "_hidden_params", None) + if hp is not None: + return hp + if isinstance(response, dict): + inner = response.get("_hidden_params") + if isinstance(inner, dict): + return inner + return {} + + +def hidden_params_to_plain_dict(hp: Any) -> dict: + """ + Normalize ``get_response_hidden_params`` output (``dict`` or ``HiddenParams``) to a + plain ``dict`` for merging into ``metadata['hidden_params']``. + """ + if not hp: + return {} + if isinstance(hp, dict): + return dict(hp) + if hasattr(hp, "model_dump"): + return hp.model_dump(exclude_none=True) + return {} + + +def strip_litellm_internal_keys_from_dict_response(response: Any) -> None: + """Remove internal keys from dict API responses before JSON serialization.""" + if isinstance(response, dict): + response.pop("_hidden_params", None) + + +def merge_hidden_params_with_logging_timings( + response: Any, + logging_obj: Any, + *, + end_time: Optional[datetime.datetime] = None, +) -> dict: + """ + Merge response hidden params with timing derived from the LiteLLM logging object. + + Used by the proxy when ``response`` does not expose ``_hidden_params`` (e.g. Anthropic + ``/v1/messages`` streaming async generators) or when timing keys are missing, so + ``x-litellm-overhead-duration-ms`` and spend metadata can still be populated. + + Mirrors :meth:`ResponseMetadata.set_timing_metrics` for overhead/cache/callback fields. + """ + hp_raw = get_response_hidden_params(response) + if isinstance(hp_raw, dict): + out: dict = dict(hp_raw) + elif hp_raw is not None and hasattr(hp_raw, "model_dump"): + out = hp_raw.model_dump(exclude_none=True) + else: + out = {} + + if out.get("litellm_overhead_time_ms") is not None: + return out + + if logging_obj is None or not hasattr(logging_obj, "model_call_details"): + return out + + mcd = logging_obj.model_call_details + start_time = getattr(logging_obj, "start_time", None) + _end = end_time or datetime.datetime.now() + + if start_time is None: + return out + + total_ms = (_end - start_time).total_seconds() * 1000 + out.setdefault("_response_ms", total_ms) + + llm_api_duration_ms = mcd.get("llm_api_duration_ms") + if llm_api_duration_ms is not None: + overhead_ms = round(total_ms - float(llm_api_duration_ms), 4) + out["litellm_overhead_time_ms"] = max(overhead_ms, 0.0) + else: + caching_details = getattr(logging_obj, "caching_details", None) + if ( + caching_details is not None + and caching_details.get("cache_hit") is True + and (cache_duration_ms := caching_details.get("cache_duration_ms")) + is not None + ): + out["litellm_overhead_time_ms"] = max( + total_ms - float(cache_duration_ms), 0.0 + ) + + callback_duration_ms = getattr(logging_obj, "callback_duration_ms", None) + if callback_duration_ms is not None: + out.setdefault( + "callback_duration_ms", round(float(callback_duration_ms), 4) + ) + + if LITELLM_DETAILED_TIMING and llm_api_duration_ms is not None: + detailed: dict = { + "timing_llm_api_ms": round(float(llm_api_duration_ms), 4), + } + msg_copy_ms = getattr(logging_obj, "message_copy_duration_ms", None) + if msg_copy_ms is not None: + detailed["timing_message_copy_ms"] = round(float(msg_copy_ms), 4) + api_call_start = mcd.get("api_call_start_time") + if api_call_start is not None and start_time is not None: + pre_ms = (api_call_start - start_time).total_seconds() * 1000 + detailed["timing_pre_processing_ms"] = round(pre_ms, 4) + post_ms = total_ms - pre_ms - float(llm_api_duration_ms) + detailed["timing_post_processing_ms"] = round(max(post_ms, 0), 4) + out.update(detailed) + + return out diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index e72dd16bc96..24f642dd2c4 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -4,6 +4,12 @@ from typing import Any, Optional, Union from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.llm_response_utils.hidden_params_timing import ( + get_response_hidden_params, + hidden_params_to_plain_dict, + merge_hidden_params_with_logging_timings, + strip_litellm_internal_keys_from_dict_response, +) from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject from litellm.types.utils import ( EmbeddingResponse, @@ -12,124 +18,14 @@ from litellm.types.utils import ( TranscriptionResponse, ) - -def get_response_hidden_params(response: Any) -> Union[HiddenParams, dict]: - """ - Read LiteLLM internal fields from ModelResponse/Streaming responses or from - dict-shaped provider responses (e.g. Anthropic ``/v1/messages`` JSON bodies). - - Dict responses store timing/cost under the ``_hidden_params`` key; the proxy - strips that key before returning JSON to clients. - """ - if response is None: - return {} - hp = getattr(response, "_hidden_params", None) - if hp is not None: - return hp - if isinstance(response, dict): - inner = response.get("_hidden_params") - if isinstance(inner, dict): - return inner - return {} - - -def hidden_params_to_plain_dict(hp: Any) -> dict: - """ - Normalize ``get_response_hidden_params`` output (``dict`` or ``HiddenParams``) to a - plain ``dict`` for merging into ``metadata['hidden_params']``. - """ - if not hp: - return {} - if isinstance(hp, dict): - return dict(hp) - if hasattr(hp, "model_dump"): - return hp.model_dump(exclude_none=True) - return {} - - -def strip_litellm_internal_keys_from_dict_response(response: Any) -> None: - """Remove internal keys from dict API responses before JSON serialization.""" - if isinstance(response, dict): - response.pop("_hidden_params", None) - - -def merge_hidden_params_with_logging_timings( - response: Any, - logging_obj: Any, - *, - end_time: Optional[datetime.datetime] = None, -) -> dict: - """ - Merge response hidden params with timing derived from the LiteLLM logging object. - - Used by the proxy when ``response`` does not expose ``_hidden_params`` (e.g. Anthropic - ``/v1/messages`` streaming async generators) or when timing keys are missing, so - ``x-litellm-overhead-duration-ms`` and spend metadata can still be populated. - - Mirrors :meth:`ResponseMetadata.set_timing_metrics` for overhead/cache/callback fields. - """ - hp_raw = get_response_hidden_params(response) - if isinstance(hp_raw, dict): - out: dict = dict(hp_raw) - elif hp_raw is not None and hasattr(hp_raw, "model_dump"): - out = hp_raw.model_dump(exclude_none=True) - else: - out = {} - - if out.get("litellm_overhead_time_ms") is not None: - return out - - if logging_obj is None or not hasattr(logging_obj, "model_call_details"): - return out - - mcd = logging_obj.model_call_details - start_time = getattr(logging_obj, "start_time", None) - _end = end_time or datetime.datetime.now() - - if start_time is None: - return out - - total_ms = (_end - start_time).total_seconds() * 1000 - out.setdefault("_response_ms", total_ms) - - llm_api_duration_ms = mcd.get("llm_api_duration_ms") - if llm_api_duration_ms is not None: - overhead_ms = round(total_ms - float(llm_api_duration_ms), 4) - out["litellm_overhead_time_ms"] = max(overhead_ms, 0.0) - else: - caching_details = getattr(logging_obj, "caching_details", None) - if ( - caching_details is not None - and caching_details.get("cache_hit") is True - and (cache_duration_ms := caching_details.get("cache_duration_ms")) - is not None - ): - out["litellm_overhead_time_ms"] = max( - total_ms - float(cache_duration_ms), 0.0 - ) - - callback_duration_ms = getattr(logging_obj, "callback_duration_ms", None) - if callback_duration_ms is not None: - out.setdefault( - "callback_duration_ms", round(float(callback_duration_ms), 4) - ) - - if LITELLM_DETAILED_TIMING and llm_api_duration_ms is not None: - detailed: dict = { - "timing_llm_api_ms": round(float(llm_api_duration_ms), 4), - } - msg_copy_ms = getattr(logging_obj, "message_copy_duration_ms", None) - if msg_copy_ms is not None: - detailed["timing_message_copy_ms"] = round(float(msg_copy_ms), 4) - api_call_start = mcd.get("api_call_start_time") - if api_call_start is not None and start_time is not None: - pre_ms = (api_call_start - start_time).total_seconds() * 1000 - detailed["timing_pre_processing_ms"] = round(pre_ms, 4) - post_ms = total_ms - pre_ms - float(llm_api_duration_ms) - detailed["timing_post_processing_ms"] = round(max(post_ms, 0), 4) - out.update(detailed) - - return out +__all__ = ( + "ResponseMetadata", + "get_response_hidden_params", + "hidden_params_to_plain_dict", + "merge_hidden_params_with_logging_timings", + "strip_litellm_internal_keys_from_dict_response", + "update_response_metadata", +) class ResponseMetadata: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6b749886379..cd55ad84c5e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -36,6 +36,10 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) +from litellm.litellm_core_utils.llm_response_utils.hidden_params_timing import ( + merge_hidden_params_with_logging_timings, + strip_litellm_internal_keys_from_dict_response, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import check_response_size_is_safe @@ -1038,10 +1042,6 @@ class ProxyBaseLLMRequestProcessing: _exception_raised = False try: - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - merge_hidden_params_with_logging_timings, - ) - # Async generators (e.g. Anthropic /v1/messages SSE) have no _hidden_params; # merge timing from logging_obj.model_call_details (llm_api_duration_ms, etc.). hidden_params = merge_hidden_params_with_logging_timings( @@ -1287,11 +1287,6 @@ class ProxyBaseLLMRequestProcessing: log_context=f"litellm_call_id={logging_obj.litellm_call_id}", ) - from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - merge_hidden_params_with_logging_timings, - strip_litellm_internal_keys_from_dict_response, - ) - # Re-merge after post_call_success_hook so dict / CSW hidden_params + logging timings align. hidden_params = merge_hidden_params_with_logging_timings( response,