diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d9d63681594..75b75f9421a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -60,9 +60,6 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - get_response_hidden_params, -) from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -1705,7 +1702,17 @@ class Logging(LiteLLMLoggingBaseClass): """ if logging_result is None: return - incoming = get_response_hidden_params(logging_result) + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + get_response_hidden_params, + hidden_params_to_plain_dict, + ) + + incoming_raw = get_response_hidden_params(logging_result) + incoming = hidden_params_to_plain_dict(incoming_raw) + if not incoming: + return + # Do not let explicit None values from model_dump wipe proxy-injected timing keys. + incoming = {k: v for k, v in incoming.items() if v is not None} if not incoming: return if self.model_call_details.get("litellm_params") is None: @@ -1719,9 +1726,6 @@ class Logging(LiteLLMLoggingBaseClass): existing = md.get("hidden_params") if not isinstance(existing, dict): existing = {} - if not isinstance(incoming, dict): - md["hidden_params"] = existing or incoming - return md["hidden_params"] = {**existing, **incoming} def _process_hidden_params_and_response_cost( @@ -1730,18 +1734,20 @@ class Logging(LiteLLMLoggingBaseClass): start_time, end_time, ): - hidden_params = get_response_hidden_params(logging_result) - if hidden_params: - if self.model_call_details.get("litellm_params") is not None: - self.model_call_details["litellm_params"].setdefault("metadata", {}) - if self.model_call_details["litellm_params"]["metadata"] is None: - self.model_call_details["litellm_params"]["metadata"] = {} - self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = get_response_hidden_params(logging_result) # type: ignore + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + get_response_hidden_params, + hidden_params_to_plain_dict, + ) + + hp_raw = get_response_hidden_params(logging_result) + if hp_raw: + self._merge_hidden_params_from_response_into_metadata(logging_result) + hp_dict = hidden_params_to_plain_dict(hp_raw) if hp_raw else {} if self.model_call_details.get("cache_hit") is True: self.model_call_details["response_cost"] = 0.0 - elif "response_cost" in hidden_params: - self.model_call_details["response_cost"] = hidden_params["response_cost"] + elif "response_cost" in hp_dict: + self.model_call_details["response_cost"] = hp_dict["response_cost"] elif self.model_call_details.get("response_cost") is not None: # Preserve response_cost if already calculated (e.g., by pass-through # handlers like Gemini/Vertex which call completion_cost directly) @@ -5316,6 +5322,10 @@ def _extract_response_obj_and_hidden_params( hidden_params = getattr(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + get_response_hidden_params, + ) + hp = get_response_hidden_params(init_response_obj) if hp: hidden_params = ( diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ffc981448f7..e72dd16bc96 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -33,6 +33,20 @@ def get_response_hidden_params(response: Any) -> Union[HiddenParams, dict]: return {} +def hidden_params_to_plain_dict(hp: Any) -> dict: + """ + Normalize ``get_response_hidden_params`` output (``dict`` or ``HiddenParams``) to a + plain ``dict`` for merging into ``metadata['hidden_params']``. + """ + if not hp: + return {} + if isinstance(hp, dict): + return dict(hp) + if hasattr(hp, "model_dump"): + return hp.model_dump(exclude_none=True) + return {} + + def strip_litellm_internal_keys_from_dict_response(response: Any) -> None: """Remove internal keys from dict API responses before JSON serialization.""" if isinstance(response, dict): diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index dea0cb3980d..6b749886379 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -36,10 +36,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - merge_hidden_params_with_logging_timings, - strip_litellm_internal_keys_from_dict_response, -) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import check_response_size_is_safe @@ -1042,6 +1038,10 @@ class ProxyBaseLLMRequestProcessing: _exception_raised = False try: + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + merge_hidden_params_with_logging_timings, + ) + # Async generators (e.g. Anthropic /v1/messages SSE) have no _hidden_params; # merge timing from logging_obj.model_call_details (llm_api_duration_ms, etc.). hidden_params = merge_hidden_params_with_logging_timings( @@ -1287,6 +1287,11 @@ class ProxyBaseLLMRequestProcessing: log_context=f"litellm_call_id={logging_obj.litellm_call_id}", ) + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + merge_hidden_params_with_logging_timings, + strip_litellm_internal_keys_from_dict_response, + ) + # Re-merge after post_call_success_hook so dict / CSW hidden_params + logging timings align. hidden_params = merge_hidden_params_with_logging_timings( response, diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 792bbdad59a..cf5976e3849 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2333,6 +2333,43 @@ def test_merge_hidden_params_from_response_preserves_proxy_injected_timing(): assert hp["model_id"] == "mid-2" +def test_merge_hidden_params_from_response_preserves_proxy_injected_timing_with_hidden_params_model(): + """Assembled streaming response may use HiddenParams object; merge must not drop fields.""" + from litellm.types.llms.base import HiddenParams + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + logging_obj = LiteLLMLoggingObj( + model="claude-3-5-sonnet", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="merge-hp-hp-model", + function_id="merge-hp-hp-model-fn", + ) + logging_obj.model_call_details = { + "litellm_params": { + "metadata": { + "hidden_params": { + "litellm_overhead_time_ms": 273.329, + "_response_ms": 1590.0, + } + } + }, + } + + class _Assembled: + _hidden_params = HiddenParams(response_cost=0.001, model_id="mid-2") + + logging_obj._merge_hidden_params_from_response_into_metadata(_Assembled()) + hp = logging_obj.model_call_details["litellm_params"]["metadata"]["hidden_params"] + assert hp["litellm_overhead_time_ms"] == 273.329 + assert hp["_response_ms"] == 1590.0 + assert hp["response_cost"] == 0.001 + assert hp["model_id"] == "mid-2" + + def test_get_standard_logging_payload_includes_metadata_hidden_params_overhead(): """Spend logs read overhead from standard_logging_object.hidden_params; merge metadata.hidden_params.""" from datetime import datetime