scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

fix(logging): avoid mutating streaming hidden params

Browse source 
Backfill calculated streaming response cost into logging payload copies so OTEL spans expose hidden_params.response_cost without mutating the response object.

Made-with: Cursor
This commit is contained in:
Milan 2026-04-27 14:56:48 +03:00
parent d4a989d16e
commit 73e15e0124
No known key found for this signature in database
2 changed files with 72 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
litellm/litellm_core_utils/litellm_logging.py
View file

@ -978,9 +978,7 @@ class Logging(LiteLLMLoggingBaseClass):
try:
# [Non-blocking Extra Debug Information in metadata]
if turn_off_message_logging is True:
_metadata[
"raw_request"
] = "redacted by litellm. \
_metadata["raw_request"] = "redacted by litellm. \
'litellm.turn_off_message_logging=True'"
else:
curl_command = self._get_request_curl_command(
@ -1015,12 +1013,8 @@ class Logging(LiteLLMLoggingBaseClass):
] = RawRequestTypedDict(
error=str(e),
)
_metadata[
"raw_request"
] = "Unable to Log \
raw request: {}".format(
str(e)
)
_metadata["raw_request"] = "Unable to Log \
raw request: {}".format(str(e))
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
@ -1703,13 +1697,20 @@ class Logging(LiteLLMLoggingBaseClass):
return
if self.model_call_details.get("litellm_params") is None:
return
metadata_hidden_params = hidden_params.copy()
response_cost = self.model_call_details.get("response_cost")
if hidden_params.get("response_cost") is None and response_cost is not None:
hidden_params["response_cost"] = response_cost
self.model_call_details["litellm_params"].setdefault("metadata", {})
if self.model_call_details["litellm_params"]["metadata"] is None:
self.model_call_details["litellm_params"]["metadata"] = {}
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = hidden_params
if (
metadata_hidden_params.get("response_cost") is None
and response_cost is not None
):
metadata_hidden_params["response_cost"] = response_cost
litellm_params = self.model_call_details["litellm_params"]
metadata = litellm_params.setdefault("metadata", {})
if metadata is None:
metadata = {}
litellm_params["metadata"] = metadata
metadata["hidden_params"] = metadata_hidden_params
def _process_hidden_params_and_response_cost(
self,
@ -5393,11 +5394,6 @@ def get_standard_logging_object_payload(
completion_start_time_float=completion_start_time_float,
stream=kwargs.get("stream", False),
)
# clean up litellm hidden params
clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
hidden_params
)
# clean up litellm metadata
clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
metadata=metadata,
@ -5431,6 +5427,17 @@ def get_standard_logging_object_payload(
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
response_cost: float = kwargs.get("response_cost", 0) or 0.0
# clean up litellm hidden params
clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
hidden_params
)
if (
clean_hidden_params["response_cost"] is None
and kwargs.get("response_cost") is not None
):
clean_hidden_params["response_cost"] = response_cost
model_cost_information = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,
@ -5439,7 +5446,6 @@ def get_standard_logging_object_payload(
init_response_obj=init_response_obj,
api_base=litellm_params.get("api_base"),
)
response_cost: float = kwargs.get("response_cost", 0) or 0.0
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,

46
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -2297,10 +2297,54 @@ def test_merge_hidden_params_from_response_into_metadata_backfills_response_cost
class _Resp:
_hidden_params = {"response_cost": None, "model_id": "mid-test"}
logging_obj._merge_hidden_params_from_response_into_metadata(_Resp())
response = _Resp()
logging_obj._merge_hidden_params_from_response_into_metadata(response)
meta = logging_obj.model_call_details["litellm_params"]["metadata"]
assert meta["hidden_params"]["response_cost"] == 0.002
assert meta["hidden_params"]["model_id"] == "mid-test"
assert response._hidden_params["response_cost"] is None
def test_standard_logging_hidden_params_backfills_response_cost_without_mutating_response():
"""Streaming standard logging payload should expose the calculated response cost."""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import Usage
logging_obj = LiteLLMLoggingObj(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="acompletion",
start_time=time.time(),
litellm_call_id="standard-hp-cost-test",
function_id="standard-hp-cost-fn",
)
logging_obj.model_call_details = {
"litellm_params": {"metadata": {}, "proxy_server_request": {}},
"litellm_call_id": "standard-hp-cost-test",
"call_type": "acompletion",
"stream": True,
"model": "gpt-4o-mini",
"custom_llm_provider": "openai",
"optional_params": {"stream": True},
"response_cost": 0.002,
}
response = ModelResponse(
id="standard-hp-cost-response",
model="gpt-4o-mini",
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
response._hidden_params = {"response_cost": None, "model_id": "mid-test"}
payload = logging_obj._build_standard_logging_payload(
response, datetime.now(), datetime.now()
)
assert payload is not None
assert payload["hidden_params"]["response_cost"] == 0.002
assert response._hidden_params["response_cost"] is None
def test_merge_hidden_params_from_response_into_metadata_preserves_response_cost():