From 6910ea644da2082fbd97bd0d44fc2b58a981094e Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Mon, 28 Sep 2026 19:55:02 +0000 Subject: [PATCH] fix(litellm): stamp cache-hit marker on HiddenParams carriers and restore probe side effects gauntlet-mini on ace57d25f verified two live gaps. TextCompletionResponse and ResponsesAPIResponse carry _hidden_params as a HiddenParams object, so the dict-gated marker write skipped them and the served hit still stamped the full price. The marker write is now container-agnostic, which it already was for the partial-embedding stamp, and a regression test exercises the carrier end to end. _uncached_response_cost also restores billed_token_rates, zero_cost_warned, and the response_cost_failure_debug_information key alongside cost_breakdown, so the probe leaves no pricing state on the call it reads. --- litellm/caching/caching_handler.py | 12 +--- litellm/litellm_core_utils/litellm_logging.py | 13 +++- .../test_response_metadata.py | 59 ++++++++++++++++++- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 0e4f444224b..1f03b83c4d0 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -963,11 +963,7 @@ class LLMCachingHandler: ) response_obj: Final = ResponsesAPIResponse(**cached_result) - if ( - hasattr(response_obj, "_hidden_params") - and response_obj._hidden_params is not None - and isinstance(response_obj._hidden_params, dict) - ): + if hasattr(response_obj, "_hidden_params") and response_obj._hidden_params is not None: response_obj._hidden_params["cache_hit"] = True if _stream_replay_requested(kwargs): @@ -980,11 +976,7 @@ class LLMCachingHandler: else: cached_result = response_obj - if ( - hasattr(cached_result, "_hidden_params") - and cached_result._hidden_params is not None - and isinstance(cached_result._hidden_params, dict) - ): + if hasattr(cached_result, "_hidden_params") and cached_result._hidden_params is not None: cached_result._hidden_params["cache_hit"] = True ######################################################### diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c086560e20a..ca563da9cca 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1976,12 +1976,23 @@ class Logging(LiteLLMLoggingBaseClass): list, ], ) -> float | None: - """Price a response as if it had missed the cache, without overwriting this call's cost breakdown""" + """Price a response as if it had missed the cache, without leaving the probe's side effects on this call""" previous_cost_breakdown: Final = self.cost_breakdown + previous_billed_token_rates: Final = self.billed_token_rates + previous_zero_cost_warned: Final = self.zero_cost_warned + debug_key: Final = "response_cost_failure_debug_information" + debug_missing: Final = object() + debug_before: Final = self.model_call_details.get(debug_key, debug_missing) try: return self._response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True) finally: self.cost_breakdown = previous_cost_breakdown + self.billed_token_rates = previous_billed_token_rates + self.zero_cost_warned = previous_zero_cost_warned + if debug_before is debug_missing: + self.model_call_details.pop(debug_key, None) + else: + self.model_call_details[debug_key] = debug_before def _record_zero_cost_diagnostic( self, diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 18ee9d6ea51..05ba5f32c1e 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -26,7 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import ModelResponse, Usage +from litellm.types.utils import ModelResponse, TextCompletionResponse, Usage class TestCallbackDurationMs: @@ -703,3 +703,60 @@ def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_result assert result._hidden_params["response_cost"] == 0.0 assert logging_obj.cost_breakdown is None + + +def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_hidden_params_carrier(monkeypatch): + """A cache hit whose _hidden_params is a HiddenParams object (text_completion's carrier) stamps 0 cost, not the full price.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id: Final = "cache-hit-deployment-hiddenparams" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openai", + "mode": "completion", + } + } + ) + start_time: Final = datetime.datetime(2026, 9, 28, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-3.5-turbo-instruct", + messages="Hi", + stream=False, + call_type="text_completion", + start_time=start_time, + litellm_call_id="cache-hit-hiddenparams", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gpt-3.5-turbo-instruct", + litellm_params={ + "metadata": {"model_info": {"id": deployment_id}}, + }, + optional_params={}, + custom_llm_provider="openai", + ) + result: Final = TextCompletionResponse( + id="cmpl-x", + object="text_completion", + created=1, + model="gpt-3.5-turbo-instruct", + choices=[{"text": "ok", "index": 0, "finish_reason": "stop"}], + usage={"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18}, + ) + result._hidden_params["cache_hit"] = True + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-3.5-turbo-instruct", + kwargs={"model_info": {"id": deployment_id}}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + + assert result._hidden_params.get("response_cost") == 0.0 + assert logging_obj.cost_breakdown is None