fix(litellm): stamp cache-hit marker on HiddenParams carriers and restore probe side effects

gauntlet-mini on ace57d25f verified two live gaps. TextCompletionResponse and
ResponsesAPIResponse carry _hidden_params as a HiddenParams object, so the
dict-gated marker write skipped them and the served hit still stamped the full
price. The marker write is now container-agnostic, which it already was for the
partial-embedding stamp, and a regression test exercises the carrier end to end.
_uncached_response_cost also restores billed_token_rates, zero_cost_warned, and
the response_cost_failure_debug_information key alongside cost_breakdown, so the
probe leaves no pricing state on the call it reads.
This commit is contained in:
shrey kharbanda 2026-09-28 19:55:02 +00:00
parent ace57d25f0
commit 6910ea644d
3 changed files with 72 additions and 12 deletions

View file

@ -963,11 +963,7 @@ class LLMCachingHandler:
)
response_obj: Final = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
if hasattr(response_obj, "_hidden_params") and response_obj._hidden_params is not None:
response_obj._hidden_params["cache_hit"] = True
if _stream_replay_requested(kwargs):
@ -980,11 +976,7 @@ class LLMCachingHandler:
else:
cached_result = response_obj
if (
hasattr(cached_result, "_hidden_params")
and cached_result._hidden_params is not None
and isinstance(cached_result._hidden_params, dict)
):
if hasattr(cached_result, "_hidden_params") and cached_result._hidden_params is not None:
cached_result._hidden_params["cache_hit"] = True
#########################################################

View file

@ -1976,12 +1976,23 @@ class Logging(LiteLLMLoggingBaseClass):
list,
],
) -> float | None:
"""Price a response as if it had missed the cache, without overwriting this call's cost breakdown"""
"""Price a response as if it had missed the cache, without leaving the probe's side effects on this call"""
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
previous_zero_cost_warned: Final = self.zero_cost_warned
debug_key: Final = "response_cost_failure_debug_information"
debug_missing: Final = object()
debug_before: Final = self.model_call_details.get(debug_key, debug_missing)
try:
return self._response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
finally:
self.cost_breakdown = previous_cost_breakdown
self.billed_token_rates = previous_billed_token_rates
self.zero_cost_warned = previous_zero_cost_warned
if debug_before is debug_missing:
self.model_call_details.pop(debug_key, None)
else:
self.model_call_details[debug_key] = debug_before
def _record_zero_cost_diagnostic(
self,

View file

@ -26,7 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.types.utils import ModelResponse, Usage
from litellm.types.utils import ModelResponse, TextCompletionResponse, Usage
class TestCallbackDurationMs:
@ -703,3 +703,60 @@ def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_result
assert result._hidden_params["response_cost"] == 0.0
assert logging_obj.cost_breakdown is None
def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_hidden_params_carrier(monkeypatch):
"""A cache hit whose _hidden_params is a HiddenParams object (text_completion's carrier) stamps 0 cost, not the full price."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
deployment_id: Final = "cache-hit-deployment-hiddenparams"
litellm.register_model(
model_cost={
deployment_id: {
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"litellm_provider": "openai",
"mode": "completion",
}
}
)
start_time: Final = datetime.datetime(2026, 9, 28, 12, 0, 0)
logging_obj: Final = Logging(
model="gpt-3.5-turbo-instruct",
messages="Hi",
stream=False,
call_type="text_completion",
start_time=start_time,
litellm_call_id="cache-hit-hiddenparams",
function_id="f",
)
logging_obj.update_environment_variables(
model="gpt-3.5-turbo-instruct",
litellm_params={
"metadata": {"model_info": {"id": deployment_id}},
},
optional_params={},
custom_llm_provider="openai",
)
result: Final = TextCompletionResponse(
id="cmpl-x",
object="text_completion",
created=1,
model="gpt-3.5-turbo-instruct",
choices=[{"text": "ok", "index": 0, "finish_reason": "stop"}],
usage={"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
)
result._hidden_params["cache_hit"] = True
update_response_metadata(
result=result,
logging_obj=logging_obj,
model="gpt-3.5-turbo-instruct",
kwargs={"model_info": {"id": deployment_id}},
start_time=start_time,
end_time=start_time + datetime.timedelta(seconds=2),
)
assert result._hidden_params.get("response_cost") == 0.0
assert logging_obj.cost_breakdown is None