mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
fix(litellm): stamp cache-hit marker on HiddenParams carriers and restore probe side effects
gauntlet-mini on ace57d25f verified two live gaps. TextCompletionResponse and
ResponsesAPIResponse carry _hidden_params as a HiddenParams object, so the
dict-gated marker write skipped them and the served hit still stamped the full
price. The marker write is now container-agnostic, which it already was for the
partial-embedding stamp, and a regression test exercises the carrier end to end.
_uncached_response_cost also restores billed_token_rates, zero_cost_warned, and
the response_cost_failure_debug_information key alongside cost_breakdown, so the
probe leaves no pricing state on the call it reads.
This commit is contained in:
parent
ace57d25f0
commit
6910ea644d
3 changed files with 72 additions and 12 deletions
|
|
@ -963,11 +963,7 @@ class LLMCachingHandler:
|
|||
)
|
||||
|
||||
response_obj: Final = ResponsesAPIResponse(**cached_result)
|
||||
if (
|
||||
hasattr(response_obj, "_hidden_params")
|
||||
and response_obj._hidden_params is not None
|
||||
and isinstance(response_obj._hidden_params, dict)
|
||||
):
|
||||
if hasattr(response_obj, "_hidden_params") and response_obj._hidden_params is not None:
|
||||
response_obj._hidden_params["cache_hit"] = True
|
||||
|
||||
if _stream_replay_requested(kwargs):
|
||||
|
|
@ -980,11 +976,7 @@ class LLMCachingHandler:
|
|||
else:
|
||||
cached_result = response_obj
|
||||
|
||||
if (
|
||||
hasattr(cached_result, "_hidden_params")
|
||||
and cached_result._hidden_params is not None
|
||||
and isinstance(cached_result._hidden_params, dict)
|
||||
):
|
||||
if hasattr(cached_result, "_hidden_params") and cached_result._hidden_params is not None:
|
||||
cached_result._hidden_params["cache_hit"] = True
|
||||
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -1976,12 +1976,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
list,
|
||||
],
|
||||
) -> float | None:
|
||||
"""Price a response as if it had missed the cache, without overwriting this call's cost breakdown"""
|
||||
"""Price a response as if it had missed the cache, without leaving the probe's side effects on this call"""
|
||||
previous_cost_breakdown: Final = self.cost_breakdown
|
||||
previous_billed_token_rates: Final = self.billed_token_rates
|
||||
previous_zero_cost_warned: Final = self.zero_cost_warned
|
||||
debug_key: Final = "response_cost_failure_debug_information"
|
||||
debug_missing: Final = object()
|
||||
debug_before: Final = self.model_call_details.get(debug_key, debug_missing)
|
||||
try:
|
||||
return self._response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
|
||||
finally:
|
||||
self.cost_breakdown = previous_cost_breakdown
|
||||
self.billed_token_rates = previous_billed_token_rates
|
||||
self.zero_cost_warned = previous_zero_cost_warned
|
||||
if debug_before is debug_missing:
|
||||
self.model_call_details.pop(debug_key, None)
|
||||
else:
|
||||
self.model_call_details[debug_key] = debug_before
|
||||
|
||||
def _record_zero_cost_diagnostic(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
|||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
from litellm.types.utils import ModelResponse, TextCompletionResponse, Usage
|
||||
|
||||
|
||||
class TestCallbackDurationMs:
|
||||
|
|
@ -703,3 +703,60 @@ def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_result
|
|||
|
||||
assert result._hidden_params["response_cost"] == 0.0
|
||||
assert logging_obj.cost_breakdown is None
|
||||
|
||||
|
||||
def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_hidden_params_carrier(monkeypatch):
|
||||
"""A cache hit whose _hidden_params is a HiddenParams object (text_completion's carrier) stamps 0 cost, not the full price."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
deployment_id: Final = "cache-hit-deployment-hiddenparams"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
deployment_id: {
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "completion",
|
||||
}
|
||||
}
|
||||
)
|
||||
start_time: Final = datetime.datetime(2026, 9, 28, 12, 0, 0)
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
messages="Hi",
|
||||
stream=False,
|
||||
call_type="text_completion",
|
||||
start_time=start_time,
|
||||
litellm_call_id="cache-hit-hiddenparams",
|
||||
function_id="f",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
litellm_params={
|
||||
"metadata": {"model_info": {"id": deployment_id}},
|
||||
},
|
||||
optional_params={},
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
result: Final = TextCompletionResponse(
|
||||
id="cmpl-x",
|
||||
object="text_completion",
|
||||
created=1,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
choices=[{"text": "ok", "index": 0, "finish_reason": "stop"}],
|
||||
usage={"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
|
||||
)
|
||||
result._hidden_params["cache_hit"] = True
|
||||
|
||||
update_response_metadata(
|
||||
result=result,
|
||||
logging_obj=logging_obj,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
kwargs={"model_info": {"id": deployment_id}},
|
||||
start_time=start_time,
|
||||
end_time=start_time + datetime.timedelta(seconds=2),
|
||||
)
|
||||
|
||||
assert result._hidden_params.get("response_cost") == 0.0
|
||||
assert logging_obj.cost_breakdown is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue