test(caching): cover cache-hit marker on HiddenParams carriers through the caching handler

This commit is contained in:
shrey kharbanda 2026-09-28 20:12:07 +00:00
parent 6910ea644d
commit cbf6975384
2 changed files with 98 additions and 4 deletions

View file

@ -2268,3 +2268,57 @@ async def test_partial_embedding_cache_hit_sends_only_misses_and_keeps_input_ord
assert len(embedder.provider_inputs) == 2, embedder.provider_inputs
assert [item["embedding"] for item in repeat.data] == [[float(len(text))] for text in mixed_input]
def test_text_completion_cache_hit_is_marked_and_stamped_zero_cost(monkeypatch):
"""A cached text_completion dict rebuilds into a HiddenParams carrier that still gets the cache_hit marker and a 0 cost."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
deployment_id = "lit8679-text-completion-deployment"
litellm.register_model(
model_cost={
deployment_id: {
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"litellm_provider": "openai",
"mode": "completion",
}
}
)
caching_handler = LLMCachingHandler(original_function=lambda: None, request_kwargs={}, start_time=datetime.now())
logging_obj = LiteLLMLogging(
litellm_call_id="lit8679-text-completion-hit",
call_type=CallTypes.text_completion.value,
model="gpt-3.5-turbo-instruct",
messages="Hello",
function_id=str(uuid.uuid4()),
stream=False,
start_time=datetime.now(),
)
logging_obj.update_environment_variables(
model="gpt-3.5-turbo-instruct",
litellm_params={"metadata": {"model_info": {"id": deployment_id}}},
optional_params={},
custom_llm_provider="openai",
)
result = caching_handler._convert_cached_result_to_model_response(
cached_result={
"id": "cmpl-lit8679",
"object": "text_completion",
"created": 1,
"model": "gpt-3.5-turbo-instruct",
"choices": [{"text": "Hello, how can I help you today?", "index": 0, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
},
call_type=CallTypes.text_completion.value,
kwargs={"model_info": {"id": deployment_id}},
logging_obj=logging_obj,
model="gpt-3.5-turbo-instruct",
args=(),
)
assert isinstance(result, TextCompletionResponse)
assert result._hidden_params.get("cache_hit") is True
assert result._hidden_params.get("response_cost") == 0.0
assert logging_obj.cost_breakdown is None

View file

@ -28,6 +28,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.litellm_logging import (
_extract_response_obj_and_hidden_params,
_get_status_fields,
get_standard_logging_object_payload,
set_callbacks,
)
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
@ -9090,12 +9091,51 @@ def test_uncached_response_cost_recomputes_price_past_a_zero_stamp():
litellm.model_cost.pop(deployment_id, None)
def test_uncached_response_cost_leaves_no_pricing_state_on_the_call():
"""The uncached price probe restores every field the priced path writes on the logging object."""
deployment_id: Final = "lit8679-probe-state-deployment"
litellm.register_model(
model_cost={
deployment_id: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
}
},
persist_across_reloads=False,
)
try:
logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-probe-state")
cached_response: Final = ModelResponse(
model="gpt-5.4-nano",
choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))],
usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30),
)
cached_response._hidden_params = {"cache_hit": True, "cache_key": "lit8679-key", "response_cost": 0.0}
state_before: Final = (
logging_obj.cost_breakdown,
logging_obj.billed_token_rates,
logging_obj.zero_cost_warned,
"response_cost_failure_debug_information" in logging_obj.model_call_details,
)
cost: Final = logging_obj._uncached_response_cost(result=cached_response)
assert cost is not None and cost > 0.0
assert (
logging_obj.cost_breakdown,
logging_obj.billed_token_rates,
logging_obj.zero_cost_warned,
"response_cost_failure_debug_information" in logging_obj.model_call_details,
) == state_before
assert state_before == (None, None, False, False)
finally:
litellm.model_cost.pop(deployment_id, None)
def test_payload_saved_cache_cost_keeps_uncached_price_when_result_stamped_zero():
"""A cache-hit payload still reports what the call would have cost, not the 0.0 stamped on the served response."""
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload,
)
deployment_id: Final = "lit8679-saved-cost-deployment"
litellm.register_model(
model_cost={