From cbf69753842ef67013a43b258026a0b45bd44627 Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Mon, 28 Sep 2026 20:12:07 +0000 Subject: [PATCH] test(caching): cover cache-hit marker on HiddenParams carriers through the caching handler --- tests/unit/caching/test_caching_handler.py | 54 +++++++++++++++++++ .../test_litellm_logging.py | 48 +++++++++++++++-- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 6cf8e901cd7..1d22859a7a3 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -2268,3 +2268,57 @@ async def test_partial_embedding_cache_hit_sends_only_misses_and_keeps_input_ord assert len(embedder.provider_inputs) == 2, embedder.provider_inputs assert [item["embedding"] for item in repeat.data] == [[float(len(text))] for text in mixed_input] + + +def test_text_completion_cache_hit_is_marked_and_stamped_zero_cost(monkeypatch): + """A cached text_completion dict rebuilds into a HiddenParams carrier that still gets the cache_hit marker and a 0 cost.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + deployment_id = "lit8679-text-completion-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openai", + "mode": "completion", + } + } + ) + caching_handler = LLMCachingHandler(original_function=lambda: None, request_kwargs={}, start_time=datetime.now()) + logging_obj = LiteLLMLogging( + litellm_call_id="lit8679-text-completion-hit", + call_type=CallTypes.text_completion.value, + model="gpt-3.5-turbo-instruct", + messages="Hello", + function_id=str(uuid.uuid4()), + stream=False, + start_time=datetime.now(), + ) + logging_obj.update_environment_variables( + model="gpt-3.5-turbo-instruct", + litellm_params={"metadata": {"model_info": {"id": deployment_id}}}, + optional_params={}, + custom_llm_provider="openai", + ) + + result = caching_handler._convert_cached_result_to_model_response( + cached_result={ + "id": "cmpl-lit8679", + "object": "text_completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "Hello, how can I help you today?", "index": 0, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18}, + }, + call_type=CallTypes.text_completion.value, + kwargs={"model_info": {"id": deployment_id}}, + logging_obj=logging_obj, + model="gpt-3.5-turbo-instruct", + args=(), + ) + + assert isinstance(result, TextCompletionResponse) + assert result._hidden_params.get("cache_hit") is True + assert result._hidden_params.get("response_cost") == 0.0 + assert logging_obj.cost_breakdown is None diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 5d8bb0a5d13..102a0fdd445 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import ( _extract_response_obj_and_hidden_params, _get_status_fields, + get_standard_logging_object_payload, set_callbacks, ) from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo @@ -9090,12 +9091,51 @@ def test_uncached_response_cost_recomputes_price_past_a_zero_stamp(): litellm.model_cost.pop(deployment_id, None) +def test_uncached_response_cost_leaves_no_pricing_state_on_the_call(): + """The uncached price probe restores every field the priced path writes on the logging object.""" + deployment_id: Final = "lit8679-probe-state-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-probe-state") + cached_response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + cached_response._hidden_params = {"cache_hit": True, "cache_key": "lit8679-key", "response_cost": 0.0} + state_before: Final = ( + logging_obj.cost_breakdown, + logging_obj.billed_token_rates, + logging_obj.zero_cost_warned, + "response_cost_failure_debug_information" in logging_obj.model_call_details, + ) + + cost: Final = logging_obj._uncached_response_cost(result=cached_response) + + assert cost is not None and cost > 0.0 + assert ( + logging_obj.cost_breakdown, + logging_obj.billed_token_rates, + logging_obj.zero_cost_warned, + "response_cost_failure_debug_information" in logging_obj.model_call_details, + ) == state_before + assert state_before == (None, None, False, False) + finally: + litellm.model_cost.pop(deployment_id, None) + + def test_payload_saved_cache_cost_keeps_uncached_price_when_result_stamped_zero(): """A cache-hit payload still reports what the call would have cost, not the 0.0 stamped on the served response.""" - from litellm.litellm_core_utils.litellm_logging import ( - get_standard_logging_object_payload, - ) - deployment_id: Final = "lit8679-saved-cost-deployment" litellm.register_model( model_cost={