fix(caching): report zero response_cost on a response-cache hit (#46471)

* fix(caching): report zero response_cost on a response-cache hit

* test(caching): cover response_cost on a cache hit

* fix(caching): pass cache_hit when stamping a cached response's metadata

litellm/caching/caching_handler.py

@@ -1088,6 +1088,7 @@ class LLMCachingHandler:
             kwargs=kwargs,
             start_time=self.start_time,
             end_time=datetime.datetime.now(),
+            cache_hit=True,
         )
         return cached_result
 

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -115,8 +115,19 @@ class ResponseMetadata:
         """Check if response type supports timing metrics"""
         return isinstance(self.result, (ModelResponse, EmbeddingResponse, TranscriptionResponse))
 
-    def set_hidden_params(self, logging_obj: LiteLLMLoggingObject, model: str | None, kwargs: dict) -> None:
-        """Set hidden parameters on the response"""
+    def set_hidden_params(
+        self,
+        logging_obj: LiteLLMLoggingObject,
+        model: str | None,
+        kwargs: dict,
+        cache_hit: bool | None = None,
+    ) -> None:
+        """Set hidden parameters on the response.
+
+        ``cache_hit`` is passed for a response served from the cache: ``logging_obj`` only
+        learns of the hit later, in the success handler, so its cost calculator would
+        otherwise price the cached response as a fresh call.
+        """
 
         ## ADD OTHER HIDDEN PARAMS
         model_info: Final = kwargs.get("model_info", {}) or {}
@@ -126,7 +137,10 @@ class ResponseMetadata:
             "api_base": get_api_base(model=model or "", optional_params=kwargs),
             "model_id": model_id,
             "response_cost": logging_obj.response_cost_calculator(
-                result=self.result, litellm_model_name=model, router_model_id=model_id
+                result=self.result,
+                litellm_model_name=model,
+                router_model_id=model_id,
+                cache_hit=cache_hit,
             ),
             "additional_headers": process_response_headers(
                 self._get_additional_headers_from_hidden_params() or {},
@@ -228,6 +242,7 @@ def update_response_metadata(
     start_time: datetime.datetime,
     end_time: datetime.datetime,
     include_overhead: bool = True,
+    cache_hit: bool | None = None,
 ) -> None:
     """
     Updates response metadata including hidden params and timing metrics
@@ -252,5 +267,5 @@ def update_response_metadata(
 
     metadata: Final = ResponseMetadata(result)
     metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
-    metadata.set_hidden_params(logging_obj, model, kwargs)
+    metadata.set_hidden_params(logging_obj, model, kwargs, cache_hit=cache_hit)
     metadata.apply()

tests/unit/caching/test_caching_handler.py

@@ -201,6 +201,22 @@ async def test_existing_provider_response_cache_still_hits(copilot_response_cach
     assert hidden_params.get("cache_hit") is True
 
 
+@pytest.mark.asyncio
+async def test_cache_hit_reports_zero_response_cost(copilot_response_cache: Cache) -> None:
+    messages: Final = [{"role": "user", "content": "cache cost repro"}]
+    first: Final = await litellm.acompletion(
+        model="gpt-4o-mini", messages=messages, mock_response="hi there " * 50, caching=True
+    )
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+    second: Final = await litellm.acompletion(
+        model="gpt-4o-mini", messages=messages, mock_response="hi there " * 50, caching=True
+    )
+
+    assert first._hidden_params["response_cost"] > 0
+    assert second._hidden_params["cache_hit"] is True
+    assert second._hidden_params["response_cost"] == 0
+
+
 chat_completion_response = litellm.ModelResponse(
     id=str(uuid.uuid4()),
     choices=[