diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 8d588896b2f..1122d9615f9 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2433,6 +2433,19 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): key=f"gen_ai.cost.{key}", value=value, ) + if standard_logging_payload.get("cache_hit") is True: + self.safe_set_attribute( + span=span, + key="litellm.cache_hit", + value=True, + ) + saved_cache_cost: Final = standard_logging_payload.get("saved_cache_cost") + if isinstance(saved_cache_cost, (int, float)): + self.safe_set_attribute( + span=span, + key="litellm.saved_cache_cost", + value=saved_cache_cost, + ) ############################################# ########## LLM Request Attributes ########### ############################################# diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5af669591f7..c086560e20a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1810,6 +1810,7 @@ class Logging(LiteLLMLoggingBaseClass): cache_hit: bool | None = None, litellm_model_name: str | None = None, router_model_id: str | None = None, + ignore_stamped_cost: bool = False, ) -> float | None: """ Calculate response cost using result + logging object variables. @@ -1844,7 +1845,9 @@ class Logging(LiteLLMLoggingBaseClass): ): hidden_params: Final = result_hidden_params if ( - "response_cost" in hidden_params and hidden_params["response_cost"] is not None + not ignore_stamped_cost + and "response_cost" in hidden_params + and hidden_params["response_cost"] is not None ): # use cost if already calculated self._record_zero_cost_diagnostic( priced_result, @@ -1950,6 +1953,36 @@ class Logging(LiteLLMLoggingBaseClass): return None + def _uncached_response_cost( + self, + result: Union[ + ModelResponse, + ModelResponseStream, + EmbeddingResponse, + ImageResponse, + TranscriptionResponse, + TextCompletionResponse, + HttpxBinaryResponseContent, + RerankResponse, + Batch, + FineTuningJob, + ResponsesAPIResponse, + ResponseCompletedEvent, + OpenAIFileObject, + LiteLLMRealtimeStreamLoggingObject, + OpenAIModerationResponse, + "SearchResponse", + dict, + list, + ], + ) -> float | None: + """Price a response as if it had missed the cache, without overwriting this call's cost breakdown""" + previous_cost_breakdown: Final = self.cost_breakdown + try: + return self._response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True) + finally: + self.cost_breakdown = previous_cost_breakdown + def _record_zero_cost_diagnostic( self, result: object, @@ -6521,13 +6554,7 @@ def get_standard_logging_object_payload( saved_cache_cost: float = 0.0 if cache_hit is True: id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id - saved_cache_cost = ( - logging_obj._response_cost_calculator( - result=init_response_obj, - cache_hit=False, - ) - or 0.0 - ) + saved_cache_cost = logging_obj._uncached_response_cost(result=init_response_obj) or 0.0 ## Get model cost information ## base_model = _get_base_model_from_metadata(model_call_details=kwargs) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 503814cc143..f312aa05f52 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -126,7 +126,10 @@ class ResponseMetadata: "api_base": get_api_base(model=model or "", optional_params=kwargs), "model_id": model_id, "response_cost": logging_obj._response_cost_calculator( - result=self.result, litellm_model_name=model, router_model_id=model_id + result=self.result, + litellm_model_name=model, + router_model_id=model_id, + cache_hit=True if self._result_is_cache_hit() else None, ), "additional_headers": process_response_headers( self._get_additional_headers_from_hidden_params() or {}, @@ -148,6 +151,10 @@ class ResponseMetadata: for key, value in new_params.items(): setattr(self._hidden_params, key, value) + def _result_is_cache_hit(self) -> bool: + """The caching handler stamps `_hidden_params["cache_hit"]` before update_response_metadata runs""" + return self._hidden_params.get("cache_hit") is True + def _get_additional_headers_from_hidden_params(self) -> httpx.Headers | dict[str, str] | None: """Get `additional_headers` from hidden params - handles when self._hidden_params is a dict or HiddenParams object""" if isinstance(self._hidden_params, dict): diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 175bd95c263..64f6649fc0d 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -401,6 +401,70 @@ class TestOpenTelemetryCostBreakdown(unittest.TestCase): assert ("gen_ai.cost.tool_usage_cost", 0.0001) not in call_args_list assert ("gen_ai.cost.original_cost", 0.004) not in call_args_list + def test_cache_hit_span_carries_cache_indicator_and_saved_cost(self): + """ + A cache-hit span reports litellm.cache_hit and litellm.saved_cache_cost so cost + attribution can tell a cache-served response apart from a billed call. + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "cache_hit": True, + "saved_cache_cost": 0.003, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + mock_span.set_attribute.assert_any_call("litellm.cache_hit", True) + mock_span.set_attribute.assert_any_call("litellm.saved_cache_cost", 0.003) + + def test_non_cache_hit_span_has_no_cache_indicator(self): + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "cache_hit": False, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + call_args_list = [call[0] for call in mock_span.set_attribute.call_args_list] + assert all(key != "litellm.cache_hit" for key, _value in call_args_list) + assert all(key != "litellm.saved_cache_cost" for key, _value in call_args_list) + class TestOpenTelemetryProviderInitialization(unittest.TestCase): """Test suite for verifying provider initialization respects existing providers""" diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 6f297e6e06a..18ee9d6ea51 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -651,3 +651,55 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_ assert result._response_ms == pytest.approx(2000) assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2) + + +def test_update_response_metadata_stamps_zero_response_cost_for_cache_hit_result(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id: Final = "cache-hit-deployment-response-metadata" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + start_time: Final = datetime.datetime(2026, 9, 28, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="cache-hit-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + litellm_params={ + "metadata": {"model_info": {"id": deployment_id}}, + }, + optional_params={}, + custom_llm_provider="openai", + ) + result: Final = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + result._hidden_params = {"cache_hit": True, "cache_key": "lit8679-cache-key"} + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-5.4-nano", + kwargs={"model_info": {"id": deployment_id}}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + + assert result._hidden_params["response_cost"] == 0.0 + assert logging_obj.cost_breakdown is None diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 2fc747e1b48..5d8bb0a5d13 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -9029,3 +9029,120 @@ def test_signoz_dispatch_requires_an_endpoint(monkeypatch): logging_module._in_memory_loggers.clear() monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) is_otel_v2_enabled.cache_clear() + + +def _lit8679_priced_logging_obj(deployment_id: str, litellm_call_id: str) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id=litellm_call_id, + function_id="fn", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + user="", + optional_params={}, + litellm_params={"metadata": {"model_group": "lit8679-group", "model_info": {"id": deployment_id}}}, + custom_llm_provider="openai", + ) + return logging_obj + + +def test_uncached_response_cost_recomputes_price_past_a_zero_stamp(): + """A probe for the price a cache-served response would have had ignores the 0.0 stamped on it.""" + deployment_id: Final = "lit8679-uncached-cost-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-uncached") + oracle_logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-oracle") + cached_response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + cached_response._hidden_params = {"cache_hit": True, "cache_key": "lit8679-key", "response_cost": 0.0} + fresh_response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + + cost: Final = logging_obj._uncached_response_cost(result=cached_response) + uncached_price: Final = oracle_logging_obj._response_cost_calculator(result=fresh_response) + + assert cost == uncached_price + assert cost is not None and cost > 0.0 + assert logging_obj.cost_breakdown is None + finally: + litellm.model_cost.pop(deployment_id, None) + + +def test_payload_saved_cache_cost_keeps_uncached_price_when_result_stamped_zero(): + """A cache-hit payload still reports what the call would have cost, not the 0.0 stamped on the served response.""" + from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, + ) + + deployment_id: Final = "lit8679-saved-cost-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-payload") + oracle_logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-payload-oracle") + response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + response._hidden_params = {"cache_hit": True, "cache_key": "lit8679-key", "response_cost": 0.0} + fresh_response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + logging_obj.model_call_details["cache_hit"] = True + now: Final = datetime.datetime.now() + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-5.4-nano", + "messages": [], + "cache_hit": True, + "litellm_params": logging_obj.litellm_params, + }, + init_response_obj=response, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + uncached_price: Final = oracle_logging_obj._response_cost_calculator(result=fresh_response) + + assert payload is not None + assert payload["saved_cache_cost"] == uncached_price + assert payload["saved_cache_cost"] > 0.0 + assert payload["response_cost"] == 0.0 + assert logging_obj.cost_breakdown is None + finally: + litellm.model_cost.pop(deployment_id, None)