diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 888c90b7661..c0cb5c88836 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -519,8 +519,12 @@ class OpenTelemetryV2(CustomLogger): # logger is a success/failure callback only, so ``pre_call`` never reaches it # and no carrier exists. The payload plus the request-level provider-handoff # stamp (``upstream_started``) is the affirmative signal of a real call; a - # gate rejection carries ``is_no_upstream_call`` and gets no span. - if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None): + # gate rejection carries ``is_no_upstream_call`` and gets no span. A cache + # hit never hands off to a provider, so ``cache_hit`` substitutes for the + # upstream stamp — the payload proves the call completed. + if carrier is None and ( + call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit) + ): return None try: return self._finish_carrier(carrier, call, start_time, end_time) diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 0ac8b47e239..4e07d090edd 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -91,6 +91,8 @@ class GenAIMapper: f"{LiteLLM.COST_PREFIX}margin_fixed_amount": lambda d: d.cost.margin_fixed_amount, f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent, f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount, + LiteLLM.CACHE_HIT: lambda d: d.cache_hit, + LiteLLM.SAVED_CACHE_COST: lambda d: d.saved_cache_cost, LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming, LiteLLM.REQUEST_ROUTE: lambda d: d.request_route, LiteLLM.REQUEST_PURPOSE: lambda d: d.request_purpose, diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 809fc794461..cccc3bf8f09 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -236,6 +236,10 @@ class LLMCallEvent: # ``InternalCallOrigin`` such as ``autorouter_classifier``), ``None`` for the # caller's own provider attempt. purpose: str | None + # True for a response served from the litellm response cache. A cache hit + # never hands off to a provider, so ``upstream_started`` stays False even + # though the call completed and is worth a span. + cache_hit: bool # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The # span is renamed from the typed payload at close (``finish_span``); this only # needs to be reasonable for a span that never gets closed (a leak). @@ -260,6 +264,7 @@ class LLMCallEvent: upstream_started=kwargs.get("api_call_start_time") is not None, upstream_start_seconds=_epoch_seconds(kwargs.get("api_call_start_time")), purpose=internal_call_origin(payload, kwargs), + cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown provisional_span_name=f"{operation.value} {model}".strip(), time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs), trace=trace, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index ccec93920c9..360b36e0417 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -438,6 +438,8 @@ class LLMCallSpanData: session_id: str | None = None embedding_output: EmbeddingOutput | None = None routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({})) + cache_hit: bool | None = None + saved_cache_cost: float | None = None @classmethod def from_standard_logging_payload( @@ -501,6 +503,8 @@ class LLMCallSpanData: session_id=session_id or None, embedding_output=embedding_output if capture_content else None, routing_attributes=routing_decision_attributes(decision), + cache_hit=as_bool(payload.get("cache_hit")), # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown + saved_cache_cost=as_float(payload.get("saved_cache_cost")), # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 38cf7266593..71b813053df 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -340,6 +340,10 @@ class LiteLLM: # semconv key for an MCP server's *name* (the convention uses ``server.address`` # for its network location), so it lives under the vendor namespace. MCP_SERVER_NAME: Final = "litellm.mcp.server.name" + # Whether the response was served from the litellm response cache instead of + # an upstream provider call, and the USD cost the hit avoided. + CACHE_HIT: Final = "litellm.cache_hit" + SAVED_CACHE_COST: Final = "litellm.saved_cache_cost" class Metric: diff --git a/tests/unit/integrations/otel/test_otel_v2_components.py b/tests/unit/integrations/otel/test_otel_v2_components.py index bc230a8bcb2..41667013763 100644 --- a/tests/unit/integrations/otel/test_otel_v2_components.py +++ b/tests/unit/integrations/otel/test_otel_v2_components.py @@ -358,6 +358,19 @@ def test_genai_mapper_cost_breakdown_absent(): assert not any(k.startswith(LiteLLM.COST_PREFIX) and k != f"{LiteLLM.COST_PREFIX}total" for k in attrs) +def test_genai_mapper_cache_hit_attrs(): + from litellm.integrations.otel.model.semconv import LiteLLM + + attrs = GenAIMapper().map(replace(_full_llm_call(), cache_hit=True, saved_cache_cost=0.002)) + assert attrs[LiteLLM.CACHE_HIT] is True + assert attrs[LiteLLM.SAVED_CACHE_COST] == 0.002 + + # The default (None) keeps the span sparse: neither key present. + uncached = GenAIMapper().map(_full_llm_call()) + assert LiteLLM.CACHE_HIT not in uncached + assert LiteLLM.SAVED_CACHE_COST not in uncached + + def test_llm_cost_from_breakdown_maps_costbreakdown_keys(): cost = LLMCost.from_breakdown( { diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 3b1dd592d08..78e43173db8 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -490,6 +490,38 @@ def test_no_span_when_pre_call_never_ran(): assert exporter.get_finished_spans() == () # no phantom LLM span +def test_cache_hit_without_pre_call_emits_zero_cost_span(): + """A response served from the litellm cache never hands off to a provider, + so ``pre_call`` never runs and ``api_call_start_time`` is never stamped — + but the completed call's payload still warrants a span, carrying the cache + attributes rather than being dropped like a gate rejection.""" + logger, exporter = _logger() + payload = _payload(cache_hit=True, response_cost=0.0, saved_cache_cost=0.002, cost_breakdown=None) + asyncio.run( + logger.async_log_success_event(_kwargs(payload=payload), None, None, None) + ) + (span,) = exporter.get_finished_spans() + assert span.attributes[LiteLLM.CACHE_HIT] is True + assert span.attributes[LiteLLM.SAVED_CACHE_COST] == 0.002 + assert span.attributes[f"{LiteLLM.COST_PREFIX}total"] == 0.0 + assert not [ + k + for k in span.attributes + if k.startswith(LiteLLM.COST_PREFIX) and k != f"{LiteLLM.COST_PREFIX}total" + ] + + +def test_cache_miss_without_pre_call_still_emits_no_span(): + """The cache-hit relaxation is not a general loosening: a success callback + with no ``pre_call`` carrier and no cache hit still emits nothing.""" + logger, exporter = _logger() + payload = _payload(cache_hit=False, response_cost=0.002) + asyncio.run( + logger.async_log_success_event(_kwargs(payload=payload), None, None, None) + ) + assert exporter.get_finished_spans() == () + + def test_real_llm_failure_still_emitted(): """A genuine LLM failure: ``pre_call`` ran (the call was attempted), so the CLIENT span is opened at the boundary and closed ERROR."""