diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index b09498f9292..3ac92b04c27 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -62,6 +62,8 @@ class GenAIMapper: GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds, GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens, GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens, + GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens, + GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens, Error.TYPE: lambda d: d.error.error_type if d.error else None, Server.ADDRESS: lambda d: d.server.address if d.server else None, Server.PORT: lambda d: d.server.port if d.server else None, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index f70c777e1a7..d35405538f6 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -95,6 +95,22 @@ class LLMUsage: input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None + cache_creation_input_tokens: int | None = None + cache_read_input_tokens: int | None = None + + @classmethod + def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage: + # Cache token counts only exist on the raw provider usage object under metadata + metadata: Final[Mapping[str, object]] = payload.get("metadata") or {} + raw_usage: Final = metadata.get("usage_object") + usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {} + return cls( + input_tokens=as_int(payload.get("prompt_tokens")), + output_tokens=as_int(payload.get("completion_tokens")), + total_tokens=as_int(payload.get("total_tokens")), + cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")), + cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")), + ) @dataclass(frozen=True) @@ -363,11 +379,7 @@ class LLMCallSpanData: response_model=context.response_model, response_id=as_str(response.get("id")), request_params=LLMRequestParams.from_model_parameters(params), - usage=LLMUsage( - input_tokens=as_int(payload.get("prompt_tokens")), - output_tokens=as_int(payload.get("completion_tokens")), - total_tokens=as_int(payload.get("total_tokens")), - ), + usage=LLMUsage.from_standard_logging_payload(payload), finish_reasons=finish_reasons, error=_parse_error(payload), response_cost=as_float(payload.get("response_cost")), diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 4ad0cb5d1b4..f7a6280f95b 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -110,6 +110,8 @@ class GenAI: # usage USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens" + USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens" + USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens" # content (opt-in, gated by capture mode) INPUT_MESSAGES: Final = "gen_ai.input.messages" OUTPUT_MESSAGES: Final = "gen_ai.output.messages" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 115e385eda4..4aa28b5abfd 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name builders, and the registry validator's failure paths. Needs the OTel SDK.""" import json +from dataclasses import replace import pytest @@ -215,6 +216,27 @@ def test_genai_mapper_all_request_params(): assert attrs["server.port"] == 443 +def test_genai_mapper_cache_token_attrs(): + cached = replace( + _full_llm_call(), + usage=LLMUsage( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cache_creation_input_tokens=7, + cache_read_input_tokens=3, + ), + ) + attrs = GenAIMapper().map(cached) + assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7 + assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3 + + # No cache usage keeps the span sparse: neither key present. + uncached = GenAIMapper().map(_full_llm_call()) + assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached + assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached + + def test_genai_mapper_stamps_input_output_messages(): data = LLMCallSpanData( operation=GenAIOperation.CHAT, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index baa72b5a7fe..ca628aa3405 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -525,6 +525,28 @@ def test_llm_call_adapter_extracts_all_fields(): assert data.identity.key_hash == "hsh" +def test_llm_call_adapter_extracts_cache_tokens_from_usage_object(): + payload = _sample_payload() + payload["metadata"] = { + **payload["metadata"], + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 3, + }, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.usage.cache_creation_input_tokens == 7 + assert data.usage.cache_read_input_tokens == 3 + + +def test_llm_call_adapter_cache_tokens_none_without_usage_object(): + data = LLMCallSpanData.from_standard_logging_payload(_sample_payload()) + assert data.usage.cache_creation_input_tokens is None + assert data.usage.cache_read_input_tokens is None + + def test_llm_call_adapter_failure_path(): payload = _sample_payload( status="failure",