From ad75134fcee0a96402066344db934b77a4304992 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:59:59 -0700 Subject: [PATCH 1/4] fix(otel): emit cache token counts on OTel v2 LLM spans (#38716) * fix(otel): emit cache token counts on OTel v2 LLM spans Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): trim comment in LLMUsage adapter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): drop casts in LLMUsage cache token adapter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(deps): bump restrictedpython to 8.3 for GHSA-ffg3-p8fm-mjx2 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 40edeaaecb0c7b0d7c0fac06a74fa67d0b57400c) --- litellm/integrations/otel/mappers/genai.py | 2 ++ litellm/integrations/otel/model/payloads.py | 22 ++++++++++++++----- litellm/integrations/otel/model/semconv.py | 2 ++ .../otel/test_otel_v2_components.py | 22 +++++++++++++++++++ .../otel/test_otel_v2_sources_of_truth.py | 22 +++++++++++++++++++ 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 5e3401cd62c..286a58ce5e5 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -62,6 +62,8 @@ class GenAIMapper: GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds, GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens, GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens, + GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens, + GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens, Error.TYPE: lambda d: d.error.error_type if d.error else None, Server.ADDRESS: lambda d: d.server.address if d.server else None, Server.PORT: lambda d: d.server.port if d.server else None, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 4e4ed4b7513..49440dab31b 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -95,6 +95,22 @@ class LLMUsage: input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None + cache_creation_input_tokens: int | None = None + cache_read_input_tokens: int | None = None + + @classmethod + def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage: + # Cache token counts only exist on the raw provider usage object under metadata + metadata: Final[Mapping[str, object]] = payload.get("metadata") or {} + raw_usage: Final = metadata.get("usage_object") + usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {} + return cls( + input_tokens=as_int(payload.get("prompt_tokens")), + output_tokens=as_int(payload.get("completion_tokens")), + total_tokens=as_int(payload.get("total_tokens")), + cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")), + cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")), + ) @dataclass(frozen=True) @@ -349,11 +365,7 @@ class LLMCallSpanData: response_model=context.response_model, response_id=as_str(response.get("id")), request_params=LLMRequestParams.from_model_parameters(params), - usage=LLMUsage( - input_tokens=as_int(payload.get("prompt_tokens")), - output_tokens=as_int(payload.get("completion_tokens")), - total_tokens=as_int(payload.get("total_tokens")), - ), + usage=LLMUsage.from_standard_logging_payload(payload), finish_reasons=finish_reasons, error=_parse_error(payload), response_cost=as_float(payload.get("response_cost")), diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 1647e0a5bd1..7497ec6c102 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -109,6 +109,8 @@ class GenAI: # usage USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens" + USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens" + USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens" # content (opt-in, gated by capture mode) INPUT_MESSAGES: Final = "gen_ai.input.messages" OUTPUT_MESSAGES: Final = "gen_ai.output.messages" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index d856d6871a3..8bb07d1f8ec 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name builders, and the registry validator's failure paths. Needs the OTel SDK.""" import json +from dataclasses import replace import pytest @@ -227,6 +228,27 @@ def test_genai_mapper_all_request_params(): assert attrs["server.port"] == 443 +def test_genai_mapper_cache_token_attrs(): + cached = replace( + _full_llm_call(), + usage=LLMUsage( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cache_creation_input_tokens=7, + cache_read_input_tokens=3, + ), + ) + attrs = GenAIMapper().map(cached) + assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7 + assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3 + + # No cache usage keeps the span sparse: neither key present. + uncached = GenAIMapper().map(_full_llm_call()) + assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached + assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached + + def test_genai_mapper_stamps_input_output_messages(): data = LLMCallSpanData( operation=GenAIOperation.CHAT, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 2a66d5ee139..38f636c2b06 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -507,6 +507,28 @@ def test_llm_call_adapter_extracts_all_fields(): assert data.identity.key_hash == "hsh" +def test_llm_call_adapter_extracts_cache_tokens_from_usage_object(): + payload = _sample_payload() + payload["metadata"] = { + **payload["metadata"], + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 3, + }, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.usage.cache_creation_input_tokens == 7 + assert data.usage.cache_read_input_tokens == 3 + + +def test_llm_call_adapter_cache_tokens_none_without_usage_object(): + data = LLMCallSpanData.from_standard_logging_payload(_sample_payload()) + assert data.usage.cache_creation_input_tokens is None + assert data.usage.cache_read_input_tokens is None + + def test_llm_call_adapter_failure_path(): payload = _sample_payload( status="failure", From ddc4d8f9d9ab0e463b06f44faef41cc12d415c4a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 1 Sep 2026 12:51:06 -0700 Subject: [PATCH 2/4] chore(deps): bump restrictedpython to 8.5 --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 628483c0117..677deaf9a97 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-19T15:53:37.294198Z" +exclude-newer = "2026-08-29T19:50:55.983368Z" exclude-newer-span = "P3D" [manifest] @@ -8261,11 +8261,11 @@ wheels = [ [[package]] name = "restrictedpython" -version = "8.1" +version = "8.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/1c/aec08bcb4ab14a1521579fbe21ceff2a634bb1f737f11cf7f9c8bb96e680/restrictedpython-8.1.tar.gz", hash = "sha256:4a69304aceacf6bee74bdf153c728221d4e3109b39acbfe00b3494927080d898", size = 838331, upload-time = "2025-10-19T14:11:32.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/3b/8e41f7cfabbb30b1013ebc7484303d6c87da2906ec432d69dea11d2f7d75/restrictedpython-8.5.tar.gz", hash = "sha256:4ed1269dbe3caa88db650d1af325198a952aeb1451eca05df0cfa65db4466215", size = 455879, upload-time = "2026-08-19T07:02:10.934Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/c0/3848f4006f7e164ee20833ca984067e4b3fc99fe7f1dfa88b4927e681299/restrictedpython-8.1-py3-none-any.whl", hash = "sha256:4769449c6cdb10f2071649ba386902befff0eff2a8fd6217989fa7b16aeae926", size = 27651, upload-time = "2025-10-19T14:11:30.201Z" }, + { url = "https://files.pythonhosted.org/packages/58/57/16ce3c721f5a33317e4110575d5c9976c0c45f7fd96ca2e0adeab06e6026/restrictedpython-8.5-py3-none-any.whl", hash = "sha256:6c70e0a3af13e830d37225788cdc8ab5804a8df4b500c135086eaef34b5c01e0", size = 30962, upload-time = "2026-08-19T07:02:09.553Z" }, ] [[package]] From d72131846b786ebc54332f27966dc9593282826d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 1 Sep 2026 12:55:43 -0700 Subject: [PATCH 3/4] =?UTF-8?q?bump:=20version=201.99.0=20=E2=86=92=201.99?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fca5c7da1e2..7e70b719cc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.99.0" +version = "1.99.1" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -310,7 +310,7 @@ members = ["enterprise", "litellm-proxy-extras"] profile = "black" [tool.commitizen] -version = "1.99.0" +version = "1.99.1" version_files = [ "pyproject.toml:^version", ] From 3ebb415b54dc66ef5dc6d2bf7460d84261b12415 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 1 Sep 2026 12:56:04 -0700 Subject: [PATCH 4/4] chore: refresh uv.lock for 1.99.1 --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 677deaf9a97..49ffc34777a 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-29T19:50:55.983368Z" +exclude-newer = "2026-08-29T19:55:59.005565Z" exclude-newer-span = "P3D" [manifest] @@ -4266,7 +4266,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.99.0" +version = "1.99.1" source = { editable = "." } dependencies = [ { name = "aiohttp" },