scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge pull request #39179 from BerriAI/litellm_backport_1_99_x_otelcache

Browse source 
chore(release): backport #38716 to stable/1.99.x and cut 1.99.1
This commit is contained in:
yuneng-jiang 2026-09-01 14:18:02 -07:00 • committed by GitHub
commit 10f4033437
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 72 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/integrations/otel/mappers/genai.py
View file

@ -62,6 +62,8 @@ class GenAIMapper:
GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens,
GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens,
Error.TYPE: lambda d: d.error.error_type if d.error else None,
Server.ADDRESS: lambda d: d.server.address if d.server else None,
Server.PORT: lambda d: d.server.port if d.server else None,

22
litellm/integrations/otel/model/payloads.py
View file

@ -95,6 +95,22 @@ class LLMUsage:
input_tokens: int | None = None
output_tokens: int | None = None
total_tokens: int | None = None
cache_creation_input_tokens: int | None = None
cache_read_input_tokens: int | None = None
@classmethod
def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage:
# Cache token counts only exist on the raw provider usage object under metadata
metadata: Final[Mapping[str, object]] = payload.get("metadata") or {}
raw_usage: Final = metadata.get("usage_object")
usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {}
return cls(
input_tokens=as_int(payload.get("prompt_tokens")),
output_tokens=as_int(payload.get("completion_tokens")),
total_tokens=as_int(payload.get("total_tokens")),
cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")),
cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")),
)
@dataclass(frozen=True)
@ -349,11 +365,7 @@ class LLMCallSpanData:
response_model=context.response_model,
response_id=as_str(response.get("id")),
request_params=LLMRequestParams.from_model_parameters(params),
usage=LLMUsage(
input_tokens=as_int(payload.get("prompt_tokens")),
output_tokens=as_int(payload.get("completion_tokens")),
total_tokens=as_int(payload.get("total_tokens")),
),
usage=LLMUsage.from_standard_logging_payload(payload),
finish_reasons=finish_reasons,
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),

2
litellm/integrations/otel/model/semconv.py
View file

@ -109,6 +109,8 @@ class GenAI:
# usage
USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens"
USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens"
# content (opt-in, gated by capture mode)
INPUT_MESSAGES: Final = "gen_ai.input.messages"
OUTPUT_MESSAGES: Final = "gen_ai.output.messages"

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm"
version = "1.99.0"
version = "1.99.1"
description = "Library to easily interface with LLM API providers"
readme = "README.md"
requires-python = ">=3.10, <3.15"
@ -310,7 +310,7 @@ members = ["enterprise", "litellm-proxy-extras"]
profile = "black"
[tool.commitizen]
version = "1.99.0"
version = "1.99.1"
version_files = [
"pyproject.toml:^version",
]

22
tests/test_litellm/integrations/otel/test_otel_v2_components.py
View file

@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name
builders, and the registry validator's failure paths. Needs the OTel SDK."""
import json
from dataclasses import replace
import pytest
@ -227,6 +228,27 @@ def test_genai_mapper_all_request_params():
assert attrs["server.port"] == 443
def test_genai_mapper_cache_token_attrs():
cached = replace(
_full_llm_call(),
usage=LLMUsage(
input_tokens=10,
output_tokens=5,
total_tokens=15,
cache_creation_input_tokens=7,
cache_read_input_tokens=3,
),
)
attrs = GenAIMapper().map(cached)
assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7
assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3
# No cache usage keeps the span sparse: neither key present.
uncached = GenAIMapper().map(_full_llm_call())
assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached
assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached
def test_genai_mapper_stamps_input_output_messages():
data = LLMCallSpanData(
operation=GenAIOperation.CHAT,

22
tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
View file

@ -507,6 +507,28 @@ def test_llm_call_adapter_extracts_all_fields():
assert data.identity.key_hash == "hsh"
def test_llm_call_adapter_extracts_cache_tokens_from_usage_object():
payload = _sample_payload()
payload["metadata"] = {
**payload["metadata"],
"usage_object": {
"prompt_tokens": 10,
"completion_tokens": 5,
"cache_creation_input_tokens": 7,
"cache_read_input_tokens": 3,
},
}
data = LLMCallSpanData.from_standard_logging_payload(payload)
assert data.usage.cache_creation_input_tokens == 7
assert data.usage.cache_read_input_tokens == 3
def test_llm_call_adapter_cache_tokens_none_without_usage_object():
data = LLMCallSpanData.from_standard_logging_payload(_sample_payload())
assert data.usage.cache_creation_input_tokens is None
assert data.usage.cache_read_input_tokens is None
def test_llm_call_adapter_failure_path():
payload = _sample_payload(
status="failure",

10
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-08-19T15:53:37.294198Z"
exclude-newer = "2026-08-29T19:55:59.005565Z"
exclude-newer-span = "P3D"
[manifest]
@ -4266,7 +4266,7 @@ wheels = [
[[package]]
name = "litellm"
version = "1.99.0"
version = "1.99.1"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
@ -8261,11 +8261,11 @@ wheels = [
[[package]]
name = "restrictedpython"
version = "8.1"
version = "8.5"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/5f/1c/aec08bcb4ab14a1521579fbe21ceff2a634bb1f737f11cf7f9c8bb96e680/restrictedpython-8.1.tar.gz", hash = "sha256:4a69304aceacf6bee74bdf153c728221d4e3109b39acbfe00b3494927080d898", size = 838331, upload-time = "2025-10-19T14:11:32.531Z" }
sdist = { url = "https://files.pythonhosted.org/packages/7c/3b/8e41f7cfabbb30b1013ebc7484303d6c87da2906ec432d69dea11d2f7d75/restrictedpython-8.5.tar.gz", hash = "sha256:4ed1269dbe3caa88db650d1af325198a952aeb1451eca05df0cfa65db4466215", size = 455879, upload-time = "2026-08-19T07:02:10.934Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1a/c0/3848f4006f7e164ee20833ca984067e4b3fc99fe7f1dfa88b4927e681299/restrictedpython-8.1-py3-none-any.whl", hash = "sha256:4769449c6cdb10f2071649ba386902befff0eff2a8fd6217989fa7b16aeae926", size = 27651, upload-time = "2025-10-19T14:11:30.201Z" },
{ url = "https://files.pythonhosted.org/packages/58/57/16ce3c721f5a33317e4110575d5c9976c0c45f7fd96ca2e0adeab06e6026/restrictedpython-8.5-py3-none-any.whl", hash = "sha256:6c70e0a3af13e830d37225788cdc8ab5804a8df4b500c135086eaef34b5c01e0", size = 30962, upload-time = "2026-08-19T07:02:09.553Z" },
]
[[package]]