mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
feat(otel): promote nested request metadata keys to litellm.metadata.* span attributes
baggage_metadata_keys entries such as requester_metadata.trace_id now resolve the caller's nested metadata.trace_id and stamp it on the LLM-call span as litellm.metadata.trace_id, in both the OTEL v2 logger and the legacy OpenTelemetry callback. Nested metadata mappings are flattened to dotted paths, only allowlisted leaves are promoted, and the requester_metadata blob itself is never promoted Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin
parent
4e996400e2
commit
17059564a8
8 changed files with 177 additions and 13 deletions
|
|
@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Mapping
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
|
||||
|
||||
import litellm
|
||||
|
|
@ -20,7 +21,9 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
|
|||
OTELSemconvCategory,
|
||||
parse_semconv_opt_in,
|
||||
)
|
||||
from litellm.integrations.otel.model.baggage import promoted_metadata
|
||||
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
|
||||
from litellm.integrations.otel.model.metadata import flatten_metadata
|
||||
from litellm.integrations.otel.model.semconv import Metric
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
|
@ -288,6 +291,7 @@ class OpenTelemetryConfig:
|
|||
# under ``litellm.team.metadata``. Empty by default so none of a team's
|
||||
# metadata leaves the process until explicitly allowlisted.
|
||||
baggage_team_metadata_keys: list[str] = field(default_factory=list)
|
||||
baggage_metadata_keys: list[str] = field(default_factory=list)
|
||||
# Prometheus-style include/exclude control over which attributes are stamped
|
||||
# on emitted metrics, to cap metric cardinality.
|
||||
attributes: OTELMetricAttributeFilter | None = None
|
||||
|
|
@ -314,6 +318,9 @@ class OpenTelemetryConfig:
|
|||
self.baggage_team_metadata_keys = _normalize_team_metadata_keys(
|
||||
self.baggage_team_metadata_keys
|
||||
) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS"))
|
||||
self.baggage_metadata_keys = _normalize_team_metadata_keys(
|
||||
self.baggage_metadata_keys
|
||||
) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_METADATA_KEYS"))
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
|
|
@ -366,11 +373,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
**kwargs,
|
||||
):
|
||||
team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None)
|
||||
metadata_keys_override: Final = kwargs.pop("baggage_metadata_keys", None)
|
||||
metric_attributes_override: Final = kwargs.pop("attributes", None)
|
||||
if config is None:
|
||||
config = OpenTelemetryConfig.from_env()
|
||||
if team_metadata_keys_override is not None:
|
||||
config.baggage_team_metadata_keys = _normalize_team_metadata_keys(team_metadata_keys_override)
|
||||
if metadata_keys_override is not None:
|
||||
config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override)
|
||||
if metric_attributes_override is not None:
|
||||
config.attributes = _build_metric_attribute_filter(metric_attributes_override)
|
||||
|
||||
|
|
@ -1542,6 +1552,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
if team_metadata:
|
||||
self.safe_set_attribute(span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata)
|
||||
|
||||
if self.config.baggage_metadata_keys:
|
||||
flat_metadata: Final = MappingProxyType(dict(flatten_metadata(metadata)))
|
||||
for key, value in promoted_metadata(flat_metadata, tuple(self.config.baggage_metadata_keys)).items():
|
||||
self.safe_set_attribute(span=span, key=key, value=value)
|
||||
|
||||
model_group: Final = standard_logging_payload.get("model_group")
|
||||
if model_group:
|
||||
self.safe_set_attribute(span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.integrations.otel.model.metadata import (
|
|||
LLMCallEvent,
|
||||
RequestIdentity,
|
||||
auth_metadata,
|
||||
metadata_from_request_data,
|
||||
model_from_request_data,
|
||||
)
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
|
|
@ -679,7 +680,12 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# / errors are the FastAPI instrumentor's job, so we don't touch it here.
|
||||
# ====================================================================== #
|
||||
|
||||
def seed_request_identity(self, user_api_key_dict: object, model: str | None = None) -> None:
|
||||
def seed_request_identity(
|
||||
self,
|
||||
user_api_key_dict: object,
|
||||
model: str | None = None,
|
||||
request_metadata: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
"""Attach request-identity Baggage to the current context + server span.
|
||||
|
||||
Seeding identity into Baggage makes **every** span emitted afterwards for
|
||||
|
|
@ -691,7 +697,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
isn't determined yet, which is correct.
|
||||
"""
|
||||
try:
|
||||
identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict)
|
||||
identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata)
|
||||
bag: Final = promoted_baggage(
|
||||
identity,
|
||||
model,
|
||||
|
|
@ -743,6 +749,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self.seed_request_identity(
|
||||
user_api_key_dict,
|
||||
model=model_from_request_data(data),
|
||||
request_metadata=metadata_from_request_data(data),
|
||||
)
|
||||
return data
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ never promoted whole.
|
|||
|
||||
import json
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.otel.model.metadata import RequestIdentity
|
||||
|
|
@ -85,13 +86,26 @@ def promoted_baggage(
|
|||
value = extract(identity, request_model, team_metadata_keys)
|
||||
if value:
|
||||
out[key] = value
|
||||
for meta_key in metadata_keys:
|
||||
value = identity.metadata.get(meta_key)
|
||||
if value:
|
||||
out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value
|
||||
out.update(promoted_metadata(identity.metadata, metadata_keys))
|
||||
return out
|
||||
|
||||
|
||||
def promoted_metadata(metadata: Mapping[str, str], metadata_keys: tuple[str, ...]) -> Mapping[str, str]:
|
||||
"""Allowlisted entries of a flattened metadata mapping under ``litellm.metadata.*``.
|
||||
|
||||
A dotted key such as ``requester_metadata.trace_id`` reads the nested value and
|
||||
is promoted under its last segment (``litellm.metadata.trace_id``), so the
|
||||
caller-facing attribute name is independent of where the proxy stored it.
|
||||
"""
|
||||
return MappingProxyType(
|
||||
{
|
||||
f"{LiteLLM.METADATA_PREFIX}{meta_key.rsplit('.', 1)[-1]}": value
|
||||
for meta_key in metadata_keys
|
||||
if (value := metadata.get(meta_key))
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _filtered_team_metadata_json(
|
||||
metadata: Mapping[str, object] | None,
|
||||
allowed_keys: tuple[str, ...],
|
||||
|
|
|
|||
|
|
@ -210,7 +210,10 @@ class OpenTelemetryV2Config(BaseSettings):
|
|||
validation_alias=AliasChoices("baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"),
|
||||
description=(
|
||||
"Metadata sub-keys promoted under the ``litellm.metadata.*`` "
|
||||
"namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
|
||||
"namespace. A dotted path such as ``requester_metadata.trace_id`` "
|
||||
"reads the caller's nested ``metadata.trace_id`` and is promoted under "
|
||||
"its last segment (``litellm.metadata.trace_id``). "
|
||||
"Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
|
||||
"env var (comma-separated) or "
|
||||
"``callback_settings.otel.baggage_metadata_keys`` in config.yaml."
|
||||
),
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ class RequestIdentity:
|
|||
model, not just the user-facing one.
|
||||
"""
|
||||
raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {})
|
||||
metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))}
|
||||
metadata: Final = MappingProxyType(dict(flatten_metadata(raw_meta)))
|
||||
return cls(
|
||||
call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")),
|
||||
# StandardLoggingMetadata's canonical key is ``user_api_key_team_id``;
|
||||
|
|
@ -95,7 +95,9 @@ class RequestIdentity:
|
|||
)
|
||||
|
||||
@classmethod
|
||||
def from_user_api_key_auth(cls, auth: object) -> RequestIdentity:
|
||||
def from_user_api_key_auth(
|
||||
cls, auth: object, request_metadata: Mapping[str, object] | None = None
|
||||
) -> RequestIdentity:
|
||||
"""Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module
|
||||
free of a proxy import).
|
||||
|
||||
|
|
@ -103,11 +105,12 @@ class RequestIdentity:
|
|||
guardrail, or service span is created — so the whole request's spans
|
||||
inherit identity, not just the LLM-call span. Metadata sub-keys use the
|
||||
``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS``
|
||||
promotes.
|
||||
promotes; ``request_metadata`` (the proxy's per-request metadata dict) is
|
||||
flattened to dotted keys so ``requester_metadata.<key>`` resolves too.
|
||||
"""
|
||||
get: Final = lambda name: getattr(auth, name, None) # noqa: E731
|
||||
metadata: Final = {
|
||||
meta_key: str(value)
|
||||
auth_meta: Final = tuple(
|
||||
(meta_key, str(value))
|
||||
for meta_key, attr in (
|
||||
("user_api_key_user_id", "user_id"),
|
||||
("user_api_key_org_id", "org_id"),
|
||||
|
|
@ -115,7 +118,9 @@ class RequestIdentity:
|
|||
("user_api_key_end_user_id", "end_user_id"),
|
||||
)
|
||||
if (value := get(attr))
|
||||
}
|
||||
)
|
||||
request_meta: Final = flatten_metadata(request_metadata) if request_metadata is not None else ()
|
||||
metadata: Final = MappingProxyType(dict((*request_meta, *auth_meta)))
|
||||
return cls(
|
||||
team_id=as_str(get("team_id")),
|
||||
team_alias=as_str(get("team_alias")),
|
||||
|
|
@ -351,6 +356,33 @@ def model_from_request_data(data: object) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def metadata_from_request_data(data: object) -> Mapping[str, object] | None:
|
||||
"""The proxy's per-request metadata dict from a pre-call ``data`` dict.
|
||||
|
||||
The proxy writes it under ``metadata`` or ``litellm_metadata`` depending on
|
||||
the route; the one carrying the ``requester_metadata`` snapshot wins.
|
||||
"""
|
||||
top: Final = _as_str_mapping(data)
|
||||
if top is None:
|
||||
return None
|
||||
candidates: Final = tuple(
|
||||
nested for name in ("metadata", "litellm_metadata") if (nested := _as_str_mapping(top.get(name))) is not None
|
||||
)
|
||||
return next(
|
||||
(c for c in candidates if isinstance(c.get("requester_metadata"), Mapping)),
|
||||
candidates[0] if candidates else None,
|
||||
)
|
||||
|
||||
|
||||
def flatten_metadata(raw: Mapping[str, object], prefix: str = "") -> Iterator[tuple[str, str]]:
|
||||
"""Scalar leaves of a nested metadata mapping, keyed by their dotted path."""
|
||||
for key, value in raw.items():
|
||||
if (nested := _as_str_mapping(value)) is not None:
|
||||
yield from flatten_metadata(nested, f"{prefix}{key}.")
|
||||
elif isinstance(value, (str, bool, int, float)):
|
||||
yield f"{prefix}{key}", str(value)
|
||||
|
||||
|
||||
def resolve_provider_model(payload: StandardLoggingPayload) -> str | None:
|
||||
"""The model litellm dispatched to the provider, from the payload.
|
||||
|
||||
|
|
|
|||
|
|
@ -168,6 +168,34 @@ def test_allowlisted_metadata_subkey_promoted_blob_excluded():
|
|||
assert all("private_note" not in k for k in span.attributes)
|
||||
|
||||
|
||||
def test_nested_metadata_key_promoted_under_leaf_name():
|
||||
"""A dotted allowlist entry reads the nested caller metadata the proxy stores
|
||||
under ``requester_metadata`` and lands on the LLM-call span as
|
||||
``litellm.metadata.<leaf>``; unlisted siblings and the blob stay out."""
|
||||
engine, exporter = _engine_and_exporter()
|
||||
payload = _payload()
|
||||
payload["metadata"]["requester_metadata"] = {
|
||||
"trace_id": "abc",
|
||||
"attempt": 0,
|
||||
"empty": "",
|
||||
"nested": {"deep": "x"},
|
||||
}
|
||||
data = LLMCallSpanData.from_standard_logging_payload(payload)
|
||||
bag = promoted_baggage(
|
||||
data.identity,
|
||||
data.request_model,
|
||||
BAGGAGE_PROMOTED_KEYS,
|
||||
metadata_keys=("requester_metadata.trace_id", "requester_metadata.attempt", "requester_metadata.empty"),
|
||||
)
|
||||
engine.emit(SpanRole.LLM_CALL, data, ctx_mod.set_request_baggage(bag))
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert span.attributes[f"{LiteLLM.METADATA_PREFIX}trace_id"] == "abc"
|
||||
assert span.attributes[f"{LiteLLM.METADATA_PREFIX}attempt"] == "0"
|
||||
assert f"{LiteLLM.METADATA_PREFIX}empty" not in span.attributes
|
||||
assert f"{LiteLLM.METADATA_PREFIX}deep" not in span.attributes
|
||||
assert not any(k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") for k in span.attributes)
|
||||
|
||||
|
||||
def test_http_attributes_never_promoted():
|
||||
"""Even if http.* is present in baggage, the processor must not stamp it on
|
||||
child spans (it belongs on the SERVER span only)."""
|
||||
|
|
|
|||
|
|
@ -1655,6 +1655,41 @@ def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans():
|
|||
assert srv.attributes[f"{LiteLLM.METADATA_PREFIX}user_api_key_user_id"] == "u1"
|
||||
|
||||
|
||||
def test_pre_call_hook_promotes_nested_request_metadata_key():
|
||||
"""``baggage_metadata_keys: [requester_metadata.trace_id]`` reads the caller's
|
||||
``metadata.trace_id`` (snapshotted by the proxy under ``requester_metadata``)
|
||||
and stamps ``litellm.metadata.trace_id`` on the server, LLM-call and service
|
||||
spans of the request; unlisted siblings are not promoted."""
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["requester_metadata.trace_id"])
|
||||
exporter = InMemorySpanExporter()
|
||||
logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter))
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
data = {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}}
|
||||
kwargs = _kwargs()
|
||||
|
||||
async def _flow():
|
||||
await logger.async_pre_call_hook(_Auth(), None, data, "completion")
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
await logger.async_log_success_event(kwargs, None, None, None)
|
||||
await logger.async_service_success_hook(payload=_ServicePayload("redis", "set"), parent_otel_span=server)
|
||||
|
||||
with trace.use_span(server, end_on_exit=False):
|
||||
asyncio.run(_flow())
|
||||
server.end()
|
||||
|
||||
spans = {s.name: s for s in exporter.get_finished_spans()}
|
||||
key = f"{LiteLLM.METADATA_PREFIX}trace_id"
|
||||
assert spans[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes[key] == "abc"
|
||||
assert spans["chat gpt-4o"].attributes[key] == "abc"
|
||||
assert spans["redis set"].attributes[key] == "abc"
|
||||
assert data == {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}}
|
||||
assert not any(
|
||||
k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") or k == f"{LiteLLM.METADATA_PREFIX}deep"
|
||||
for s in spans.values()
|
||||
for k in s.attributes
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Service hooks (Phase 3)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
|
|||
|
|
@ -5581,6 +5581,31 @@ class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase):
|
|||
otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
|
||||
assert "http.route" not in self._attr(span, exp)
|
||||
|
||||
def test_nested_metadata_key_promoted_under_leaf_name(self):
|
||||
"""``baggage_metadata_keys: [requester_metadata.trace_id]`` stamps the
|
||||
caller's nested metadata value as ``litellm.metadata.trace_id``; unlisted
|
||||
siblings stay inside the ``metadata.requester_metadata`` blob."""
|
||||
otel = OpenTelemetry(config=OpenTelemetryConfig(baggage_metadata_keys=["requester_metadata.trace_id"]))
|
||||
kwargs = self._kwargs()
|
||||
kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {
|
||||
"trace_id": "abc",
|
||||
"nested": {"deep": "x"},
|
||||
}
|
||||
span, exp = self._span()
|
||||
otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
|
||||
attrs = self._attr(span, exp)
|
||||
assert attrs["litellm.metadata.trace_id"] == "abc"
|
||||
assert "litellm.metadata.deep" not in attrs
|
||||
assert not any(k.startswith("litellm.metadata.requester_metadata") for k in attrs)
|
||||
|
||||
def test_metadata_keys_default_to_none_promoted(self):
|
||||
otel = OpenTelemetry()
|
||||
kwargs = self._kwargs()
|
||||
kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {"trace_id": "abc"}
|
||||
span, exp = self._span()
|
||||
otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
|
||||
assert not any(k.startswith("litellm.metadata.") for k in self._attr(span, exp))
|
||||
|
||||
def test_team_metadata_json_helper(self):
|
||||
keys = ["a", "b"]
|
||||
assert OpenTelemetry._team_metadata_json(None, keys) is None
|
||||
|
|
@ -5631,6 +5656,11 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase):
|
|||
cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"])
|
||||
assert cfg.baggage_team_metadata_keys == ["from_arg"]
|
||||
|
||||
def test_metadata_keys_from_kwargs_and_env(self):
|
||||
with patch.dict("os.environ", {"LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.trace_id, a.b"}):
|
||||
assert OpenTelemetryConfig().baggage_metadata_keys == ["requester_metadata.trace_id", "a.b"]
|
||||
assert OpenTelemetry(baggage_metadata_keys="x.y").config.baggage_metadata_keys == ["x.y"]
|
||||
|
||||
|
||||
class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase):
|
||||
"""LIT-3600: include/exclude control over which attributes are stamped on
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue