From 17059564a8efea88a0a47fff19f0e720a399d32c Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 16 Sep 2026 18:08:14 +0000 Subject: [PATCH] feat(otel): promote nested request metadata keys to litellm.metadata.* span attributes baggage_metadata_keys entries such as requester_metadata.trace_id now resolve the caller's nested metadata.trace_id and stamp it on the LLM-call span as litellm.metadata.trace_id, in both the OTEL v2 logger and the legacy OpenTelemetry callback. Nested metadata mappings are flattened to dotted paths, only allowlisted leaves are promoted, and the requester_metadata blob itself is never promoted Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/opentelemetry.py | 15 +++++++ litellm/integrations/otel/logger.py | 11 ++++- litellm/integrations/otel/model/baggage.py | 22 ++++++++-- litellm/integrations/otel/model/config.py | 5 ++- litellm/integrations/otel/model/metadata.py | 44 ++++++++++++++++--- .../integrations/otel/test_otel_v2_baggage.py | 28 ++++++++++++ .../integrations/otel/test_otel_v2_logger.py | 35 +++++++++++++++ .../integrations/test_opentelemetry.py | 30 +++++++++++++ 8 files changed, 177 insertions(+), 13 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index d4e7fcb577e..9456817a205 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypedDict, cast import litellm @@ -20,7 +21,9 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( OTELSemconvCategory, parse_semconv_opt_in, ) +from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.db_endpoint import db_span_attributes +from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -288,6 +291,7 @@ class OpenTelemetryConfig: # under ``litellm.team.metadata``. Empty by default so none of a team's # metadata leaves the process until explicitly allowlisted. baggage_team_metadata_keys: list[str] = field(default_factory=list) + baggage_metadata_keys: list[str] = field(default_factory=list) # Prometheus-style include/exclude control over which attributes are stamped # on emitted metrics, to cap metric cardinality. attributes: OTELMetricAttributeFilter | None = None @@ -314,6 +318,9 @@ class OpenTelemetryConfig: self.baggage_team_metadata_keys = _normalize_team_metadata_keys( self.baggage_team_metadata_keys ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS")) + self.baggage_metadata_keys = _normalize_team_metadata_keys( + self.baggage_metadata_keys + ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_METADATA_KEYS")) @classmethod def from_env(cls): @@ -366,11 +373,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): **kwargs, ): team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None) + metadata_keys_override: Final = kwargs.pop("baggage_metadata_keys", None) metric_attributes_override: Final = kwargs.pop("attributes", None) if config is None: config = OpenTelemetryConfig.from_env() if team_metadata_keys_override is not None: config.baggage_team_metadata_keys = _normalize_team_metadata_keys(team_metadata_keys_override) + if metadata_keys_override is not None: + config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override) if metric_attributes_override is not None: config.attributes = _build_metric_attribute_filter(metric_attributes_override) @@ -1542,6 +1552,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if team_metadata: self.safe_set_attribute(span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata) + if self.config.baggage_metadata_keys: + flat_metadata: Final = MappingProxyType(dict(flatten_metadata(metadata))) + for key, value in promoted_metadata(flat_metadata, tuple(self.config.baggage_metadata_keys)).items(): + self.safe_set_attribute(span=span, key=key, value=value) + model_group: Final = standard_logging_payload.get("model_group") if model_group: self.safe_set_attribute(span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 9ac748b231c..285a5c3aa97 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -33,6 +33,7 @@ from litellm.integrations.otel.model.metadata import ( LLMCallEvent, RequestIdentity, auth_metadata, + metadata_from_request_data, model_from_request_data, ) from litellm.integrations.otel.model.payloads import ( @@ -679,7 +680,12 @@ class OpenTelemetryV2(CustomLogger): # / errors are the FastAPI instrumentor's job, so we don't touch it here. # ====================================================================== # - def seed_request_identity(self, user_api_key_dict: object, model: str | None = None) -> None: + def seed_request_identity( + self, + user_api_key_dict: object, + model: str | None = None, + request_metadata: Mapping[str, object] | None = None, + ) -> None: """Attach request-identity Baggage to the current context + server span. Seeding identity into Baggage makes **every** span emitted afterwards for @@ -691,7 +697,7 @@ class OpenTelemetryV2(CustomLogger): isn't determined yet, which is correct. """ try: - identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict) + identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata) bag: Final = promoted_baggage( identity, model, @@ -743,6 +749,7 @@ class OpenTelemetryV2(CustomLogger): self.seed_request_identity( user_api_key_dict, model=model_from_request_data(data), + request_metadata=metadata_from_request_data(data), ) return data diff --git a/litellm/integrations/otel/model/baggage.py b/litellm/integrations/otel/model/baggage.py index 2be9bb36def..0511eadaa8b 100644 --- a/litellm/integrations/otel/model/baggage.py +++ b/litellm/integrations/otel/model/baggage.py @@ -15,6 +15,7 @@ never promoted whole. import json from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final from litellm.integrations.otel.model.metadata import RequestIdentity @@ -85,13 +86,26 @@ def promoted_baggage( value = extract(identity, request_model, team_metadata_keys) if value: out[key] = value - for meta_key in metadata_keys: - value = identity.metadata.get(meta_key) - if value: - out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value + out.update(promoted_metadata(identity.metadata, metadata_keys)) return out +def promoted_metadata(metadata: Mapping[str, str], metadata_keys: tuple[str, ...]) -> Mapping[str, str]: + """Allowlisted entries of a flattened metadata mapping under ``litellm.metadata.*``. + + A dotted key such as ``requester_metadata.trace_id`` reads the nested value and + is promoted under its last segment (``litellm.metadata.trace_id``), so the + caller-facing attribute name is independent of where the proxy stored it. + """ + return MappingProxyType( + { + f"{LiteLLM.METADATA_PREFIX}{meta_key.rsplit('.', 1)[-1]}": value + for meta_key in metadata_keys + if (value := metadata.get(meta_key)) + } + ) + + def _filtered_team_metadata_json( metadata: Mapping[str, object] | None, allowed_keys: tuple[str, ...], diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index bd542ddc20c..e5a8132dc71 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -210,7 +210,10 @@ class OpenTelemetryV2Config(BaseSettings): validation_alias=AliasChoices("baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"), description=( "Metadata sub-keys promoted under the ``litellm.metadata.*`` " - "namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " + "namespace. A dotted path such as ``requester_metadata.trace_id`` " + "reads the caller's nested ``metadata.trace_id`` and is promoted under " + "its last segment (``litellm.metadata.trace_id``). " + "Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " "env var (comma-separated) or " "``callback_settings.otel.baggage_metadata_keys`` in config.yaml." ), diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index cc81b689708..d1fb3beae20 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -78,7 +78,7 @@ class RequestIdentity: model, not just the user-facing one. """ raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {}) - metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))} + metadata: Final = MappingProxyType(dict(flatten_metadata(raw_meta))) return cls( call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")), # StandardLoggingMetadata's canonical key is ``user_api_key_team_id``; @@ -95,7 +95,9 @@ class RequestIdentity: ) @classmethod - def from_user_api_key_auth(cls, auth: object) -> RequestIdentity: + def from_user_api_key_auth( + cls, auth: object, request_metadata: Mapping[str, object] | None = None + ) -> RequestIdentity: """Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module free of a proxy import). @@ -103,11 +105,12 @@ class RequestIdentity: guardrail, or service span is created — so the whole request's spans inherit identity, not just the LLM-call span. Metadata sub-keys use the ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS`` - promotes. + promotes; ``request_metadata`` (the proxy's per-request metadata dict) is + flattened to dotted keys so ``requester_metadata.`` resolves too. """ get: Final = lambda name: getattr(auth, name, None) # noqa: E731 - metadata: Final = { - meta_key: str(value) + auth_meta: Final = tuple( + (meta_key, str(value)) for meta_key, attr in ( ("user_api_key_user_id", "user_id"), ("user_api_key_org_id", "org_id"), @@ -115,7 +118,9 @@ class RequestIdentity: ("user_api_key_end_user_id", "end_user_id"), ) if (value := get(attr)) - } + ) + request_meta: Final = flatten_metadata(request_metadata) if request_metadata is not None else () + metadata: Final = MappingProxyType(dict((*request_meta, *auth_meta))) return cls( team_id=as_str(get("team_id")), team_alias=as_str(get("team_alias")), @@ -351,6 +356,33 @@ def model_from_request_data(data: object) -> str | None: return None +def metadata_from_request_data(data: object) -> Mapping[str, object] | None: + """The proxy's per-request metadata dict from a pre-call ``data`` dict. + + The proxy writes it under ``metadata`` or ``litellm_metadata`` depending on + the route; the one carrying the ``requester_metadata`` snapshot wins. + """ + top: Final = _as_str_mapping(data) + if top is None: + return None + candidates: Final = tuple( + nested for name in ("metadata", "litellm_metadata") if (nested := _as_str_mapping(top.get(name))) is not None + ) + return next( + (c for c in candidates if isinstance(c.get("requester_metadata"), Mapping)), + candidates[0] if candidates else None, + ) + + +def flatten_metadata(raw: Mapping[str, object], prefix: str = "") -> Iterator[tuple[str, str]]: + """Scalar leaves of a nested metadata mapping, keyed by their dotted path.""" + for key, value in raw.items(): + if (nested := _as_str_mapping(value)) is not None: + yield from flatten_metadata(nested, f"{prefix}{key}.") + elif isinstance(value, (str, bool, int, float)): + yield f"{prefix}{key}", str(value) + + def resolve_provider_model(payload: StandardLoggingPayload) -> str | None: """The model litellm dispatched to the provider, from the payload. diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py b/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py index b379b8bebc9..78fdd251d18 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py @@ -168,6 +168,34 @@ def test_allowlisted_metadata_subkey_promoted_blob_excluded(): assert all("private_note" not in k for k in span.attributes) +def test_nested_metadata_key_promoted_under_leaf_name(): + """A dotted allowlist entry reads the nested caller metadata the proxy stores + under ``requester_metadata`` and lands on the LLM-call span as + ``litellm.metadata.``; unlisted siblings and the blob stay out.""" + engine, exporter = _engine_and_exporter() + payload = _payload() + payload["metadata"]["requester_metadata"] = { + "trace_id": "abc", + "attempt": 0, + "empty": "", + "nested": {"deep": "x"}, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + bag = promoted_baggage( + data.identity, + data.request_model, + BAGGAGE_PROMOTED_KEYS, + metadata_keys=("requester_metadata.trace_id", "requester_metadata.attempt", "requester_metadata.empty"), + ) + engine.emit(SpanRole.LLM_CALL, data, ctx_mod.set_request_baggage(bag)) + (span,) = exporter.get_finished_spans() + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}trace_id"] == "abc" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}attempt"] == "0" + assert f"{LiteLLM.METADATA_PREFIX}empty" not in span.attributes + assert f"{LiteLLM.METADATA_PREFIX}deep" not in span.attributes + assert not any(k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") for k in span.attributes) + + def test_http_attributes_never_promoted(): """Even if http.* is present in baggage, the processor must not stamp it on child spans (it belongs on the SERVER span only).""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 2869c804c07..aa78e3b7c4d 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -1655,6 +1655,41 @@ def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans(): assert srv.attributes[f"{LiteLLM.METADATA_PREFIX}user_api_key_user_id"] == "u1" +def test_pre_call_hook_promotes_nested_request_metadata_key(): + """``baggage_metadata_keys: [requester_metadata.trace_id]`` reads the caller's + ``metadata.trace_id`` (snapshotted by the proxy under ``requester_metadata``) + and stamps ``litellm.metadata.trace_id`` on the server, LLM-call and service + spans of the request; unlisted siblings are not promoted.""" + cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["requester_metadata.trace_id"]) + exporter = InMemorySpanExporter() + logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter)) + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + data = {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}} + kwargs = _kwargs() + + async def _flow(): + await logger.async_pre_call_hook(_Auth(), None, data, "completion") + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + await logger.async_log_success_event(kwargs, None, None, None) + await logger.async_service_success_hook(payload=_ServicePayload("redis", "set"), parent_otel_span=server) + + with trace.use_span(server, end_on_exit=False): + asyncio.run(_flow()) + server.end() + + spans = {s.name: s for s in exporter.get_finished_spans()} + key = f"{LiteLLM.METADATA_PREFIX}trace_id" + assert spans[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes[key] == "abc" + assert spans["chat gpt-4o"].attributes[key] == "abc" + assert spans["redis set"].attributes[key] == "abc" + assert data == {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}} + assert not any( + k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") or k == f"{LiteLLM.METADATA_PREFIX}deep" + for s in spans.values() + for k in s.attributes + ) + + # --------------------------------------------------------------------------- # # Service hooks (Phase 3) # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 9ec8489f784..e25fb3964b8 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -5581,6 +5581,31 @@ class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase): otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) assert "http.route" not in self._attr(span, exp) + def test_nested_metadata_key_promoted_under_leaf_name(self): + """``baggage_metadata_keys: [requester_metadata.trace_id]`` stamps the + caller's nested metadata value as ``litellm.metadata.trace_id``; unlisted + siblings stay inside the ``metadata.requester_metadata`` blob.""" + otel = OpenTelemetry(config=OpenTelemetryConfig(baggage_metadata_keys=["requester_metadata.trace_id"])) + kwargs = self._kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = { + "trace_id": "abc", + "nested": {"deep": "x"}, + } + span, exp = self._span() + otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) + attrs = self._attr(span, exp) + assert attrs["litellm.metadata.trace_id"] == "abc" + assert "litellm.metadata.deep" not in attrs + assert not any(k.startswith("litellm.metadata.requester_metadata") for k in attrs) + + def test_metadata_keys_default_to_none_promoted(self): + otel = OpenTelemetry() + kwargs = self._kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {"trace_id": "abc"} + span, exp = self._span() + otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) + assert not any(k.startswith("litellm.metadata.") for k in self._attr(span, exp)) + def test_team_metadata_json_helper(self): keys = ["a", "b"] assert OpenTelemetry._team_metadata_json(None, keys) is None @@ -5631,6 +5656,11 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase): cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"]) assert cfg.baggage_team_metadata_keys == ["from_arg"] + def test_metadata_keys_from_kwargs_and_env(self): + with patch.dict("os.environ", {"LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.trace_id, a.b"}): + assert OpenTelemetryConfig().baggage_metadata_keys == ["requester_metadata.trace_id", "a.b"] + assert OpenTelemetry(baggage_metadata_keys="x.y").config.baggage_metadata_keys == ["x.y"] + class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): """LIT-3600: include/exclude control over which attributes are stamped on