Merge pull request #41462 from BerriAI/litellm_otel_promote_nested_request_metadata_keys

feat(otel): promote nested request metadata keys to litellm.metadata.* span attributes

(cherry picked from commit 79fc5153d3)

litellm/integrations/opentelemetry.py

@@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Mapping
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
 
 import litellm
@@ -20,7 +21,9 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
     OTELSemconvCategory,
     parse_semconv_opt_in,
 )
+from litellm.integrations.otel.model.baggage import promoted_metadata
 from litellm.integrations.otel.model.db_endpoint import db_span_attributes
+from litellm.integrations.otel.model.metadata import flatten_metadata
 from litellm.integrations.otel.model.semconv import Metric
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -288,6 +291,7 @@ class OpenTelemetryConfig:
     # under ``litellm.team.metadata``. Empty by default so none of a team's
     # metadata leaves the process until explicitly allowlisted.
     baggage_team_metadata_keys: list[str] = field(default_factory=list)
+    baggage_metadata_keys: list[str] = field(default_factory=list)
     # Prometheus-style include/exclude control over which attributes are stamped
     # on emitted metrics, to cap metric cardinality.
     attributes: OTELMetricAttributeFilter | None = None
@@ -314,6 +318,9 @@ class OpenTelemetryConfig:
         self.baggage_team_metadata_keys = _normalize_team_metadata_keys(
             self.baggage_team_metadata_keys
         ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS"))
+        self.baggage_metadata_keys = _normalize_team_metadata_keys(
+            self.baggage_metadata_keys
+        ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_METADATA_KEYS"))
 
     @classmethod
     def from_env(cls):
@@ -366,11 +373,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         **kwargs,
     ):
         team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None)
+        metadata_keys_override: Final = kwargs.pop("baggage_metadata_keys", None)
         metric_attributes_override: Final = kwargs.pop("attributes", None)
         if config is None:
             config = OpenTelemetryConfig.from_env()
         if team_metadata_keys_override is not None:
             config.baggage_team_metadata_keys = _normalize_team_metadata_keys(team_metadata_keys_override)
+        if metadata_keys_override is not None:
+            config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override)
         if metric_attributes_override is not None:
             config.attributes = _build_metric_attribute_filter(metric_attributes_override)
 
@@ -1542,6 +1552,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         if team_metadata:
             self.safe_set_attribute(span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata)
 
+        if self.config.baggage_metadata_keys:
+            flat_metadata: Final = MappingProxyType(dict(flatten_metadata(metadata)))
+            for key, value in promoted_metadata(flat_metadata, tuple(self.config.baggage_metadata_keys)).items():
+                self.safe_set_attribute(span=span, key=key, value=value)
+
         model_group: Final = standard_logging_payload.get("model_group")
         if model_group:
             self.safe_set_attribute(span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group)

litellm/integrations/otel/logger.py

@@ -33,6 +33,7 @@ from litellm.integrations.otel.model.metadata import (
     LLMCallEvent,
     RequestIdentity,
     auth_metadata,
+    metadata_from_request_data,
     model_from_request_data,
 )
 from litellm.integrations.otel.model.payloads import (
@@ -679,7 +680,12 @@ class OpenTelemetryV2(CustomLogger):
     #  / errors are the FastAPI instrumentor's job, so we don't touch it here.
     # ====================================================================== #
 
-    def seed_request_identity(self, user_api_key_dict: object, model: str | None = None) -> None:
+    def seed_request_identity(
+        self,
+        user_api_key_dict: object,
+        model: str | None = None,
+        request_metadata: Mapping[str, object] | None = None,
+    ) -> None:
         """Attach request-identity Baggage to the current context + server span.
 
         Seeding identity into Baggage makes **every** span emitted afterwards for
@@ -691,7 +697,7 @@ class OpenTelemetryV2(CustomLogger):
         isn't determined yet, which is correct.
         """
         try:
-            identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict)
+            identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata)
             bag: Final = promoted_baggage(
                 identity,
                 model,
@@ -743,6 +749,7 @@ class OpenTelemetryV2(CustomLogger):
         self.seed_request_identity(
             user_api_key_dict,
             model=model_from_request_data(data),
+            request_metadata=metadata_from_request_data(data),
         )
         return data
 

litellm/integrations/otel/model/baggage.py

@@ -15,9 +15,10 @@ never promoted whole.
 
 import json
 from collections.abc import Callable, Mapping
+from types import MappingProxyType
 from typing import Final
 
-from litellm.integrations.otel.model.metadata import RequestIdentity
+from litellm.integrations.otel.model.metadata import REQUESTER_METADATA_PATH, RequestIdentity
 from litellm.integrations.otel.model.semconv import GenAI, LiteLLM
 
 # Attribute key -> value extractor over (identity, request_model,
@@ -79,17 +80,23 @@ def promoted_baggage(
     ``team_metadata_keys`` selects sub-keys of the team's metadata to promote
     under ``litellm.team.metadata``. Empty values are dropped.
     """
-    out: Final[dict[str, str]] = {}
-    for key, extract in _PROMOTABLE.items():
-        if key in promoted_keys:
-            value = extract(identity, request_model, team_metadata_keys)
-            if value:
-                out[key] = value
-    for meta_key in metadata_keys:
-        value = identity.metadata.get(meta_key)
-        if value:
-            out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value
-    return out
+    identity_values: Final = {
+        key: value
+        for key, extract in _PROMOTABLE.items()
+        if key in promoted_keys and (value := extract(identity, request_model, team_metadata_keys))
+    }
+    return {**identity_values, **promoted_metadata(identity.metadata, metadata_keys)}
+
+
+def promoted_metadata(metadata: Mapping[str, str], metadata_keys: tuple[str, ...]) -> Mapping[str, str]:
+    """Allowlisted entries of a flattened metadata mapping under ``litellm.metadata.*``."""
+    return MappingProxyType(
+        {
+            f"{LiteLLM.METADATA_PREFIX}{meta_key.removeprefix(REQUESTER_METADATA_PATH)}": value
+            for meta_key in metadata_keys
+            if (value := metadata.get(meta_key))
+        }
+    )
 
 
 def _filtered_team_metadata_json(

litellm/integrations/otel/model/config.py

@@ -210,7 +210,10 @@ class OpenTelemetryV2Config(BaseSettings):
         validation_alias=AliasChoices("baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"),
         description=(
             "Metadata sub-keys promoted under the ``litellm.metadata.*`` "
-            "namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
+            "namespace. A dotted path such as ``requester_metadata.trace_id`` "
+            "reads the caller's nested ``metadata.trace_id`` and is promoted as "
+            "``litellm.metadata.trace_id``; other dotted keys keep their full path. "
+            "Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
             "env var (comma-separated) or "
             "``callback_settings.otel.baggage_metadata_keys`` in config.yaml."
         ),

litellm/integrations/otel/model/metadata.py

@@ -49,6 +49,8 @@ if TYPE_CHECKING:
     from litellm.types.utils import StandardLoggingPayload
 
 LANGFUSE_TRACE_NAME_HEADER: Final = "langfuse_trace_name"
+REQUESTER_METADATA_KEY: Final = "requester_metadata"
+REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}."
 
 
 @dataclass(frozen=True)
@@ -78,7 +80,7 @@ class RequestIdentity:
         model, not just the user-facing one.
         """
         raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {})
-        metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))}
+        metadata: Final = MappingProxyType(dict(flatten_metadata(raw_meta)))
         return cls(
             call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")),
             # StandardLoggingMetadata's canonical key is ``user_api_key_team_id``;
@@ -95,7 +97,9 @@ class RequestIdentity:
         )
 
     @classmethod
-    def from_user_api_key_auth(cls, auth: object) -> RequestIdentity:
+    def from_user_api_key_auth(
+        cls, auth: object, request_metadata: Mapping[str, object] | None = None
+    ) -> RequestIdentity:
         """Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module
         free of a proxy import).
 
@@ -103,11 +107,13 @@ class RequestIdentity:
         guardrail, or service span is created — so the whole request's spans
         inherit identity, not just the LLM-call span. Metadata sub-keys use the
         ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS``
-        promotes.
+        promotes; ``request_metadata`` (the caller's ``requester_metadata``
+        snapshot) is flattened to dotted keys so ``requester_metadata.<key>``
+        resolves too.
         """
         get: Final = lambda name: getattr(auth, name, None)  # noqa: E731
-        metadata: Final = {
-            meta_key: str(value)
+        auth_meta: Final = tuple(
+            (meta_key, str(value))
             for meta_key, attr in (
                 ("user_api_key_user_id", "user_id"),
                 ("user_api_key_org_id", "org_id"),
@@ -115,7 +121,9 @@ class RequestIdentity:
                 ("user_api_key_end_user_id", "end_user_id"),
             )
             if (value := get(attr))
-        }
+        )
+        request_meta: Final = flatten_metadata(request_metadata) if request_metadata is not None else ()
+        metadata: Final = MappingProxyType(dict((*request_meta, *auth_meta)))
         return cls(
             team_id=as_str(get("team_id")),
             team_alias=as_str(get("team_alias")),
@@ -351,6 +359,35 @@ def model_from_request_data(data: object) -> str | None:
     return None
 
 
+def metadata_from_request_data(data: object) -> Mapping[str, object] | None:
+    """The caller's ``requester_metadata`` snapshot from a pre-call ``data`` dict, keyed under its wrapper.
+
+    The proxy stores it under ``metadata`` or ``litellm_metadata`` depending on the route;
+    the proxy-owned siblings (``user_api_key_*``, ``requester_ip_address``) are not read.
+    """
+    top: Final = _as_str_mapping(data)
+    if top is None:
+        return None
+    snapshots: Final = tuple(
+        snapshot
+        for name in ("metadata", "litellm_metadata")
+        if (nested := _as_str_mapping(top.get(name))) is not None
+        and (snapshot := _as_str_mapping(nested.get(REQUESTER_METADATA_KEY))) is not None
+    )
+    return MappingProxyType({REQUESTER_METADATA_KEY: snapshots[0]}) if snapshots else None
+
+
+def flatten_metadata(raw: Mapping[str, object]) -> Iterator[tuple[str, str]]:
+    """Scalar leaves of a nested metadata mapping, keyed by their dotted path."""
+    stack: Final = list(tuple(raw.items())[::-1])  # mutable-ok: iterative worklist keeps the walk off the call stack
+    while stack:
+        key, value = stack.pop()
+        if (nested := _as_str_mapping(value)) is not None:
+            stack.extend(tuple((f"{key}.{sub_key}", sub_value) for sub_key, sub_value in nested.items())[::-1])
+        elif isinstance(value, (str, bool, int, float)):
+            yield key, str(value)
+
+
 def resolve_provider_model(payload: StandardLoggingPayload) -> str | None:
     """The model litellm dispatched to the provider, from the payload.
 

tests/test_litellm/integrations/otel/test_otel_v2_baggage.py

@@ -168,6 +168,46 @@ def test_allowlisted_metadata_subkey_promoted_blob_excluded():
     assert all("private_note" not in k for k in span.attributes)
 
 
+def test_nested_metadata_key_promoted_under_caller_path():
+    """A dotted allowlist entry reads the nested caller metadata the proxy stores
+    under ``requester_metadata`` and lands on the LLM-call span under the caller's
+    own path (``litellm.metadata.trace_id``, ``litellm.metadata.nested.deep``);
+    a pre-existing flat dotted key keeps its full name, and unlisted siblings and
+    the blob stay out."""
+    engine, exporter = _engine_and_exporter()
+    payload = _payload()
+    payload["metadata"]["a.b"] = "flat"
+    payload["metadata"]["requester_metadata"] = {
+        "trace_id": "abc",
+        "attempt": 0,
+        "empty": "",
+        "nested": {"deep": "x", "skipped": "y"},
+    }
+    data = LLMCallSpanData.from_standard_logging_payload(payload)
+    bag = promoted_baggage(
+        data.identity,
+        data.request_model,
+        BAGGAGE_PROMOTED_KEYS,
+        metadata_keys=(
+            "requester_metadata.trace_id",
+            "requester_metadata.attempt",
+            "requester_metadata.empty",
+            "requester_metadata.nested.deep",
+            "a.b",
+        ),
+    )
+    engine.emit(SpanRole.LLM_CALL, data, ctx_mod.set_request_baggage(bag))
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes[f"{LiteLLM.METADATA_PREFIX}trace_id"] == "abc"
+    assert span.attributes[f"{LiteLLM.METADATA_PREFIX}attempt"] == "0"
+    assert span.attributes[f"{LiteLLM.METADATA_PREFIX}nested.deep"] == "x"
+    assert span.attributes[f"{LiteLLM.METADATA_PREFIX}a.b"] == "flat"
+    assert f"{LiteLLM.METADATA_PREFIX}empty" not in span.attributes
+    assert f"{LiteLLM.METADATA_PREFIX}deep" not in span.attributes
+    assert f"{LiteLLM.METADATA_PREFIX}nested.skipped" not in span.attributes
+    assert not any(k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") for k in span.attributes)
+
+
 def test_http_attributes_never_promoted():
     """Even if http.* is present in baggage, the processor must not stamp it on
     child spans (it belongs on the SERVER span only)."""

tests/test_litellm/integrations/otel/test_otel_v2_logger.py

@@ -1623,17 +1623,22 @@ def test_provider_model_and_team_metadata_on_real_boundary_flow():
 def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans():
     """The pre-call hook seeds identity Baggage in the request context so the
     server span (stamped directly) AND later child spans (service here, via the
-    Baggage processor) carry identity — not just the LLM-call span."""
+    Baggage processor) carry identity — not just the LLM-call span. Only the
+    caller's ``requester_metadata`` is read from the request dict, so a proxy-owned
+    sibling such as ``requester_ip_address`` is not stamped from here even though
+    the default allowlist names it, and an unlisted caller key is not promoted."""
     logger, exporter = _logger()
     server = logger._emitter.start_span(
         SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
     )
+    data = {
+        "model": "gpt-4o",
+        "metadata": {"requester_ip_address": "127.0.0.1", "requester_metadata": {"trace_id": "abc"}},
+    }
 
     async def _flow():
         # pre-call seeds baggage + stamps the active server span
-        await logger.async_pre_call_hook(
-            _Auth(), None, {"model": "gpt-4o"}, "completion"
-        )
+        await logger.async_pre_call_hook(_Auth(), None, data, "completion")
         # a later service call (same task) must inherit the identity
         await logger.async_service_success_hook(
             payload=_ServicePayload("redis", "set"), parent_otel_span=server
@@ -1653,6 +1658,46 @@ def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans():
         srv.attributes[LiteLLM.TEAM_ID] == "t1"
     )  # stamped directly on the server span
     assert srv.attributes[f"{LiteLLM.METADATA_PREFIX}user_api_key_user_id"] == "u1"
+    assert not any(
+        k in (f"{LiteLLM.METADATA_PREFIX}requester_ip_address", f"{LiteLLM.METADATA_PREFIX}trace_id")
+        for s in (redis, srv)
+        for k in s.attributes
+    )
+
+
+def test_pre_call_hook_promotes_nested_request_metadata_key():
+    """``baggage_metadata_keys: [requester_metadata.trace_id]`` reads the caller's
+    ``metadata.trace_id`` (snapshotted by the proxy under ``requester_metadata``)
+    and stamps ``litellm.metadata.trace_id`` on the server, LLM-call and service
+    spans of the request; unlisted siblings are not promoted."""
+    cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["requester_metadata.trace_id"])
+    exporter = InMemorySpanExporter()
+    logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter))
+    server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
+    data = {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}}
+    kwargs = _kwargs()
+
+    async def _flow():
+        await logger.async_pre_call_hook(_Auth(), None, data, "completion")
+        logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
+        await logger.async_log_success_event(kwargs, None, None, None)
+        await logger.async_service_success_hook(payload=_ServicePayload("redis", "set"), parent_otel_span=server)
+
+    with trace.use_span(server, end_on_exit=False):
+        asyncio.run(_flow())
+    server.end()
+
+    spans = {s.name: s for s in exporter.get_finished_spans()}
+    key = f"{LiteLLM.METADATA_PREFIX}trace_id"
+    assert spans[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes[key] == "abc"
+    assert spans["chat gpt-4o"].attributes[key] == "abc"
+    assert spans["redis set"].attributes[key] == "abc"
+    assert data == {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}}
+    assert not any(
+        k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") or k.endswith("deep")
+        for s in spans.values()
+        for k in s.attributes
+    )
 
 
 # --------------------------------------------------------------------------- #

tests/test_litellm/integrations/test_opentelemetry.py

@@ -5581,6 +5581,38 @@ class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase):
         otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
         assert "http.route" not in self._attr(span, exp)
 
+    def test_nested_metadata_key_promoted_under_caller_path(self):
+        """``baggage_metadata_keys: [requester_metadata.trace_id]`` stamps the
+        caller's nested metadata value as ``litellm.metadata.trace_id`` and a deeper
+        path keeps its dotted name; unlisted siblings stay inside the
+        ``metadata.requester_metadata`` blob."""
+        otel = OpenTelemetry(
+            config=OpenTelemetryConfig(
+                baggage_metadata_keys=["requester_metadata.trace_id", "requester_metadata.nested.deep"]
+            )
+        )
+        kwargs = self._kwargs()
+        kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {
+            "trace_id": "abc",
+            "nested": {"deep": "x", "skipped": "y"},
+        }
+        span, exp = self._span()
+        otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
+        attrs = self._attr(span, exp)
+        assert attrs["litellm.metadata.trace_id"] == "abc"
+        assert attrs["litellm.metadata.nested.deep"] == "x"
+        assert "litellm.metadata.deep" not in attrs
+        assert "litellm.metadata.nested.skipped" not in attrs
+        assert not any(k.startswith("litellm.metadata.requester_metadata") for k in attrs)
+
+    def test_metadata_keys_default_to_none_promoted(self):
+        otel = OpenTelemetry()
+        kwargs = self._kwargs()
+        kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {"trace_id": "abc"}
+        span, exp = self._span()
+        otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"})
+        assert not any(k.startswith("litellm.metadata.") for k in self._attr(span, exp))
+
     def test_team_metadata_json_helper(self):
         keys = ["a", "b"]
         assert OpenTelemetry._team_metadata_json(None, keys) is None
@@ -5631,6 +5663,11 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase):
             cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"])
             assert cfg.baggage_team_metadata_keys == ["from_arg"]
 
+    def test_metadata_keys_from_kwargs_and_env(self):
+        with patch.dict("os.environ", {"LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.trace_id, a.b"}):
+            assert OpenTelemetryConfig().baggage_metadata_keys == ["requester_metadata.trace_id", "a.b"]
+        assert OpenTelemetry(baggage_metadata_keys="x.y").config.baggage_metadata_keys == ["x.y"]
+
 
 class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase):
     """LIT-3600: include/exclude control over which attributes are stamped on