fix(otel): stamp allowlisted late request metadata on boundary-born LLM spans

OTel v2 opens the LLM-call span at the pre_call boundary, so the Baggage processor stamps only the identity seeded at auth. Metadata that first appears in the final StandardLoggingPayload and is allowlisted via baggage_metadata_keys never reached that span, so litellm.metadata.<key> was missing whenever the span was born at the boundary rather than deferred. finish_span now re-applies promoted_baggage from the typed payload identity, making a boundary-born span carry exactly the allowlisted keys a deferred span already gets, and nothing outside the allowlist

litellm/integrations/otel/emitter.py

@@ -7,9 +7,11 @@ from opentelemetry.context import Context
 from opentelemetry.trace import Link, Span, Tracer
 from opentelemetry.trace.status import Status, StatusCode
 
+from litellm.integrations.otel.model.baggage import promoted_baggage
 from litellm.integrations.otel.model.config import OpenTelemetryV2Config
 from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData
+from litellm.integrations.otel.model.metadata import RequestIdentity
 from litellm.integrations.otel.model.payloads import (
     GuardrailSpanData,
     LLMCallSpanData,
@@ -51,6 +53,24 @@ _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = {
 _DEDUP_CACHE_MAX = 10_000
 
 
+def _span_identity(data: SpanData) -> RequestIdentity | None:
+    """The request identity a span carries, or ``None`` for span kinds that have
+    none (service / guardrail spans). Only identity-bearing spans get the
+    allowlisted request metadata re-applied at close."""
+    match data:
+        case LLMCallSpanData() | MCPToolCallSpanData() | MCPListToolsSpanData():
+            return data.identity
+        case _:
+            return None
+
+
+def _span_request_model(data: SpanData) -> str | None:
+    """The user-facing request model promoted into Baggage as ``gen_ai.request.model``.
+    Only the LLM-call span carries one; MCP spans promote identity without a model,
+    matching how the deferred path seeds their Baggage."""
+    return data.request_model if isinstance(data, LLMCallSpanData) else None
+
+
 def _stamp_otel_error_attributes(span: Span, error_type: str, resolved_message: str) -> None:
     """Stamp the OTel-semconv error attributes (``error.type`` + ``error.message``).
     ``error_type`` and ``resolved_message`` are ``finish_span``'s already-computed
@@ -234,6 +254,7 @@ class SpanEmitter:
         for mapper in self._mappers:
             for key, value in mapper.map(data).items():
                 span.set_attribute(key, value)
+        self._stamp_identity_baggage(span, data)
         error = (
             data.error
             if isinstance(
@@ -264,3 +285,29 @@ class SpanEmitter:
         # span-level health signal litellm doesn't actually evaluate. Only a
         # genuine error sets a status.
         span.end(end_time=end_time_ns)
+
+    def _stamp_identity_baggage(self, span: Span, data: SpanData) -> None:
+        """Re-apply the operator-allowlisted request identity — including the
+        ``litellm.metadata.*`` allowlist — from the final typed payload onto ``span``.
+
+        ``LiteLLMBaggageSpanProcessor`` only stamps whatever was in Baggage when a
+        span *started*. A span opened at the ``pre_call`` boundary starts before the
+        request's full metadata is known — only the auth-time identity is seeded — so
+        late request/routing metadata (and any identity not resolved until routing)
+        never lands on it. Applying ``promoted_baggage`` here, from ``identity.metadata``
+        parsed out of the payload, makes a boundary-born span carry exactly the keys a
+        deferred span already gets from Baggage, and nothing outside the allowlist. It
+        is a no-op-equivalent overlay for the deferred / MCP paths, where the same
+        values were already stamped at start, so both paths end up identical.
+        """
+        identity = _span_identity(data)
+        if identity is None:
+            return
+        for key, value in promoted_baggage(
+            identity,
+            _span_request_model(data),
+            promoted_keys=tuple(self._config.baggage_promoted_keys),
+            metadata_keys=tuple(self._config.baggage_metadata_keys),
+            team_metadata_keys=tuple(self._config.baggage_team_metadata_keys),
+        ).items():
+            span.set_attribute(key, value)

tests/test_litellm/integrations/otel/test_otel_v2_logger.py

@@ -104,11 +104,12 @@ def _kwargs(payload=None):
     }
 
 
-def _logger(legacy_compat=True, team_metadata_keys=None):
+def _logger(legacy_compat=True, team_metadata_keys=None, metadata_keys=None):
     cfg = OpenTelemetryV2Config(
         exporter="in_memory",
         legacy_compat=legacy_compat,
         baggage_team_metadata_keys=team_metadata_keys or [],
+        **({"baggage_metadata_keys": metadata_keys} if metadata_keys is not None else {}),
     )
     exporter = InMemorySpanExporter()
     tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter)
@@ -1320,6 +1321,69 @@ def test_provider_model_and_team_metadata_on_real_boundary_flow():
     assert json.loads(srv.attributes[LiteLLM.TEAM_METADATA]) == expected
 
 
+def test_allowlisted_late_metadata_stamped_on_boundary_born_span():
+    """Regression for #35191: metadata allowlisted via ``baggage_metadata_keys``
+    that is only present in the final payload — not in the auth-time identity —
+    must land on the LLM-call span even when that span was opened at the
+    ``pre_call`` boundary.
+
+    ``request_label`` isn't known at auth, so it never rides the identity Baggage
+    seeded there onto the server span or into the span the boundary opener starts;
+    the close path has to apply the allowlist from the typed payload. This mirrors
+    how ``litellm.provider.model`` reaches the boundary-born span. Unlisted
+    metadata (``private_note``) stays off the span, and the late label is absent
+    from the server span, which started before it was known.
+    """
+    logger, exporter = _logger(metadata_keys=["request_label"])
+    server = logger._emitter.start_span(
+        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
+    )
+    payload = _payload(
+        metadata={
+            "user_api_key_team_id": "t1",
+            "request_label": "canary",
+            "private_note": "do-not-promote",
+        },
+    )
+    kwargs = _kwargs(payload=payload)
+    with trace.use_span(server, end_on_exit=False):
+        logger.seed_request_identity(_Auth(), model="gpt-4o")
+        logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
+        assert logger._open_llm_calls["call_1"].span is not None  # born at the boundary
+        asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
+    server.end()
+
+    spans = {s.name: s for s in exporter.get_finished_spans()}
+    llm = spans["chat gpt-4o"]
+    srv = spans[LITELLM_PROXY_REQUEST_SPAN_NAME]
+    label = f"{LiteLLM.METADATA_PREFIX}request_label"
+    assert llm.attributes[label] == "canary"
+    # not known at auth, so it never reached the server span (started first)
+    assert label not in srv.attributes
+    # unlisted metadata is never stamped, on either span
+    assert f"{LiteLLM.METADATA_PREFIX}private_note" not in llm.attributes
+    assert f"{LiteLLM.METADATA_PREFIX}private_note" not in srv.attributes
+
+
+def test_allowlisted_late_metadata_stamped_on_deferred_span():
+    """Parity with the boundary-born path (#35191): the deferred (SDK / thread-pool)
+    close must stamp the same allowlisted late metadata, so a request carries
+    ``litellm.metadata.request_label`` regardless of which path opened its span."""
+    logger, exporter = _logger(metadata_keys=["request_label"])
+    payload = _payload(
+        metadata={
+            "user_api_key_team_id": "t1",
+            "request_label": "canary",
+            "private_note": "do-not-promote",
+        },
+    )
+    _emit_llm(logger, kwargs=_kwargs(payload=payload))
+    (span,) = exporter.get_finished_spans()
+    assert span.parent is None  # deferred → root span
+    assert span.attributes[f"{LiteLLM.METADATA_PREFIX}request_label"] == "canary"
+    assert f"{LiteLLM.METADATA_PREFIX}private_note" not in span.attributes
+
+
 def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans():
     """The pre-call hook seeds identity Baggage in the request context so the
     server span (stamped directly) AND later child spans (service here, via the