From 134f63e2a258289a1f29b10117c10f4c2e9c6cf4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:10:29 +0000 Subject: [PATCH] fix(otel): stamp allowlisted late request metadata on boundary-born LLM spans OTel v2 opens the LLM-call span at the pre_call boundary, so the Baggage processor stamps only the identity seeded at auth. Metadata that first appears in the final StandardLoggingPayload and is allowlisted via baggage_metadata_keys never reached that span, so litellm.metadata. was missing whenever the span was born at the boundary rather than deferred. finish_span now re-applies promoted_baggage from the typed payload identity, making a boundary-born span carry exactly the allowlisted keys a deferred span already gets, and nothing outside the allowlist --- litellm/integrations/otel/emitter.py | 47 +++++++++++++ .../integrations/otel/test_otel_v2_logger.py | 66 ++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 8651cf586cd..d7db8ebedee 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -7,9 +7,11 @@ from opentelemetry.context import Context from opentelemetry.trace import Link, Span, Tracer from opentelemetry.trace.status import Status, StatusCode +from litellm.integrations.otel.model.baggage import promoted_baggage from litellm.integrations.otel.model.config import OpenTelemetryV2Config from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData +from litellm.integrations.otel.model.metadata import RequestIdentity from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, @@ -51,6 +53,24 @@ _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = { _DEDUP_CACHE_MAX = 10_000 +def _span_identity(data: SpanData) -> RequestIdentity | None: + """The request identity a span carries, or ``None`` for span kinds that have + none (service / guardrail spans). Only identity-bearing spans get the + allowlisted request metadata re-applied at close.""" + match data: + case LLMCallSpanData() | MCPToolCallSpanData() | MCPListToolsSpanData(): + return data.identity + case _: + return None + + +def _span_request_model(data: SpanData) -> str | None: + """The user-facing request model promoted into Baggage as ``gen_ai.request.model``. + Only the LLM-call span carries one; MCP spans promote identity without a model, + matching how the deferred path seeds their Baggage.""" + return data.request_model if isinstance(data, LLMCallSpanData) else None + + def _stamp_otel_error_attributes(span: Span, error_type: str, resolved_message: str) -> None: """Stamp the OTel-semconv error attributes (``error.type`` + ``error.message``). ``error_type`` and ``resolved_message`` are ``finish_span``'s already-computed @@ -234,6 +254,7 @@ class SpanEmitter: for mapper in self._mappers: for key, value in mapper.map(data).items(): span.set_attribute(key, value) + self._stamp_identity_baggage(span, data) error = ( data.error if isinstance( @@ -264,3 +285,29 @@ class SpanEmitter: # span-level health signal litellm doesn't actually evaluate. Only a # genuine error sets a status. span.end(end_time=end_time_ns) + + def _stamp_identity_baggage(self, span: Span, data: SpanData) -> None: + """Re-apply the operator-allowlisted request identity — including the + ``litellm.metadata.*`` allowlist — from the final typed payload onto ``span``. + + ``LiteLLMBaggageSpanProcessor`` only stamps whatever was in Baggage when a + span *started*. A span opened at the ``pre_call`` boundary starts before the + request's full metadata is known — only the auth-time identity is seeded — so + late request/routing metadata (and any identity not resolved until routing) + never lands on it. Applying ``promoted_baggage`` here, from ``identity.metadata`` + parsed out of the payload, makes a boundary-born span carry exactly the keys a + deferred span already gets from Baggage, and nothing outside the allowlist. It + is a no-op-equivalent overlay for the deferred / MCP paths, where the same + values were already stamped at start, so both paths end up identical. + """ + identity = _span_identity(data) + if identity is None: + return + for key, value in promoted_baggage( + identity, + _span_request_model(data), + promoted_keys=tuple(self._config.baggage_promoted_keys), + metadata_keys=tuple(self._config.baggage_metadata_keys), + team_metadata_keys=tuple(self._config.baggage_team_metadata_keys), + ).items(): + span.set_attribute(key, value) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 41c02501acc..1a039240667 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -104,11 +104,12 @@ def _kwargs(payload=None): } -def _logger(legacy_compat=True, team_metadata_keys=None): +def _logger(legacy_compat=True, team_metadata_keys=None, metadata_keys=None): cfg = OpenTelemetryV2Config( exporter="in_memory", legacy_compat=legacy_compat, baggage_team_metadata_keys=team_metadata_keys or [], + **({"baggage_metadata_keys": metadata_keys} if metadata_keys is not None else {}), ) exporter = InMemorySpanExporter() tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter) @@ -1320,6 +1321,69 @@ def test_provider_model_and_team_metadata_on_real_boundary_flow(): assert json.loads(srv.attributes[LiteLLM.TEAM_METADATA]) == expected +def test_allowlisted_late_metadata_stamped_on_boundary_born_span(): + """Regression for #35191: metadata allowlisted via ``baggage_metadata_keys`` + that is only present in the final payload — not in the auth-time identity — + must land on the LLM-call span even when that span was opened at the + ``pre_call`` boundary. + + ``request_label`` isn't known at auth, so it never rides the identity Baggage + seeded there onto the server span or into the span the boundary opener starts; + the close path has to apply the allowlist from the typed payload. This mirrors + how ``litellm.provider.model`` reaches the boundary-born span. Unlisted + metadata (``private_note``) stays off the span, and the late label is absent + from the server span, which started before it was known. + """ + logger, exporter = _logger(metadata_keys=["request_label"]) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + payload = _payload( + metadata={ + "user_api_key_team_id": "t1", + "request_label": "canary", + "private_note": "do-not-promote", + }, + ) + kwargs = _kwargs(payload=payload) + with trace.use_span(server, end_on_exit=False): + logger.seed_request_identity(_Auth(), model="gpt-4o") + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + assert logger._open_llm_calls["call_1"].span is not None # born at the boundary + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + + spans = {s.name: s for s in exporter.get_finished_spans()} + llm = spans["chat gpt-4o"] + srv = spans[LITELLM_PROXY_REQUEST_SPAN_NAME] + label = f"{LiteLLM.METADATA_PREFIX}request_label" + assert llm.attributes[label] == "canary" + # not known at auth, so it never reached the server span (started first) + assert label not in srv.attributes + # unlisted metadata is never stamped, on either span + assert f"{LiteLLM.METADATA_PREFIX}private_note" not in llm.attributes + assert f"{LiteLLM.METADATA_PREFIX}private_note" not in srv.attributes + + +def test_allowlisted_late_metadata_stamped_on_deferred_span(): + """Parity with the boundary-born path (#35191): the deferred (SDK / thread-pool) + close must stamp the same allowlisted late metadata, so a request carries + ``litellm.metadata.request_label`` regardless of which path opened its span.""" + logger, exporter = _logger(metadata_keys=["request_label"]) + payload = _payload( + metadata={ + "user_api_key_team_id": "t1", + "request_label": "canary", + "private_note": "do-not-promote", + }, + ) + _emit_llm(logger, kwargs=_kwargs(payload=payload)) + (span,) = exporter.get_finished_spans() + assert span.parent is None # deferred → root span + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}request_label"] == "canary" + assert f"{LiteLLM.METADATA_PREFIX}private_note" not in span.attributes + + def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans(): """The pre-call hook seeds identity Baggage in the request context so the server span (stamped directly) AND later child spans (service here, via the