diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 9fc268d90fe..498a4f1daf0 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -157,7 +157,7 @@ class OpenTelemetryV2(CustomLogger): event_recorder=self._init_events(logger_provider), ) self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) - self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() + self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() # mutable-ok: bounded LRU of open call spans self._closed_call_ids: OrderedDict[str, None] = OrderedDict() # mutable-ok: bounded LRU of emitted call ids self._init_otel_logger_on_litellm_proxy() diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index 8a50312c7d4..81517077594 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -22,11 +22,21 @@ if TYPE_CHECKING: _PROPAGATOR = TraceContextTextMapPropagator() -# The request's root span (the FastAPI-owned SERVER span), captured once so request-level -# spans (LLM call, guardrails) parent to it explicitly rather than to whatever span is active — -# which under the ``auth`` phase span or a detached success task would misnest or orphan them. -# A ``ContextVar`` so it rides the request task and its ``create_task`` children (the async -# logging callbacks that close the span). +# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the +# proxy first resolves it, so request-level spans (the LLM call, guardrails) can +# parent to it EXPLICITLY instead of to whatever span happens to be active at the +# instant they are emitted. Ambient-only parenting (``get_current_span()``) is +# wrong at two boundaries: +# * inside the ``auth`` phase span the active span is the auth span, so an LLM / +# guardrail span emitted there would nest under auth instead of being its +# sibling; and +# * in a detached success task (pass-through logs success from a fire-and-forget +# ``asyncio.create_task``) the server span may not be active at all, orphaning +# the span into a brand-new trace. +# A ``ContextVar`` (not a request attribute) so it rides the request task's context +# and is inherited by ``asyncio.create_task`` children — i.e. the async logging +# callbacks that close the span. It is never reset: the contextvar dies with the +# request task, so there is nothing to leak. _request_root_span: "ContextVar[Span | None]" = ContextVar("litellm_otel_request_root_span", default=None) _request_destinations: 'ContextVar[tuple["OtelDestination", ...]]' = ContextVar( @@ -34,9 +44,7 @@ _request_destinations: 'ContextVar[tuple["OtelDestination", ...]]' = ContextVar( ) -def set_request_destinations( - destinations: 'tuple["OtelDestination", ...]', -) -> None: +def set_request_destinations(destinations: 'tuple["OtelDestination", ...]') -> None: """Anchor the admin-resolved destinations for this request.""" _request_destinations.set(tuple(destinations)) @@ -49,7 +57,9 @@ def request_destinations() -> 'tuple["OtelDestination", ...]': def set_request_root_span(span: Span) -> None: """Anchor the request's root (server) span for explicit child parenting. - No-ops for a non-recordable span (a bad capture can't replace a good one); idempotent. + No-ops for a non-recordable span so a bad capture can never replace a good one + with a phantom parent. Idempotent — the proxy captures the same server span at + more than one entry point. """ if is_recordable_span(span): _request_root_span.set(span) @@ -61,9 +71,11 @@ def request_root_span() -> "Span | None": return span if is_recordable_span(span) else None -# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the MCP client -# propagated in the current request's ``params._meta``; set per message. A ``ContextVar`` so it -# rides the request task and is readable by the inline success-logging callback. +# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the +# MCP client propagated in the current request's ``params._meta``. The MCP gateway +# sets it per message so the MCP span can parent to the client's span rather than +# to the transport. A ``ContextVar`` because, like the root-span anchor, it must +# ride the request task and be readable by the inline success-logging callback. _mcp_message_trace_carrier: "ContextVar[Mapping[str, str] | None]" = ContextVar( "litellm_otel_mcp_message_trace_carrier", default=None ) @@ -85,10 +97,17 @@ def reset_mcp_message_trace_carrier(token: "Token[Mapping[str, str] | None]") -> # The transport span of the HTTP request carrying the CURRENT MCP message. -# ``_request_root_span`` can't serve MCP: a stateful streamable-HTTP session runs every message -# on the task of its ``initialize`` POST, so that anchor is frozen at ``initialize`` and never -# sees later ``tools/call`` POSTs. The gateway instead resolves each message's transport span on -# the request task and the handler publishes it here. +# +# ``_request_root_span`` above cannot be used for MCP: a *stateful* streamable-HTTP +# session runs every message on the single task spawned by that session's +# ``initialize`` POST, so the ContextVar the ASGI request task writes at auth time +# is frozen at ``initialize`` there and never sees the later ``tools/call`` POSTs. +# Reading it from the message handler parents every tool call in the session to the +# first request's server span and aims that call's ``error.*`` at it — a span that +# ended long ago, so the SDK drops the write and the failure reaches no request at +# all. The gateway instead resolves the current message's transport span on the +# request task and hands it over the same way it hands over per-request auth, and +# the handler publishes it here for the span emitter and the failure hook. _mcp_message_transport_span: "ContextVar[Span | None]" = ContextVar( "litellm_otel_mcp_message_transport_span", default=None ) @@ -97,10 +116,21 @@ _mcp_message_transport_span: "ContextVar[Span | None]" = ContextVar( def set_mcp_message_transport_span(span: object) -> "Token[Span | None]": """Publish the transport span of the request carrying the current MCP message. - Also re-anchors the request root so everything the message emits lands on this request, not - the one that opened the session; only a transport still open for writes is anchored. Takes - ``object`` (untyped ASGI-scope value); a non-span is stored as ``None``. Returns the reset - token, which the caller must reset once the message is handled to avoid leaking it. + Also re-anchors the request root, so everything else the message emits or stamps + — the identity attributes seeded onto the server span, a guardrail span, a + proxy-level failure — lands on this request instead of on the one that opened + the session. The MCP SDK dispatches each message on its own task, so the anchor + is scoped to this message; the handler re-publishes it for the next one either + way. Only a transport still open for writes is anchored: replacing the anchor + with a request that already answered would just move the dropped writes from one + finished span to another. + + Takes ``object`` because the gateway reads it back out of the ASGI scope, whose + values are untyped; anything that is not a usable span is stored as ``None`` + rather than trusted. + + Returns the reset token; the caller must reset it once the message is handled + so the transport never leaks to the next message on the same session task. """ transport = span if isinstance(span, Span) and is_recordable_span(span) else None if transport is not None and transport.is_recording(): @@ -113,11 +143,15 @@ def reset_mcp_message_transport_span(token: "Token[Span | None]") -> None: def mcp_message_transport_span() -> "Span | None": - """The published transport span, only while it is still recording (open for writes). + """The published transport span, only while it is still open for writes. - Recording — not merely valid — is the bar because this span is the target of ``error.*`` - stamping from another task, and a finished span keeps a valid context forever but would - refuse the write. Returns ``None`` for a transport that has already answered. + Recording — not merely valid — is the bar here because this span is the target + of ``error.*`` stamping from another task, and the publisher's validity check + cannot speak for a span that has since ended. A finished span keeps a valid + context forever, so it would otherwise be handed back for a write the SDK then + refuses. The POST carrying a ``tools/call`` stays open until the result is + written, so it is recording for the life of the call; a notification POST can + answer first, and this returns ``None`` for it rather than writing into the void. """ span = _mcp_message_transport_span.get() if span is None or not span.is_recording(): @@ -128,9 +162,11 @@ def mcp_message_transport_span() -> "Span | None": def _mcp_transport_span_context() -> "SpanContext | None": """The transport span an MCP message span should attach to. - Prefers the transport the gateway published for this message; falls back to the ambient - request anchor for paths that emit an MCP span on the request task (REST MCP endpoints, SDK). - Only the immutable context is needed, so unlike ``mcp_message_transport_span`` this does not + Prefers the transport the gateway published for this specific message; falls + back to the ambient request anchor for paths that emit an MCP span on the + request task itself (the REST MCP endpoints, the SDK). Parenting and linking + only need the immutable context, and unlike ``mcp_message_transport_span`` they + stay correct against a transport that has already finished, so this does not require the span to still be recording. """ published = _mcp_message_transport_span.get() @@ -161,10 +197,16 @@ def context_from_span(span: Span, context: Context | None = None) -> Context: def resolve_parent_context(threaded: Span | None = None) -> Context: """The context a child span should parent under. - Ambient-first: parent to the active OTel context, falling back to an explicitly passed - ``threaded`` span only when the ambient context has no recordable span (a background service - call with no request on the stack); when neither is recordable the ambient context is returned - unchanged, so the span starts a new root trace. Only service/DB spans pass ``threaded``. + Ambient-first: parent to the active OTel context (the server span, restored + by the logging worker or active in the request task), falling back to a span + passed explicitly (``threaded``) only when the ambient context has no + recordable span — e.g. a background service call with no request on the + stack. When neither is recordable the ambient context is returned unchanged, + so the span starts a new root trace. + + Only service/DB spans pass ``threaded`` (the ``parent_otel_span`` handed to + the service hook). Request-level spans — the LLM call and guardrails — are + created where the server span is genuinely ambient, so they never need it. """ ctx = get_current() if is_recordable_span(threaded) and not is_recordable_span(get_current_span(ctx)): @@ -175,10 +217,15 @@ def resolve_parent_context(threaded: Span | None = None) -> Context: def resolve_request_span_context() -> Context: """The parent context for a request-level span (the LLM call, a guardrail). - These are direct children of the request's root server span, never nested under whatever span - is momentarily active, so prefer the anchored root span; fall back to ambient only on the - SDK/no-proxy path with no anchor. Unlike :func:`resolve_parent_context`, this never returns - the active span when an anchor exists. + These are direct children of the request's root server span — siblings of the + ``auth`` phase span and of each other, never nested under whatever span is + momentarily active. So prefer the explicitly anchored root span; fall back to + ambient context only when there is no anchor (the SDK / no-proxy path), where + the span legitimately starts its own root trace. + + Unlike :func:`resolve_parent_context` (used by DB/service spans, which DO want + to nest under the active phase span, e.g. an auth DB lookup under ``auth``), + this never returns the active span when an anchor exists. """ root = request_root_span() if root is not None: @@ -191,17 +238,33 @@ def resolve_mcp_span_context( ) -> "tuple[Context, tuple[Link, ...]]": """Parent context + links for an MCP message span. - Per the OTel GenAI MCP semconv: when the client propagates W3C trace context in - ``params._meta`` (SEP-414), parent to that remote context and link the transport span. - Almost no client implements SEP-414, so with no remote parent, parent to this message's - transport span (from :func:`_mcp_transport_span_context`, the current message's POST, so a - long-lived session doesn't glue every message under its first request) and add no link. With - neither, the span starts its own root trace. + When the client propagates W3C trace context in the request's ``params._meta`` + (SEP-414), MCP and the underlying transport are independent lifecycles — one + streamable-HTTP session multiplexes many messages, and the client's own span is + the truthful parent. So, per the OTel GenAI MCP semconv: - Only ``traceparent``/``tracestate`` is extracted, never the client's Baggage: ``params._meta`` - is caller-controlled and honoring remote baggage would let a client spoof identity attribution. - The extraction base context is empty so a malformed ``traceparent`` can't fall through to the - ambient (stale session) span. + * parent to the trace context the client propagated (a *remote* parent), and + * record the transport span as a *link*, never the parent. + + Almost no client implements SEP-414 yet, so in practice nothing is propagated. + Rooting the span there splits a single tool call into two disconnected traces + joined only by a link, which is how it surfaces in APM: the ``POST`` transaction + and the ``tools/call`` span share no trace. With no remote parent to honor, + parent to the transport span of the request carrying this message instead, so + the call stays in one trace; no link is added since the transport is now the + real parent. The transport comes from :func:`_mcp_transport_span_context`, which + is the *current message's* POST rather than whatever request happened to open + the session, so a long-lived session does not glue every message under its + first request. With neither a remote parent nor a transport the returned context + carries no span and the span legitimately starts its own root trace. + + Only trace context (``traceparent``/``tracestate``) is extracted, never the + client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel + baggage processor stamps allowlisted baggage keys (``litellm.team.id``, + ``litellm.metadata.*``, ...) onto the span as attributes, so honoring remote + baggage would let a client spoof a span's identity attribution. The base context + for extraction is explicitly empty so an absent or malformed ``traceparent`` can + never fall through to the ambient (stale session) span. """ source = carrier if carrier is not None else _mcp_message_trace_carrier.get() parent = _PROPAGATOR.extract(dict(source or {}), context=Context()) diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 96b69863d1b..54b72591b1a 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -272,7 +272,7 @@ class TenantTracerCache: ) kind = self._owned_otlp_kind() - appended = [ + appended = tuple( ExporterSpec( kind=kind, endpoint=d.endpoint, @@ -280,8 +280,8 @@ class TenantTracerCache: owner=None, ) for d in destinations - ] - base_exporters = [*self._config.exporters] if include_base_exporters else [] + ) + base_exporters = (*self._config.exporters,) if include_base_exporters else () merged_resource_attrs = { **self._config.resource_attributes, **{key: value for d in destinations for key, value in destination_resource_attrs(d).items()},